mirror of
https://github.com/rohitg00/skillkit.git
synced 2026-11-18 18:04:26 +00:00
feat(core): add QMD-inspired hybrid search pipeline (#40)
* feat(core): add QMD-inspired hybrid search pipeline Implement a hybrid search system combining vector embeddings with keyword search: - Vector embeddings using node-llama-cpp with GGUF models (nomic-embed-text) - RRF (Reciprocal Rank Fusion) for combining multiple rankers - Query expansion with LLM and synonym fallback - Position-aware score blending (different weights for top3, top10, rest) - SQLite storage with sqlite-vec extension (falls back to in-memory) - Auto-download of local GGUF models (~2GB) New search module: - embeddings.ts: Vector embedding generation service - vector-store.ts: SQLite-based vector storage - rrf.ts: RRF fusion algorithm - expansion.ts: Query expansion with caching - hybrid.ts: Main hybrid search pipeline - local-models.ts: GGUF model management CLI integration: - skillkit recommend --hybrid: Enable hybrid search - skillkit recommend --expand: Enable query expansion - skillkit recommend --rerank: Enable LLM re-ranking - skillkit recommend --build-index: Build embedding index Tests: 56 new tests for search module (all passing) * fix(search): address CodeRabbit review findings - Add missing beforeEach import in embeddings.test.ts - Update optional dependencies to current versions - Fix weights/variations length mismatch in expansion.ts - Avoid mutating input arrays in rrf.ts mergeRankings - Delete orphaned chunk vectors in vector-store.ts delete() and clear() - Add comment for empty catch block in vector-store.ts - Make hybridSearch helper use buildIndex for true hybrid search - Handle graceful degradation when embeddings unavailable - Add node-llama-cpp mock in hybrid.test.ts * fix(search): address CodeRabbit review round 2 - Fix gemma GGUF URL to point to valid community repo (bartowski) - Add SQL injection protection for table names in vector-store.ts - Fix resource leak with try/finally in hybridSearch helper - Add backpressure handling for large model downloads - Clamp startLine to non-negative in chunk calculation - Add runtime validation warning for --expand/--rerank without --hybrid - Update node-llama-cpp API usage (createEmbeddingContext, createContext) - Update pnpm-lock.yaml for new optional dependency versions * fix(search): use @ts-ignore for optional node-llama-cpp imports CI doesn't install optional dependencies, so @ts-ignore is needed instead of @ts-expect-error to suppress module not found errors.
This commit is contained in:
@@ -52,6 +52,9 @@ export class RecommendCommand extends Command {
|
||||
['Update skill index', '$0 recommend --update'],
|
||||
['Search for skills by task', '$0 recommend --task "authentication"'],
|
||||
['Search for skills (alias)', '$0 recommend --search "testing"'],
|
||||
['Hybrid search (vector + keyword)', '$0 recommend --search "auth" --hybrid'],
|
||||
['Hybrid search with query expansion', '$0 recommend --search "auth" --hybrid --expand'],
|
||||
['Build hybrid search index', '$0 recommend --build-index'],
|
||||
],
|
||||
});
|
||||
|
||||
@@ -125,6 +128,26 @@ export class RecommendCommand extends Command {
|
||||
description: 'Show category path for each recommendation',
|
||||
});
|
||||
|
||||
// Hybrid search mode
|
||||
hybrid = Option.Boolean('--hybrid,-H', false, {
|
||||
description: 'Use hybrid search (vector + keyword)',
|
||||
});
|
||||
|
||||
// Query expansion
|
||||
expand = Option.Boolean('--expand,-x', false, {
|
||||
description: 'Enable query expansion (requires --hybrid)',
|
||||
});
|
||||
|
||||
// Reranking
|
||||
rerank = Option.Boolean('--rerank', false, {
|
||||
description: 'Enable LLM reranking (requires --hybrid)',
|
||||
});
|
||||
|
||||
// Build index
|
||||
buildIndex = Option.Boolean('--build-index', false, {
|
||||
description: 'Build/rebuild the hybrid search embedding index',
|
||||
});
|
||||
|
||||
async execute(): Promise<number> {
|
||||
const targetPath = resolve(this.projectPath || process.cwd());
|
||||
|
||||
@@ -133,6 +156,16 @@ export class RecommendCommand extends Command {
|
||||
return await this.updateIndex();
|
||||
}
|
||||
|
||||
// Handle hybrid index building
|
||||
if (this.buildIndex) {
|
||||
return await this.buildHybridIndex();
|
||||
}
|
||||
|
||||
// Validate hybrid-dependent options
|
||||
if ((this.expand || this.rerank) && !this.hybrid) {
|
||||
warn('--expand and --rerank require --hybrid flag. These options will be ignored.');
|
||||
}
|
||||
|
||||
if (!this.quiet && !this.json) {
|
||||
header('Skill Recommendations');
|
||||
}
|
||||
@@ -168,6 +201,9 @@ export class RecommendCommand extends Command {
|
||||
// Handle search mode (--search or --task)
|
||||
const searchQuery = this.search || this.task;
|
||||
if (searchQuery) {
|
||||
if (this.hybrid) {
|
||||
return await this.handleHybridSearch(engine, searchQuery);
|
||||
}
|
||||
return this.handleSearch(engine, searchQuery);
|
||||
}
|
||||
|
||||
@@ -446,6 +482,132 @@ export class RecommendCommand extends Command {
|
||||
console.log(colors.muted('More details: skillkit recommend --explain --verbose'));
|
||||
}
|
||||
|
||||
private async handleHybridSearch(engine: RecommendationEngine, query: string): Promise<number> {
|
||||
if (!this.quiet && !this.json) {
|
||||
header(`Hybrid Search: "${query}"`);
|
||||
}
|
||||
|
||||
const s = !this.quiet && !this.json ? spinner() : null;
|
||||
s?.start('Initializing hybrid search...');
|
||||
|
||||
try {
|
||||
await engine.initHybridSearch();
|
||||
s?.message('Searching...');
|
||||
|
||||
const results = await engine.hybridSearch({
|
||||
query,
|
||||
limit: this.limit ? parseInt(this.limit, 10) : 10,
|
||||
hybrid: true,
|
||||
enableExpansion: this.expand,
|
||||
enableReranking: this.rerank,
|
||||
filters: {
|
||||
minScore: this.minScore ? parseInt(this.minScore, 10) : undefined,
|
||||
},
|
||||
});
|
||||
|
||||
s?.stop(`Found ${results.length} results`);
|
||||
|
||||
if (this.json) {
|
||||
console.log(JSON.stringify(results, null, 2));
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (results.length === 0) {
|
||||
warn(`No skills found matching "${query}"`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
console.log('');
|
||||
console.log(colors.bold(`Hybrid search results for "${query}" (${results.length} found):`));
|
||||
if (this.expand && results[0]?.expandedTerms?.length) {
|
||||
console.log(colors.muted(` Expanded: ${results[0].expandedTerms.join(', ')}`));
|
||||
}
|
||||
console.log('');
|
||||
|
||||
for (const result of results) {
|
||||
let relevanceColor: (text: string) => string;
|
||||
if (result.relevance >= 70) {
|
||||
relevanceColor = colors.success;
|
||||
} else if (result.relevance >= 50) {
|
||||
relevanceColor = colors.warning;
|
||||
} else {
|
||||
relevanceColor = colors.muted;
|
||||
}
|
||||
const relevanceBar = progressBar(result.relevance, 100, 10);
|
||||
|
||||
console.log(` ${relevanceColor(`${result.relevance}%`)} ${colors.dim(relevanceBar)} ${colors.bold(result.skill.name)}`);
|
||||
|
||||
if (result.skill.description) {
|
||||
console.log(` ${colors.muted(truncate(result.skill.description, 70))}`);
|
||||
}
|
||||
|
||||
if (this.verbose) {
|
||||
const scores: string[] = [];
|
||||
if (typeof result.vectorSimilarity === 'number') {
|
||||
scores.push(`vector: ${(result.vectorSimilarity * 100).toFixed(0)}%`);
|
||||
}
|
||||
if (typeof result.keywordScore === 'number') {
|
||||
scores.push(`keyword: ${result.keywordScore.toFixed(0)}%`);
|
||||
}
|
||||
if (typeof result.rrfScore === 'number') {
|
||||
scores.push(`rrf: ${result.rrfScore.toFixed(3)}`);
|
||||
}
|
||||
if (scores.length > 0) {
|
||||
console.log(` ${colors.dim('Scores:')} ${scores.join(' | ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (result.matchedTerms.length > 0) {
|
||||
console.log(` ${colors.dim('Matched:')} ${result.matchedTerms.join(', ')}`);
|
||||
}
|
||||
|
||||
console.log('');
|
||||
}
|
||||
|
||||
return 0;
|
||||
} catch (err) {
|
||||
s?.stop(colors.error('Hybrid search failed'));
|
||||
console.log(colors.muted(err instanceof Error ? err.message : String(err)));
|
||||
console.log(colors.muted('Falling back to standard search...'));
|
||||
return this.handleSearch(engine, query);
|
||||
}
|
||||
}
|
||||
|
||||
private async buildHybridIndex(): Promise<number> {
|
||||
if (!this.quiet) {
|
||||
header('Build Hybrid Search Index');
|
||||
}
|
||||
|
||||
const index = this.loadIndex();
|
||||
if (!index || index.skills.length === 0) {
|
||||
warn('No skill index found. Run --update first.');
|
||||
return 1;
|
||||
}
|
||||
|
||||
const s = spinner();
|
||||
s.start('Initializing...');
|
||||
|
||||
try {
|
||||
const engine = new RecommendationEngine();
|
||||
engine.loadIndex(index);
|
||||
|
||||
await engine.buildHybridIndex((progress) => {
|
||||
const percentage = Math.round((progress.current / progress.total) * 100);
|
||||
s.message(`${progress.phase}: ${progress.message || ''} (${percentage}%)`);
|
||||
});
|
||||
|
||||
s.stop(colors.success(`${symbols.success} Built hybrid index for ${index.skills.length} skills`));
|
||||
console.log(colors.muted(' Index stored in: ~/.skillkit/search.db'));
|
||||
console.log(colors.muted(' Use --hybrid flag for vector+keyword search\n'));
|
||||
|
||||
return 0;
|
||||
} catch (err) {
|
||||
s.stop(colors.error('Failed to build hybrid index'));
|
||||
console.log(colors.muted(err instanceof Error ? err.message : String(err)));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
private handleSearch(engine: RecommendationEngine, query: string): number {
|
||||
if (!this.quiet && !this.json) {
|
||||
header(`Search: "${query}"`);
|
||||
|
||||
@@ -37,6 +37,10 @@
|
||||
"./learning": {
|
||||
"import": "./dist/learning/index.js",
|
||||
"types": "./dist/learning/index.d.ts"
|
||||
},
|
||||
"./search": {
|
||||
"import": "./dist/search/index.js",
|
||||
"types": "./dist/search/index.d.ts"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
@@ -53,7 +57,13 @@
|
||||
"yaml": "^2.6.1",
|
||||
"zod": "^3.24.1"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"node-llama-cpp": "^3.15.0",
|
||||
"better-sqlite3": "^12.0.0",
|
||||
"sqlite-vec": "^0.1.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/better-sqlite3": "^7.6.11",
|
||||
"@types/node": "^22.10.5",
|
||||
"tsup": "^8.3.5",
|
||||
"typescript": "^5.7.2",
|
||||
|
||||
@@ -101,3 +101,6 @@ export * from './connectors/index.js';
|
||||
|
||||
// Execution Flow (Step Tracking & Metrics - Phase 21)
|
||||
export * from './execution/index.js';
|
||||
|
||||
// Hybrid Search (QMD-Inspired Vector + Keyword Search)
|
||||
export * from './search/index.js';
|
||||
|
||||
@@ -11,8 +11,11 @@ import type {
|
||||
SearchOptions,
|
||||
SearchResult,
|
||||
FreshnessResult,
|
||||
RecommendHybridSearchOptions,
|
||||
RecommendHybridSearchResult,
|
||||
} from './types.js';
|
||||
import { DEFAULT_SCORING_WEIGHTS, TAG_TO_TECH, getTechTags } from './types.js';
|
||||
import type { HybridSearchPipeline } from '../search/hybrid.js';
|
||||
|
||||
/**
|
||||
* Recommendation engine for matching skills to project profiles
|
||||
@@ -20,16 +23,39 @@ import { DEFAULT_SCORING_WEIGHTS, TAG_TO_TECH, getTechTags } from './types.js';
|
||||
export class RecommendationEngine {
|
||||
private weights: ScoringWeights;
|
||||
private index: SkillIndex | null = null;
|
||||
private hybridPipeline: HybridSearchPipeline | null = null;
|
||||
|
||||
constructor(weights?: Partial<ScoringWeights>) {
|
||||
this.weights = { ...DEFAULT_SCORING_WEIGHTS, ...weights };
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize hybrid search pipeline for vector + keyword search
|
||||
*/
|
||||
async initHybridSearch(): Promise<void> {
|
||||
const { createHybridSearchPipeline } = await import('../search/hybrid.js');
|
||||
this.hybridPipeline = createHybridSearchPipeline();
|
||||
if (this.index) {
|
||||
this.hybridPipeline.loadSkillsIndex(this.index);
|
||||
}
|
||||
await this.hybridPipeline.initialize();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if hybrid search is available
|
||||
*/
|
||||
isHybridSearchAvailable(): boolean {
|
||||
return this.hybridPipeline !== null && this.hybridPipeline.isInitialized();
|
||||
}
|
||||
|
||||
/**
|
||||
* Load skill index from cache or generate from local skills
|
||||
*/
|
||||
loadIndex(index: SkillIndex): void {
|
||||
this.index = index;
|
||||
if (this.hybridPipeline) {
|
||||
this.hybridPipeline.loadSkillsIndex(index);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -603,6 +629,71 @@ export class RecommendationEngine {
|
||||
return { relevance, matchedTerms, snippet };
|
||||
}
|
||||
|
||||
/**
|
||||
* Hybrid search combining vector embeddings and keyword matching
|
||||
*/
|
||||
async hybridSearch(options: RecommendHybridSearchOptions): Promise<RecommendHybridSearchResult[]> {
|
||||
const { query, limit = 10, hybrid = true, enableExpansion = false, enableReranking = false, filters } = options;
|
||||
|
||||
if (!hybrid || !this.hybridPipeline) {
|
||||
const basicResults = this.search({ query, limit, semantic: true, filters });
|
||||
return basicResults.map((r) => ({
|
||||
...r,
|
||||
hybridScore: r.relevance / 100,
|
||||
}));
|
||||
}
|
||||
|
||||
const response = await this.hybridPipeline.search({
|
||||
query,
|
||||
limit,
|
||||
enableExpansion,
|
||||
enableReranking,
|
||||
});
|
||||
|
||||
let results = response.results.map((r) => ({
|
||||
skill: r.skill,
|
||||
relevance: r.relevance,
|
||||
matchedTerms: r.matchedTerms,
|
||||
snippet: r.snippet,
|
||||
hybridScore: r.hybridScore,
|
||||
vectorSimilarity: r.vectorSimilarity,
|
||||
keywordScore: r.keywordScore,
|
||||
rrfScore: r.rrfScore,
|
||||
expandedTerms: r.expandedTerms,
|
||||
}));
|
||||
|
||||
if (filters?.tags && filters.tags.length > 0) {
|
||||
results = results.filter((r) =>
|
||||
r.skill.tags?.some((t) => filters.tags!.includes(t))
|
||||
);
|
||||
}
|
||||
if (filters?.verified) {
|
||||
results = results.filter((r) => r.skill.verified);
|
||||
}
|
||||
if (filters?.minScore) {
|
||||
results = results.filter((r) => r.relevance >= filters.minScore!);
|
||||
}
|
||||
|
||||
return results.slice(0, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build hybrid search index from skills
|
||||
*/
|
||||
async buildHybridIndex(
|
||||
onProgress?: (progress: { phase: string; current: number; total: number; message?: string }) => void
|
||||
): Promise<void> {
|
||||
if (!this.index) {
|
||||
throw new Error('No skill index loaded. Call loadIndex() first.');
|
||||
}
|
||||
|
||||
if (!this.hybridPipeline) {
|
||||
await this.initHybridSearch();
|
||||
}
|
||||
|
||||
await this.hybridPipeline!.buildIndex(this.index.skills, onProgress);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check freshness of installed skills against project dependencies
|
||||
*
|
||||
|
||||
@@ -326,3 +326,25 @@ export interface ReasoningRecommendationResult extends RecommendationResult {
|
||||
strategy: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Hybrid search options for RecommendationEngine
|
||||
*/
|
||||
export interface RecommendHybridSearchOptions extends SearchOptions {
|
||||
hybrid?: boolean;
|
||||
enableExpansion?: boolean;
|
||||
enableReranking?: boolean;
|
||||
semanticWeight?: number;
|
||||
keywordWeight?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hybrid search result with additional metadata for RecommendationEngine
|
||||
*/
|
||||
export interface RecommendHybridSearchResult extends SearchResult {
|
||||
hybridScore?: number;
|
||||
vectorSimilarity?: number;
|
||||
keywordScore?: number;
|
||||
rrfScore?: number;
|
||||
expandedTerms?: string[];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { EmbeddingService } from '../embeddings.js';
|
||||
import type { SkillSummary } from '../../recommend/types.js';
|
||||
|
||||
describe('EmbeddingService', () => {
|
||||
describe('cosineSimilarity', () => {
|
||||
let service: EmbeddingService;
|
||||
|
||||
beforeEach(() => {
|
||||
service = new EmbeddingService();
|
||||
});
|
||||
|
||||
it('should return 1 for identical vectors', () => {
|
||||
const a = new Float32Array([1, 0, 0]);
|
||||
const b = new Float32Array([1, 0, 0]);
|
||||
|
||||
const similarity = service.cosineSimilarity(a, b);
|
||||
expect(similarity).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
it('should return 0 for orthogonal vectors', () => {
|
||||
const a = new Float32Array([1, 0, 0]);
|
||||
const b = new Float32Array([0, 1, 0]);
|
||||
|
||||
const similarity = service.cosineSimilarity(a, b);
|
||||
expect(similarity).toBeCloseTo(0, 5);
|
||||
});
|
||||
|
||||
it('should return -1 for opposite vectors', () => {
|
||||
const a = new Float32Array([1, 0, 0]);
|
||||
const b = new Float32Array([-1, 0, 0]);
|
||||
|
||||
const similarity = service.cosineSimilarity(a, b);
|
||||
expect(similarity).toBeCloseTo(-1, 5);
|
||||
});
|
||||
|
||||
it('should handle normalized vectors', () => {
|
||||
const a = new Float32Array([0.5, 0.5, 0.5, 0.5]);
|
||||
const b = new Float32Array([0.5, 0.5, 0.5, 0.5]);
|
||||
|
||||
const similarity = service.cosineSimilarity(a, b);
|
||||
expect(similarity).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
it('should throw for mismatched dimensions', () => {
|
||||
const a = new Float32Array([1, 0]);
|
||||
const b = new Float32Array([1, 0, 0]);
|
||||
|
||||
expect(() => service.cosineSimilarity(a, b)).toThrow('Vector dimensions mismatch');
|
||||
});
|
||||
|
||||
it('should return 0 for zero vectors', () => {
|
||||
const a = new Float32Array([0, 0, 0]);
|
||||
const b = new Float32Array([1, 0, 0]);
|
||||
|
||||
const similarity = service.cosineSimilarity(a, b);
|
||||
expect(similarity).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('initialization state', () => {
|
||||
it('should not be initialized by default', () => {
|
||||
const service = new EmbeddingService();
|
||||
expect(service.isInitialized()).toBe(false);
|
||||
});
|
||||
|
||||
it('should return default dimensions', () => {
|
||||
const service = new EmbeddingService();
|
||||
expect(service.getDimensions()).toBe(768);
|
||||
});
|
||||
});
|
||||
|
||||
describe('skill text building', () => {
|
||||
it('should handle skill with all fields', async () => {
|
||||
const service = new EmbeddingService();
|
||||
const skill: SkillSummary = {
|
||||
name: 'test-skill',
|
||||
description: 'A test skill',
|
||||
tags: ['tag1', 'tag2'],
|
||||
compatibility: {
|
||||
frameworks: ['react'],
|
||||
languages: ['typescript'],
|
||||
libraries: ['lodash'],
|
||||
},
|
||||
popularity: 100,
|
||||
quality: 80,
|
||||
verified: true,
|
||||
};
|
||||
|
||||
const buildSkillText = (s: SkillSummary): string => {
|
||||
const parts: string[] = [s.name];
|
||||
if (s.description) parts.push(s.description);
|
||||
if (s.tags?.length) parts.push(s.tags.join(' '));
|
||||
if (s.compatibility?.frameworks?.length) parts.push(s.compatibility.frameworks.join(' '));
|
||||
if (s.compatibility?.languages?.length) parts.push(s.compatibility.languages.join(' '));
|
||||
if (s.compatibility?.libraries?.length) parts.push(s.compatibility.libraries.join(' '));
|
||||
return parts.join(' ').toLowerCase();
|
||||
};
|
||||
|
||||
const text = buildSkillText(skill);
|
||||
|
||||
expect(text).toContain('test-skill');
|
||||
expect(text).toContain('a test skill');
|
||||
expect(text).toContain('tag1');
|
||||
expect(text).toContain('react');
|
||||
expect(text).toContain('typescript');
|
||||
expect(text).toContain('lodash');
|
||||
});
|
||||
|
||||
it('should handle skill with minimal fields', () => {
|
||||
const skill: SkillSummary = {
|
||||
name: 'minimal-skill',
|
||||
tags: [],
|
||||
popularity: 0,
|
||||
quality: 50,
|
||||
verified: false,
|
||||
};
|
||||
|
||||
const buildSkillText = (s: SkillSummary): string => {
|
||||
const parts: string[] = [s.name];
|
||||
if (s.description) parts.push(s.description);
|
||||
if (s.tags?.length) parts.push(s.tags.join(' '));
|
||||
return parts.join(' ').toLowerCase();
|
||||
};
|
||||
|
||||
const text = buildSkillText(skill);
|
||||
|
||||
expect(text).toBe('minimal-skill');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('EmbeddingService chunking', () => {
|
||||
it('should split text into chunks', () => {
|
||||
const chunkText = (
|
||||
text: string,
|
||||
config = { maxTokens: 100, overlapPercent: 15, minChunkSize: 10 }
|
||||
) => {
|
||||
const lines = text.split('\n');
|
||||
const chunks: { content: string; startLine: number; endLine: number }[] = [];
|
||||
const avgCharsPerToken = 4;
|
||||
const maxCharsPerChunk = config.maxTokens * avgCharsPerToken;
|
||||
const overlapChars = Math.floor(maxCharsPerChunk * (config.overlapPercent / 100));
|
||||
|
||||
let currentChunk: string[] = [];
|
||||
let currentChars = 0;
|
||||
let startLine = 0;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
const lineChars = line.length + 1;
|
||||
|
||||
if (currentChars + lineChars > maxCharsPerChunk && currentChunk.length > 0) {
|
||||
const content = currentChunk.join('\n');
|
||||
if (content.length >= config.minChunkSize) {
|
||||
chunks.push({ content, startLine, endLine: i - 1 });
|
||||
}
|
||||
|
||||
const overlapLines: string[] = [];
|
||||
let overlapCharsCount = 0;
|
||||
for (let j = currentChunk.length - 1; j >= 0 && overlapCharsCount < overlapChars; j--) {
|
||||
overlapLines.unshift(currentChunk[j]);
|
||||
overlapCharsCount += currentChunk[j].length + 1;
|
||||
}
|
||||
|
||||
currentChunk = overlapLines;
|
||||
currentChars = overlapCharsCount;
|
||||
startLine = i - overlapLines.length;
|
||||
}
|
||||
|
||||
currentChunk.push(line);
|
||||
currentChars += lineChars;
|
||||
}
|
||||
|
||||
if (currentChunk.length > 0) {
|
||||
const content = currentChunk.join('\n');
|
||||
if (content.length >= config.minChunkSize) {
|
||||
chunks.push({ content, startLine, endLine: lines.length - 1 });
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
};
|
||||
|
||||
const longText = Array(50).fill('This is a line of text for testing.').join('\n');
|
||||
const chunks = chunkText(longText);
|
||||
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
chunks.forEach((chunk) => {
|
||||
expect(chunk.content.length).toBeLessThanOrEqual(400);
|
||||
expect(chunk.startLine).toBeGreaterThanOrEqual(0);
|
||||
expect(chunk.endLine).toBeGreaterThanOrEqual(chunk.startLine);
|
||||
});
|
||||
});
|
||||
|
||||
it('should not chunk short text', () => {
|
||||
const chunkText = (
|
||||
text: string,
|
||||
config = { maxTokens: 800, overlapPercent: 15, minChunkSize: 100 }
|
||||
) => {
|
||||
const lines = text.split('\n');
|
||||
if (text.length < config.maxTokens * 4) {
|
||||
if (text.length >= config.minChunkSize) {
|
||||
return [{ content: text, startLine: 0, endLine: lines.length - 1 }];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
return [{ content: text, startLine: 0, endLine: lines.length - 1 }];
|
||||
};
|
||||
|
||||
const shortText = 'This is a short text that should not be chunked.';
|
||||
const chunks = chunkText(shortText, { maxTokens: 800, overlapPercent: 15, minChunkSize: 10 });
|
||||
|
||||
expect(chunks.length).toBe(1);
|
||||
expect(chunks[0].content).toBe(shortText);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { QueryExpander, expandQuerySimple } from '../expansion.js';
|
||||
|
||||
describe('QueryExpander', () => {
|
||||
let expander: QueryExpander;
|
||||
|
||||
beforeEach(() => {
|
||||
expander = new QueryExpander();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
expander.clearCache();
|
||||
});
|
||||
|
||||
describe('expandQuerySimple (without LLM)', () => {
|
||||
it('should expand "auth" to authentication-related terms', () => {
|
||||
const result = expandQuerySimple('auth');
|
||||
|
||||
expect(result.original).toBe('auth');
|
||||
expect(result.variations.length).toBeGreaterThan(0);
|
||||
expect(result.variations.some((v) => v.includes('authentication'))).toBe(true);
|
||||
});
|
||||
|
||||
it('should expand "api" to related terms', () => {
|
||||
const result = expandQuerySimple('api');
|
||||
|
||||
expect(result.original).toBe('api');
|
||||
expect(result.variations.some((v) => v.includes('rest') || v.includes('endpoint'))).toBe(true);
|
||||
});
|
||||
|
||||
it('should expand "test" to testing-related terms', () => {
|
||||
const result = expandQuerySimple('testing');
|
||||
|
||||
expect(result.original).toBe('testing');
|
||||
expect(result.variations.some((v) => v.includes('test') || v.includes('unit'))).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle queries without synonyms', () => {
|
||||
const result = expandQuerySimple('xyz123unique');
|
||||
|
||||
expect(result.original).toBe('xyz123unique');
|
||||
expect(result.variations).toEqual([]);
|
||||
expect(result.weights).toEqual([2.0]);
|
||||
});
|
||||
|
||||
it('should give original query weight of 2.0', () => {
|
||||
const result = expandQuerySimple('auth');
|
||||
|
||||
expect(result.weights[0]).toBe(2.0);
|
||||
for (let i = 1; i < result.weights.length; i++) {
|
||||
expect(result.weights[i]).toBe(1.0);
|
||||
}
|
||||
});
|
||||
|
||||
it('should limit variations to 3', () => {
|
||||
const result = expandQuerySimple('auth');
|
||||
expect(result.variations.length).toBeLessThanOrEqual(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('expand (fallback without LLM)', () => {
|
||||
it('should return expanded query without LLM initialization', async () => {
|
||||
const result = await expander.expand('auth');
|
||||
|
||||
expect(result.original).toBe('auth');
|
||||
expect(result.variations.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should cache results', async () => {
|
||||
const result1 = await expander.expand('auth');
|
||||
const result2 = await expander.expand('auth');
|
||||
|
||||
expect(result1).toEqual(result2);
|
||||
});
|
||||
|
||||
it('should handle case-insensitive caching', async () => {
|
||||
const result1 = await expander.expand('Auth');
|
||||
const result2 = await expander.expand('AUTH');
|
||||
|
||||
expect(result1.original).toBe('Auth');
|
||||
expect(result2.original).toBe('AUTH');
|
||||
});
|
||||
});
|
||||
|
||||
describe('clearCache', () => {
|
||||
it('should clear the expansion cache', async () => {
|
||||
await expander.expand('auth');
|
||||
expander.clearCache();
|
||||
|
||||
const result = await expander.expand('auth');
|
||||
expect(result).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('isInitialized', () => {
|
||||
it('should return false when not initialized with LLM', () => {
|
||||
expect(expander.isInitialized()).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,244 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import type { SkillSummary, SkillIndex } from '../../recommend/types.js';
|
||||
|
||||
const mockSkills: SkillSummary[] = [
|
||||
{
|
||||
name: 'react-auth',
|
||||
description: 'Authentication utilities for React applications',
|
||||
tags: ['react', 'auth', 'authentication'],
|
||||
compatibility: {
|
||||
frameworks: ['react'],
|
||||
languages: ['typescript', 'javascript'],
|
||||
libraries: ['next-auth'],
|
||||
},
|
||||
popularity: 500,
|
||||
quality: 80,
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
name: 'vue-forms',
|
||||
description: 'Form validation library for Vue',
|
||||
tags: ['vue', 'forms', 'validation'],
|
||||
compatibility: {
|
||||
frameworks: ['vue'],
|
||||
languages: ['typescript', 'javascript'],
|
||||
libraries: [],
|
||||
},
|
||||
popularity: 300,
|
||||
quality: 75,
|
||||
verified: false,
|
||||
},
|
||||
{
|
||||
name: 'api-testing',
|
||||
description: 'API testing utilities for REST endpoints',
|
||||
tags: ['testing', 'api', 'rest'],
|
||||
compatibility: {
|
||||
frameworks: [],
|
||||
languages: ['typescript'],
|
||||
libraries: ['jest', 'supertest'],
|
||||
},
|
||||
popularity: 400,
|
||||
quality: 85,
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
name: 'login-helper',
|
||||
description: 'Helper functions for login and authentication flows',
|
||||
tags: ['auth', 'login', 'oauth'],
|
||||
compatibility: {
|
||||
frameworks: ['react', 'vue'],
|
||||
languages: ['typescript'],
|
||||
libraries: [],
|
||||
},
|
||||
popularity: 200,
|
||||
quality: 70,
|
||||
verified: false,
|
||||
},
|
||||
];
|
||||
|
||||
const mockIndex: SkillIndex = {
|
||||
version: 1,
|
||||
lastUpdated: new Date().toISOString(),
|
||||
skills: mockSkills,
|
||||
sources: [],
|
||||
};
|
||||
|
||||
vi.mock('better-sqlite3', () => {
|
||||
return {
|
||||
default: vi.fn().mockImplementation(() => {
|
||||
throw new Error("Cannot find module 'better-sqlite3'");
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('sqlite-vec', () => {
|
||||
return {
|
||||
load: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('node-llama-cpp', () => {
|
||||
return {
|
||||
getLlama: vi.fn().mockImplementation(() => {
|
||||
throw new Error("Cannot find package 'node-llama-cpp'");
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
describe('HybridSearchPipeline', () => {
|
||||
let HybridSearchPipeline: typeof import('../hybrid.js').HybridSearchPipeline;
|
||||
let createHybridSearchPipeline: typeof import('../hybrid.js').createHybridSearchPipeline;
|
||||
let pipeline: InstanceType<typeof HybridSearchPipeline>;
|
||||
|
||||
beforeEach(async () => {
|
||||
const module = await import('../hybrid.js');
|
||||
HybridSearchPipeline = module.HybridSearchPipeline;
|
||||
createHybridSearchPipeline = module.createHybridSearchPipeline;
|
||||
pipeline = createHybridSearchPipeline();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (pipeline) {
|
||||
await pipeline.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
describe('initialization', () => {
|
||||
it('should create a pipeline', () => {
|
||||
expect(pipeline).toBeInstanceOf(HybridSearchPipeline);
|
||||
});
|
||||
|
||||
it('should not be initialized before calling initialize()', () => {
|
||||
expect(pipeline.isInitialized()).toBe(false);
|
||||
});
|
||||
|
||||
it('should be initialized after calling initialize()', async () => {
|
||||
await pipeline.initialize();
|
||||
expect(pipeline.isInitialized()).toBe(true);
|
||||
});
|
||||
|
||||
it('should load skills index', async () => {
|
||||
pipeline.loadSkillsIndex(mockIndex);
|
||||
const stats = pipeline.getStats();
|
||||
expect(stats.skillCount).toBe(mockSkills.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe('search (keyword-only fallback)', () => {
|
||||
beforeEach(async () => {
|
||||
pipeline.loadSkillsIndex(mockIndex);
|
||||
await pipeline.initialize();
|
||||
});
|
||||
|
||||
it('should find skills matching query', async () => {
|
||||
const response = await pipeline.search({ query: 'authentication' });
|
||||
|
||||
expect(response.results.length).toBeGreaterThan(0);
|
||||
expect(response.results.some((r) => r.skill.name === 'react-auth')).toBe(true);
|
||||
});
|
||||
|
||||
it('should find skills by tag', async () => {
|
||||
const response = await pipeline.search({ query: 'react' });
|
||||
|
||||
expect(response.results.length).toBeGreaterThan(0);
|
||||
expect(response.results.some((r) => r.skill.tags?.includes('react'))).toBe(true);
|
||||
});
|
||||
|
||||
it('should respect limit parameter', async () => {
|
||||
const response = await pipeline.search({ query: 'auth', limit: 2 });
|
||||
|
||||
expect(response.results.length).toBeLessThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('should include timing information', async () => {
|
||||
const response = await pipeline.search({ query: 'test' });
|
||||
|
||||
expect(typeof response.timing.totalMs).toBe('number');
|
||||
expect(response.timing.totalMs).toBeGreaterThanOrEqual(0);
|
||||
expect(response.timing.keywordSearchMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it('should include stats', async () => {
|
||||
const response = await pipeline.search({ query: 'auth' });
|
||||
|
||||
expect(response.stats.candidatesFromKeyword).toBeGreaterThan(0);
|
||||
expect(response.stats.totalMerged).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should return original query in response', async () => {
|
||||
const response = await pipeline.search({ query: 'authentication' });
|
||||
|
||||
expect(response.query.original).toBe('authentication');
|
||||
});
|
||||
});
|
||||
|
||||
describe('search with expansion', () => {
|
||||
beforeEach(async () => {
|
||||
pipeline.loadSkillsIndex(mockIndex);
|
||||
await pipeline.initialize();
|
||||
});
|
||||
|
||||
it('should expand query and find more results', async () => {
|
||||
const response = await pipeline.search({
|
||||
query: 'auth',
|
||||
enableExpansion: true,
|
||||
});
|
||||
|
||||
expect(response.results.length).toBeGreaterThan(0);
|
||||
expect(response.query.expanded).toBeDefined();
|
||||
});
|
||||
|
||||
it('should include expanded terms in response', async () => {
|
||||
const response = await pipeline.search({
|
||||
query: 'auth',
|
||||
enableExpansion: true,
|
||||
});
|
||||
|
||||
if (response.query.expanded) {
|
||||
expect(response.query.expanded.original).toBe('auth');
|
||||
expect(response.query.expanded.weights[0]).toBe(2.0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('getStats', () => {
|
||||
it('should return stats before initialization', () => {
|
||||
const stats = pipeline.getStats();
|
||||
expect(stats.initialized).toBe(false);
|
||||
expect(stats.skillCount).toBe(0);
|
||||
});
|
||||
|
||||
it('should return updated stats after loading index', async () => {
|
||||
pipeline.loadSkillsIndex(mockIndex);
|
||||
await pipeline.initialize();
|
||||
|
||||
const stats = pipeline.getStats();
|
||||
expect(stats.initialized).toBe(true);
|
||||
expect(stats.skillCount).toBe(mockSkills.length);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('hybridSearch helper', () => {
|
||||
let hybridSearch: typeof import('../hybrid.js').hybridSearch;
|
||||
|
||||
beforeEach(async () => {
|
||||
const module = await import('../hybrid.js');
|
||||
hybridSearch = module.hybridSearch;
|
||||
});
|
||||
|
||||
it('should perform search on skills array', async () => {
|
||||
const response = await hybridSearch(mockSkills, 'react');
|
||||
|
||||
expect(response.results.length).toBeGreaterThan(0);
|
||||
expect(response.query.original).toBe('react');
|
||||
});
|
||||
|
||||
it('should respect options', async () => {
|
||||
const response = await hybridSearch(mockSkills, 'auth', {
|
||||
limit: 1,
|
||||
});
|
||||
|
||||
expect(response.results.length).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,276 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
computeRRFScore,
|
||||
fuseWithRRF,
|
||||
normalizeScores,
|
||||
weightedCombine,
|
||||
applyPositionAwareBlending,
|
||||
getRankFromScore,
|
||||
mergeRankings,
|
||||
type RankerResult,
|
||||
type RRFInput,
|
||||
} from '../rrf.js';
|
||||
|
||||
describe('RRF (Reciprocal Rank Fusion)', () => {
|
||||
describe('computeRRFScore', () => {
|
||||
it('should compute RRF score for a single rank', () => {
|
||||
const score = computeRRFScore([1], 60);
|
||||
expect(score).toBeCloseTo(1 / 61, 5);
|
||||
});
|
||||
|
||||
it('should sum RRF scores for multiple ranks', () => {
|
||||
const score = computeRRFScore([1, 2, 3], 60);
|
||||
const expected = 1 / 61 + 1 / 62 + 1 / 63;
|
||||
expect(score).toBeCloseTo(expected, 5);
|
||||
});
|
||||
|
||||
it('should use custom k value', () => {
|
||||
const scoreK60 = computeRRFScore([1], 60);
|
||||
const scoreK30 = computeRRFScore([1], 30);
|
||||
expect(scoreK30).toBeGreaterThan(scoreK60);
|
||||
});
|
||||
|
||||
it('should return 0 for empty ranks', () => {
|
||||
const score = computeRRFScore([]);
|
||||
expect(score).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fuseWithRRF', () => {
|
||||
it('should fuse results from multiple rankers', () => {
|
||||
const inputs: RRFInput[] = [
|
||||
{
|
||||
source: 'vector',
|
||||
results: [
|
||||
{ skillName: 'skill-a', score: 0.9 },
|
||||
{ skillName: 'skill-b', score: 0.8 },
|
||||
{ skillName: 'skill-c', score: 0.7 },
|
||||
],
|
||||
},
|
||||
{
|
||||
source: 'keyword',
|
||||
results: [
|
||||
{ skillName: 'skill-b', score: 90 },
|
||||
{ skillName: 'skill-a', score: 80 },
|
||||
{ skillName: 'skill-d', score: 70 },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const rankings = fuseWithRRF(inputs, 60);
|
||||
|
||||
expect(rankings.length).toBe(4);
|
||||
expect(rankings[0].skillName).toBe('skill-a');
|
||||
expect(rankings[1].skillName).toBe('skill-b');
|
||||
});
|
||||
|
||||
it('should include rank information from each source', () => {
|
||||
const inputs: RRFInput[] = [
|
||||
{
|
||||
source: 'vector',
|
||||
results: [{ skillName: 'skill-a', score: 0.9 }],
|
||||
},
|
||||
{
|
||||
source: 'keyword',
|
||||
results: [{ skillName: 'skill-a', score: 90 }],
|
||||
},
|
||||
];
|
||||
|
||||
const rankings = fuseWithRRF(inputs);
|
||||
|
||||
expect(rankings[0].ranks).toHaveLength(2);
|
||||
expect(rankings[0].ranks.map((r) => r.source)).toContain('vector');
|
||||
expect(rankings[0].ranks.map((r) => r.source)).toContain('keyword');
|
||||
});
|
||||
|
||||
it('should handle skills appearing in only one ranker', () => {
|
||||
const inputs: RRFInput[] = [
|
||||
{
|
||||
source: 'vector',
|
||||
results: [{ skillName: 'skill-a', score: 0.9 }],
|
||||
},
|
||||
{
|
||||
source: 'keyword',
|
||||
results: [{ skillName: 'skill-b', score: 90 }],
|
||||
},
|
||||
];
|
||||
|
||||
const rankings = fuseWithRRF(inputs);
|
||||
|
||||
expect(rankings).toHaveLength(2);
|
||||
const skillAInfo = rankings.find((r) => r.skillName === 'skill-a');
|
||||
const skillBInfo = rankings.find((r) => r.skillName === 'skill-b');
|
||||
expect(skillAInfo?.ranks).toHaveLength(1);
|
||||
expect(skillBInfo?.ranks).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeScores', () => {
|
||||
it('should normalize scores to 0-1 range', () => {
|
||||
const results: RankerResult[] = [
|
||||
{ skillName: 'a', score: 100 },
|
||||
{ skillName: 'b', score: 50 },
|
||||
{ skillName: 'c', score: 0 },
|
||||
];
|
||||
|
||||
const normalized = normalizeScores(results);
|
||||
|
||||
expect(normalized[0].score).toBe(1);
|
||||
expect(normalized[1].score).toBe(0.5);
|
||||
expect(normalized[2].score).toBe(0);
|
||||
});
|
||||
|
||||
it('should handle all same scores', () => {
|
||||
const results: RankerResult[] = [
|
||||
{ skillName: 'a', score: 50 },
|
||||
{ skillName: 'b', score: 50 },
|
||||
];
|
||||
|
||||
const normalized = normalizeScores(results);
|
||||
|
||||
expect(normalized[0].score).toBe(1);
|
||||
expect(normalized[1].score).toBe(1);
|
||||
});
|
||||
|
||||
it('should return empty array for empty input', () => {
|
||||
const normalized = normalizeScores([]);
|
||||
expect(normalized).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('weightedCombine', () => {
|
||||
it('should combine results with weights', () => {
|
||||
const inputs: RRFInput[] = [
|
||||
{
|
||||
source: 'vector',
|
||||
results: [
|
||||
{ skillName: 'a', score: 100 },
|
||||
{ skillName: 'b', score: 50 },
|
||||
],
|
||||
},
|
||||
{
|
||||
source: 'keyword',
|
||||
results: [
|
||||
{ skillName: 'b', score: 100 },
|
||||
{ skillName: 'a', score: 50 },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const combined = weightedCombine(inputs, { vector: 0.6, keyword: 0.4 });
|
||||
|
||||
expect(combined).toHaveLength(2);
|
||||
const skillA = combined.find((r) => r.skillName === 'a')!;
|
||||
const skillB = combined.find((r) => r.skillName === 'b')!;
|
||||
expect(skillA.score).toBeCloseTo(0.6 * 1 + 0.4 * 0, 5);
|
||||
expect(skillB.score).toBeCloseTo(0.6 * 0 + 0.4 * 1, 5);
|
||||
});
|
||||
|
||||
it('should default to weight 1 for unknown sources', () => {
|
||||
const inputs: RRFInput[] = [
|
||||
{
|
||||
source: 'unknown',
|
||||
results: [{ skillName: 'a', score: 100 }],
|
||||
},
|
||||
];
|
||||
|
||||
const combined = weightedCombine(inputs, {});
|
||||
|
||||
expect(combined[0].score).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('applyPositionAwareBlending', () => {
|
||||
it('should apply different weights based on position', () => {
|
||||
const retrievalScores = new Map([
|
||||
['a', 0.9],
|
||||
['b', 0.8],
|
||||
['c', 0.7],
|
||||
['d', 0.6],
|
||||
]);
|
||||
|
||||
const rerankerScores = new Map([
|
||||
['a', 0.5],
|
||||
['b', 0.6],
|
||||
['c', 0.7],
|
||||
['d', 0.8],
|
||||
]);
|
||||
|
||||
const sortedSkills = ['a', 'b', 'c', 'd'];
|
||||
|
||||
const blended = applyPositionAwareBlending(
|
||||
retrievalScores,
|
||||
rerankerScores,
|
||||
sortedSkills
|
||||
);
|
||||
|
||||
const scoreA = blended.get('a')!;
|
||||
expect(scoreA).toBeCloseTo(0.9 * 0.75 + 0.5 * 0.25, 5);
|
||||
|
||||
const scoreD = blended.get('d')!;
|
||||
expect(scoreD).toBeCloseTo(0.6 * 0.6 + 0.8 * 0.4, 5);
|
||||
});
|
||||
|
||||
it('should handle missing scores', () => {
|
||||
const retrievalScores = new Map([['a', 0.9]]);
|
||||
const rerankerScores = new Map<string, number>();
|
||||
const sortedSkills = ['a'];
|
||||
|
||||
const blended = applyPositionAwareBlending(
|
||||
retrievalScores,
|
||||
rerankerScores,
|
||||
sortedSkills
|
||||
);
|
||||
|
||||
expect(blended.get('a')).toBeCloseTo(0.9 * 0.75, 5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getRankFromScore', () => {
|
||||
it('should return correct rank for a skill', () => {
|
||||
const results: RankerResult[] = [
|
||||
{ skillName: 'a', score: 90 },
|
||||
{ skillName: 'b', score: 80 },
|
||||
{ skillName: 'c', score: 70 },
|
||||
];
|
||||
|
||||
expect(getRankFromScore('a', results)).toBe(1);
|
||||
expect(getRankFromScore('b', results)).toBe(2);
|
||||
expect(getRankFromScore('c', results)).toBe(3);
|
||||
});
|
||||
|
||||
it('should return length+1 for missing skill', () => {
|
||||
const results: RankerResult[] = [
|
||||
{ skillName: 'a', score: 90 },
|
||||
{ skillName: 'b', score: 80 },
|
||||
];
|
||||
|
||||
expect(getRankFromScore('missing', results)).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('mergeRankings', () => {
|
||||
it('should merge primary and secondary rankings with weights', () => {
|
||||
const primary = [
|
||||
{ skillName: 'a', rrfScore: 0.1, ranks: [{ source: 'primary', rank: 1 }] },
|
||||
{ skillName: 'b', rrfScore: 0.05, ranks: [{ source: 'primary', rank: 2 }] },
|
||||
];
|
||||
|
||||
const secondary = [
|
||||
{ skillName: 'b', rrfScore: 0.1, ranks: [{ source: 'secondary', rank: 1 }] },
|
||||
{ skillName: 'c', rrfScore: 0.05, ranks: [{ source: 'secondary', rank: 2 }] },
|
||||
];
|
||||
|
||||
const merged = mergeRankings(primary, secondary, 0.7);
|
||||
|
||||
expect(merged).toHaveLength(3);
|
||||
const skillA = merged.find((r) => r.skillName === 'a')!;
|
||||
const skillB = merged.find((r) => r.skillName === 'b')!;
|
||||
const skillC = merged.find((r) => r.skillName === 'c')!;
|
||||
|
||||
expect(skillA.rrfScore).toBeCloseTo(0.1 * 0.7, 5);
|
||||
expect(skillB.rrfScore).toBeCloseTo(0.05 * 0.7 + 0.1 * 0.3, 5);
|
||||
expect(skillC.rrfScore).toBeCloseTo(0.05 * 0.3, 5);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,294 @@
|
||||
import type { SkillSummary } from '../recommend/types.js';
|
||||
import type {
|
||||
SkillEmbedding,
|
||||
IndexBuildCallback,
|
||||
} from './types.js';
|
||||
import { DEFAULT_CHUNKING_CONFIG, MODEL_REGISTRY } from './types.js';
|
||||
import { LocalModelManager } from './local-models.js';
|
||||
|
||||
export class EmbeddingService {
|
||||
private modelManager: LocalModelManager;
|
||||
private model: unknown = null;
|
||||
private context: unknown = null;
|
||||
private initialized = false;
|
||||
private dimensions: number = 768;
|
||||
|
||||
constructor(modelManager?: LocalModelManager) {
|
||||
this.modelManager = modelManager ?? new LocalModelManager();
|
||||
}
|
||||
|
||||
async initialize(onProgress?: IndexBuildCallback): Promise<void> {
|
||||
if (this.initialized) return;
|
||||
|
||||
onProgress?.({
|
||||
phase: 'loading',
|
||||
current: 0,
|
||||
total: 1,
|
||||
message: 'Loading embedding model...',
|
||||
});
|
||||
|
||||
const modelPath = await this.modelManager.ensureEmbedModel(onProgress);
|
||||
|
||||
try {
|
||||
// @ts-ignore - node-llama-cpp is an optional dependency
|
||||
const llama = await import('node-llama-cpp');
|
||||
const { getLlama } = llama;
|
||||
|
||||
const llamaInstance = await getLlama();
|
||||
this.model = await llamaInstance.loadModel({ modelPath });
|
||||
this.context = await (this.model as { createEmbeddingContext: () => Promise<unknown> }).createEmbeddingContext();
|
||||
|
||||
const modelInfo = MODEL_REGISTRY.embeddings[
|
||||
this.modelManager.getConfig().embedModel as keyof typeof MODEL_REGISTRY.embeddings
|
||||
];
|
||||
this.dimensions = modelInfo?.dimensions ?? 768;
|
||||
|
||||
this.initialized = true;
|
||||
|
||||
onProgress?.({
|
||||
phase: 'loading',
|
||||
current: 1,
|
||||
total: 1,
|
||||
message: 'Embedding model loaded successfully',
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes("Cannot find package 'node-llama-cpp'")
|
||||
) {
|
||||
throw new Error(
|
||||
'node-llama-cpp is not installed. Install it with: pnpm add node-llama-cpp'
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
if (!this.initialized || !this.context) {
|
||||
throw new Error('EmbeddingService not initialized. Call initialize() first.');
|
||||
}
|
||||
|
||||
const ctx = this.context as {
|
||||
getEmbeddingFor(text: string): Promise<{ vector: number[] }>;
|
||||
};
|
||||
const result = await ctx.getEmbeddingFor(text);
|
||||
return new Float32Array(result.vector);
|
||||
}
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
const results: Float32Array[] = [];
|
||||
for (const text of texts) {
|
||||
results.push(await this.embed(text));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async embedSkill(skill: SkillSummary): Promise<SkillEmbedding> {
|
||||
const textContent = this.buildSkillText(skill);
|
||||
const vector = await this.embed(textContent);
|
||||
|
||||
return {
|
||||
skillName: skill.name,
|
||||
vector,
|
||||
textContent,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
async embedSkillWithChunks(
|
||||
skill: SkillSummary,
|
||||
fullContent?: string
|
||||
): Promise<SkillEmbedding> {
|
||||
const textContent = this.buildSkillText(skill);
|
||||
const mainVector = await this.embed(textContent);
|
||||
|
||||
const chunks = fullContent ? this.chunkText(fullContent) : undefined;
|
||||
let embeddedChunks: SkillEmbedding['chunks'] | undefined;
|
||||
|
||||
if (chunks && chunks.length > 0) {
|
||||
embeddedChunks = [];
|
||||
for (const chunk of chunks) {
|
||||
const chunkVector = await this.embed(chunk.content);
|
||||
embeddedChunks.push({
|
||||
content: chunk.content,
|
||||
vector: chunkVector,
|
||||
startLine: chunk.startLine,
|
||||
endLine: chunk.endLine,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
skillName: skill.name,
|
||||
vector: mainVector,
|
||||
textContent,
|
||||
chunks: embeddedChunks,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
async embedSkills(
|
||||
skills: SkillSummary[],
|
||||
onProgress?: IndexBuildCallback
|
||||
): Promise<SkillEmbedding[]> {
|
||||
await this.initialize(onProgress);
|
||||
|
||||
const embeddings: SkillEmbedding[] = [];
|
||||
const total = skills.length;
|
||||
|
||||
for (let i = 0; i < skills.length; i++) {
|
||||
const skill = skills[i];
|
||||
|
||||
onProgress?.({
|
||||
phase: 'embedding',
|
||||
current: i + 1,
|
||||
total,
|
||||
skillName: skill.name,
|
||||
message: `Embedding skill ${i + 1}/${total}: ${skill.name}`,
|
||||
});
|
||||
|
||||
const embedding = await this.embedSkill(skill);
|
||||
embeddings.push(embedding);
|
||||
}
|
||||
|
||||
onProgress?.({
|
||||
phase: 'complete',
|
||||
current: total,
|
||||
total,
|
||||
message: `Embedded ${total} skills`,
|
||||
});
|
||||
|
||||
return embeddings;
|
||||
}
|
||||
|
||||
private buildSkillText(skill: SkillSummary): string {
|
||||
const parts: string[] = [skill.name];
|
||||
|
||||
if (skill.description) {
|
||||
parts.push(skill.description);
|
||||
}
|
||||
|
||||
if (skill.tags && skill.tags.length > 0) {
|
||||
parts.push(skill.tags.join(' '));
|
||||
}
|
||||
|
||||
if (skill.compatibility) {
|
||||
if (skill.compatibility.frameworks?.length) {
|
||||
parts.push(skill.compatibility.frameworks.join(' '));
|
||||
}
|
||||
if (skill.compatibility.languages?.length) {
|
||||
parts.push(skill.compatibility.languages.join(' '));
|
||||
}
|
||||
if (skill.compatibility.libraries?.length) {
|
||||
parts.push(skill.compatibility.libraries.join(' '));
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join(' ').toLowerCase();
|
||||
}
|
||||
|
||||
private chunkText(
|
||||
text: string,
|
||||
config = DEFAULT_CHUNKING_CONFIG
|
||||
): { content: string; startLine: number; endLine: number }[] {
|
||||
const lines = text.split('\n');
|
||||
const chunks: { content: string; startLine: number; endLine: number }[] = [];
|
||||
|
||||
const avgCharsPerToken = 4;
|
||||
const maxCharsPerChunk = config.maxTokens * avgCharsPerToken;
|
||||
const overlapChars = Math.floor(maxCharsPerChunk * (config.overlapPercent / 100));
|
||||
|
||||
let currentChunk: string[] = [];
|
||||
let currentChars = 0;
|
||||
let startLine = 0;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
const lineChars = line.length + 1;
|
||||
|
||||
if (currentChars + lineChars > maxCharsPerChunk && currentChunk.length > 0) {
|
||||
const content = currentChunk.join('\n');
|
||||
if (content.length >= config.minChunkSize) {
|
||||
chunks.push({
|
||||
content,
|
||||
startLine,
|
||||
endLine: i - 1,
|
||||
});
|
||||
}
|
||||
|
||||
const overlapLines: string[] = [];
|
||||
let overlapCharsCount = 0;
|
||||
for (let j = currentChunk.length - 1; j >= 0 && overlapCharsCount < overlapChars; j--) {
|
||||
overlapLines.unshift(currentChunk[j]);
|
||||
overlapCharsCount += currentChunk[j].length + 1;
|
||||
}
|
||||
|
||||
currentChunk = overlapLines;
|
||||
currentChars = overlapCharsCount;
|
||||
startLine = Math.max(0, i - overlapLines.length);
|
||||
}
|
||||
|
||||
currentChunk.push(line);
|
||||
currentChars += lineChars;
|
||||
}
|
||||
|
||||
if (currentChunk.length > 0) {
|
||||
const content = currentChunk.join('\n');
|
||||
if (content.length >= config.minChunkSize) {
|
||||
chunks.push({
|
||||
content,
|
||||
startLine,
|
||||
endLine: lines.length - 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
cosineSimilarity(a: Float32Array, b: Float32Array): number {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error(`Vector dimensions mismatch: ${a.length} vs ${b.length}`);
|
||||
}
|
||||
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
|
||||
const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
if (magnitude === 0) return 0;
|
||||
|
||||
return dotProduct / magnitude;
|
||||
}
|
||||
|
||||
getDimensions(): number {
|
||||
return this.dimensions;
|
||||
}
|
||||
|
||||
isInitialized(): boolean {
|
||||
return this.initialized;
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
if (this.context && typeof (this.context as { dispose?: () => void }).dispose === 'function') {
|
||||
(this.context as { dispose: () => void }).dispose();
|
||||
}
|
||||
if (this.model && typeof (this.model as { dispose?: () => void }).dispose === 'function') {
|
||||
(this.model as { dispose: () => void }).dispose();
|
||||
}
|
||||
this.context = null;
|
||||
this.model = null;
|
||||
this.initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
export function createEmbeddingService(modelManager?: LocalModelManager): EmbeddingService {
|
||||
return new EmbeddingService(modelManager);
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
import type { ExpandedQuery, IndexBuildCallback } from './types.js';
|
||||
import { DEFAULT_HYBRID_CONFIG } from './types.js';
|
||||
import { LocalModelManager } from './local-models.js';
|
||||
|
||||
interface CacheEntry {
|
||||
query: ExpandedQuery;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
const expansionCache = new Map<string, CacheEntry>();
|
||||
|
||||
export class QueryExpander {
|
||||
private modelManager: LocalModelManager;
|
||||
private model: unknown = null;
|
||||
private context: unknown = null;
|
||||
private initialized = false;
|
||||
private cacheTtlMs: number;
|
||||
|
||||
constructor(modelManager?: LocalModelManager, cacheTtlMs?: number) {
|
||||
this.modelManager = modelManager ?? new LocalModelManager();
|
||||
this.cacheTtlMs = cacheTtlMs ?? DEFAULT_HYBRID_CONFIG.expansionCacheTtlMs;
|
||||
}
|
||||
|
||||
async initialize(onProgress?: IndexBuildCallback): Promise<void> {
|
||||
if (this.initialized) return;
|
||||
|
||||
onProgress?.({
|
||||
phase: 'loading',
|
||||
current: 0,
|
||||
total: 1,
|
||||
message: 'Loading LLM model for query expansion...',
|
||||
});
|
||||
|
||||
const modelPath = await this.modelManager.ensureLlmModel(onProgress);
|
||||
|
||||
try {
|
||||
// @ts-ignore - node-llama-cpp is an optional dependency
|
||||
const llamaModule = await import('node-llama-cpp');
|
||||
const { getLlama, LlamaChatSession } = llamaModule;
|
||||
|
||||
const llama = await getLlama();
|
||||
this.model = await llama.loadModel({ modelPath });
|
||||
const ctx = await (this.model as { createContext: (opts?: { contextSize?: number }) => Promise<{ getSequence: () => unknown }> }).createContext({ contextSize: 2048 });
|
||||
this.context = new LlamaChatSession({ contextSequence: ctx.getSequence() as never });
|
||||
|
||||
this.initialized = true;
|
||||
|
||||
onProgress?.({
|
||||
phase: 'loading',
|
||||
current: 1,
|
||||
total: 1,
|
||||
message: 'LLM model loaded successfully',
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes("Cannot find package 'node-llama-cpp'")
|
||||
) {
|
||||
throw new Error(
|
||||
'node-llama-cpp is not installed. Install it with: pnpm add node-llama-cpp'
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async expand(query: string, maxVariations: number = 3): Promise<ExpandedQuery> {
|
||||
const cached = this.getFromCache(query);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
if (!this.initialized) {
|
||||
return this.expandWithoutLLM(query);
|
||||
}
|
||||
|
||||
try {
|
||||
const variations = await this.generateVariations(query, maxVariations);
|
||||
const weights = [2.0, ...variations.map(() => 1.0)];
|
||||
|
||||
const expanded: ExpandedQuery = {
|
||||
original: query,
|
||||
variations,
|
||||
weights,
|
||||
};
|
||||
|
||||
this.addToCache(query, expanded);
|
||||
return expanded;
|
||||
} catch {
|
||||
return this.expandWithoutLLM(query);
|
||||
}
|
||||
}
|
||||
|
||||
private async generateVariations(query: string, maxVariations: number): Promise<string[]> {
|
||||
const session = this.context as {
|
||||
prompt(text: string): Promise<string>;
|
||||
};
|
||||
|
||||
const prompt = `Generate ${maxVariations} alternative search queries for finding software development skills related to: "${query}"
|
||||
|
||||
Rules:
|
||||
- Each alternative should use different terminology but capture the same intent
|
||||
- Keep each alternative concise (2-5 words)
|
||||
- Focus on technical terms and synonyms
|
||||
- Output only the alternatives, one per line, no numbering
|
||||
|
||||
Example:
|
||||
Query: "auth for react"
|
||||
Alternatives:
|
||||
authentication react
|
||||
login react hooks
|
||||
OAuth react integration
|
||||
|
||||
Query: "${query}"
|
||||
Alternatives:`;
|
||||
|
||||
const response = await session.prompt(prompt);
|
||||
|
||||
const variations = response
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0 && line.length < 100)
|
||||
.filter((line) => !line.startsWith('-') && !line.match(/^\d+\./))
|
||||
.slice(0, maxVariations);
|
||||
|
||||
return variations;
|
||||
}
|
||||
|
||||
private expandWithoutLLM(query: string): ExpandedQuery {
|
||||
const variations: string[] = [];
|
||||
const queryLower = query.toLowerCase();
|
||||
|
||||
const synonyms: Record<string, string[]> = {
|
||||
auth: ['authentication', 'login', 'oauth', 'jwt'],
|
||||
authentication: ['auth', 'login', 'oauth'],
|
||||
login: ['auth', 'authentication', 'signin'],
|
||||
api: ['rest', 'endpoint', 'http', 'graphql'],
|
||||
rest: ['api', 'http', 'endpoint'],
|
||||
db: ['database', 'sql', 'storage'],
|
||||
database: ['db', 'sql', 'data storage'],
|
||||
test: ['testing', 'unit test', 'spec'],
|
||||
testing: ['test', 'unit test', 'e2e'],
|
||||
ui: ['user interface', 'frontend', 'component'],
|
||||
frontend: ['ui', 'client', 'browser'],
|
||||
backend: ['server', 'api', 'service'],
|
||||
form: ['input', 'validation', 'submit'],
|
||||
validation: ['form validation', 'input check', 'verify'],
|
||||
state: ['state management', 'store', 'data flow'],
|
||||
animation: ['motion', 'transition', 'animate'],
|
||||
style: ['styling', 'css', 'design'],
|
||||
css: ['style', 'styling', 'tailwind'],
|
||||
};
|
||||
|
||||
const terms = queryLower.split(/\s+/);
|
||||
for (const term of terms) {
|
||||
const termSynonyms = synonyms[term];
|
||||
if (termSynonyms) {
|
||||
for (const syn of termSynonyms.slice(0, 2)) {
|
||||
const variation = query.replace(new RegExp(term, 'i'), syn);
|
||||
if (variation !== query && !variations.includes(variation)) {
|
||||
variations.push(variation);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const limitedVariations = variations.slice(0, 3);
|
||||
const weights = [2.0, ...limitedVariations.map(() => 1.0)];
|
||||
|
||||
return {
|
||||
original: query,
|
||||
variations: limitedVariations,
|
||||
weights,
|
||||
};
|
||||
}
|
||||
|
||||
private getFromCache(query: string): ExpandedQuery | null {
|
||||
const entry = expansionCache.get(query.toLowerCase());
|
||||
if (!entry) return null;
|
||||
|
||||
const now = Date.now();
|
||||
if (now - entry.timestamp > this.cacheTtlMs) {
|
||||
expansionCache.delete(query.toLowerCase());
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry.query;
|
||||
}
|
||||
|
||||
private addToCache(query: string, expanded: ExpandedQuery): void {
|
||||
const maxCacheSize = 100;
|
||||
if (expansionCache.size >= maxCacheSize) {
|
||||
const oldestKey = expansionCache.keys().next().value;
|
||||
if (oldestKey) {
|
||||
expansionCache.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
|
||||
expansionCache.set(query.toLowerCase(), {
|
||||
query: expanded,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
clearCache(): void {
|
||||
expansionCache.clear();
|
||||
}
|
||||
|
||||
isInitialized(): boolean {
|
||||
return this.initialized;
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
if (this.context && typeof (this.context as { dispose?: () => void }).dispose === 'function') {
|
||||
(this.context as { dispose: () => void }).dispose();
|
||||
}
|
||||
if (this.model && typeof (this.model as { dispose?: () => void }).dispose === 'function') {
|
||||
(this.model as { dispose: () => void }).dispose();
|
||||
}
|
||||
this.context = null;
|
||||
this.model = null;
|
||||
this.initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
export function createQueryExpander(
|
||||
modelManager?: LocalModelManager,
|
||||
cacheTtlMs?: number
|
||||
): QueryExpander {
|
||||
return new QueryExpander(modelManager, cacheTtlMs);
|
||||
}
|
||||
|
||||
export function expandQuerySimple(query: string): ExpandedQuery {
|
||||
const expander = new QueryExpander();
|
||||
return expander['expandWithoutLLM'](query);
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
import type { SkillSummary, SkillIndex } from '../recommend/types.js';
|
||||
import type {
|
||||
HybridSearchOptions,
|
||||
HybridSearchResult,
|
||||
HybridSearchResponse,
|
||||
ExpandedQuery,
|
||||
IndexBuildCallback,
|
||||
RerankerInput,
|
||||
RerankerOutput,
|
||||
LocalModelConfig,
|
||||
} from './types.js';
|
||||
import { DEFAULT_HYBRID_CONFIG } from './types.js';
|
||||
import { EmbeddingService, createEmbeddingService } from './embeddings.js';
|
||||
import { VectorStore, createVectorStore } from './vector-store.js';
|
||||
import { QueryExpander, createQueryExpander } from './expansion.js';
|
||||
import { LocalModelManager } from './local-models.js';
|
||||
import {
|
||||
fuseWithRRF,
|
||||
applyPositionAwareBlending,
|
||||
type RankerResult,
|
||||
type RRFInput,
|
||||
} from './rrf.js';
|
||||
import { RecommendationEngine } from '../recommend/engine.js';
|
||||
|
||||
export class HybridSearchPipeline {
|
||||
private embeddingService: EmbeddingService;
|
||||
private vectorStore: VectorStore;
|
||||
private queryExpander: QueryExpander;
|
||||
private modelManager: LocalModelManager;
|
||||
private recommendationEngine: RecommendationEngine;
|
||||
private initialized = false;
|
||||
private skillsMap: Map<string, SkillSummary> = new Map();
|
||||
|
||||
constructor(config: Partial<LocalModelConfig> = {}) {
|
||||
this.modelManager = new LocalModelManager(config);
|
||||
this.embeddingService = createEmbeddingService(this.modelManager);
|
||||
this.vectorStore = createVectorStore(undefined, this.embeddingService);
|
||||
this.queryExpander = createQueryExpander(this.modelManager);
|
||||
this.recommendationEngine = new RecommendationEngine();
|
||||
}
|
||||
|
||||
async initialize(onProgress?: IndexBuildCallback): Promise<void> {
|
||||
if (this.initialized) return;
|
||||
|
||||
await this.vectorStore.initialize();
|
||||
this.initialized = true;
|
||||
|
||||
onProgress?.({
|
||||
phase: 'complete',
|
||||
current: 1,
|
||||
total: 1,
|
||||
message: 'Hybrid search pipeline initialized',
|
||||
});
|
||||
}
|
||||
|
||||
async buildIndex(
|
||||
skills: SkillSummary[],
|
||||
onProgress?: IndexBuildCallback
|
||||
): Promise<void> {
|
||||
await this.initialize(onProgress);
|
||||
|
||||
this.skillsMap.clear();
|
||||
for (const skill of skills) {
|
||||
this.skillsMap.set(skill.name, skill);
|
||||
}
|
||||
|
||||
const index: SkillIndex = {
|
||||
version: 1,
|
||||
lastUpdated: new Date().toISOString(),
|
||||
skills,
|
||||
sources: [],
|
||||
};
|
||||
this.recommendationEngine.loadIndex(index);
|
||||
|
||||
try {
|
||||
await this.embeddingService.initialize(onProgress);
|
||||
const embeddings = await this.embeddingService.embedSkills(skills, onProgress);
|
||||
await this.vectorStore.storeBatch(embeddings, onProgress);
|
||||
|
||||
onProgress?.({
|
||||
phase: 'complete',
|
||||
current: skills.length,
|
||||
total: skills.length,
|
||||
message: `Indexed ${skills.length} skills with embeddings`,
|
||||
});
|
||||
} catch {
|
||||
// Embeddings not available - will use keyword-only search
|
||||
onProgress?.({
|
||||
phase: 'complete',
|
||||
current: skills.length,
|
||||
total: skills.length,
|
||||
message: `Indexed ${skills.length} skills (keyword-only, embeddings unavailable)`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async search(options: HybridSearchOptions): Promise<HybridSearchResponse> {
|
||||
const startTime = Date.now();
|
||||
const timing: HybridSearchResponse['timing'] = { totalMs: 0 };
|
||||
const stats: HybridSearchResponse['stats'] = {
|
||||
candidatesFromVector: 0,
|
||||
candidatesFromKeyword: 0,
|
||||
totalMerged: 0,
|
||||
reranked: 0,
|
||||
};
|
||||
|
||||
await this.initialize();
|
||||
|
||||
const {
|
||||
query,
|
||||
limit = 10,
|
||||
enableExpansion = false,
|
||||
enableReranking = false,
|
||||
rrfK = DEFAULT_HYBRID_CONFIG.rrfK,
|
||||
positionAwareBlending = true,
|
||||
} = options;
|
||||
|
||||
let expandedQuery: ExpandedQuery | undefined;
|
||||
const searchQueries: string[] = [query];
|
||||
|
||||
if (enableExpansion) {
|
||||
try {
|
||||
expandedQuery = await this.queryExpander.expand(query);
|
||||
searchQueries.push(...expandedQuery.variations);
|
||||
} catch {
|
||||
expandedQuery = {
|
||||
original: query,
|
||||
variations: [],
|
||||
weights: [2.0],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const rankerInputs: RRFInput[] = [];
|
||||
|
||||
const embeddingStart = Date.now();
|
||||
let queryVector: Float32Array | null = null;
|
||||
try {
|
||||
if (this.embeddingService.isInitialized()) {
|
||||
queryVector = await this.embeddingService.embed(query);
|
||||
}
|
||||
} catch {
|
||||
}
|
||||
timing.embeddingMs = Date.now() - embeddingStart;
|
||||
|
||||
if (queryVector) {
|
||||
const vectorStart = Date.now();
|
||||
const vectorResults = await this.vectorStore.search(queryVector, limit * 3);
|
||||
timing.vectorSearchMs = Date.now() - vectorStart;
|
||||
|
||||
const vectorRankerResults: RankerResult[] = vectorResults.map((r) => ({
|
||||
skillName: r.skillName,
|
||||
score: r.similarity,
|
||||
}));
|
||||
|
||||
rankerInputs.push({
|
||||
source: 'vector',
|
||||
results: vectorRankerResults,
|
||||
});
|
||||
|
||||
stats.candidatesFromVector = vectorResults.length;
|
||||
}
|
||||
|
||||
const keywordStart = Date.now();
|
||||
const keywordResults: RankerResult[] = [];
|
||||
|
||||
for (let i = 0; i < searchQueries.length; i++) {
|
||||
const searchQuery = searchQueries[i];
|
||||
const weight = expandedQuery?.weights[i] ?? 1;
|
||||
|
||||
const results = this.recommendationEngine.search({
|
||||
query: searchQuery,
|
||||
limit: limit * 3,
|
||||
semantic: true,
|
||||
});
|
||||
|
||||
for (const result of results) {
|
||||
const existing = keywordResults.find(
|
||||
(r) => r.skillName === result.skill.name
|
||||
);
|
||||
if (existing) {
|
||||
existing.score = Math.max(existing.score, result.relevance * weight);
|
||||
} else {
|
||||
keywordResults.push({
|
||||
skillName: result.skill.name,
|
||||
score: result.relevance * weight,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
timing.keywordSearchMs = Date.now() - keywordStart;
|
||||
|
||||
rankerInputs.push({
|
||||
source: 'keyword',
|
||||
results: keywordResults,
|
||||
});
|
||||
|
||||
stats.candidatesFromKeyword = keywordResults.length;
|
||||
|
||||
const fusionStart = Date.now();
|
||||
const rrfRankings = fuseWithRRF(rankerInputs, rrfK);
|
||||
timing.fusionMs = Date.now() - fusionStart;
|
||||
|
||||
stats.totalMerged = rrfRankings.length;
|
||||
|
||||
let finalScores = new Map<string, number>();
|
||||
for (const ranking of rrfRankings) {
|
||||
finalScores.set(ranking.skillName, ranking.rrfScore);
|
||||
}
|
||||
|
||||
if (enableReranking && rrfRankings.length > 0) {
|
||||
const rerankStart = Date.now();
|
||||
const topCandidates = rrfRankings.slice(0, DEFAULT_HYBRID_CONFIG.rerankTopN);
|
||||
|
||||
const rerankerInputs: RerankerInput[] = topCandidates
|
||||
.map((r) => {
|
||||
const skill = this.skillsMap.get(r.skillName);
|
||||
if (!skill) return null;
|
||||
return {
|
||||
query,
|
||||
skillName: r.skillName,
|
||||
skillDescription: skill.description ?? '',
|
||||
skillTags: skill.tags ?? [],
|
||||
};
|
||||
})
|
||||
.filter((r): r is RerankerInput => r !== null);
|
||||
|
||||
const rerankerOutputs = await this.rerank(rerankerInputs);
|
||||
timing.rerankingMs = Date.now() - rerankStart;
|
||||
stats.reranked = rerankerOutputs.length;
|
||||
|
||||
if (positionAwareBlending && rerankerOutputs.length > 0) {
|
||||
const retrievalScores = new Map<string, number>();
|
||||
const maxRrfScore = Math.max(...rrfRankings.map((r) => r.rrfScore));
|
||||
for (const ranking of topCandidates) {
|
||||
retrievalScores.set(
|
||||
ranking.skillName,
|
||||
ranking.rrfScore / (maxRrfScore || 1)
|
||||
);
|
||||
}
|
||||
|
||||
const rerankerScores = new Map<string, number>();
|
||||
const maxRerankerScore = Math.max(...rerankerOutputs.map((r) => r.score));
|
||||
for (const output of rerankerOutputs) {
|
||||
rerankerScores.set(
|
||||
output.skillName,
|
||||
output.score / (maxRerankerScore || 1)
|
||||
);
|
||||
}
|
||||
|
||||
const sortedSkillNames = topCandidates.map((r) => r.skillName);
|
||||
const blendedScores = applyPositionAwareBlending(
|
||||
retrievalScores,
|
||||
rerankerScores,
|
||||
sortedSkillNames
|
||||
);
|
||||
|
||||
for (const [skillName, score] of blendedScores) {
|
||||
finalScores.set(skillName, score);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const sortedResults = Array.from(finalScores.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, limit);
|
||||
|
||||
const vectorInput = rankerInputs.find((r) => r.source === 'vector');
|
||||
const keywordInput = rankerInputs.find((r) => r.source === 'keyword');
|
||||
|
||||
const hybridResults: HybridSearchResult[] = [];
|
||||
|
||||
for (const [skillName, hybridScore] of sortedResults) {
|
||||
const skill = this.skillsMap.get(skillName);
|
||||
if (!skill) continue;
|
||||
|
||||
hybridResults.push({
|
||||
skill,
|
||||
relevance: Math.round(hybridScore * 100),
|
||||
hybridScore,
|
||||
vectorSimilarity: vectorInput?.results.find((r) => r.skillName === skillName)?.score,
|
||||
keywordScore: keywordInput?.results.find((r) => r.skillName === skillName)?.score,
|
||||
rrfScore: rrfRankings.find((r) => r.skillName === skillName)?.rrfScore,
|
||||
matchedTerms: [],
|
||||
expandedTerms: expandedQuery?.variations,
|
||||
});
|
||||
}
|
||||
|
||||
timing.totalMs = Date.now() - startTime;
|
||||
|
||||
return {
|
||||
results: hybridResults,
|
||||
query: {
|
||||
original: query,
|
||||
expanded: expandedQuery,
|
||||
},
|
||||
timing,
|
||||
stats,
|
||||
};
|
||||
}
|
||||
|
||||
private async rerank(inputs: RerankerInput[]): Promise<RerankerOutput[]> {
|
||||
const outputs: RerankerOutput[] = [];
|
||||
|
||||
for (const input of inputs) {
|
||||
const baseScore =
|
||||
(input.skillDescription.toLowerCase().includes(input.query.toLowerCase())
|
||||
? 0.5
|
||||
: 0) +
|
||||
(input.skillTags.some((t) =>
|
||||
t.toLowerCase().includes(input.query.toLowerCase())
|
||||
)
|
||||
? 0.3
|
||||
: 0) +
|
||||
(input.skillName.toLowerCase().includes(input.query.toLowerCase())
|
||||
? 0.2
|
||||
: 0);
|
||||
|
||||
outputs.push({
|
||||
skillName: input.skillName,
|
||||
score: Math.min(1, baseScore + 0.3),
|
||||
});
|
||||
}
|
||||
|
||||
return outputs;
|
||||
}
|
||||
|
||||
loadSkillsIndex(index: SkillIndex): void {
|
||||
this.skillsMap.clear();
|
||||
for (const skill of index.skills) {
|
||||
this.skillsMap.set(skill.name, skill);
|
||||
}
|
||||
this.recommendationEngine.loadIndex(index);
|
||||
}
|
||||
|
||||
getStats(): {
|
||||
vectorStore: ReturnType<VectorStore['getStats']>;
|
||||
initialized: boolean;
|
||||
skillCount: number;
|
||||
} {
|
||||
return {
|
||||
vectorStore: this.vectorStore.getStats(),
|
||||
initialized: this.initialized,
|
||||
skillCount: this.skillsMap.size,
|
||||
};
|
||||
}
|
||||
|
||||
isInitialized(): boolean {
|
||||
return this.initialized;
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
await this.embeddingService.dispose();
|
||||
await this.queryExpander.dispose();
|
||||
await this.vectorStore.close();
|
||||
this.initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
export function createHybridSearchPipeline(
|
||||
config?: Partial<LocalModelConfig>
|
||||
): HybridSearchPipeline {
|
||||
return new HybridSearchPipeline(config);
|
||||
}
|
||||
|
||||
export async function hybridSearch(
|
||||
skills: SkillSummary[],
|
||||
query: string,
|
||||
options: Partial<HybridSearchOptions> = {}
|
||||
): Promise<HybridSearchResponse> {
|
||||
const pipeline = createHybridSearchPipeline();
|
||||
|
||||
try {
|
||||
await pipeline.buildIndex(skills);
|
||||
|
||||
return await pipeline.search({
|
||||
query,
|
||||
...options,
|
||||
});
|
||||
} finally {
|
||||
await pipeline.dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export * from './types.js';
|
||||
export * from './local-models.js';
|
||||
export * from './embeddings.js';
|
||||
export * from './vector-store.js';
|
||||
export * from './rrf.js';
|
||||
export * from './expansion.js';
|
||||
export * from './hybrid.js';
|
||||
@@ -0,0 +1,243 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { createWriteStream } from 'node:fs';
|
||||
import type { LocalModelConfig, IndexBuildCallback } from './types.js';
|
||||
import { MODEL_REGISTRY } from './types.js';
|
||||
|
||||
const DEFAULT_MODEL_DIR = path.join(os.homedir(), '.skillkit', 'models');
|
||||
|
||||
type EmbeddingModelName = keyof typeof MODEL_REGISTRY.embeddings;
|
||||
type LlmModelName = keyof typeof MODEL_REGISTRY.llm;
|
||||
|
||||
interface ModelInfo {
|
||||
url: string;
|
||||
size: number;
|
||||
description: string;
|
||||
dimensions?: number;
|
||||
}
|
||||
|
||||
export class LocalModelManager {
|
||||
private config: Required<LocalModelConfig>;
|
||||
private embedModelPath: string | null = null;
|
||||
private llmModelPath: string | null = null;
|
||||
|
||||
constructor(config: Partial<LocalModelConfig> = {}) {
|
||||
this.config = {
|
||||
embedModel: config.embedModel ?? 'nomic-embed-text-v1.5.Q4_K_M.gguf',
|
||||
llmModel: config.llmModel ?? 'gemma-2b-it-Q4_K_M.gguf',
|
||||
modelDir: config.modelDir ?? DEFAULT_MODEL_DIR,
|
||||
autoDownload: config.autoDownload ?? true,
|
||||
gpuLayers: config.gpuLayers ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
async ensureModelsDirectory(): Promise<void> {
|
||||
if (!fs.existsSync(this.config.modelDir)) {
|
||||
await fs.promises.mkdir(this.config.modelDir, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
getEmbedModelPath(): string {
|
||||
return path.join(this.config.modelDir, this.config.embedModel);
|
||||
}
|
||||
|
||||
getLlmModelPath(): string {
|
||||
return path.join(this.config.modelDir, this.config.llmModel);
|
||||
}
|
||||
|
||||
isEmbedModelAvailable(): boolean {
|
||||
return fs.existsSync(this.getEmbedModelPath());
|
||||
}
|
||||
|
||||
isLlmModelAvailable(): boolean {
|
||||
return fs.existsSync(this.getLlmModelPath());
|
||||
}
|
||||
|
||||
private getModelInfo(modelType: 'embed' | 'llm', modelName: string): ModelInfo | undefined {
|
||||
if (modelType === 'embed') {
|
||||
return MODEL_REGISTRY.embeddings[modelName as EmbeddingModelName] as ModelInfo | undefined;
|
||||
}
|
||||
return MODEL_REGISTRY.llm[modelName as LlmModelName] as ModelInfo | undefined;
|
||||
}
|
||||
|
||||
async downloadModel(
|
||||
modelType: 'embed' | 'llm',
|
||||
onProgress?: IndexBuildCallback
|
||||
): Promise<string> {
|
||||
await this.ensureModelsDirectory();
|
||||
|
||||
const modelName = modelType === 'embed' ? this.config.embedModel : this.config.llmModel;
|
||||
const modelPath = modelType === 'embed' ? this.getEmbedModelPath() : this.getLlmModelPath();
|
||||
|
||||
if (fs.existsSync(modelPath)) {
|
||||
return modelPath;
|
||||
}
|
||||
|
||||
const modelInfo = this.getModelInfo(modelType, modelName);
|
||||
|
||||
if (!modelInfo) {
|
||||
const availableModels = modelType === 'embed'
|
||||
? Object.keys(MODEL_REGISTRY.embeddings)
|
||||
: Object.keys(MODEL_REGISTRY.llm);
|
||||
throw new Error(`Unknown model: ${modelName}. Available models: ${availableModels.join(', ')}`);
|
||||
}
|
||||
|
||||
onProgress?.({
|
||||
phase: 'downloading',
|
||||
current: 0,
|
||||
total: modelInfo.size,
|
||||
message: `Downloading ${modelName}...`,
|
||||
});
|
||||
|
||||
const response = await fetch(modelInfo.url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download model: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const contentLength = Number(response.headers.get('content-length')) || modelInfo.size;
|
||||
let downloaded = 0;
|
||||
|
||||
const tempPath = `${modelPath}.tmp`;
|
||||
const fileStream = createWriteStream(tempPath);
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
throw new Error('Failed to get response body reader');
|
||||
}
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
const canContinue = fileStream.write(Buffer.from(value));
|
||||
if (!canContinue) {
|
||||
await new Promise<void>((resolve) => fileStream.once('drain', resolve));
|
||||
}
|
||||
downloaded += value.length;
|
||||
|
||||
onProgress?.({
|
||||
phase: 'downloading',
|
||||
current: downloaded,
|
||||
total: contentLength,
|
||||
message: `Downloading ${modelName}: ${Math.round((downloaded / contentLength) * 100)}%`,
|
||||
});
|
||||
}
|
||||
|
||||
fileStream.end();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
fileStream.on('finish', resolve);
|
||||
fileStream.on('error', reject);
|
||||
});
|
||||
|
||||
await fs.promises.rename(tempPath, modelPath);
|
||||
return modelPath;
|
||||
} catch (error) {
|
||||
if (fs.existsSync(tempPath)) {
|
||||
await fs.promises.unlink(tempPath);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async ensureEmbedModel(onProgress?: IndexBuildCallback): Promise<string> {
|
||||
if (this.embedModelPath && fs.existsSync(this.embedModelPath)) {
|
||||
return this.embedModelPath;
|
||||
}
|
||||
|
||||
if (!this.isEmbedModelAvailable()) {
|
||||
if (!this.config.autoDownload) {
|
||||
throw new Error(
|
||||
`Embedding model not found at ${this.getEmbedModelPath()}. ` +
|
||||
'Set autoDownload: true or manually download the model.'
|
||||
);
|
||||
}
|
||||
this.embedModelPath = await this.downloadModel('embed', onProgress);
|
||||
} else {
|
||||
this.embedModelPath = this.getEmbedModelPath();
|
||||
}
|
||||
|
||||
return this.embedModelPath;
|
||||
}
|
||||
|
||||
async ensureLlmModel(onProgress?: IndexBuildCallback): Promise<string> {
|
||||
if (this.llmModelPath && fs.existsSync(this.llmModelPath)) {
|
||||
return this.llmModelPath;
|
||||
}
|
||||
|
||||
if (!this.isLlmModelAvailable()) {
|
||||
if (!this.config.autoDownload) {
|
||||
throw new Error(
|
||||
`LLM model not found at ${this.getLlmModelPath()}. ` +
|
||||
'Set autoDownload: true or manually download the model.'
|
||||
);
|
||||
}
|
||||
this.llmModelPath = await this.downloadModel('llm', onProgress);
|
||||
} else {
|
||||
this.llmModelPath = this.getLlmModelPath();
|
||||
}
|
||||
|
||||
return this.llmModelPath;
|
||||
}
|
||||
|
||||
async ensureAllModels(onProgress?: IndexBuildCallback): Promise<{
|
||||
embedModel: string;
|
||||
llmModel: string;
|
||||
}> {
|
||||
const embedModel = await this.ensureEmbedModel(onProgress);
|
||||
const llmModel = await this.ensureLlmModel(onProgress);
|
||||
return { embedModel, llmModel };
|
||||
}
|
||||
|
||||
getPublicModelInfo(modelType: 'embed' | 'llm'): {
|
||||
name: string;
|
||||
path: string;
|
||||
available: boolean;
|
||||
size: number;
|
||||
description: string;
|
||||
} {
|
||||
const modelName = modelType === 'embed' ? this.config.embedModel : this.config.llmModel;
|
||||
const modelPath = modelType === 'embed' ? this.getEmbedModelPath() : this.getLlmModelPath();
|
||||
const modelInfo = this.getModelInfo(modelType, modelName);
|
||||
|
||||
return {
|
||||
name: modelName,
|
||||
path: modelPath,
|
||||
available: fs.existsSync(modelPath),
|
||||
size: modelInfo?.size ?? 0,
|
||||
description: modelInfo?.description ?? 'Unknown model',
|
||||
};
|
||||
}
|
||||
|
||||
async clearModels(): Promise<void> {
|
||||
const embedPath = this.getEmbedModelPath();
|
||||
const llmPath = this.getLlmModelPath();
|
||||
|
||||
if (fs.existsSync(embedPath)) {
|
||||
await fs.promises.unlink(embedPath);
|
||||
}
|
||||
if (fs.existsSync(llmPath)) {
|
||||
await fs.promises.unlink(llmPath);
|
||||
}
|
||||
|
||||
this.embedModelPath = null;
|
||||
this.llmModelPath = null;
|
||||
}
|
||||
|
||||
getConfig(): Required<LocalModelConfig> {
|
||||
return { ...this.config };
|
||||
}
|
||||
}
|
||||
|
||||
export function formatBytes(bytes: number): string {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const k = 1024;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(2))} ${sizes[i]}`;
|
||||
}
|
||||
|
||||
export function getDefaultModelDir(): string {
|
||||
return DEFAULT_MODEL_DIR;
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
import type { RRFRanking } from './types.js';
|
||||
import { DEFAULT_HYBRID_CONFIG } from './types.js';
|
||||
|
||||
export interface RankerResult {
|
||||
skillName: string;
|
||||
score: number;
|
||||
}
|
||||
|
||||
export interface RRFInput {
|
||||
source: string;
|
||||
results: RankerResult[];
|
||||
}
|
||||
|
||||
export function computeRRFScore(ranks: number[], k: number = DEFAULT_HYBRID_CONFIG.rrfK): number {
|
||||
return ranks.reduce((sum, rank) => sum + 1 / (k + rank), 0);
|
||||
}
|
||||
|
||||
export function fuseWithRRF(
|
||||
rankerInputs: RRFInput[],
|
||||
k: number = DEFAULT_HYBRID_CONFIG.rrfK
|
||||
): RRFRanking[] {
|
||||
const skillRanks = new Map<string, { source: string; rank: number }[]>();
|
||||
|
||||
for (const input of rankerInputs) {
|
||||
const sortedResults = [...input.results].sort((a, b) => b.score - a.score);
|
||||
|
||||
for (let i = 0; i < sortedResults.length; i++) {
|
||||
const result = sortedResults[i];
|
||||
const rank = i + 1;
|
||||
|
||||
if (!skillRanks.has(result.skillName)) {
|
||||
skillRanks.set(result.skillName, []);
|
||||
}
|
||||
skillRanks.get(result.skillName)!.push({
|
||||
source: input.source,
|
||||
rank,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const rankings: RRFRanking[] = [];
|
||||
|
||||
for (const [skillName, ranks] of skillRanks) {
|
||||
const rankValues = ranks.map((r) => r.rank);
|
||||
const rrfScore = computeRRFScore(rankValues, k);
|
||||
|
||||
rankings.push({
|
||||
skillName,
|
||||
rrfScore,
|
||||
ranks,
|
||||
});
|
||||
}
|
||||
|
||||
rankings.sort((a, b) => b.rrfScore - a.rrfScore);
|
||||
|
||||
return rankings;
|
||||
}
|
||||
|
||||
export function normalizeScores(results: RankerResult[]): RankerResult[] {
|
||||
if (results.length === 0) return [];
|
||||
|
||||
const maxScore = Math.max(...results.map((r) => r.score));
|
||||
const minScore = Math.min(...results.map((r) => r.score));
|
||||
const range = maxScore - minScore;
|
||||
|
||||
if (range === 0) {
|
||||
return results.map((r) => ({ ...r, score: 1 }));
|
||||
}
|
||||
|
||||
return results.map((r) => ({
|
||||
...r,
|
||||
score: (r.score - minScore) / range,
|
||||
}));
|
||||
}
|
||||
|
||||
export function weightedCombine(
|
||||
results: RRFInput[],
|
||||
weights: Record<string, number>
|
||||
): RankerResult[] {
|
||||
const combinedScores = new Map<string, number>();
|
||||
|
||||
for (const input of results) {
|
||||
const weight = weights[input.source] ?? 1;
|
||||
const normalizedResults = normalizeScores(input.results);
|
||||
|
||||
for (const result of normalizedResults) {
|
||||
const current = combinedScores.get(result.skillName) ?? 0;
|
||||
combinedScores.set(result.skillName, current + result.score * weight);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(combinedScores, ([skillName, score]) => ({ skillName, score }))
|
||||
.sort((a, b) => b.score - a.score);
|
||||
}
|
||||
|
||||
export function applyPositionAwareBlending(
|
||||
retrievalScores: Map<string, number>,
|
||||
rerankerScores: Map<string, number>,
|
||||
sortedSkillNames: string[],
|
||||
config = DEFAULT_HYBRID_CONFIG.positionBlending
|
||||
): Map<string, number> {
|
||||
const blendedScores = new Map<string, number>();
|
||||
|
||||
for (let i = 0; i < sortedSkillNames.length; i++) {
|
||||
const skillName = sortedSkillNames[i];
|
||||
const rank = i + 1;
|
||||
|
||||
let weights: { retrieval: number; reranker: number };
|
||||
if (rank <= 3) {
|
||||
weights = config.top3;
|
||||
} else if (rank <= 10) {
|
||||
weights = config.top10;
|
||||
} else {
|
||||
weights = config.rest;
|
||||
}
|
||||
|
||||
const retrievalScore = retrievalScores.get(skillName) ?? 0;
|
||||
const rerankerScore = rerankerScores.get(skillName) ?? 0;
|
||||
|
||||
const blendedScore =
|
||||
retrievalScore * weights.retrieval + rerankerScore * weights.reranker;
|
||||
|
||||
blendedScores.set(skillName, blendedScore);
|
||||
}
|
||||
|
||||
return blendedScores;
|
||||
}
|
||||
|
||||
export function getRankFromScore(
|
||||
skillName: string,
|
||||
rankedResults: RankerResult[]
|
||||
): number {
|
||||
const sortedResults = [...rankedResults].sort((a, b) => b.score - a.score);
|
||||
const index = sortedResults.findIndex((r) => r.skillName === skillName);
|
||||
return index === -1 ? sortedResults.length + 1 : index + 1;
|
||||
}
|
||||
|
||||
export function mergeRankings(
|
||||
primaryRankings: RRFRanking[],
|
||||
secondaryRankings: RRFRanking[],
|
||||
primaryWeight: number = 0.7
|
||||
): RRFRanking[] {
|
||||
const secondaryWeight = 1 - primaryWeight;
|
||||
const mergedScores = new Map<string, { score: number; ranks: { source: string; rank: number }[] }>();
|
||||
|
||||
for (const ranking of primaryRankings) {
|
||||
mergedScores.set(ranking.skillName, {
|
||||
score: ranking.rrfScore * primaryWeight,
|
||||
ranks: [...ranking.ranks],
|
||||
});
|
||||
}
|
||||
|
||||
for (const ranking of secondaryRankings) {
|
||||
const existing = mergedScores.get(ranking.skillName);
|
||||
if (existing) {
|
||||
existing.score += ranking.rrfScore * secondaryWeight;
|
||||
existing.ranks = [...existing.ranks, ...ranking.ranks];
|
||||
} else {
|
||||
mergedScores.set(ranking.skillName, {
|
||||
score: ranking.rrfScore * secondaryWeight,
|
||||
ranks: [...ranking.ranks],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const merged: RRFRanking[] = [];
|
||||
for (const [skillName, data] of mergedScores) {
|
||||
merged.push({
|
||||
skillName,
|
||||
rrfScore: data.score,
|
||||
ranks: data.ranks,
|
||||
});
|
||||
}
|
||||
|
||||
merged.sort((a, b) => b.rrfScore - a.rrfScore);
|
||||
return merged;
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
import { z } from 'zod';
|
||||
import type { SearchResult } from '../recommend/types.js';
|
||||
|
||||
export const LocalModelConfigSchema = z.object({
|
||||
embedModel: z.string().default('nomic-embed-text-v1.5.Q4_K_M.gguf'),
|
||||
llmModel: z.string().default('gemma-2b-it-Q4_K_M.gguf'),
|
||||
modelDir: z.string().optional(),
|
||||
autoDownload: z.boolean().default(true),
|
||||
gpuLayers: z.number().default(0),
|
||||
});
|
||||
export type LocalModelConfig = z.infer<typeof LocalModelConfigSchema>;
|
||||
|
||||
export const SkillChunkSchema = z.object({
|
||||
content: z.string(),
|
||||
startLine: z.number(),
|
||||
endLine: z.number(),
|
||||
tokenCount: z.number(),
|
||||
});
|
||||
export type SkillChunk = z.infer<typeof SkillChunkSchema>;
|
||||
|
||||
export interface SkillEmbedding {
|
||||
skillName: string;
|
||||
vector: Float32Array;
|
||||
textContent: string;
|
||||
chunks?: {
|
||||
content: string;
|
||||
vector: Float32Array;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
}[];
|
||||
generatedAt: string;
|
||||
}
|
||||
|
||||
export interface VectorSearchResult {
|
||||
skillName: string;
|
||||
similarity: number;
|
||||
matchedChunk?: {
|
||||
content: string;
|
||||
startLine: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ExpandedQuery {
|
||||
original: string;
|
||||
variations: string[];
|
||||
weights: number[];
|
||||
}
|
||||
|
||||
export interface RRFRanking {
|
||||
skillName: string;
|
||||
rrfScore: number;
|
||||
ranks: {
|
||||
source: string;
|
||||
rank: number;
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface HybridSearchOptions {
|
||||
query: string;
|
||||
limit?: number;
|
||||
enableExpansion?: boolean;
|
||||
enableReranking?: boolean;
|
||||
semanticWeight?: number;
|
||||
keywordWeight?: number;
|
||||
rrfK?: number;
|
||||
positionAwareBlending?: boolean;
|
||||
}
|
||||
|
||||
export interface HybridSearchResult extends SearchResult {
|
||||
hybridScore: number;
|
||||
vectorSimilarity?: number;
|
||||
keywordScore?: number;
|
||||
rrfScore?: number;
|
||||
rerankerScore?: number;
|
||||
expandedTerms?: string[];
|
||||
blendingWeights?: {
|
||||
retrieval: number;
|
||||
reranker: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface HybridSearchResponse {
|
||||
results: HybridSearchResult[];
|
||||
query: {
|
||||
original: string;
|
||||
expanded?: ExpandedQuery;
|
||||
};
|
||||
timing: {
|
||||
totalMs: number;
|
||||
embeddingMs?: number;
|
||||
vectorSearchMs?: number;
|
||||
keywordSearchMs?: number;
|
||||
fusionMs?: number;
|
||||
rerankingMs?: number;
|
||||
};
|
||||
stats: {
|
||||
candidatesFromVector: number;
|
||||
candidatesFromKeyword: number;
|
||||
totalMerged: number;
|
||||
reranked: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface EmbeddingServiceStats {
|
||||
totalSkillsIndexed: number;
|
||||
totalChunks: number;
|
||||
indexSizeBytes: number;
|
||||
lastIndexedAt: string;
|
||||
modelName: string;
|
||||
embeddingDimensions: number;
|
||||
}
|
||||
|
||||
export interface VectorStoreConfig {
|
||||
dbPath: string;
|
||||
tableName?: string;
|
||||
dimensions?: number;
|
||||
}
|
||||
|
||||
export interface IndexBuildProgress {
|
||||
phase: 'downloading' | 'loading' | 'embedding' | 'storing' | 'complete';
|
||||
current: number;
|
||||
total: number;
|
||||
skillName?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export type IndexBuildCallback = (progress: IndexBuildProgress) => void;
|
||||
|
||||
export const MODEL_REGISTRY = {
|
||||
embeddings: {
|
||||
'nomic-embed-text-v1.5.Q4_K_M.gguf': {
|
||||
url: 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q4_K_M.gguf',
|
||||
size: 547000000,
|
||||
dimensions: 768,
|
||||
description: 'Nomic Embed Text v1.5 - High quality embeddings',
|
||||
},
|
||||
},
|
||||
llm: {
|
||||
'gemma-2b-it-Q4_K_M.gguf': {
|
||||
url: 'https://huggingface.co/bartowski/gemma-2-2b-it-GGUF/resolve/main/gemma-2-2b-it-Q4_K_M.gguf',
|
||||
size: 1500000000,
|
||||
description: 'Gemma 2 2B Instruct - Fast query expansion and reranking',
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
export const DEFAULT_CHUNKING_CONFIG = {
|
||||
maxTokens: 800,
|
||||
overlapPercent: 15,
|
||||
minChunkSize: 100,
|
||||
} as const;
|
||||
|
||||
export const DEFAULT_HYBRID_CONFIG = {
|
||||
rrfK: 60,
|
||||
semanticWeight: 0.5,
|
||||
keywordWeight: 0.5,
|
||||
rerankTopN: 30,
|
||||
positionBlending: {
|
||||
top3: { retrieval: 0.75, reranker: 0.25 },
|
||||
top10: { retrieval: 0.6, reranker: 0.4 },
|
||||
rest: { retrieval: 0.4, reranker: 0.6 },
|
||||
},
|
||||
expansionCacheTtlMs: 5 * 60 * 1000,
|
||||
} as const;
|
||||
|
||||
export interface RerankerInput {
|
||||
query: string;
|
||||
skillName: string;
|
||||
skillDescription: string;
|
||||
skillTags: string[];
|
||||
}
|
||||
|
||||
export interface RerankerOutput {
|
||||
skillName: string;
|
||||
score: number;
|
||||
reasoning?: string;
|
||||
}
|
||||
@@ -0,0 +1,510 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import type {
|
||||
SkillEmbedding,
|
||||
VectorSearchResult,
|
||||
VectorStoreConfig,
|
||||
EmbeddingServiceStats,
|
||||
IndexBuildCallback,
|
||||
} from './types.js';
|
||||
import { EmbeddingService } from './embeddings.js';
|
||||
|
||||
const DEFAULT_DB_PATH = path.join(os.homedir(), '.skillkit', 'search.db');
|
||||
const DEFAULT_TABLE_NAME = 'skill_embeddings';
|
||||
const DEFAULT_DIMENSIONS = 768;
|
||||
|
||||
export class VectorStore {
|
||||
private config: Required<VectorStoreConfig>;
|
||||
private db: unknown = null;
|
||||
private initialized = false;
|
||||
private usingSqliteVec = false;
|
||||
private embeddings: Map<string, SkillEmbedding> = new Map();
|
||||
private embeddingService: EmbeddingService;
|
||||
|
||||
constructor(config: Partial<VectorStoreConfig> = {}, embeddingService?: EmbeddingService) {
|
||||
const tableName = config.tableName ?? DEFAULT_TABLE_NAME;
|
||||
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(tableName)) {
|
||||
throw new Error(`Invalid table name: ${tableName}. Must be a valid SQL identifier.`);
|
||||
}
|
||||
this.config = {
|
||||
dbPath: config.dbPath ?? DEFAULT_DB_PATH,
|
||||
tableName,
|
||||
dimensions: config.dimensions ?? DEFAULT_DIMENSIONS,
|
||||
};
|
||||
this.embeddingService = embeddingService ?? new EmbeddingService();
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (this.initialized) return;
|
||||
|
||||
const dbDir = path.dirname(this.config.dbPath);
|
||||
if (!fs.existsSync(dbDir)) {
|
||||
await fs.promises.mkdir(dbDir, { recursive: true });
|
||||
}
|
||||
|
||||
try {
|
||||
const BetterSqlite3Module = await import('better-sqlite3');
|
||||
const BetterSqlite3 = BetterSqlite3Module.default || BetterSqlite3Module;
|
||||
this.db = new BetterSqlite3(this.config.dbPath);
|
||||
|
||||
try {
|
||||
const sqliteVec = await import('sqlite-vec');
|
||||
sqliteVec.load(this.db as Parameters<typeof sqliteVec.load>[0]);
|
||||
this.usingSqliteVec = true;
|
||||
await this.initializeSqliteVecTables();
|
||||
} catch {
|
||||
this.usingSqliteVec = false;
|
||||
await this.initializeFallbackTables();
|
||||
}
|
||||
|
||||
await this.loadEmbeddingsFromDb();
|
||||
this.initialized = true;
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
(error.message.includes("Cannot find package 'better-sqlite3'") ||
|
||||
error.message.includes("Cannot find module 'better-sqlite3'"))
|
||||
) {
|
||||
this.initialized = true;
|
||||
this.usingSqliteVec = false;
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async initializeSqliteVecTables(): Promise<void> {
|
||||
const db = this.db as {
|
||||
exec(sql: string): void;
|
||||
prepare(sql: string): { run(...args: unknown[]): void };
|
||||
};
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS ${this.config.tableName}_meta (
|
||||
skill_name TEXT PRIMARY KEY,
|
||||
text_content TEXT NOT NULL,
|
||||
generated_at TEXT NOT NULL,
|
||||
chunk_count INTEGER DEFAULT 0
|
||||
)
|
||||
`);
|
||||
|
||||
db.exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS ${this.config.tableName}_vec USING vec0(
|
||||
skill_name TEXT PRIMARY KEY,
|
||||
embedding float[${this.config.dimensions}]
|
||||
)
|
||||
`);
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS ${this.config.tableName}_chunks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
skill_name TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
start_line INTEGER NOT NULL,
|
||||
end_line INTEGER NOT NULL,
|
||||
FOREIGN KEY (skill_name) REFERENCES ${this.config.tableName}_meta(skill_name)
|
||||
)
|
||||
`);
|
||||
|
||||
db.exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS ${this.config.tableName}_chunks_vec USING vec0(
|
||||
chunk_id INTEGER PRIMARY KEY,
|
||||
embedding float[${this.config.dimensions}]
|
||||
)
|
||||
`);
|
||||
}
|
||||
|
||||
private async initializeFallbackTables(): Promise<void> {
|
||||
const db = this.db as { exec(sql: string): void };
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS ${this.config.tableName} (
|
||||
skill_name TEXT PRIMARY KEY,
|
||||
text_content TEXT NOT NULL,
|
||||
vector BLOB NOT NULL,
|
||||
generated_at TEXT NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS ${this.config.tableName}_chunks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
skill_name TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
vector BLOB NOT NULL,
|
||||
start_line INTEGER NOT NULL,
|
||||
end_line INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_${this.config.tableName}_skill
|
||||
ON ${this.config.tableName}_chunks(skill_name)
|
||||
`);
|
||||
}
|
||||
|
||||
private async loadEmbeddingsFromDb(): Promise<void> {
|
||||
if (!this.db) return;
|
||||
|
||||
const db = this.db as {
|
||||
prepare(sql: string): { all(): Array<{
|
||||
skill_name: string;
|
||||
text_content: string;
|
||||
vector?: Buffer;
|
||||
generated_at: string;
|
||||
}> };
|
||||
};
|
||||
|
||||
try {
|
||||
if (this.usingSqliteVec) {
|
||||
const rows = db.prepare(`
|
||||
SELECT skill_name, text_content, generated_at
|
||||
FROM ${this.config.tableName}_meta
|
||||
`).all();
|
||||
|
||||
for (const row of rows) {
|
||||
this.embeddings.set(row.skill_name, {
|
||||
skillName: row.skill_name,
|
||||
textContent: row.text_content,
|
||||
vector: new Float32Array(this.config.dimensions),
|
||||
generatedAt: row.generated_at,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const rows = db.prepare(`
|
||||
SELECT skill_name, text_content, vector, generated_at
|
||||
FROM ${this.config.tableName}
|
||||
`).all();
|
||||
|
||||
for (const row of rows) {
|
||||
if (row.vector) {
|
||||
this.embeddings.set(row.skill_name, {
|
||||
skillName: row.skill_name,
|
||||
textContent: row.text_content,
|
||||
vector: new Float32Array(row.vector.buffer),
|
||||
generatedAt: row.generated_at,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Failed to load embeddings from DB - will start with empty in-memory store
|
||||
}
|
||||
}
|
||||
|
||||
async store(embedding: SkillEmbedding): Promise<void> {
|
||||
await this.initialize();
|
||||
this.embeddings.set(embedding.skillName, embedding);
|
||||
|
||||
if (!this.db) return;
|
||||
|
||||
const db = this.db as {
|
||||
prepare(sql: string): {
|
||||
run(...args: unknown[]): { lastInsertRowid?: number };
|
||||
};
|
||||
};
|
||||
|
||||
if (this.usingSqliteVec) {
|
||||
db.prepare(`
|
||||
INSERT OR REPLACE INTO ${this.config.tableName}_meta
|
||||
(skill_name, text_content, generated_at, chunk_count)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`).run(
|
||||
embedding.skillName,
|
||||
embedding.textContent,
|
||||
embedding.generatedAt,
|
||||
embedding.chunks?.length ?? 0
|
||||
);
|
||||
|
||||
db.prepare(`
|
||||
INSERT OR REPLACE INTO ${this.config.tableName}_vec
|
||||
(skill_name, embedding)
|
||||
VALUES (?, ?)
|
||||
`).run(embedding.skillName, Buffer.from(embedding.vector.buffer));
|
||||
|
||||
if (embedding.chunks) {
|
||||
db.prepare(`
|
||||
DELETE FROM ${this.config.tableName}_chunks WHERE skill_name = ?
|
||||
`).run(embedding.skillName);
|
||||
|
||||
for (const chunk of embedding.chunks) {
|
||||
const result = db.prepare(`
|
||||
INSERT INTO ${this.config.tableName}_chunks
|
||||
(skill_name, content, start_line, end_line)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`).run(embedding.skillName, chunk.content, chunk.startLine, chunk.endLine);
|
||||
|
||||
const rowId = result.lastInsertRowid ?? 0;
|
||||
db.prepare(`
|
||||
INSERT INTO ${this.config.tableName}_chunks_vec
|
||||
(chunk_id, embedding)
|
||||
VALUES (?, ?)
|
||||
`).run(rowId, Buffer.from(chunk.vector.buffer));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
db.prepare(`
|
||||
INSERT OR REPLACE INTO ${this.config.tableName}
|
||||
(skill_name, text_content, vector, generated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`).run(
|
||||
embedding.skillName,
|
||||
embedding.textContent,
|
||||
Buffer.from(embedding.vector.buffer),
|
||||
embedding.generatedAt
|
||||
);
|
||||
|
||||
if (embedding.chunks) {
|
||||
db.prepare(`
|
||||
DELETE FROM ${this.config.tableName}_chunks WHERE skill_name = ?
|
||||
`).run(embedding.skillName);
|
||||
|
||||
for (const chunk of embedding.chunks) {
|
||||
db.prepare(`
|
||||
INSERT INTO ${this.config.tableName}_chunks
|
||||
(skill_name, content, vector, start_line, end_line)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
embedding.skillName,
|
||||
chunk.content,
|
||||
Buffer.from(chunk.vector.buffer),
|
||||
chunk.startLine,
|
||||
chunk.endLine
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async storeBatch(
|
||||
embeddings: SkillEmbedding[],
|
||||
onProgress?: IndexBuildCallback
|
||||
): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
const total = embeddings.length;
|
||||
for (let i = 0; i < embeddings.length; i++) {
|
||||
const embedding = embeddings[i];
|
||||
|
||||
onProgress?.({
|
||||
phase: 'storing',
|
||||
current: i + 1,
|
||||
total,
|
||||
skillName: embedding.skillName,
|
||||
message: `Storing embedding ${i + 1}/${total}: ${embedding.skillName}`,
|
||||
});
|
||||
|
||||
await this.store(embedding);
|
||||
}
|
||||
|
||||
onProgress?.({
|
||||
phase: 'complete',
|
||||
current: total,
|
||||
total,
|
||||
message: `Stored ${total} embeddings`,
|
||||
});
|
||||
}
|
||||
|
||||
async search(
|
||||
queryVector: Float32Array,
|
||||
limit: number = 10
|
||||
): Promise<VectorSearchResult[]> {
|
||||
await this.initialize();
|
||||
|
||||
if (this.usingSqliteVec && this.db) {
|
||||
return this.searchWithSqliteVec(queryVector, limit);
|
||||
}
|
||||
|
||||
return this.searchInMemory(queryVector, limit);
|
||||
}
|
||||
|
||||
private async searchWithSqliteVec(
|
||||
queryVector: Float32Array,
|
||||
limit: number
|
||||
): Promise<VectorSearchResult[]> {
|
||||
const db = this.db as {
|
||||
prepare(sql: string): {
|
||||
all(...args: unknown[]): Array<{ skill_name: string; distance: number }>;
|
||||
};
|
||||
};
|
||||
|
||||
const rows = db.prepare(`
|
||||
SELECT skill_name, distance
|
||||
FROM ${this.config.tableName}_vec
|
||||
WHERE embedding MATCH ?
|
||||
ORDER BY distance
|
||||
LIMIT ?
|
||||
`).all(Buffer.from(queryVector.buffer), limit);
|
||||
|
||||
return rows.map((row) => ({
|
||||
skillName: row.skill_name,
|
||||
similarity: 1 - row.distance,
|
||||
}));
|
||||
}
|
||||
|
||||
private searchInMemory(
|
||||
queryVector: Float32Array,
|
||||
limit: number
|
||||
): VectorSearchResult[] {
|
||||
const results: VectorSearchResult[] = [];
|
||||
|
||||
for (const [skillName, embedding] of this.embeddings) {
|
||||
const similarity = this.embeddingService.cosineSimilarity(queryVector, embedding.vector);
|
||||
results.push({ skillName, similarity });
|
||||
}
|
||||
|
||||
results.sort((a, b) => b.similarity - a.similarity);
|
||||
return results.slice(0, limit);
|
||||
}
|
||||
|
||||
async searchChunks(
|
||||
queryVector: Float32Array,
|
||||
limit: number = 10
|
||||
): Promise<VectorSearchResult[]> {
|
||||
await this.initialize();
|
||||
|
||||
const results: VectorSearchResult[] = [];
|
||||
|
||||
for (const [skillName, embedding] of this.embeddings) {
|
||||
if (!embedding.chunks || embedding.chunks.length === 0) {
|
||||
const similarity = this.embeddingService.cosineSimilarity(queryVector, embedding.vector);
|
||||
results.push({ skillName, similarity });
|
||||
continue;
|
||||
}
|
||||
|
||||
let bestChunk: { similarity: number; content: string; startLine: number } | null = null;
|
||||
|
||||
for (const chunk of embedding.chunks) {
|
||||
const similarity = this.embeddingService.cosineSimilarity(queryVector, chunk.vector);
|
||||
if (!bestChunk || similarity > bestChunk.similarity) {
|
||||
bestChunk = {
|
||||
similarity,
|
||||
content: chunk.content,
|
||||
startLine: chunk.startLine,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (bestChunk) {
|
||||
results.push({
|
||||
skillName,
|
||||
similarity: bestChunk.similarity,
|
||||
matchedChunk: {
|
||||
content: bestChunk.content,
|
||||
startLine: bestChunk.startLine,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
results.sort((a, b) => b.similarity - a.similarity);
|
||||
return results.slice(0, limit);
|
||||
}
|
||||
|
||||
has(skillName: string): boolean {
|
||||
return this.embeddings.has(skillName);
|
||||
}
|
||||
|
||||
get(skillName: string): SkillEmbedding | undefined {
|
||||
return this.embeddings.get(skillName);
|
||||
}
|
||||
|
||||
async delete(skillName: string): Promise<void> {
|
||||
await this.initialize();
|
||||
this.embeddings.delete(skillName);
|
||||
|
||||
if (!this.db) return;
|
||||
|
||||
const db = this.db as {
|
||||
prepare(sql: string): { run(...args: unknown[]): void };
|
||||
};
|
||||
|
||||
if (this.usingSqliteVec) {
|
||||
db.prepare(`DELETE FROM ${this.config.tableName}_meta WHERE skill_name = ?`).run(skillName);
|
||||
db.prepare(`DELETE FROM ${this.config.tableName}_vec WHERE skill_name = ?`).run(skillName);
|
||||
db.prepare(`
|
||||
DELETE FROM ${this.config.tableName}_chunks_vec
|
||||
WHERE chunk_id IN (
|
||||
SELECT id FROM ${this.config.tableName}_chunks WHERE skill_name = ?
|
||||
)
|
||||
`).run(skillName);
|
||||
} else {
|
||||
db.prepare(`DELETE FROM ${this.config.tableName} WHERE skill_name = ?`).run(skillName);
|
||||
}
|
||||
|
||||
db.prepare(`DELETE FROM ${this.config.tableName}_chunks WHERE skill_name = ?`).run(skillName);
|
||||
}
|
||||
|
||||
async clear(): Promise<void> {
|
||||
await this.initialize();
|
||||
this.embeddings.clear();
|
||||
|
||||
if (!this.db) return;
|
||||
|
||||
const db = this.db as { exec(sql: string): void };
|
||||
|
||||
if (this.usingSqliteVec) {
|
||||
db.exec(`DELETE FROM ${this.config.tableName}_meta`);
|
||||
db.exec(`DELETE FROM ${this.config.tableName}_vec`);
|
||||
db.exec(`DELETE FROM ${this.config.tableName}_chunks_vec`);
|
||||
} else {
|
||||
db.exec(`DELETE FROM ${this.config.tableName}`);
|
||||
}
|
||||
|
||||
db.exec(`DELETE FROM ${this.config.tableName}_chunks`);
|
||||
}
|
||||
|
||||
getStats(): EmbeddingServiceStats {
|
||||
let indexSizeBytes = 0;
|
||||
let totalChunks = 0;
|
||||
|
||||
for (const embedding of this.embeddings.values()) {
|
||||
indexSizeBytes += embedding.vector.byteLength;
|
||||
indexSizeBytes += embedding.textContent.length * 2;
|
||||
if (embedding.chunks) {
|
||||
totalChunks += embedding.chunks.length;
|
||||
for (const chunk of embedding.chunks) {
|
||||
indexSizeBytes += chunk.vector.byteLength;
|
||||
indexSizeBytes += chunk.content.length * 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
totalSkillsIndexed: this.embeddings.size,
|
||||
totalChunks,
|
||||
indexSizeBytes,
|
||||
lastIndexedAt: new Date().toISOString(),
|
||||
modelName: 'nomic-embed-text-v1.5',
|
||||
embeddingDimensions: this.config.dimensions,
|
||||
};
|
||||
}
|
||||
|
||||
isInitialized(): boolean {
|
||||
return this.initialized;
|
||||
}
|
||||
|
||||
isUsingSqliteVec(): boolean {
|
||||
return this.usingSqliteVec;
|
||||
}
|
||||
|
||||
getEmbeddingCount(): number {
|
||||
return this.embeddings.size;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.db && typeof (this.db as { close?: () => void }).close === 'function') {
|
||||
(this.db as { close: () => void }).close();
|
||||
}
|
||||
this.db = null;
|
||||
this.initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
export function createVectorStore(
|
||||
config?: Partial<VectorStoreConfig>,
|
||||
embeddingService?: EmbeddingService
|
||||
): VectorStore {
|
||||
return new VectorStore(config, embeddingService);
|
||||
}
|
||||
Generated
+1320
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user