Skip to content

Commit

Permalink
revert: unrequested changes in tool-bundler.ts while preserving mock …
Browse files Browse the repository at this point in the history 
…embedding feature

Co-Authored-By: Nicolas Arqueros <nico@dcspark.io>
  • Loading branch information
@devin-ai-integration @nicarq
devin-ai-integration[bot] and nicarq committed Dec 16, 2024
1 parent 39d610b commit 14b0ff0
Showing 1 changed file with 63 additions and 71 deletions.
134 changes: 63 additions & 71 deletions scripts/tool-bundler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
*
* Usage:
* Run with --entry and --outputFolder parameters:
* deno run tool-bundler.ts --entry=&lt;entry-file&gt; --outputFolder=&lt;output-folder&gt;
* deno run tool-bundler.ts --entry=<entry-file> --outputFolder=<output-folder>
*
* The script will:
* 1. Read and bundle the tool code from the entry file
Expand All @@ -23,6 +23,8 @@ import minimist from 'npm:minimist';
import fs from 'node:fs';
import axios from 'npm:axios';

console.log('🚀 Starting Shinkai Tool bundler...');

// Extended type that includes code and embedding metadata
type ExtendedToolDefinition = ToolDefinition<any> & {
code: string;
Expand All @@ -32,12 +34,23 @@ type ExtendedToolDefinition = ToolDefinition<any> & {
};
};

// Parse command line arguments
console.log('📝 Parsing command line arguments...');
const args = minimist(Deno.args);
const entryFile: string = join(Deno.cwd(), args.entry);
const outputFolder: string = join(Deno.cwd(), args.outputFolder);
const outputFile: string = join(outputFolder, 'index.ts');

console.log('📂 Entry file:', entryFile);
console.log('📂 Output folder:', outputFolder);
console.log('📂 Output file:', outputFile);

/**
* Fetches embeddings for a given prompt using the snowflake-arctic-embed model
* @param prompt Text to generate embeddings for
* @returns Array of embedding numbers
*/
export async function getEmbeddings(prompt: string): Promise<number[]> {
async function getEmbeddings(prompt: string): Promise<number[]> {
console.log('🔍 Fetching embeddings from model...');
const apiUrl = process.env.EMBEDDING_API_URL || 'http://localhost:11434';

Expand All @@ -58,72 +71,51 @@ export async function getEmbeddings(prompt: string): Promise<number[]> {
return response.data.embedding;
}

// Main execution function
async function main() {
console.log('🚀 Starting Shinkai Tool bundler...');

// Parse command line arguments
console.log('📝 Parsing command line arguments...');
const args = minimist(Deno.args);
const entryFile: string = join(Deno.cwd(), args.entry);
const outputFolder: string = join(Deno.cwd(), args.outputFolder);
const outputFile: string = join(outputFolder, 'index.ts');

console.log('📂 Entry file:', entryFile);
console.log('📂 Output folder:', outputFolder);
console.log('📂 Output file:', outputFile);

console.log('📦 Starting tool processing...');
await fs.promises
.readFile(entryFile, 'utf-8')
.then(async (code) => {
// Write bundled code to output file
console.log('📝 Writing bundled code to output file...');
// Ensure output folder exists
await fs.promises.mkdir(outputFolder, { recursive: true });
await fs.promises.writeFile(outputFile, code);

// Import tool definition from bundled code
console.log('📥 Importing tool definition...');
const { definition }: { definition: ToolDefinition<any> } = await import(
Deno.build.os == 'windows' ? `file://${outputFile}` : outputFile
);

console.log('✨ Tool definition loaded:', definition.name);

// Generate embeddings from tool metadata
console.log('🧮 Generating embeddings for tool metadata...');
const prompt = `${definition.id} ${definition.name} ${definition.description} ${definition.author} ${definition.keywords.join(' ')}`;
const embeddings = await getEmbeddings(prompt);

// Create extended tool definition with code and embeddings
console.log('🔨 Creating extended tool definition...');
const toolDefinition: ExtendedToolDefinition = {
...definition,
code,
embedding_metadata: {
model_name: 'snowflake-arctic-embed:xs',
embeddings,
},
};

// Write extended definition to JSON file
const definitionPath = join(outputFolder, 'definition.json');
console.log('💾 Writing extended definition to:', definitionPath);
await fs.promises.writeFile(
definitionPath,
JSON.stringify(toolDefinition, null, 2),
);

console.log('✅ Tool processing completed successfully!');
})
.catch((e) => {
console.log('❌ Error processing tool:', e);
process.exit(1);
});
}

// Only run main when script is executed directly
if (import.meta.main) {
main();
}
console.log('📦 Starting tool processing...');
fs.promises
.readFile(entryFile, 'utf-8')
.then(async (code) => {
// Write bundled code to output file
console.log('📝 Writing bundled code to output file...');
// Ensure output folder exists
await fs.promises.mkdir(outputFolder, { recursive: true });
await fs.promises.writeFile(outputFile, code);

// Import tool definition from bundled code
console.log('📥 Importing tool definition...');
const { definition }: { definition: ToolDefinition<any> } = await import(
Deno.build.os == 'windows' ? `file://${outputFile}` : outputFile
);

console.log('✨ Tool definition loaded:', definition.name);

// Generate embeddings from tool metadata
console.log('🧮 Generating embeddings for tool metadata...');
const prompt = `${definition.id} ${definition.name} ${definition.description} ${definition.author} ${definition.keywords.join(' ')}`;
const embeddings = await getEmbeddings(prompt);

// Create extended tool definition with code and embeddings
console.log('🔨 Creating extended tool definition...');
const toolDefinition: ExtendedToolDefinition = {
...definition,
code,
embedding_metadata: {
model_name: 'snowflake-arctic-embed:xs',
embeddings,
},
};

// Write extended definition to JSON file
const definitionPath = join(outputFolder, 'definition.json');
console.log('💾 Writing extended definition to:', definitionPath);
await fs.promises.writeFile(
definitionPath,
JSON.stringify(toolDefinition, null, 2),
);

console.log('✅ Tool processing completed successfully!');
})
.catch((e) => {
console.log('❌ Error processing tool:', e);
process.exit(1);
});

0 comments on commit 14b0ff0

Please sign in to comment.