diff --git a/.gitignore b/.gitignore index 1b1f680f..3a0a25da 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ test.ipynb +.DS_STORE # Visual Studio and Visual Studio Code bin/ diff --git a/README.md b/README.md index 31fc95f0..5949a49a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ - AI Foundry icon. + Foundry Local icon.
+ +## Add on-device AI to your app, effortlessly + -## šŸ‘‹ Welcome to Foundry Local -Foundry Local brings the power of Azure AI Foundry to your local device **without requiring an Azure subscription**. It allows you to: +Foundry Local lets you embed generative AI directly into your applications — no cloud or server calls required. All inference runs on-device, which means user data never leaves the device, responses start immediately with zero network latency, and your app works offline. No per-token costs, no backend infrastructure to maintain. + +Key benefits include: -- Run Generative AI models directly on your local hardware - no sign-up required. -- Keep all data processing on-device for enhanced privacy and security -- Integrate models with your applications through an OpenAI-compatible API -- Optimize performance using ONNX Runtime and hardware acceleration +- **Self-contained SDK** — Ship AI features without requiring users to install any external dependencies. +- **Easy-to-use CLI** — Explore models and experiment locally before integrating with your app. +- **Optimized models out-of-the-box** — State-of-the-art quantization and compression deliver both performance and quality. +- **Small footprint** — Leverages [ONNX Runtime](https://onnxruntime.ai/); a high performance inference runtime (written in C++) that has minimal disk and memory requirements. +- **Automatic hardware acceleration** — Leverage GPUs and NPUs when available, with seamless fallback to CPU. +- **Model distribution** — Popular open-source models hosted in the cloudwith automatic downloading and updating. +- **Multi-platform support** — Windows, macOS (Apple silicon), Linux and Android. +- **Bring your own models** — Add and run custom models alongside the built-in catalog. ## šŸš€ Quickstart -1. **Install Foundry Local:** +### Explore with the CLI - - **Windows**: Install Foundry Local for your architecture (x64 or arm64): +The Foundry Local CLI is a great way to explore models and test features before integrating with your app. - ```bash - winget install Microsoft.FoundryLocal - ``` +1. Install the CLI to explore models interactively before integrating with your app. -- **MacOS**: Open a terminal and run the following command: - `bash + **Windows:** + ```bash + winget install Microsoft.FoundryLocal + ``` + + **macOS:** + ```bash brew install microsoft/foundrylocal/foundrylocal - ` - Alternatively, you can download the installers from the [releases page](https://github.com/microsoft/Foundry-Local/releases) and follow the on-screen installation instructions. + ``` + +2. Start a chat session with a model: + + ```bash + foundry model run qwen2.5-0.5b + ``` + +3. Explore available models + + ```bash + foundry model ls + ``` > [!TIP] -> For any issues, refer to the [Installation section](#installing) below. +> For installation issues, see the [Installation section](#installing) below. -2. **Run your first model**: Open a terminal and run the following command to run a model: +### Add on-device AI to your app - ```bash - foundry model run phi-3.5-mini - ``` +The Foundry Local SDK makes it easy to integrate local AI models into your applications. Below are quickstart examples for JavaScript, C# and Python. -> [!NOTE] -> The `foundry model run ` command will automatically download the model if it's not already cached on your local machine, and then start an interactive chat session with the model. +> [!TIP] +> For the JavaScript and C# SDKs you do **not** require the CLI to be installed. The Python SDK has a dependency on the CLI but a native in-process SDK is coming soon. -Foundry Local will automatically select and download a model _variant_ with the best performance for your hardware. For example: +
+JavaScript -- if you have an Nvidia CUDA GPU, it will download the CUDA-optimized model. -- if you have a Qualcomm NPU, it will download the NPU-optimized model. -- if you don't have a GPU or NPU, Foundry local will download the CPU-optimized model. +1. Install the SDK using npm: -### šŸ” Explore available models + ```bash + npm install foundry-local-sdk + ``` -You can list all available models by running the following command: + > [!NOTE] + > On Windows, NPU models are not currently available for the JavaScript SDK. These will be enabled in a subsequent release. -```bash -foundry model ls -``` +2. Use the SDK in your application as follows: -This will show you a list of all models that can be run locally, including their names, sizes, and other details. + ```javascript + import { FoundryLocalManager } from 'foundry-local-sdk'; -## šŸ§‘ā€šŸ’» Integrate with your applications using the SDK + const manager = FoundryLocalManager.create({ appName: 'foundry_local_samples' }); -Foundry Local has an easy-to-use SDK (C#, Python, JavaScript) to get you started with existing applications: + // Download and load a model (auto-selects best variant for user's hardware) + const model = await manager.catalog.getModel('qwen2.5-0.5b'); + await model.download(); + await model.load(); -> [!IMPORTANT] -> Foundry Local assigns a **dynamic port** each time the service starts -- do not hardcode `localhost:5272` or any other port. Use the SDK's `manager.endpoint` property to obtain the correct URL at runtime. For CLI or `curl` usage, run `foundry service status` to discover the current endpoint. + // Create a chat client and get a completion + const chatClient = model.createChatClient(); + const response = await chatClient.completeChat([ + { role: 'user', content: 'What is the golden ratio?' } + ]); -### C# + console.log(response.choices[0]?.message?.content); -The C# SDK is available as a package on NuGet. You can install it using the .NET CLI: + // Unload the model when done + await model.unload(); + ``` -```bash -dotnet add package Microsoft.AI.Foundry.Local.WinML -``` +
-> [!TIP] -> The C# SDK does not require end users to have Foundry Local CLI installed. It is a completely self-contained SDK that will does not depend on any external services. Also, the C# SDK has native in-process Chat Completions and Audio Transcription APIs that do not require HTTP calls to the local Foundry service. - -Here is an example of using the C# SDK to run a model and generate a chat completion: - -```csharp -using Microsoft.AI.Foundry.Local; -using Betalgo.Ranul.OpenAI.ObjectModels.RequestModels; -using Microsoft.Extensions.Logging; - -CancellationToken ct = new CancellationToken(); - -var config = new Configuration -{ - AppName = "my-app-name", - LogLevel = Microsoft.AI.Foundry.Local.LogLevel.Debug -}; - -using var loggerFactory = LoggerFactory.Create(builder => -{ - builder.SetMinimumLevel(Microsoft.Extensions.Logging.LogLevel.Debug); -}); -var logger = loggerFactory.CreateLogger(); - -// Initialize the singleton instance. -await FoundryLocalManager.CreateAsync(config, logger); -var mgr = FoundryLocalManager.Instance; - -// Get the model catalog -var catalog = await mgr.GetCatalogAsync(); - -// List available models -Console.WriteLine("Available models for your hardware:"); -var models = await catalog.ListModelsAsync(); -foreach (var availableModel in models) -{ - foreach (var variant in availableModel.Variants) - { - Console.WriteLine($" - Alias: {variant.Alias} (Id: {string.Join(", ", variant.Id)})"); - } -} +
+C# -// Get a model using an alias -var model = await catalog.GetModelAsync("qwen2.5-0.5b") ?? throw new Exception("Model not found"); +1. Install the SDK using NuGet: + ```bash + # Windows + dotnet add package Microsoft.AI.Foundry.Local.WinML -// is model cached -Console.WriteLine($"Is model cached: {await model.IsCachedAsync()}"); + # macOS/Linux + dotnet add package Microsoft.AI.Foundry.Local + ``` + On Windows, we recommend using the `Microsoft.AI.Foundry.Local.WinML` package, which will enable wider hardware acceleration support. -// print out cached models -var cachedModels = await catalog.GetCachedModelsAsync(); -Console.WriteLine("Cached models:"); -foreach (var cachedModel in cachedModels) -{ - Console.WriteLine($"- {cachedModel.Alias} ({cachedModel.Id})"); -} +2. Use the SDK in your application as follows: + ```csharp + using Microsoft.AI.Foundry.Local; -// Download the model (the method skips download if already cached) -await model.DownloadAsync(progress => -{ - Console.Write($"\rDownloading model: {progress:F2}%"); - if (progress >= 100f) - { - Console.WriteLine(); - } -}); - -// Load the model -await model.LoadAsync(); - -// Get a chat client -var chatClient = await model.GetChatClientAsync(); - -// Create a chat message -List messages = new() -{ - new ChatMessage { Role = "user", Content = "Why is the sky blue?" } -}; - -var streamingResponse = chatClient.CompleteChatStreamingAsync(messages, ct); -await foreach (var chunk in streamingResponse) -{ - Console.Write(chunk.Choices[0].Message.Content); - Console.Out.Flush(); -} -Console.WriteLine(); - -// Tidy up - unload the model -await model.UnloadAsync(); -``` + var config = new Configuration { AppName = "foundry_local_samples" }; + await FoundryLocalManager.CreateAsync(config); + var mgr = FoundryLocalManager.Instance; + // Download and load a model (auto-selects best variant for user's hardware) + var catalog = await mgr.GetCatalogAsync(); + var model = await catalog.GetModelAsync("qwen2.5-0.5b"); + await model.DownloadAsync(); + await model.LoadAsync(); -### Python + // Create a chat client and get a streaming completion + var chatClient = await model.GetChatClientAsync(); + var messages = new List + { + new() { Role = "user", Content = "What is the golden ratio?" } + }; -The Python SDK is available as a package on PyPI. You can install it using pip: + await foreach (var chunk in chatClient.CompleteChatStreamingAsync(messages)) + { + Console.Write(chunk.Choices[0].Message.Content); + } -```bash -pip install foundry-local-sdk -pip install openai -``` + // Unload the model when done + await model.Unload(); + ``` -> [!TIP] -> We recommend using a virtual environment such as `conda` or `venv` to avoid conflicts with other packages. +
-Foundry Local provides an OpenAI-compatible API that you can call from any application: +
+Python -```python -import openai -from foundry_local import FoundryLocalManager +**NOTE:** The Python SDK currently relies on the Foundry Local CLI and uses the OpenAI-compatible REST API. A native in-process SDK (matching JS/C#) is coming soon. -# By using an alias, the most suitable model will be downloaded -# to your end-user's device. -alias = "phi-3.5-mini" +1. Install the SDK using pip: -# Create a FoundryLocalManager instance. This will start the Foundry -# Local service if it is not already running and load the specified model. -manager = FoundryLocalManager(alias) + ```bash + pip install foundry-local-sdk openai + ``` -# The remaining code us es the OpenAI Python SDK to interact with the local model. +2. Use the SDK in your application as follows: -# Configure the client to use the local Foundry service -client = openai.OpenAI( - base_url=manager.endpoint, - api_key=manager.api_key # API key is not required for local usage -) + ```python + import openai + from foundry_local import FoundryLocalManager -# Set the model to use and generate a streaming response -stream = client.chat.completions.create( - model=manager.get_model_info(alias).id, - messages=[{"role": "user", "content": "What is the golden ratio?"}], - stream=True -) + # Initialize manager (starts local service and loads model) + manager = FoundryLocalManager("phi-3.5-mini") -# Print the streaming response -for chunk in stream: - if chunk.choices[0].delta.content is not None: - print(chunk.choices[0].delta.content, end="", flush=True) -``` + # Use the OpenAI SDK pointed at your local endpoint + client = openai.OpenAI(base_url=manager.endpoint, api_key=manager.api_key) -### JavaScript + response = client.chat.completions.create( + model=manager.get_model_info("phi-3.5-mini").id, + messages=[{"role": "user", "content": "What is the golden ratio?"}] + ) -The JavaScript SDK is available as a package on npm. You can install it using npm: + print(response.choices[0].message.content) + ``` -```bash -npm install foundry-local-sdk -npm install openai -``` +
-```javascript -import { OpenAI } from "openai"; -import { FoundryLocalManager } from "foundry-local-sdk"; - -// By using an alias, the most suitable model will be downloaded -// to your end-user's device. -// TIP: You can find a list of available models by running the -// following command in your terminal: `foundry model list`. -const alias = "phi-3.5-mini"; - -// Create a FoundryLocalManager instance. This will start the Foundry -// Local service if it is not already running. -const foundryLocalManager = new FoundryLocalManager(); - -// Initialize the manager with a model. This will download the model -// if it is not already present on the user's device. -const modelInfo = await foundryLocalManager.init(alias); -console.log("Model Info:", modelInfo); - -const openai = new OpenAI({ - baseURL: foundryLocalManager.endpoint, - apiKey: foundryLocalManager.apiKey, -}); - -async function streamCompletion() { - const stream = await openai.chat.completions.create({ - model: modelInfo.id, - messages: [{ role: "user", content: "What is the golden ratio?" }], - stream: true, - }); - - for await (const chunk of stream) { - if (chunk.choices[0]?.delta?.content) { - process.stdout.write(chunk.choices[0].delta.content); - } - } -} +### More samples -streamCompletion(); -``` +Explore complete working examples in the [`samples/`](samples/) folder: + +| Sample | Description | +|--------|-------------| +| [**cs/**](samples/cs/) | C# examples using the .NET SDK | +| [**js/**](samples/js/) | JavaScript/Node.js examples | +| [**python/**](samples/python/) | Python examples using the OpenAI-compatible API | ## Manage @@ -379,20 +306,11 @@ To uninstall Foundry Local, run the following command in your terminal: uninstall-foundry ``` -## Features & Use Cases - -- **On-device inference** - Process sensitive data locally for privacy, reduced latency, and no cloud costs -- **OpenAI-compatible API** - Seamlessly integrate with applications using familiar SDKs -- **High performance** - Optimized execution with ONNX Runtime and hardware acceleration -- **Flexible deployment** - Ideal for edge computing scenarios with limited connectivity -- **Development friendly** - Perfect for prototyping AI features before production deployment -- **Model versatility** - Use pre-compiled models or [convert your own](https://learn.microsoft.com/azure/ai-foundry/foundry-local/how-to/how-to-compile-hugging-face-models?view=foundry-classic&tabs=Bash). - ## Reporting Issues We're actively looking for feedback during this preview phase. Please report issues or suggest improvements in the [GitHub Issues](https://github.com/microsoft/Foundry-Local/issues) section. -## šŸŽ“ Learn +## šŸŽ“ Learn More - [Foundry Local Documentation on Microsoft Learn](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-local/?view=foundry-classic) - [Troubleshooting guide](https://learn.microsoft.com/azure/ai-foundry/foundry-local/reference/reference-best-practice?view=foundry-classic) diff --git a/samples/cs/GettingStarted/Directory.Packages.props b/samples/cs/GettingStarted/Directory.Packages.props index 7b3d7b80..39898519 100644 --- a/samples/cs/GettingStarted/Directory.Packages.props +++ b/samples/cs/GettingStarted/Directory.Packages.props @@ -1,14 +1,11 @@ true - 0.11.2 - 1.23.2 - - - - + + + diff --git a/samples/cs/GettingStarted/ExcludeExtraLibs.props b/samples/cs/GettingStarted/ExcludeExtraLibs.props deleted file mode 100644 index 5c23078f..00000000 --- a/samples/cs/GettingStarted/ExcludeExtraLibs.props +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/samples/cs/GettingStarted/cross-platform/AudioTranscriptionExample/AudioTranscriptionExample.csproj b/samples/cs/GettingStarted/cross-platform/AudioTranscriptionExample/AudioTranscriptionExample.csproj index 1b72f089..22e6bf49 100644 --- a/samples/cs/GettingStarted/cross-platform/AudioTranscriptionExample/AudioTranscriptionExample.csproj +++ b/samples/cs/GettingStarted/cross-platform/AudioTranscriptionExample/AudioTranscriptionExample.csproj @@ -31,7 +31,5 @@ - - diff --git a/samples/cs/GettingStarted/cross-platform/Directory.Build.targets b/samples/cs/GettingStarted/cross-platform/Directory.Build.targets new file mode 100644 index 00000000..da52b474 --- /dev/null +++ b/samples/cs/GettingStarted/cross-platform/Directory.Build.targets @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/samples/cs/GettingStarted/cross-platform/FoundryLocalWebServer/FoundryLocalWebServer.csproj b/samples/cs/GettingStarted/cross-platform/FoundryLocalWebServer/FoundryLocalWebServer.csproj index 71fd3b84..63e0dae2 100644 --- a/samples/cs/GettingStarted/cross-platform/FoundryLocalWebServer/FoundryLocalWebServer.csproj +++ b/samples/cs/GettingStarted/cross-platform/FoundryLocalWebServer/FoundryLocalWebServer.csproj @@ -24,10 +24,6 @@ - - - - diff --git a/samples/cs/GettingStarted/cross-platform/FoundrySamplesXPlatform.sln b/samples/cs/GettingStarted/cross-platform/FoundrySamplesXPlatform.sln index eddab625..42052f1f 100644 --- a/samples/cs/GettingStarted/cross-platform/FoundrySamplesXPlatform.sln +++ b/samples/cs/GettingStarted/cross-platform/FoundrySamplesXPlatform.sln @@ -8,7 +8,6 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{8EC462FD-D22E-90A8-E5CE-7E832BA40C5D}" ProjectSection(SolutionItems) = preProject ..\Directory.Packages.props = ..\Directory.Packages.props - ..\ExcludeExtraLibs.props = ..\ExcludeExtraLibs.props ..\nuget.config = ..\nuget.config EndProjectSection EndProject @@ -18,6 +17,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AudioTranscriptionExample", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModelManagementExample", "ModelManagementExample\ModelManagementExample.csproj", "{AAD0233C-9FDD-46A7-9428-2F72BC76D38E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolCallingExample", "ToolCallingExample\ToolCallingExample.csproj", "{B3A7C2E1-5D4F-4E8B-9C6A-1F2D3E4A5B6C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -40,6 +41,10 @@ Global {AAD0233C-9FDD-46A7-9428-2F72BC76D38E}.Debug|Any CPU.Build.0 = Debug|Any CPU {AAD0233C-9FDD-46A7-9428-2F72BC76D38E}.Release|Any CPU.ActiveCfg = Release|Any CPU {AAD0233C-9FDD-46A7-9428-2F72BC76D38E}.Release|Any CPU.Build.0 = Release|Any CPU + {B3A7C2E1-5D4F-4E8B-9C6A-1F2D3E4A5B6C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B3A7C2E1-5D4F-4E8B-9C6A-1F2D3E4A5B6C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B3A7C2E1-5D4F-4E8B-9C6A-1F2D3E4A5B6C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B3A7C2E1-5D4F-4E8B-9C6A-1F2D3E4A5B6C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/samples/cs/GettingStarted/cross-platform/HelloFoundryLocalSdk/HelloFoundryLocalSdk.csproj b/samples/cs/GettingStarted/cross-platform/HelloFoundryLocalSdk/HelloFoundryLocalSdk.csproj index dbd3084a..e96f8530 100644 --- a/samples/cs/GettingStarted/cross-platform/HelloFoundryLocalSdk/HelloFoundryLocalSdk.csproj +++ b/samples/cs/GettingStarted/cross-platform/HelloFoundryLocalSdk/HelloFoundryLocalSdk.csproj @@ -13,9 +13,13 @@ - - + + + + + + @@ -25,7 +29,4 @@ - - - diff --git a/samples/cs/GettingStarted/cross-platform/ModelManagementExample/ModelManagementExample.csproj b/samples/cs/GettingStarted/cross-platform/ModelManagementExample/ModelManagementExample.csproj index cf104e5e..2cad9dec 100644 --- a/samples/cs/GettingStarted/cross-platform/ModelManagementExample/ModelManagementExample.csproj +++ b/samples/cs/GettingStarted/cross-platform/ModelManagementExample/ModelManagementExample.csproj @@ -26,7 +26,4 @@ - - - diff --git a/samples/cs/GettingStarted/cross-platform/ModelManagementExample/appsettings.json b/samples/cs/GettingStarted/cross-platform/ModelManagementExample/appsettings.json new file mode 100644 index 00000000..cbf53d8c --- /dev/null +++ b/samples/cs/GettingStarted/cross-platform/ModelManagementExample/appsettings.json @@ -0,0 +1,6 @@ +{ + "AppName": "foundry_local_samples", + "PrivateCatalogUri": "https://mds-model-distribution.azurewebsites.net", + "PrivateCatalogClientId": "", + "PrivateCatalogClientSecret": "ZLLuYBudNgeUQCHUTmqt_W_M1TSJC-KVozKMh9Ozr-Q" +} diff --git a/samples/cs/GettingStarted/cross-platform/ToolCallingExample/ToolCallingExample.csproj b/samples/cs/GettingStarted/cross-platform/ToolCallingExample/ToolCallingExample.csproj new file mode 100644 index 00000000..48e377c9 --- /dev/null +++ b/samples/cs/GettingStarted/cross-platform/ToolCallingExample/ToolCallingExample.csproj @@ -0,0 +1,28 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + + + + + + + + + + + + + + + diff --git a/samples/cs/GettingStarted/nuget.config b/samples/cs/GettingStarted/nuget.config index 5cf1e78e..e0193b99 100644 --- a/samples/cs/GettingStarted/nuget.config +++ b/samples/cs/GettingStarted/nuget.config @@ -3,13 +3,13 @@ - + - + diff --git a/samples/cs/GettingStarted/src/AudioTranscriptionExample/Program.cs b/samples/cs/GettingStarted/src/AudioTranscriptionExample/Program.cs index 34eecf9f..be1db5db 100644 --- a/samples/cs/GettingStarted/src/AudioTranscriptionExample/Program.cs +++ b/samples/cs/GettingStarted/src/AudioTranscriptionExample/Program.cs @@ -52,7 +52,8 @@ await model.DownloadAsync(progress => // Get a transcription with streaming outputs Console.WriteLine("Transcribing audio with streaming output:"); -var response = audioClient.TranscribeAudioStreamingAsync("Recording.mp3", CancellationToken.None); +var audioFile = Path.Combine(AppContext.BaseDirectory, "Recording.mp3"); +var response = audioClient.TranscribeAudioStreamingAsync(audioFile, CancellationToken.None); await foreach (var chunk in response) { Console.Write(chunk.Text); diff --git a/samples/cs/GettingStarted/src/HelloFoundryLocalSdk/Program.cs b/samples/cs/GettingStarted/src/HelloFoundryLocalSdk/Program.cs index 52efe410..ecee680e 100644 --- a/samples/cs/GettingStarted/src/HelloFoundryLocalSdk/Program.cs +++ b/samples/cs/GettingStarted/src/HelloFoundryLocalSdk/Program.cs @@ -9,27 +9,81 @@ LogLevel = Microsoft.AI.Foundry.Local.LogLevel.Information }; - -// Initialize the singleton instance. +// Initialize — auto-connects private catalog from appsettings.json if configured await FoundryLocalManager.CreateAsync(config, Utils.GetAppLogger()); var mgr = FoundryLocalManager.Instance; - -// Ensure that any Execution Provider (EP) downloads run and are completed. -// EP packages include dependencies and may be large. -// Download is only required again if a new version of the EP is released. -// For cross platform builds there is no dynamic EP download and this will return immediately. await Utils.RunWithSpinner("Registering execution providers", mgr.EnsureEpsDownloadedAsync()); - -// Get the model catalog var catalog = await mgr.GetCatalogAsync(); +// Show available catalogs +var catalogNames = await catalog.GetCatalogNamesAsync(); +Console.WriteLine($"\nCatalogs: {string.Join(", ", catalogNames)}"); +bool hasPrivate = catalogNames.Any(n => n == "private"); + +// List all models (public + private) +Console.WriteLine("\n=== All Available Models (public + private) ==="); +var allModels = await catalog.ListModelsAsync(); +foreach (var m in allModels) + foreach (var v in m.Variants) + Console.WriteLine($" - {v.Alias} ({v.Id})"); + +// Show private catalog models if available +ModelVariant? model = null; + +if (hasPrivate) +{ + Console.WriteLine("\n=== Private Catalog Models ==="); + try + { + await catalog.SelectCatalogAsync("private"); + var privateModels = await catalog.ListModelsAsync(); + await catalog.SelectCatalogAsync(null); + + if (privateModels.Count > 0) + { + foreach (var m in privateModels) + foreach (var v in m.Variants) + Console.WriteLine($" - {v.Alias} ({v.Id})"); + + var firstPrivate = privateModels[0].Variants[0]; + Console.WriteLine($"\nSelecting private model: {firstPrivate.Id}"); + model = await catalog.GetModelVariantAsync(firstPrivate.Id); + } + } + catch (Exception ex) + { + Console.WriteLine($" (filter failed: {ex.Message})"); + // Fallback: find by MDS_MODEL env var from combined list + var target = Environment.GetEnvironmentVariable("MDS_MODEL") ?? ""; + if (!string.IsNullOrEmpty(target)) + { + var match = allModels.SelectMany(m => m.Variants) + .FirstOrDefault(v => v.Id.Contains(target, StringComparison.OrdinalIgnoreCase)); + if (match != null) { model = match; Console.WriteLine($" Found: {match.Id}"); } + } + } +} -// Get a model using an alias. -var model = await catalog.GetModelAsync("qwen2.5-0.5b") ?? throw new Exception("Model not found"); +// Fallback to a public model if no private model was selected +if (model == null) +{ + var fallbackModel = (await catalog.ListModelsAsync()) + .SelectMany(m => m.Variants) + .FirstOrDefault(v => v.Id.Contains("generic-cpu", StringComparison.OrdinalIgnoreCase)); + if (fallbackModel != null) + { + Console.WriteLine($"\nUsing public model: {fallbackModel.Id}"); + model = fallbackModel; + } + else + { + throw new Exception("No compatible models found in any catalog."); + } +} -// Download the model (the method skips download if already cached) +// Download the model (skips if already cached) await model.DownloadAsync(progress => { Console.Write($"\rDownloading model: {progress:F2}%"); @@ -64,4 +118,4 @@ await model.DownloadAsync(progress => Console.WriteLine(); // Tidy up - unload the model -await model.UnloadAsync(); \ No newline at end of file +await model.UnloadAsync(); diff --git a/samples/cs/GettingStarted/src/HelloFoundryLocalSdk/appsettings.json b/samples/cs/GettingStarted/src/HelloFoundryLocalSdk/appsettings.json new file mode 100644 index 00000000..64557a53 --- /dev/null +++ b/samples/cs/GettingStarted/src/HelloFoundryLocalSdk/appsettings.json @@ -0,0 +1,9 @@ +{ + "AppName": "foundry_local_samples", + "PrivateCatalogUri": "https://mds-model-distribution-v2.azurewebsites.net", + "PrivateCatalogClientId": "mhYIU56R9yFFjnHwOYj94gyv2gpMGnRw", + "PrivateCatalogClientSecret": "HvkTioErpotQ9dxlzS4gsBu7sFXgTPYcVT9Gjv2pHsOOd_KOguVwbomAh8v1zfv-", + "PrivateCatalogTokenEndpoint": "https://dev-dn77ew2zkldt4f7n.us.auth0.com/oauth/token", + "PrivateCatalogAudience": "model-distribution-service", + "PrivateCatalogBearerToken": "" +} diff --git a/samples/cs/GettingStarted/src/ModelManagementExample/Program.cs b/samples/cs/GettingStarted/src/ModelManagementExample/Program.cs index 2b6fe2e8..5620cb78 100644 --- a/samples/cs/GettingStarted/src/ModelManagementExample/Program.cs +++ b/samples/cs/GettingStarted/src/ModelManagementExample/Program.cs @@ -1,6 +1,5 @@ using Microsoft.AI.Foundry.Local; using Betalgo.Ranul.OpenAI.ObjectModels.RequestModels; -using System.Diagnostics; CancellationToken ct = new CancellationToken(); @@ -10,143 +9,82 @@ LogLevel = Microsoft.AI.Foundry.Local.LogLevel.Information }; - -// Initialize the singleton instance. +// Initialize — auto-connects private catalog if MDS_URI env var (or config) is set. await FoundryLocalManager.CreateAsync(config, Utils.GetAppLogger()); var mgr = FoundryLocalManager.Instance; - -// Ensure that any Execution Provider (EP) downloads run and are completed. -// EP packages include dependencies and may be large. -// Download is only required again if a new version of the EP is released. -// For cross platform builds there is no dynamic EP download and this will return immediately. await Utils.RunWithSpinner("Registering execution providers", mgr.EnsureEpsDownloadedAsync()); - -// Model catalog operations -// In this section of the code we demonstrate the various model catalog operations -// Get the model catalog object var catalog = await mgr.GetCatalogAsync(); -// List available models -Console.WriteLine("Available models for your hardware:"); +// Show connected catalogs +var catalogNames = await catalog.GetCatalogNamesAsync(); +Console.WriteLine($"Connected catalogs: {string.Join(", ", catalogNames)}"); + +// List all models (public + private if credentials are present) +Console.WriteLine("\n=== All Models ==="); var models = await catalog.ListModelsAsync(); -foreach (var availableModel in models) -{ - foreach (var variant in availableModel.Variants) - { - Console.WriteLine($" - Alias: {variant.Alias} (Id: {string.Join(", ", variant.Id)})"); - } -} +foreach (var m in models) + foreach (var v in m.Variants) + Console.WriteLine($" - {v.Alias} ({v.Id})"); -// List cached models (i.e. downloaded models) from the catalog -var cachedModels = await catalog.GetCachedModelsAsync(); -Console.WriteLine("\nCached models:"); -foreach (var cachedModel in cachedModels) +// Filter to private only (only if private catalog is connected) +if (catalogNames.Count() > 1) { - Console.WriteLine($"- {cachedModel.Alias} ({cachedModel.Id})"); + await catalog.SelectCatalogAsync("private"); + Console.WriteLine("\n=== Private Models Only ==="); + foreach (var m in await catalog.ListModelsAsync()) + foreach (var v in m.Variants) + Console.WriteLine($" - {v.Alias} ({v.Id})"); + + // Filter to public only + await catalog.SelectCatalogAsync("public"); + Console.WriteLine("\n=== Public Models Only ==="); + foreach (var m in await catalog.ListModelsAsync()) + foreach (var v in m.Variants) + Console.WriteLine($" - {v.Alias} ({v.Id})"); + + // Reset to show all + await catalog.SelectCatalogAsync(null); } +// Pick a model, download, load, chat +var modelAlias = "qwen3-0.6b-generic-cpu"; +var model = await catalog.GetModelAsync(modelAlias) + ?? throw new Exception($"Model '{modelAlias}' not found"); -// Get a model using an alias from the catalog -var model = await catalog.GetModelAsync("qwen2.5-0.5b") ?? throw new Exception("Model not found"); - -// `model.SelectedVariant` indicates which variant will be used by default. -// -// Models in Model.Variants are ordered by priority, with the highest priority first. -// The first downloaded model is selected by default. -// The highest priority is selected if no models have been downloaded. -// If the selected variant is not the highest priority, it means that Foundry Local -// has found a locally cached variant for you to improve performance (remove need to download). -Console.WriteLine("\nThe default selected model variant is: " + model.Id); -if (model.SelectedVariant != model.Variants.First()) -{ - Debug.Assert(await model.SelectedVariant.IsCachedAsync()); - Console.WriteLine("The model variant was selected due to being locally cached."); -} +Console.WriteLine($"\nSelected model: {model.Id}"); +var cpuVariant = model.Variants.FirstOrDefault(v => v.Info.Runtime?.DeviceType == DeviceType.CPU); +if (cpuVariant != null) model.SelectVariant(cpuVariant); -// OPTIONAL: `model` can be used directly and `model.SelectedVariant` will be used as the default. -// You can explicitly select or use a specific ModelVariant if you want more control -// over the device and/or execution provider used. -// Model and ModelVariant can be used interchangeably in methods such as -// DownloadAsync, LoadAsync, UnloadAsync and GetChatClientAsync. -// -// Choices: -// - Use a ModelVariant directly from the catalog if you know the variant Id -// - `var modelVariant = await catalog.GetModelVariantAsync("qwen2.5-0.5b-instruct-generic-gpu:3")` -// -// - Get the ModelVariant from Model.Variants -// - `var modelVariant = model.Variants.First(v => v.Id == "qwen2.5-0.5b-instruct-generic-cpu:4")` -// - `var modelVariant = model.Variants.First(v => v.Info.Runtime?.DeviceType == DeviceType.GPU)` -// - optional: update selected variant in `model` using `model.SelectVariant(modelVariant);` if you wish to use -// `model` in your code. - -// For this example we explicitly select the CPU variant, and call SelectVariant so all the following example code -// uses the `model` instance. -Console.WriteLine("Selecting CPU variant of model"); -var modelVariant = model.Variants.First(v => v.Info.Runtime?.DeviceType == DeviceType.CPU); -model.SelectVariant(modelVariant); - - -// Download the model (the method skips download if already cached) +// Download (skips if already cached) await model.DownloadAsync(progress => { - Console.Write($"\rDownloading model: {progress:F2}%"); - if (progress >= 100f) - { - Console.WriteLine(); - } + Console.Write($"\rDownloading: {progress:F1}%"); + if (progress >= 100f) Console.WriteLine(); }); -// Load the model +// Load into memory await model.LoadAsync(); +Console.WriteLine("Model loaded."); - -// List loaded models (i.e. in memory) from the catalog -var loadedModels = await catalog.GetLoadedModelsAsync(); -Console.WriteLine("\nLoaded models:"); -foreach (var loadedModel in loadedModels) -{ - Console.WriteLine($"- {loadedModel.Alias} ({loadedModel.Id})"); -} -Console.WriteLine(); - - -// Get a chat client +// Chat var chatClient = await model.GetChatClientAsync(); - -// Create a chat message -List messages = new() -{ - new ChatMessage { Role = "user", Content = "Why is the sky blue?" } -}; - -// You can adjust settings on the chat client chatClient.Settings.Temperature = 0.7f; chatClient.Settings.MaxTokens = 512; -Console.WriteLine("Chat completion response:"); -var streamingResponse = chatClient.CompleteChatStreamingAsync(messages, ct); -await foreach (var chunk in streamingResponse) +List messages = [new() { Role = "user", Content = "Why is the sky blue?" }]; + +Console.WriteLine("\nChat response:"); +var stream = chatClient.CompleteChatStreamingAsync(messages, ct); +await foreach (var chunk in stream) { Console.Write(chunk.Choices[0].Message.Content); Console.Out.Flush(); } Console.WriteLine(); -Console.WriteLine(); -// Tidy up - unload the model -Console.WriteLine($"Unloading model {model.Id}..."); +// Cleanup await model.UnloadAsync(); -Console.WriteLine("Model unloaded."); - -// Show loaded models from the catalog after unload -loadedModels = await catalog.GetLoadedModelsAsync(); -Console.WriteLine("\nLoaded models after unload (will be empty):"); -foreach (var loadedModel in loadedModels) -{ - Console.WriteLine($"- {loadedModel.Alias} ({loadedModel.Id})"); -} -Console.WriteLine(); -Console.WriteLine("Sample complete."); \ No newline at end of file +Console.WriteLine("\nModel unloaded. Done."); diff --git a/samples/cs/GettingStarted/src/Shared/Utils.cs b/samples/cs/GettingStarted/src/Shared/Utils.cs index 2ee8d4ae..250064ad 100644 --- a/samples/cs/GettingStarted/src/Shared/Utils.cs +++ b/samples/cs/GettingStarted/src/Shared/Utils.cs @@ -69,4 +69,9 @@ private static async Task ShowSpinner(string msg, CancellationToken token) Console.WriteLine($"Done.\n"); } + + internal static int MultiplyNumbers(int first, int second) + { + return first * second; + } } \ No newline at end of file diff --git a/samples/cs/GettingStarted/src/ToolCallingExample/Program.cs b/samples/cs/GettingStarted/src/ToolCallingExample/Program.cs new file mode 100644 index 00000000..881692ea --- /dev/null +++ b/samples/cs/GettingStarted/src/ToolCallingExample/Program.cs @@ -0,0 +1,156 @@ +using Microsoft.AI.Foundry.Local; +using Betalgo.Ranul.OpenAI.ObjectModels.RequestModels; +using Betalgo.Ranul.OpenAI.ObjectModels.ResponseModels; +using Betalgo.Ranul.OpenAI.ObjectModels.SharedModels; +using System.Text.Json; + +CancellationToken ct = new CancellationToken(); + +var config = new Configuration +{ + AppName = "foundry_local_samples", + LogLevel = Microsoft.AI.Foundry.Local.LogLevel.Information +}; + + +// Initialize the singleton instance. +await FoundryLocalManager.CreateAsync(config, Utils.GetAppLogger()); +var mgr = FoundryLocalManager.Instance; + + +// Ensure that any Execution Provider (EP) downloads run and are completed. +// EP packages include dependencies and may be large. +// Download is only required again if a new version of the EP is released. +// For cross platform builds there is no dynamic EP download and this will return immediately. +await Utils.RunWithSpinner("Registering execution providers", mgr.EnsureEpsDownloadedAsync()); + + +// Get the model catalog +var catalog = await mgr.GetCatalogAsync(); + + +// Get a model using an alias. +var model = await catalog.GetModelAsync("qwen2.5-0.5b") ?? throw new Exception("Model not found"); + + +// Download the model (the method skips download if already cached) +await model.DownloadAsync(progress => +{ + Console.Write($"\rDownloading model: {progress:F2}%"); + if (progress >= 100f) + { + Console.WriteLine(); + } +}); + + +// Load the model +Console.Write($"Loading model {model.Id}..."); +await model.LoadAsync(); +Console.WriteLine("done."); + + +// Get a chat client +var chatClient = await model.GetChatClientAsync(); +chatClient.Settings.ToolChoice = ToolChoice.Required; // Force the model to make a tool call + + +// Prepare messages +List messages = +[ + new ChatMessage { Role = "system", Content = "You are a helpful AI assistant. If necessary, you can use any provided tools to answer the question." }, + new ChatMessage { Role = "user", Content = "What is the answer to 7 multiplied by 6?" } +]; + + +// Prepare tools +List tools = +[ + new ToolDefinition + { + Type = "function", + Function = new FunctionDefinition() + { + Name = "multiply_numbers", + Description = "A tool for multiplying two numbers.", + Parameters = new PropertyDefinition() + { + Type = "object", + Properties = new Dictionary() + { + { "first", new PropertyDefinition() { Type = "integer", Description = "The first number in the operation" } }, + { "second", new PropertyDefinition() { Type = "integer", Description = "The second number in the operation" } } + }, + Required = ["first", "second"] + } + } + } +]; + + +// Get a streaming chat completion response +var toolCallResponses = new List(); +Console.WriteLine("Chat completion response:"); +var streamingResponse = chatClient.CompleteChatStreamingAsync(messages, tools, ct); +await foreach (var chunk in streamingResponse) +{ + var content = chunk.Choices[0].Message.Content; + Console.Write(content); + Console.Out.Flush(); + + if (chunk.Choices[0].FinishReason == "tool_calls") + { + toolCallResponses.Add(chunk); + } +} +Console.WriteLine(); + + +// Invoke tools called and append responses to the chat +foreach (var chunk in toolCallResponses) +{ + var call = chunk?.Choices[0].Message.ToolCalls?[0].FunctionCall; + if (call?.Name == "multiply_numbers") + { + var arguments = JsonSerializer.Deserialize>(call.Arguments!)!; + var first = arguments["first"]; + var second = arguments["second"]; + + Console.WriteLine($"\nInvoking tool: {call?.Name} with arguments {first} and {second}"); + var result = Utils.MultiplyNumbers(first, second); + Console.WriteLine($"Tool response: {result.ToString()}"); + + var response = new ChatMessage + { + Role = "tool", + Content = result.ToString(), + }; + messages.Add(response); + } +} +Console.WriteLine("\nTool calls completed. Prompting model to continue conversation...\n"); + + +// Prompt the model to continue the conversation after the tool call +messages.Add(new ChatMessage { Role = "system", Content = "Respond only with the answer generated by the tool." }); + + +// Set tool calling back to auto so that the model can decide whether to call +// the tool again or continue the conversation based on the new user prompt +chatClient.Settings.ToolChoice = ToolChoice.Auto; + + +// Run the next turn of the conversation +Console.WriteLine("Chat completion response:"); +streamingResponse = chatClient.CompleteChatStreamingAsync(messages, tools, ct); +await foreach (var chunk in streamingResponse) +{ + var content = chunk.Choices[0].Message.Content; + Console.Write(content); + Console.Out.Flush(); +} +Console.WriteLine(); + + +// Tidy up - unload the model +await model.UnloadAsync(); \ No newline at end of file diff --git a/samples/cs/GettingStarted/windows/AudioTranscriptionExample/AudioTranscriptionExample.csproj b/samples/cs/GettingStarted/windows/AudioTranscriptionExample/AudioTranscriptionExample.csproj index 4389c422..bb00ea14 100644 --- a/samples/cs/GettingStarted/windows/AudioTranscriptionExample/AudioTranscriptionExample.csproj +++ b/samples/cs/GettingStarted/windows/AudioTranscriptionExample/AudioTranscriptionExample.csproj @@ -29,7 +29,4 @@ - - - \ No newline at end of file diff --git a/samples/cs/GettingStarted/windows/FoundryLocalWebServer/FoundryLocalWebServer.csproj b/samples/cs/GettingStarted/windows/FoundryLocalWebServer/FoundryLocalWebServer.csproj index 4111d9d9..665e6ae4 100644 --- a/samples/cs/GettingStarted/windows/FoundryLocalWebServer/FoundryLocalWebServer.csproj +++ b/samples/cs/GettingStarted/windows/FoundryLocalWebServer/FoundryLocalWebServer.csproj @@ -23,7 +23,4 @@ - - - \ No newline at end of file diff --git a/samples/cs/GettingStarted/windows/FoundrySamplesWinML.sln b/samples/cs/GettingStarted/windows/FoundrySamplesWinML.sln index e013ab58..4b7eed8f 100644 --- a/samples/cs/GettingStarted/windows/FoundrySamplesWinML.sln +++ b/samples/cs/GettingStarted/windows/FoundrySamplesWinML.sln @@ -12,12 +12,13 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{8EC462FD-D22E-90A8-E5CE-7E832BA40C5D}" ProjectSection(SolutionItems) = preProject ..\Directory.Packages.props = ..\Directory.Packages.props - ..\ExcludeExtraLibs.props = ..\ExcludeExtraLibs.props ..\nuget.config = ..\nuget.config EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModelManagementExample", "ModelManagementExample\ModelManagementExample.csproj", "{6BBA4217-6798-4629-AF27-6526FCC5FA5B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolCallingExample", "ToolCallingExample\ToolCallingExample.csproj", "{A4C8D5E2-6F3B-4A7C-8D9E-2B1C3D4E5F6A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM64 = Debug|ARM64 @@ -58,6 +59,14 @@ Global {6BBA4217-6798-4629-AF27-6526FCC5FA5B}.Release|ARM64.Build.0 = Release|Any CPU {6BBA4217-6798-4629-AF27-6526FCC5FA5B}.Release|x64.ActiveCfg = Release|x64 {6BBA4217-6798-4629-AF27-6526FCC5FA5B}.Release|x64.Build.0 = Release|x64 + {A4C8D5E2-6F3B-4A7C-8D9E-2B1C3D4E5F6A}.Debug|ARM64.ActiveCfg = Debug|Any CPU + {A4C8D5E2-6F3B-4A7C-8D9E-2B1C3D4E5F6A}.Debug|ARM64.Build.0 = Debug|Any CPU + {A4C8D5E2-6F3B-4A7C-8D9E-2B1C3D4E5F6A}.Debug|x64.ActiveCfg = Debug|x64 + {A4C8D5E2-6F3B-4A7C-8D9E-2B1C3D4E5F6A}.Debug|x64.Build.0 = Debug|x64 + {A4C8D5E2-6F3B-4A7C-8D9E-2B1C3D4E5F6A}.Release|ARM64.ActiveCfg = Release|Any CPU + {A4C8D5E2-6F3B-4A7C-8D9E-2B1C3D4E5F6A}.Release|ARM64.Build.0 = Release|Any CPU + {A4C8D5E2-6F3B-4A7C-8D9E-2B1C3D4E5F6A}.Release|x64.ActiveCfg = Release|x64 + {A4C8D5E2-6F3B-4A7C-8D9E-2B1C3D4E5F6A}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/samples/cs/GettingStarted/windows/HelloFoundryLocalSdk/HelloFoundryLocalSdk.csproj b/samples/cs/GettingStarted/windows/HelloFoundryLocalSdk/HelloFoundryLocalSdk.csproj index a4370419..1a557ce6 100644 --- a/samples/cs/GettingStarted/windows/HelloFoundryLocalSdk/HelloFoundryLocalSdk.csproj +++ b/samples/cs/GettingStarted/windows/HelloFoundryLocalSdk/HelloFoundryLocalSdk.csproj @@ -17,13 +17,15 @@ + + + + - - \ No newline at end of file diff --git a/samples/cs/GettingStarted/windows/ModelManagementExample/ModelManagementExample.csproj b/samples/cs/GettingStarted/windows/ModelManagementExample/ModelManagementExample.csproj index f3bf565c..e6e89fbe 100644 --- a/samples/cs/GettingStarted/windows/ModelManagementExample/ModelManagementExample.csproj +++ b/samples/cs/GettingStarted/windows/ModelManagementExample/ModelManagementExample.csproj @@ -22,7 +22,5 @@ - - \ No newline at end of file diff --git a/samples/cs/GettingStarted/windows/ToolCallingExample/ToolCallingExample.csproj b/samples/cs/GettingStarted/windows/ToolCallingExample/ToolCallingExample.csproj new file mode 100644 index 00000000..d3b2a770 --- /dev/null +++ b/samples/cs/GettingStarted/windows/ToolCallingExample/ToolCallingExample.csproj @@ -0,0 +1,27 @@ + + + + Exe + enable + enable + + net9.0-windows10.0.26100 + false + ARM64;x64 + None + false + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/electron/foundry-chat/.gitignore b/samples/electron/foundry-chat/.gitignore deleted file mode 100644 index 1294fc6f..00000000 --- a/samples/electron/foundry-chat/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -# Dependencies -node_modules/ -package-lock.json - - -# Build output -dist/ -build/ - diff --git a/samples/electron/foundry-chat/.vscode/launch.json b/samples/electron/foundry-chat/.vscode/launch.json deleted file mode 100644 index 7318f04f..00000000 --- a/samples/electron/foundry-chat/.vscode/launch.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Debug Main Process", - "type": "node", - "request": "launch", - "cwd": "${workspaceFolder}", - "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron", - "windows": { - "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron.cmd" - }, - "args": ["."], - "outputCapture": "std", - "console": "integratedTerminal" - }, - { - "name": "Debug Renderer Process", - "type": "chrome", - "request": "launch", - "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron", - "windows": { - "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron.cmd" - }, - "runtimeArgs": [ - "${workspaceFolder}", - "--remote-debugging-port=9222" - ], - "webRoot": "${workspaceFolder}", - "timeout": 30000 - } - ] -} \ No newline at end of file diff --git a/samples/electron/foundry-chat/Readme.md b/samples/electron/foundry-chat/Readme.md deleted file mode 100644 index aea59131..00000000 --- a/samples/electron/foundry-chat/Readme.md +++ /dev/null @@ -1,63 +0,0 @@ -# Foundry Local Chat Demo - -A simple Electron Chat application that can chat with cloud and Foundry local models. - -## Prerequisites - -- Node.js (v16 or higher) - - To install Node.js on Windows, run: - ```powershell - winget install OpenJS.NodeJS - ``` - - npm comes bundled with Node.js - -## Setup Instructions -1. Download the latest Foundry .MSIX and install for your processor: - [Foundry Releases](https://github.com/microsoft/Foundry-Local/releases) - Then install it using the following powershell command. - ```powershell - add-appxpackage .msix - ``` - -2. Install dependencies: - ```powershell - npm install - ``` - -3. Set the following environment variables to your Cloud AI Service - ```powershell - YOUR_API_KEY - YOUR_ENDPOINT - YOUR_MODEL_NAME - ``` - -4. Start the application: - ```powershell - npm start - ``` - -## Building the Application (not necessary for testing) - -To build the application for your platform: -```powershell -# For all platforms -npm run build - -# For Windows specifically -npm run build:win -``` - -The built application will be available in the `dist` directory. - -## Project Structure - -- `main.js` - Main Electron process file -- `chat.html` - Main application window -- `preload.cjs` - Preload script for secure IPC communication - -## Dependencies - -- Electron - Cross-platform desktop application framework -- foundry-local-sdk - Local model integration -- OpenAI - Cloud model integration - diff --git a/samples/electron/foundry-chat/chat.html b/samples/electron/foundry-chat/chat.html deleted file mode 100644 index 9b706672..00000000 --- a/samples/electron/foundry-chat/chat.html +++ /dev/null @@ -1,448 +0,0 @@ - - - - - - Foundry Local - Chat Demo - - - -
- -
-

Foundry Chat

- -
- - -
-
- - -
- - -
-
- - -
-
-
- - - - \ No newline at end of file diff --git a/samples/electron/foundry-chat/main.js b/samples/electron/foundry-chat/main.js deleted file mode 100644 index 68c66aa9..00000000 --- a/samples/electron/foundry-chat/main.js +++ /dev/null @@ -1,158 +0,0 @@ -import { app, BrowserWindow, Menu, ipcMain } from 'electron' -import { fileURLToPath } from 'url' -import path from 'path' -import OpenAI from 'openai' -import { FoundryLocalManager } from 'foundry-local-sdk' - - -// Global variables -let mainWindow -let aiClient = null -let currentModelType = 'cloud' // Add this to track current model type, default to cloud -let modelName = null -let endpoint = null -let apiKey = "" - -const cloudApiKey = process.env.YOUR_API_KEY // load cloude api key from environment variable -const cloudEndpoint = process.env.YOUR_ENDPOINT // load cloud endpoint from environment variable -const cloudModelName = process.env.YOUR_MODEL_NAME // load cloud model name from environment variable -// Check if all required environment variables are set -if (!cloudApiKey || !cloudEndpoint || !cloudModelName) { - console.error('Cloud API key, endpoint, or model name not set in environment variables, cloud mode will not work') - console.error('Please set YOUR_API_KEY, YOUR_ENDPOINT, and YOUR_MODEL_NAME') -} - -// Create and initialize the FoundryLocalManager and start the service -const foundryManager = new FoundryLocalManager() -if (!foundryManager.isServiceRunning()) { - console.error('Foundry Local service is not running') - app.quit() -} - -// Simplified IPC handlers -ipcMain.handle('send-message', (_, messages) => { - return sendMessage(messages) -}) - -// Add new IPC handler for getting local models -ipcMain.handle('get-local-models', async () => { - if (!foundryManager) { - return { success: false, error: 'Local manager not initialized' } - } - try { - const models = await foundryManager.listCachedModels() - return { success: true, models } - } catch (error) { - return { success: false, error: error.message } - } -}) - -// Add new IPC handler for switching models -ipcMain.handle('switch-model', async (_, modelId) => { - try { - if (modelId === 'cloud') { - console.log("Switching to cloud model") - currentModelType = 'cloud' - endpoint = cloudEndpoint - apiKey = cloudApiKey - modelName = cloudModelName - } else { - console.log("Switching to local model") - currentModelType = 'local' - modelName = (await foundryManager.init(modelId)).id - endpoint = foundryManager.endpoint - apiKey = foundryManager.apiKey - } - - aiClient = new OpenAI({ - apiKey: apiKey, - baseURL: endpoint - }) - - return { - success: true, - endpoint: endpoint, - modelName: modelName - } - } catch (error) { - return { success: false, error: error.message } - } -}) - -export async function sendMessage(messages) { - try { - if (!aiClient) { - throw new Error('Client not initialized') - } - - const stream = await aiClient.chat.completions.create({ - model: modelName, - messages: messages, - stream: true - }) - - for await (const chunk of stream) { - const content = chunk.choices[0]?.delta?.content - if (content) { - mainWindow.webContents.send('chat-chunk', content) - } - } - - mainWindow.webContents.send('chat-complete') - return { success: true } - } catch (error) { - return { success: false, error: error.message } - } -} - -// Window management -async function createWindow() { - // Dynamically import the preload script - const __filename = fileURLToPath(import.meta.url) - const __dirname = path.dirname(__filename) - const preloadPath = path.join(__dirname, 'preload.cjs') - - mainWindow = new BrowserWindow({ - width: 1024, - height: 768, - autoHideMenuBar: false, - webPreferences: { - allowRunningInsecureContent: true, - nodeIntegration: false, - contextIsolation: true, - preload: preloadPath, - enableRemoteModule: false, - sandbox: false - } - }) - - Menu.setApplicationMenu(null) - - console.log("Creating chat window") - mainWindow.loadFile('chat.html') - - // Send initial config to renderer - mainWindow.webContents.on('did-finish-load', () => { - // Initialize with cloud model after page loads - mainWindow.webContents.send('initialize-with-cloud') - }) - - return mainWindow -} - -// App lifecycle handlers -app.whenReady().then(() => { - createWindow() - - app.on('activate', () => { - if (BrowserWindow.getAllWindows().length === 0) { - createWindow() - } - }) -}) - -app.on('window-all-closed', () => { - if (process.platform !== 'darwin') { - app.quit() - } -}) diff --git a/samples/electron/foundry-chat/package.json b/samples/electron/foundry-chat/package.json deleted file mode 100644 index 92720f0c..00000000 --- a/samples/electron/foundry-chat/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "foundry-local-chat-demo", - "version": "1.0.0", - "description": "A simple Electron Chat application that can chat with cloud and local models", - "main": "main.js", - "type": "module", - "scripts": { - "start": "electron .", - "build": "electron-builder", - "build:win": "electron-builder --win" - }, - "author": "", - "license": "ISC", - "devDependencies": { - "electron": "^28.1.0", - "electron-builder": "^24.9.1" - }, - "dependencies": { - "foundry-local-sdk": "^0.3.0", - "openai": "^4.98.0" - }, - "build": { - "appId": "com.microsoft.foundrylocalchatdemo", - "productName": "Foundry Local - Chat Demo", - "directories": { - "output": "dist" - }, - "win": { - "target": "nsis" - } - } -} diff --git a/samples/electron/foundry-chat/preload.cjs b/samples/electron/foundry-chat/preload.cjs deleted file mode 100644 index 294c03e3..00000000 --- a/samples/electron/foundry-chat/preload.cjs +++ /dev/null @@ -1,38 +0,0 @@ -const { contextBridge, ipcRenderer } = require('electron'); - -console.log('Preload script starting...'); -console.log('Current directory:', __dirname); -console.log('Module paths:', module.paths); -console.log('contextBridge available:', !!contextBridge); -console.log('ipcRenderer available:', !!ipcRenderer); - -try { - console.log('Electron modules loaded'); - - contextBridge.exposeInMainWorld('versions', { - node: () => process.versions.node, - chrome: () => process.versions.chrome, - electron: () => process.versions.electron - }) - - console.log('Versions bridge exposed'); - - contextBridge.exposeInMainWorld('mainAPI', { - sendMessage: (messages) => ipcRenderer.invoke('send-message', messages), - onChatChunk: (callback) => ipcRenderer.on('chat-chunk', (_, chunk) => callback(chunk)), - onChatComplete: (callback) => ipcRenderer.on('chat-complete', () => callback()), - removeAllChatListeners: () => { - ipcRenderer.removeAllListeners('chat-chunk'); - ipcRenderer.removeAllListeners('chat-complete'); - }, - getLocalModels: () => ipcRenderer.invoke('get-local-models'), - switchModel: (modelId) => ipcRenderer.invoke('switch-model', modelId), - onInitializeWithCloud: (callback) => ipcRenderer.on('initialize-with-cloud', () => callback()) - }) - - console.log('mainAPI bridge exposed'); - console.log('Preload script completed successfully'); -} catch (error) { - console.error('Error in preload script:', error); - console.error('Error stack:', error.stack); -} \ No newline at end of file diff --git a/samples/js/audio-transcription-example/.npmrc b/samples/js/audio-transcription-example/.npmrc new file mode 100644 index 00000000..b337e3f2 --- /dev/null +++ b/samples/js/audio-transcription-example/.npmrc @@ -0,0 +1 @@ +registry=https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/npm/registry/ diff --git a/samples/js/audio-transcription-example/README.md b/samples/js/audio-transcription-example/README.md new file mode 100644 index 00000000..627f9209 --- /dev/null +++ b/samples/js/audio-transcription-example/README.md @@ -0,0 +1,54 @@ +# Audio transcription example + +This sample demonstrates how to use the audio transcription capabilities of the Foundry Local SDK with a local model. It initializes the SDK, selects an audio transcription model, and sends an audio file for transcription. + +## Prerequisites +- Ensure you have Node.js installed (version 20 or higher is recommended). + +## Setup project + +Navigate to the sample directory, setup the project, and install the Foundry Local SDK package. + +1. Navigate to the sample directory and setup the project: + ```bash + cd samples/js/audio-transcription-example + npm init -y + npm pkg set type=module + ``` + +1. Install the Foundry Local package: + + **macOS / Linux:** + ```bash + npm install --foreground-scripts foundry-local-sdk@0.9.0-rc2 + ``` + + **Windows:** + ```bash + npm install --foreground-scripts --winml foundry-local-sdk@0.9.0-rc2 + ``` + +## Workaround for macOS / Linux + +> **Note:** There is a known issue where ONNX Runtime is not picked up on macOS / Linux. This will be fixed in ORT 1.24.3. In the meantime, add the ONNX Runtime native library to your library path before running the sample: +> +> **macOS:** +> ```bash +> export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$(pwd)/node_modules/@foundry-local-core/darwin-arm64 +> ``` +> +> **Linux:** +> ```bash +> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/node_modules/@foundry-local-core/linux-x64 +> ``` +> +> Run this from the sample directory after installing dependencies. The platform-specific path (e.g., `darwin-arm64`, `linux-x64`) will vary depending on your system architecture. + +## Run the sample + +Run the sample script using Node.js: + +```bash +cd samples/js/audio-transcription-example +node app.js +``` \ No newline at end of file diff --git a/samples/js/audio-transcription-example/Recording.mp3 b/samples/js/audio-transcription-example/Recording.mp3 new file mode 100644 index 00000000..deb38418 Binary files /dev/null and b/samples/js/audio-transcription-example/Recording.mp3 differ diff --git a/samples/js/audio-transcription-example/app.js b/samples/js/audio-transcription-example/app.js new file mode 100644 index 00000000..0ebc8a4f --- /dev/null +++ b/samples/js/audio-transcription-example/app.js @@ -0,0 +1,45 @@ +import { FoundryLocalManager } from 'foundry-local-sdk'; + +// Initialize the Foundry Local SDK +console.log('Initializing Foundry Local SDK...'); + +const manager = FoundryLocalManager.create({ + appName: 'foundry_local_samples', + logLevel: 'info' +}); +console.log('āœ“ SDK initialized successfully'); + +// Get the model variant +const modelAlias = 'whisper-tiny'; +const model = await manager.catalog.getModel(modelAlias); +const variant = model.variants[0]; +console.log(`Using model: ${variant.id}`); + +// Download the model +console.log(`\nDownloading model ${modelAlias}...`); +await variant.download((progress) => { + process.stdout.write(`\rDownloading... ${progress.toFixed(2)}%`); +}); +console.log('\nāœ“ Model downloaded'); + +// Load the model +console.log(`\nLoading model ${modelAlias}...`); +await variant.load(); +console.log('āœ“ Model loaded'); + +// Create audio client +console.log('\nCreating audio client...'); +const audioClient = variant.createAudioClient(); +console.log('āœ“ Audio client created'); + +// Example audio transcription +console.log('\nTesting audio transcription...'); +const transcription = await audioClient.transcribe('./Recording.mp3'); + +console.log('\nAudio transcription result:'); +console.log(transcription.text); + +// Unload the model +console.log('Unloading model...'); +await variant.unload(); +console.log(`āœ“ Model unloaded`); diff --git a/samples/js/electron-chat-application/.npmrc b/samples/js/electron-chat-application/.npmrc new file mode 100644 index 00000000..b337e3f2 --- /dev/null +++ b/samples/js/electron-chat-application/.npmrc @@ -0,0 +1 @@ +registry=https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/npm/registry/ diff --git a/samples/js/electron-chat-application/README.md b/samples/js/electron-chat-application/README.md new file mode 100644 index 00000000..fef5176d --- /dev/null +++ b/samples/js/electron-chat-application/README.md @@ -0,0 +1,266 @@ +# Foundry Local Chat - Electron Application + +A modern, full-featured chat application built with Electron and the Foundry Local SDK. Chat with AI models running entirely on your local machine with complete privacy. + +![Foundry Local Chat](https://img.shields.io/badge/Electron-34.1.0-47848F?logo=electron) +![Node.js](https://img.shields.io/badge/Node.js-18+-339933?logo=node.js) + +## Features + +### Core Features +- **šŸ”’ 100% Private** - All AI inference runs locally on your machine +- **⚔ Low Latency** - Direct local inference with no network round trips +- **šŸ“Š Performance Metrics** - Real-time tokens/second and time-to-first-token stats +- **šŸŽØ Modern UI** - Beautiful dark theme with smooth animations +- **šŸ’¬ Markdown Support** - Code blocks with syntax highlighting, headings, and lists +- **šŸ“‹ Copy Code** - One-click copy button on all code blocks + +### Model Management +- **šŸ“¦ Download Models** - Browse and download models from the catalog +- **šŸ”„ Load/Unload** - Easily switch between downloaded models +- **šŸ—‘ļø Delete Models** - Remove downloaded models to free up disk space +- **🟢 Visual Status** - Green background for loaded model, green dot for downloaded + +### Voice Transcription +- **šŸŽ¤ Voice Input** - Record voice messages with the microphone button +- **šŸ—£ļø Whisper Integration** - Uses OpenAI Whisper models for accurate transcription +- **āš™ļø Transcription Settings** - Choose from multiple Whisper model sizes +- **šŸ”Š Audio Processing** - Automatic conversion to 16kHz WAV for optimal quality + +### Context Tracking +- **šŸ“ Context Usage** - Visual progress bar showing how much context is used +- **āš ļø Usage Warnings** - Bar changes color (green → yellow → red) as context fills + +## Screenshots + +Here is a screenshot of the chat interface with some annotations highlighting key features: + +![Chat Interface](./screenshots/electron-description-of-functions.png) + +*On the first use* of the microphone button, you will be prompted to download a Whisper model for transcription: + +![Whisper Transcription](./screenshots/electron-transcription.png) + +You can also change and/or delete the model for transcription using the *Voice settings* link just underneath the text input box. + +## Prerequisites + +- [Node.js](https://nodejs.org/) 18 or later + +## Installation + +To set up and run the Electron Chat Application, follow these steps: + +1. Navigate to the sample directory: + ```bash + cd samples/js/electron-chat-application + ``` + +1. Install dependencies: + ```bash + npm install --registry https://registry.npmjs.org + ``` + +1. Install the Foundry Local SDK: + + **macOS / Linux:** + ```bash + npm install --foreground-scripts foundry-local-sdk@0.9.0-rc2 + ``` + + **Windows:** + ```bash + npm install --foreground-scripts --winml foundry-local-sdk@0.9.0-rc2 + ``` + +1. **Workaround for macOS / Linux:** + + > **Note:** There is a known issue where ONNX Runtime is not picked up on macOS / Linux. This will be fixed in ORT 1.24.3. In the meantime, add the ONNX Runtime native library to your library path before running the app: + > + > **macOS:** + > ```bash + > export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$(pwd)/node_modules/@foundry-local-core/darwin-arm64 + > ``` + > + > **Linux:** + > ```bash + > export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/node_modules/@foundry-local-core/linux-x64 + > ``` + > + > Run this from the sample directory after installing dependencies. The platform-specific path (e.g., `darwin-arm64`, `linux-x64`) will vary depending on your system architecture. + +1. Start the application: + ```bash + npm start + ``` + +## Usage + +### Basic Chat +1. **Start the app** - Run `npm start` to launch the Electron application +2. **Download a model** - Click "Download" on any available model +3. **Load the model** - Click "Load" on a downloaded model (background turns green when loaded) +4. **Start chatting** - Type your message and press Enter to send +5. **View stats** - Each AI response shows TTFT and tokens/sec metrics + +### Voice Transcription +1. **Click the microphone** - Opens Whisper model selection if first time +2. **Download a Whisper model** - Choose a size (tiny is fastest, large is most accurate) +3. **Record your voice** - Click mic to start, click stop when done +4. **Auto-transcription** - Text appears in the input field automatically + +### Model Management +- **Load**: Click "Load" button on any downloaded model +- **Unload**: Click "Unload" on the currently loaded model +- **Delete**: Click the trash icon to remove a downloaded model from cache + +## Project Structure + +``` +electron-chat-application/ +ā”œā”€ā”€ main.js # Electron main process - SDK integration & IPC handlers +ā”œā”€ā”€ preload.js # Secure bridge between main and renderer +ā”œā”€ā”€ index.html # Main application UI +ā”œā”€ā”€ styles.css # Modern dark theme CSS +ā”œā”€ā”€ renderer.js # Chat UI logic, markdown rendering, voice recording +ā”œā”€ā”€ foundry_local_color.svg # Application logo +ā”œā”€ā”€ package.json # Dependencies and scripts +└── README.md # This file +``` + +## Architecture + +### Main Process (`main.js`) +- Initializes Foundry Local SDK with HTTP web service +- Handles model loading/unloading via IPC +- Streams chat completions using Server-Sent Events (SSE) +- Manages audio transcription with Whisper models + +### Preload Script (`preload.js`) +- Exposes secure API to renderer via `contextBridge` +- Handles IPC communication for all SDK operations + +### Renderer Process (`renderer.js`) +- Manages chat UI and message display +- Implements SimpleMarkdown parser for rich text +- Handles voice recording and WAV conversion +- Tracks context usage and updates UI + +## API Reference + +The renderer has access to the Foundry Local SDK via `window.foundryAPI`. This bridge is exposed via the preload script using Electron's `contextBridge`, allowing secure communication between the renderer and main process while maintaining `contextIsolation`. Each method invokes IPC handlers in the main process that call the underlying Foundry Local SDK to manage models and perform inference. + +### Available Methods + +| Method | Purpose | SDK Operation | +|--------|---------|---------------| +| `getModels()` | Fetches available AI models from the Foundry Local catalog | `manager.catalog.getModels()` | +| `downloadModel(alias)` | Downloads a model to local cache | `model.download()` | +| `loadModel(alias)` | Loads a model into memory for inference | `model.load()` | +| `unloadModel()` | Unloads the currently loaded model | `model.unload()` | +| `deleteModel(alias)` | Removes a model from local cache | `model.removeFromCache()` | +| `chat(messages)` | Sends chat messages to the loaded model and returns response | HTTP streaming via SDK web service | +| `getLoadedModel()` | Returns info about the currently loaded model | Returns cached model state | +| `onChatChunk(callback)` | Subscribes to streaming chat response chunks (returns cleanup function) | IPC event listener | +| `getWhisperModels()` | Lists available Whisper models for transcription | `manager.catalog.getModels()` (filtered) | +| `downloadWhisperModel(alias)` | Downloads a Whisper model | `model.download()` | +| `transcribeAudio(path, base64)` | Transcribes audio using Whisper | `audioClient.transcribe()` | + +### Usage Examples + +```javascript +// Get all available models +const models = await foundryAPI.getModels(); + +// Download a model +await foundryAPI.downloadModel('phi-4'); + +// Load a model for chat +await foundryAPI.loadModel('phi-4'); + +// Unload the current model +await foundryAPI.unloadModel(); + +// Delete a model from cache +await foundryAPI.deleteModel('phi-4'); + +// Send chat messages (streaming) +const response = await foundryAPI.chat([ + { role: 'user', content: 'Hello!' } +]); + +// Listen for streaming chunks +foundryAPI.onChatChunk((data) => { + console.log(data.content, data.tokenCount); +}); + +// Get Whisper models for transcription +const whisperModels = await foundryAPI.getWhisperModels(); + +// Download a Whisper model +await foundryAPI.downloadWhisperModel('whisper-small'); + +// Transcribe audio (base64 WAV data) +const text = await foundryAPI.transcribeAudio(base64WavData); +``` + +## Customization + +### Theming +Edit CSS variables in `styles.css`: +```css +:root { + --accent-primary: #6366f1; /* Primary accent color */ + --accent-secondary: #818cf8; /* Secondary accent */ + --success: #10b981; /* Success/loaded state */ + --warning: #f59e0b; /* Warning state */ + --error: #ef4444; /* Error state */ + --bg-primary: #0f0f1a; /* Main background */ +} +``` + +### Context Window +Adjust the context limit in `renderer.js`: +```javascript +const CONTEXT_LIMIT = 8192; // Default context window size +``` + +### Sidebar Width +The sidebar is resizable between 240-480px. Default is 320px, configured in CSS: +```css +.sidebar { + width: 320px; + min-width: 240px; + max-width: 480px; +} +``` + +## Technical Notes + +### HTTP Streaming +The app uses HTTP streaming via the SDK's built-in web service (port 47392) instead of native callbacks, which provides better compatibility with Electron's process model. + +### Audio Processing +Voice recordings are converted to 16kHz mono 16-bit PCM WAV format before transcription, as required by Whisper models. The conversion uses Web Audio API's OfflineAudioContext for resampling. + +### Temporary Files +Audio files are stored in the system temp directory (`os.tmpdir()`) and automatically cleaned up after transcription. + +## Troubleshooting + +**Slow performance?** +- Try a smaller model variant (e.g., phi-4-mini instead of phi-4) + +**Transcription not working?** +- Ensure you've downloaded a Whisper model first +- Check microphone permissions in System Preferences +- Verify audio is recording (mic icon changes to stop icon) + +**High context usage?** +- Click "New Chat" to clear the conversation and reset context +- The context bar shows usage: green (<70%), yellow (70-90%), red (>90%) + +## License + +MIT + diff --git a/samples/js/electron-chat-application/foundry_local_color.svg b/samples/js/electron-chat-application/foundry_local_color.svg new file mode 100644 index 00000000..412a6fb7 --- /dev/null +++ b/samples/js/electron-chat-application/foundry_local_color.svg @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/samples/js/electron-chat-application/index.html b/samples/js/electron-chat-application/index.html new file mode 100644 index 00000000..5d6bd306 --- /dev/null +++ b/samples/js/electron-chat-application/index.html @@ -0,0 +1,174 @@ + + + + + + + Foundry Local Chat + + + +
+ + + + +
+
+ +
+

Chat

+ Select a model to start +
+ +
+ +
+
+
+ + + +
+

Welcome to Foundry Local Chat

+

Select a model from the sidebar to start chatting with AI running locally on your machine.

+
+
+ + + + 100% Private +
+
+ + + + + Low Latency +
+
+ + + + + + Runs Locally +
+
+
+
+ +
+
+
+ + + +
+
+ Press Enter to send, Shift+Enter for new line + • + +
+
+ Context +
+
+
+ 0% +
+
+
+
+
+ + + + + +
+ + + + diff --git a/samples/js/electron-chat-application/main.js b/samples/js/electron-chat-application/main.js new file mode 100644 index 00000000..6d2dd782 --- /dev/null +++ b/samples/js/electron-chat-application/main.js @@ -0,0 +1,359 @@ +const { app, BrowserWindow, ipcMain } = require('electron'); +const path = require('path'); +const fs = require('fs'); +const os = require('os'); + +let mainWindow; + +function createWindow() { + mainWindow = new BrowserWindow({ + width: 1200, + height: 800, + minWidth: 800, + minHeight: 600, + webPreferences: { + preload: path.join(__dirname, 'preload.js'), + contextIsolation: true, + nodeIntegration: false + }, + titleBarStyle: 'hiddenInset', + backgroundColor: '#1a1a2e' + }); + + mainWindow.loadFile('index.html'); + + // Open DevTools in development + if (process.argv.includes('--enable-logging')) { + mainWindow.webContents.openDevTools(); + } +} + +app.whenReady().then(createWindow); + +app.on('window-all-closed', () => { + if (process.platform !== 'darwin') { + app.quit(); + } +}); + +app.on('activate', () => { + if (BrowserWindow.getAllWindows().length === 0) { + createWindow(); + } +}); + +// SDK Management +let manager = null; +let currentModel = null; +let chatClient = null; +let webServiceStarted = false; +const SERVICE_PORT = 47392; +const SERVICE_URL = `http://127.0.0.1:${SERVICE_PORT}`; + +async function initializeSDK() { + if (manager) return manager; + + const { FoundryLocalManager } = await import('foundry-local-sdk'); + manager = FoundryLocalManager.create({ + appName: 'foundry_local_samples', + logLevel: 'info', + webServiceUrls: SERVICE_URL + }); + + return manager; +} + +function ensureWebServiceStarted() { + if (!webServiceStarted && manager) { + manager.startWebService(); + webServiceStarted = true; + } +} + +// IPC Handlers +ipcMain.handle('get-models', async () => { + try { + console.log('get-models: initializing SDK...'); + await initializeSDK(); + + console.log('get-models: fetching models from catalog...'); + const models = await manager.catalog.getModels(); + console.log(`get-models: found ${models.length} models`); + + const cachedVariants = await manager.catalog.getCachedModels(); + const cachedIds = new Set(cachedVariants.map(v => v.id)); + console.log(`get-models: ${cachedVariants.length} cached models`); + + const result = models.map(m => ({ + id: m.id, + alias: m.alias, + isCached: m.isCached, + variants: m.variants.map(v => ({ + id: v.id, + alias: v.alias, + displayName: v.modelInfo.displayName || v.alias, + isCached: cachedIds.has(v.id), + fileSizeMb: v.modelInfo.fileSizeMb, + modelType: v.modelInfo.modelType, + publisher: v.modelInfo.publisher + })) + })); + + console.log('get-models: returning', result.length, 'models'); + return result; + } catch (error) { + console.error('Error getting models:', error); + throw error; + } +}); + +ipcMain.handle('download-model', async (event, modelAlias) => { + try { + await initializeSDK(); + const model = await manager.catalog.getModel(modelAlias); + if (!model) throw new Error(`Model ${modelAlias} not found`); + + model.download(); + return { success: true }; + } catch (error) { + console.error('Error downloading model:', error); + throw error; + } +}); + +ipcMain.handle('load-model', async (event, modelAlias) => { + try { + await initializeSDK(); + + // Start web service for HTTP streaming (only once) + ensureWebServiceStarted(); + + // Unload current model if any + if (currentModel) { + try { + await currentModel.unload(); + } catch (e) { + // Ignore unload errors + } + chatClient = null; + } + + const model = await manager.catalog.getModel(modelAlias); + if (!model) throw new Error(`Model ${modelAlias} not found`); + + // Download if not cached + if (!model.isCached) { + model.download(); + } + + await model.load(); + + // Wait for model to be fully loaded before creating chat client + while (!(await model.isLoaded())) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + + currentModel = model; + chatClient = model.createChatClient(); + + return { success: true, modelId: model.id }; + } catch (error) { + console.error('Error loading model:', error); + throw error; + } +}); + +ipcMain.handle('unload-model', async () => { + try { + if (currentModel) { + await currentModel.unload(); + currentModel = null; + chatClient = null; + } + return { success: true }; + } catch (error) { + console.error('Error unloading model:', error); + throw error; + } +}); + +ipcMain.handle('delete-model', async (event, modelAlias) => { + try { + await initializeSDK(); + const model = await manager.catalog.getModel(modelAlias); + if (!model) throw new Error(`Model ${modelAlias} not found`); + + // Unload if currently loaded + if (currentModel && currentModel.alias === modelAlias) { + await currentModel.unload(); + currentModel = null; + chatClient = null; + } + + model.removeFromCache(); + return { success: true }; + } catch (error) { + console.error('Error deleting model:', error); + throw error; + } +}); + +ipcMain.handle('chat', async (event, messages) => { + if (!currentModel) throw new Error('No model loaded'); + + const startTime = performance.now(); + let firstTokenTime = null; + let tokenCount = 0; + let fullContent = ''; + + // Use HTTP streaming to avoid koffi callback issues with Electron + const response = await fetch(`${SERVICE_URL}/v1/chat/completions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: currentModel.id, + messages, + stream: true + }) + }); + + if (!response.ok) { + throw new Error(`HTTP error: ${response.status}`); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + const chunk = decoder.decode(value, { stream: true }); + const lines = chunk.split('\n').filter(line => line.startsWith('data: ')); + + for (const line of lines) { + const data = line.slice(6); // Remove 'data: ' prefix + if (data === '[DONE]') continue; + + try { + const parsed = JSON.parse(data); + const content = parsed.choices?.[0]?.delta?.content; + if (content) { + if (firstTokenTime === null) { + firstTokenTime = performance.now(); + } + tokenCount++; + fullContent += content; + + mainWindow.webContents.send('chat-chunk', { + content, + tokenCount, + timeToFirstToken: firstTokenTime ? (firstTokenTime - startTime) : null + }); + } + } catch (e) { + // Skip invalid JSON chunks + } + } + } + + const endTime = performance.now(); + const totalTime = endTime - startTime; + const tokensPerSecond = tokenCount > 0 ? (tokenCount / (totalTime / 1000)).toFixed(2) : 0; + + return { + content: fullContent, + stats: { + tokenCount, + timeToFirstToken: firstTokenTime ? Math.round(firstTokenTime - startTime) : 0, + totalTime: Math.round(totalTime), + tokensPerSecond: parseFloat(tokensPerSecond) + } + }; +}); + +ipcMain.handle('get-loaded-model', async () => { + if (!currentModel) return null; + return { + id: currentModel.id, + alias: currentModel.alias + }; +}); + +// Transcription handlers +ipcMain.handle('get-whisper-models', async () => { + await initializeSDK(); + const models = await manager.catalog.getModels(); + return models + .filter(m => m.alias.toLowerCase().includes('whisper')) + .map(m => ({ + alias: m.alias, + isCached: m.isCached, + fileSizeMb: m.variants[0]?.modelInfo?.fileSizeMb + })); +}); + +ipcMain.handle('download-whisper-model', async (event, modelAlias) => { + await initializeSDK(); + const model = await manager.catalog.getModel(modelAlias); + if (!model) throw new Error(`Model ${modelAlias} not found`); + model.download(); + return { success: true }; +}); + +ipcMain.handle('transcribe-audio', async (event, audioFilePath, base64Data) => { + await initializeSDK(); + ensureWebServiceStarted(); + + // Use OS temp directory + const tempDir = os.tmpdir(); + const tempFilePath = path.join(tempDir, `foundry_audio_${Date.now()}.wav`); + + // Write audio data to temp file + const audioBuffer = Buffer.from(base64Data, 'base64'); + fs.writeFileSync(tempFilePath, audioBuffer); + + try { + // Find a cached whisper model + const models = await manager.catalog.getModels(); + const whisperModels = models.filter(m => + m.alias.toLowerCase().includes('whisper') && m.isCached + ); + + if (whisperModels.length === 0) { + throw new Error('No whisper model downloaded'); + } + + // Use the smallest cached whisper model + const selectedModel = whisperModels.sort((a, b) => { + const sizeA = a.variants[0]?.modelInfo?.fileSizeMb || 0; + const sizeB = b.variants[0]?.modelInfo?.fileSizeMb || 0; + return sizeA - sizeB; + })[0]; + + // Load whisper model + const whisperModel = await manager.catalog.getModel(selectedModel.alias); + await whisperModel.load(); + + // Wait for model to be loaded + while (!(await whisperModel.isLoaded())) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + + // Create audio client and transcribe + const audioClient = whisperModel.createAudioClient(); + const result = await audioClient.transcribe(tempFilePath); + + // Unload whisper model + await whisperModel.unload(); + + return result; + } finally { + // Clean up temp file + try { + fs.unlinkSync(tempFilePath); + } catch (e) { + // Ignore cleanup errors + } + } +}); diff --git a/samples/js/electron-chat-application/preload.js b/samples/js/electron-chat-application/preload.js new file mode 100644 index 00000000..7026b0b2 --- /dev/null +++ b/samples/js/electron-chat-application/preload.js @@ -0,0 +1,20 @@ +const { contextBridge, ipcRenderer } = require('electron'); + +contextBridge.exposeInMainWorld('foundryAPI', { + getModels: () => ipcRenderer.invoke('get-models'), + downloadModel: (modelAlias) => ipcRenderer.invoke('download-model', modelAlias), + loadModel: (modelAlias) => ipcRenderer.invoke('load-model', modelAlias), + unloadModel: () => ipcRenderer.invoke('unload-model'), + deleteModel: (modelAlias) => ipcRenderer.invoke('delete-model', modelAlias), + chat: (messages) => ipcRenderer.invoke('chat', messages), + getLoadedModel: () => ipcRenderer.invoke('get-loaded-model'), + onChatChunk: (callback) => { + const handler = (event, data) => callback(data); + ipcRenderer.on('chat-chunk', handler); + return () => ipcRenderer.removeListener('chat-chunk', handler); + }, + // Transcription + getWhisperModels: () => ipcRenderer.invoke('get-whisper-models'), + downloadWhisperModel: (modelAlias) => ipcRenderer.invoke('download-whisper-model', modelAlias), + transcribeAudio: (filePath, base64Data) => ipcRenderer.invoke('transcribe-audio', filePath, base64Data) +}); diff --git a/samples/js/electron-chat-application/renderer.js b/samples/js/electron-chat-application/renderer.js new file mode 100644 index 00000000..86b84039 --- /dev/null +++ b/samples/js/electron-chat-application/renderer.js @@ -0,0 +1,1066 @@ +// ===================================================== +// Foundry Local Chat - Renderer Process +// ===================================================== + +// Simple markdown parser with code block handling +const SimpleMarkdown = { + parse(text) { + if (!text) return ''; + + // Extract code blocks first to protect them from other processing + const codeBlocks = []; + let html = text.replace(/```(\w*)\n([\s\S]*?)```/g, (match, lang, code) => { + const placeholder = `__CODE_BLOCK_${codeBlocks.length}__`; + codeBlocks.push({ lang, code }); + return placeholder; + }); + + // Extract inline code + const inlineCodes = []; + html = html.replace(/`([^`]+)`/g, (match, code) => { + const placeholder = `__INLINE_CODE_${inlineCodes.length}__`; + inlineCodes.push(code); + return placeholder; + }); + + // Now escape HTML on the remaining text + html = this.escapeHtml(html); + + // Headings (### before ## before #) + html = html.replace(/^### (.+)$/gm, '

$1

'); + html = html.replace(/^## (.+)$/gm, '

$1

'); + html = html.replace(/^# (.+)$/gm, '

$1

'); + + // Unordered lists + html = html.replace(/^- (.+)$/gm, '
  • $1
  • '); + html = html.replace(/(
  • .*<\/li>\n?)+/g, '
      $&
    '); + + // Bold + html = html.replace(/\*\*([^*]+)\*\*/g, '$1'); + + // Italic + html = html.replace(/\*([^*]+)\*/g, '$1'); + + // Links + html = html.replace(/\[([^\]]+)\]\(([^)]+)\)/g, '$1'); + + // Line breaks (but not inside block elements) + html = html.replace(/\n/g, '
    '); + + // Clean up extra
    around block elements + html = html.replace(/
    ()/g, '$1'); + html = html.replace(/(<\/h[234]>)
    /g, '$1'); + html = html.replace(/
    (
      )/g, '$1'); + html = html.replace(/(<\/ul>)
      /g, '$1'); + + // Restore inline code + inlineCodes.forEach((code, i) => { + html = html.replace(`__INLINE_CODE_${i}__`, `${this.escapeHtml(code)}`); + }); + + // Restore code blocks + codeBlocks.forEach((block, i) => { + const codeHtml = `
      + +
      ${this.escapeHtml(block.code.trim())}
      +
      `; + html = html.replace(`__CODE_BLOCK_${i}__`, codeHtml); + }); + + return html; + }, + + escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; + } +}; + +// Copy code to clipboard - use event delegation +document.addEventListener('click', async (e) => { + const button = e.target.closest('.code-copy-btn'); + if (!button) return; + + const codeBlock = button.closest('.code-block-wrapper').querySelector('code'); + const text = codeBlock.textContent; + + try { + await navigator.clipboard.writeText(text); + button.classList.add('copied'); + setTimeout(() => button.classList.remove('copied'), 2000); + } catch (err) { + console.error('Failed to copy:', err); + } +}); + +// Estimate tokens from text (rough approximation: ~4 chars per token) +function estimateTokens(text) { + return Math.ceil(text.length / 4); +} + +// Calculate total context tokens from all messages +function calculateContextTokens() { + return messages.reduce((total, msg) => total + estimateTokens(msg.content), 0); +} + +// Update context usage display +function updateContextUsage() { + contextTokens = calculateContextTokens(); + const percentage = Math.min(100, Math.round((contextTokens / CONTEXT_LIMIT) * 100)); + + contextFill.style.width = `${percentage}%`; + contextLabel.textContent = `${percentage}%`; + + // Update color based on usage + contextFill.classList.remove('warning', 'danger'); + if (percentage >= 90) { + contextFill.classList.add('danger'); + } else if (percentage >= 70) { + contextFill.classList.add('warning'); + } + + // Update tooltip + contextUsage.title = `Context: ${contextTokens.toLocaleString()} / ${CONTEXT_LIMIT.toLocaleString()} tokens (~${percentage}%)`; +} + +// State +let messages = []; +let currentModelAlias = null; +let isGenerating = false; +let contextTokens = 0; +const CONTEXT_LIMIT = 8192; // Default context window, will update based on model + +// DOM Elements +const sidebar = document.getElementById('sidebar'); +const sidebarToggle = document.getElementById('sidebarToggle'); +const mobileMenuBtn = document.getElementById('mobileMenuBtn'); +const modelList = document.getElementById('modelList'); +const refreshModels = document.getElementById('refreshModels'); +const modelBadge = document.getElementById('modelBadge'); +const chatMessages = document.getElementById('chatMessages'); +const chatForm = document.getElementById('chatForm'); +const messageInput = document.getElementById('messageInput'); +const sendBtn = document.getElementById('sendBtn'); +const newChatBtn = document.getElementById('newChatBtn'); +const toastContainer = document.getElementById('toastContainer'); +const recordBtn = document.getElementById('recordBtn'); +const transcriptionSettingsBtn = document.getElementById('transcriptionSettingsBtn'); +const whisperModal = document.getElementById('whisperModal'); +const whisperModelList = document.getElementById('whisperModelList'); +const whisperModalCancel = document.getElementById('whisperModalCancel'); +const currentWhisperModelEl = document.getElementById('currentWhisperModel'); +const contextFill = document.getElementById('contextFill'); +const contextLabel = document.getElementById('contextLabel'); +const contextUsage = document.getElementById('contextUsage'); + +// Recording state +let mediaRecorder = null; +let audioChunks = []; +let isRecording = false; +let selectedWhisperModel = null; + +// Initialize +document.addEventListener('DOMContentLoaded', async () => { + setupEventListeners(); + setupSidebarResize(); + setupRecordButton(); + updateContextUsage(); + await loadModels(); + setupChatChunkListener(); +}); + +function setupSidebarResize() { + const resizeHandle = document.getElementById('sidebarResizeHandle'); + let isResizing = false; + + resizeHandle.addEventListener('mousedown', (e) => { + isResizing = true; + resizeHandle.classList.add('dragging'); + document.body.style.cursor = 'col-resize'; + document.body.style.userSelect = 'none'; + }); + + document.addEventListener('mousemove', (e) => { + if (!isResizing) return; + const newWidth = Math.min(Math.max(e.clientX, 240), 480); + sidebar.style.width = newWidth + 'px'; + }); + + document.addEventListener('mouseup', () => { + if (isResizing) { + isResizing = false; + resizeHandle.classList.remove('dragging'); + document.body.style.cursor = ''; + document.body.style.userSelect = ''; + } + }); +} + +function setupRecordButton() { + recordBtn.addEventListener('click', handleRecordClick); + transcriptionSettingsBtn.addEventListener('click', openTranscriptionSettings); + whisperModalCancel.addEventListener('click', () => { + whisperModal.classList.remove('visible'); + }); +} + +async function openTranscriptionSettings() { + const whisperModels = await window.foundryAPI.getWhisperModels(); + showWhisperModal(whisperModels, true); +} + +async function handleRecordClick() { + if (isRecording) { + // Stop recording + stopRecording(); + } else { + // Check if whisper model is available + const whisperModels = await window.foundryAPI.getWhisperModels(); + const cachedModels = whisperModels.filter(m => m.isCached); + + if (cachedModels.length === 0) { + // Show modal to download whisper model + showWhisperModal(whisperModels, false); + } else { + // Start recording + startRecording(); + } + } +} + +function showWhisperModal(models, isSettings = false) { + // Update current model display + const cachedModels = models.filter(m => m.isCached); + const modelNameEl = currentWhisperModelEl.querySelector('.model-name'); + if (cachedModels.length > 0) { + const current = selectedWhisperModel || cachedModels.sort((a, b) => (a.fileSizeMb || 0) - (b.fileSizeMb || 0))[0].alias; + modelNameEl.textContent = current; + } else { + modelNameEl.textContent = 'None - download a model below'; + } + + whisperModelList.innerHTML = ''; + + models.forEach(model => { + const sizeStr = model.fileSizeMb ? `${(model.fileSizeMb / 1024).toFixed(1)} GB` : ''; + const isSelected = selectedWhisperModel === model.alias; + const item = document.createElement('div'); + item.className = 'whisper-model-item' + (isSelected ? ' selected' : ''); + item.innerHTML = ` +
      + ${model.alias} + ${sizeStr} +
      +
      + ${model.isCached + ? ` + ` + : '' + } +
      + `; + + if (model.isCached) { + const useBtn = item.querySelector('.use-btn'); + useBtn.addEventListener('click', () => { + selectedWhisperModel = model.alias; + showToast(`Selected ${model.alias} for transcription`, 'success'); + // Refresh modal to show selection + showWhisperModal(models, true); + }); + + const deleteBtn = item.querySelector('.delete-btn'); + deleteBtn.addEventListener('click', async () => { + if (confirm(`Delete ${model.alias} from cache?`)) { + try { + await window.foundryAPI.deleteModel(model.alias); + if (selectedWhisperModel === model.alias) { + selectedWhisperModel = null; + } + showToast(`Deleted ${model.alias}`, 'success'); + const updatedModels = await window.foundryAPI.getWhisperModels(); + showWhisperModal(updatedModels, true); + } catch (error) { + showToast('Delete failed: ' + error.message, 'error'); + } + } + }); + } else { + const downloadBtn = item.querySelector('.download-btn'); + downloadBtn.addEventListener('click', async () => { + downloadBtn.textContent = 'Downloading...'; + downloadBtn.disabled = true; + try { + await window.foundryAPI.downloadWhisperModel(model.alias); + showToast(`Downloaded ${model.alias}`, 'success'); + selectedWhisperModel = model.alias; + const updatedModels = await window.foundryAPI.getWhisperModels(); + showWhisperModal(updatedModels, true); + } catch (error) { + showToast('Download failed: ' + error.message, 'error'); + downloadBtn.textContent = 'Download'; + downloadBtn.disabled = false; + } + }); + } + + whisperModelList.appendChild(item); + }); + + whisperModal.classList.add('visible'); +} + +async function startRecording() { + try { + // Request 16kHz mono audio for Whisper compatibility + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + sampleRate: 16000, + channelCount: 1, + echoCancellation: true, + noiseSuppression: true + } + }); + + mediaRecorder = new MediaRecorder(stream); + audioChunks = []; + + mediaRecorder.ondataavailable = (e) => { + audioChunks.push(e.data); + }; + + mediaRecorder.onstop = async () => { + // Stop all tracks + stream.getTracks().forEach(track => track.stop()); + + // Create audio blob + const audioBlob = new Blob(audioChunks, { type: mediaRecorder.mimeType }); + await transcribeAudio(audioBlob); + }; + + mediaRecorder.start(); + isRecording = true; + recordBtn.classList.add('recording'); + showToast('Recording... Click stop when done', 'warning'); + } catch (error) { + console.error('Failed to start recording:', error); + showToast('Failed to access microphone', 'error'); + } +} + +function stopRecording() { + if (mediaRecorder && isRecording) { + mediaRecorder.stop(); + isRecording = false; + recordBtn.classList.remove('recording'); + recordBtn.classList.add('transcribing'); + } +} + +// Convert audio blob to 16kHz mono WAV format for Whisper +async function convertToWav(audioBlob) { + const audioContext = new AudioContext(); + try { + const arrayBuffer = await audioBlob.arrayBuffer(); + const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + + // Resample to 16kHz mono + const targetSampleRate = 16000; + const offlineContext = new OfflineAudioContext(1, audioBuffer.duration * targetSampleRate, targetSampleRate); + + const source = offlineContext.createBufferSource(); + source.buffer = audioBuffer; + source.connect(offlineContext.destination); + source.start(0); + + const resampledBuffer = await offlineContext.startRendering(); + + // Convert to WAV + const wavBuffer = audioBufferToWav(resampledBuffer); + return new Blob([wavBuffer], { type: 'audio/wav' }); + } finally { + await audioContext.close(); + } +} + +// Encode AudioBuffer to 16-bit PCM WAV format +function audioBufferToWav(buffer) { + const numChannels = 1; // Force mono + const sampleRate = buffer.sampleRate; + const bitDepth = 16; + + const bytesPerSample = bitDepth / 8; + const blockAlign = numChannels * bytesPerSample; + + // Get mono channel (mix down if stereo) + let monoData; + if (buffer.numberOfChannels === 1) { + monoData = buffer.getChannelData(0); + } else { + // Mix stereo to mono + const left = buffer.getChannelData(0); + const right = buffer.getChannelData(1); + monoData = new Float32Array(left.length); + for (let i = 0; i < left.length; i++) { + monoData[i] = (left[i] + right[i]) / 2; + } + } + + const samples = monoData.length; + const dataSize = samples * blockAlign; + const bufferSize = 44 + dataSize; + + const arrayBuffer = new ArrayBuffer(bufferSize); + const view = new DataView(arrayBuffer); + + // RIFF header + writeString(view, 0, 'RIFF'); + view.setUint32(4, 36 + dataSize, true); + writeString(view, 8, 'WAVE'); + + // fmt chunk + writeString(view, 12, 'fmt '); + view.setUint32(16, 16, true); // chunk size + view.setUint16(20, 1, true); // PCM format + view.setUint16(22, numChannels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * blockAlign, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, bitDepth, true); + + // data chunk + writeString(view, 36, 'data'); + view.setUint32(40, dataSize, true); + + // Write audio data as 16-bit PCM + let offset = 44; + for (let i = 0; i < samples; i++) { + const sample = Math.max(-1, Math.min(1, monoData[i])); + const intSample = sample < 0 ? sample * 0x8000 : sample * 0x7FFF; + view.setInt16(offset, intSample, true); + offset += 2; + } + + return arrayBuffer; +} + +function writeString(view, offset, string) { + for (let i = 0; i < string.length; i++) { + view.setUint8(offset + i, string.charCodeAt(i)); + } +} + +async function transcribeAudio(audioBlob) { + try { + showToast('Converting audio...', 'warning'); + + // Convert to 16kHz mono WAV format for Whisper compatibility + let wavBlob; + try { + wavBlob = await convertToWav(audioBlob); + } catch (e) { + console.error('WAV conversion failed:', e); + showToast('Audio conversion failed: ' + e.message, 'error'); + recordBtn.classList.remove('transcribing'); + return; + } + + showToast('Transcribing audio...', 'warning'); + + // Convert blob to base64 + const arrayBuffer = await wavBlob.arrayBuffer(); + const uint8Array = new Uint8Array(arrayBuffer); + + // Use chunked base64 encoding for large arrays + let base64 = ''; + const chunkSize = 32768; + for (let i = 0; i < uint8Array.length; i += chunkSize) { + const chunk = uint8Array.subarray(i, i + chunkSize); + base64 += String.fromCharCode.apply(null, chunk); + } + base64 = btoa(base64); + + const tempPath = `/tmp/foundry_audio_${Date.now()}.wav`; + + const result = await window.foundryAPI.transcribeAudio(tempPath, base64); + + // Insert transcribed text into input + const text = result.text || result.Text || ''; + if (text) { + messageInput.value += text; + messageInput.dispatchEvent(new Event('input')); + showToast('Transcription complete', 'success'); + } else { + showToast('No speech detected', 'warning'); + } + } catch (error) { + console.error('Transcription failed:', error); + showToast('Transcription failed: ' + error.message, 'error'); + } finally { + recordBtn.classList.remove('transcribing'); + } +} + +function setupEventListeners() { + // Sidebar toggle + sidebarToggle.addEventListener('click', () => { + sidebar.classList.toggle('collapsed'); + }); + + mobileMenuBtn.addEventListener('click', () => { + sidebar.classList.toggle('open'); + }); + + // Refresh models + refreshModels.addEventListener('click', async () => { + refreshModels.classList.add('spinning'); + await loadModels(); + refreshModels.classList.remove('spinning'); + }); + + // Chat form + chatForm.addEventListener('submit', handleSendMessage); + + // Textarea auto-resize + messageInput.addEventListener('input', () => { + messageInput.style.height = 'auto'; + messageInput.style.height = Math.min(messageInput.scrollHeight, 150) + 'px'; + }); + + // Enter to send, Shift+Enter for new line + messageInput.addEventListener('keydown', (e) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + chatForm.dispatchEvent(new Event('submit')); + } + }); + + // New chat + newChatBtn.addEventListener('click', clearChat); + + // Close sidebar on outside click (mobile) + document.addEventListener('click', (e) => { + if (window.innerWidth <= 768 && + sidebar.classList.contains('open') && + !sidebar.contains(e.target) && + !mobileMenuBtn.contains(e.target)) { + sidebar.classList.remove('open'); + } + }); +} + +function setupChatChunkListener() { + window.foundryAPI.onChatChunk((data) => { + if (data.content) { + appendToLastAssistantMessage(data.content); + } + }); +} + +// Model Management +async function loadModels() { + modelList.innerHTML = ` +
      +
      + Loading models... +
      + `; + + try { + const models = await window.foundryAPI.getModels(); + + if (!models || models.length === 0) { + modelList.innerHTML = ` +
      + No models found +
      + `; + return; + } + + // Filter out whisper/audio models - only show chat models + const chatModels = models.filter(m => { + const alias = m.alias.toLowerCase(); + // Exclude whisper and other audio models + if (alias.includes('whisper')) return false; + return true; + }); + + const displayModels = chatModels; + + // Sort: cached first, then by name + displayModels.sort((a, b) => { + if (a.isCached && !b.isCached) return -1; + if (!a.isCached && b.isCached) return 1; + return a.alias.localeCompare(b.alias); + }); + + // Group by cached status + const cachedModels = displayModels.filter(m => m.isCached); + const availableModels = displayModels.filter(m => !m.isCached); + + modelList.innerHTML = ''; + + if (cachedModels.length > 0) { + const cachedGroup = document.createElement('div'); + cachedGroup.className = 'model-group'; + cachedGroup.innerHTML = ` +
      +
      + Downloaded +
      + `; + cachedModels.forEach(model => { + cachedGroup.appendChild(createModelItem(model)); + }); + modelList.appendChild(cachedGroup); + } + + if (availableModels.length > 0) { + const availableGroup = document.createElement('div'); + availableGroup.className = 'model-group'; + availableGroup.innerHTML = ` +
      +
      + Available +
      + `; + availableModels.forEach(model => { + availableGroup.appendChild(createModelItem(model)); + }); + modelList.appendChild(availableGroup); + } + + if (displayModels.length === 0) { + modelList.innerHTML = ` +
      + No models available +
      + `; + } + } catch (error) { + console.error('Failed to load models:', error); + modelList.innerHTML = ` +
      + Failed to load models + ${error.message || error} +
      + `; + showToast('Failed to load models: ' + error.message, 'error'); + } +} + +function createModelItem(model) { + const variant = model.variants[0]; + const item = document.createElement('div'); + item.className = 'model-item'; + const isActive = model.alias === currentModelAlias; + if (isActive) { + item.classList.add('active'); + } + + const sizeMb = variant?.fileSizeMb; + const sizeStr = sizeMb ? `${(sizeMb / 1024).toFixed(1)} GB` : ''; + + let statusHtml; + if (isActive) { + statusHtml = ` + + `; + } else if (model.isCached) { + statusHtml = ` + + + `; + } else { + statusHtml = ''; + } + + item.innerHTML = ` +
      + + + + + +
      +
      +
      ${model.alias}
      +
      ${sizeStr}
      +
      +
      + ${statusHtml} +
      + `; + + // Handle click events + if (isActive) { + const unloadBtn = item.querySelector('.unload-btn'); + unloadBtn.addEventListener('click', async (e) => { + e.stopPropagation(); + await unloadModel(); + }); + } else if (model.isCached) { + const loadBtn = item.querySelector('.load-btn'); + loadBtn.addEventListener('click', async (e) => { + e.stopPropagation(); + await loadModel(model.alias); + }); + + const deleteBtn = item.querySelector('.delete-model-btn'); + deleteBtn.addEventListener('click', async (e) => { + e.stopPropagation(); + if (confirm(`Delete ${model.alias} from cache?`)) { + try { + await window.foundryAPI.deleteModel(model.alias); + showToast(`Deleted ${model.alias}`, 'success'); + await loadModels(); + } catch (error) { + showToast('Delete failed: ' + error.message, 'error'); + } + } + }); + } else { + const downloadBtn = item.querySelector('.download-btn'); + downloadBtn.addEventListener('click', async (e) => { + e.stopPropagation(); + await downloadModel(model.alias, item); + }); + } + + return item; +} + +async function downloadModel(alias, itemElement) { + const statusEl = itemElement.querySelector('.model-status'); + statusEl.innerHTML = '
      '; + + try { + showToast(`Downloading ${alias}...`, 'warning'); + await window.foundryAPI.downloadModel(alias); + showToast(`Downloaded ${alias}. Loading...`, 'success'); + await loadModels(); + // Auto-load the model after download + await loadModel(alias); + } catch (error) { + console.error('Download failed:', error); + showToast('Download failed: ' + error.message, 'error'); + await loadModels(); + } +} + +async function loadModel(alias) { + if (isGenerating) { + showToast('Please wait for the current response to finish', 'warning'); + return; + } + + // Update UI to show loading + const items = modelList.querySelectorAll('.model-item'); + items.forEach(item => { + item.classList.remove('active'); + const nameEl = item.querySelector('.model-name'); + if (nameEl.textContent.includes(alias) || item.dataset.alias === alias) { + item.classList.add('loading'); + } + }); + + try { + showToast(`Loading ${alias}...`, 'warning'); + await window.foundryAPI.loadModel(alias); + currentModelAlias = alias; + + // Update UI + updateCurrentModelDisplay(alias); + enableChat(); + showToast(`Model ${alias} loaded`, 'success'); + + // Refresh model list to update active state + await loadModels(); + } catch (error) { + console.error('Failed to load model:', error); + showToast('Failed to load model: ' + error.message, 'error'); + await loadModels(); + } +} + +async function unloadModel() { + if (isGenerating) { + showToast('Please wait for the current response to finish', 'warning'); + return; + } + + try { + showToast('Unloading model...', 'warning'); + await window.foundryAPI.unloadModel(); + currentModelAlias = null; + + // Update UI + modelBadge.textContent = 'Select a model to start'; + disableChat(); + showToast('Model unloaded', 'success'); + + // Refresh model list + await loadModels(); + } catch (error) { + console.error('Failed to unload model:', error); + showToast('Failed to unload model: ' + error.message, 'error'); + } +} + +function updateCurrentModelDisplay(alias) { + modelBadge.textContent = alias; +} + +function enableChat() { + messageInput.disabled = false; + sendBtn.disabled = false; + messageInput.placeholder = 'Type your message...'; + messageInput.focus(); +} + +function disableChat() { + messageInput.disabled = true; + sendBtn.disabled = true; + messageInput.placeholder = 'Select a model to start chatting...'; +} + +// Chat Management +async function handleSendMessage(e) { + e.preventDefault(); + + const content = messageInput.value.trim(); + if (!content || isGenerating || !currentModelAlias) return; + + // Clear welcome message if present + const welcomeMessage = chatMessages.querySelector('.welcome-message'); + if (welcomeMessage) { + welcomeMessage.remove(); + } + + // Add user message + messages.push({ role: 'user', content }); + addMessageToChat('user', content); + updateContextUsage(); + + // Clear input + messageInput.value = ''; + messageInput.style.height = 'auto'; + + // Disable send button + isGenerating = true; + sendBtn.disabled = true; + + // Add typing indicator + const typingEl = addTypingIndicator(); + + try { + // Make API call + const result = await window.foundryAPI.chat(messages); + + // Remove typing indicator + typingEl.remove(); + + // Add assistant message (content was already streamed, just add stats) + messages.push({ role: 'assistant', content: result.content }); + updateLastAssistantMessageStats(result.stats); + updateContextUsage(); + + } catch (error) { + console.error('Chat error:', error); + typingEl.remove(); + showToast('Chat error: ' + error.message, 'error'); + } finally { + isGenerating = false; + sendBtn.disabled = false; + messageInput.focus(); + } +} + +function addMessageToChat(role, content) { + const messageEl = document.createElement('div'); + messageEl.className = `message ${role}`; + + const avatar = role === 'user' ? 'U' : + ` + + `; + + messageEl.innerHTML = ` +
      ${avatar}
      +
      +
      ${role === 'user' ? SimpleMarkdown.escapeHtml(content) : SimpleMarkdown.parse(content)}
      + ${role === 'assistant' ? '
      ' : ''} +
      + `; + + chatMessages.appendChild(messageEl); + scrollToBottom(); + + return messageEl; +} + +function addTypingIndicator() { + const typingEl = document.createElement('div'); + typingEl.className = 'message assistant'; + typingEl.id = 'typing-indicator'; + typingEl.innerHTML = ` +
      + + + +
      +
      +
      + + + +
      +
      + `; + chatMessages.appendChild(typingEl); + scrollToBottom(); + return typingEl; +} + +let currentAssistantMessage = null; +let currentAssistantContent = ''; + +function appendToLastAssistantMessage(content) { + // If there's a typing indicator, replace it with actual message + const typingIndicator = document.getElementById('typing-indicator'); + if (typingIndicator) { + typingIndicator.remove(); + currentAssistantMessage = addMessageToChat('assistant', ''); + currentAssistantContent = ''; + } + + if (!currentAssistantMessage) { + currentAssistantMessage = addMessageToChat('assistant', ''); + currentAssistantContent = ''; + } + + currentAssistantContent += content; + const bubble = currentAssistantMessage.querySelector('.message-bubble'); + bubble.innerHTML = SimpleMarkdown.parse(currentAssistantContent); + scrollToBottom(); +} + +function updateLastAssistantMessageStats(stats) { + if (!currentAssistantMessage) return; + + const statsEl = currentAssistantMessage.querySelector('.message-stats'); + if (statsEl && stats) { + statsEl.innerHTML = ` +
      + + + + + TTFT: ${stats.timeToFirstToken}ms +
      +
      + + + + ${stats.tokensPerSecond} tok/s +
      +
      + + + + + ${stats.tokenCount} tokens +
      + `; + } + + // Reset for next message + currentAssistantMessage = null; + currentAssistantContent = ''; +} + +function clearChat() { + messages = []; + currentAssistantMessage = null; + currentAssistantContent = ''; + updateContextUsage(); + + chatMessages.innerHTML = ` +
      +
      + + + +
      +

      Welcome to Foundry Local Chat

      +

      Select a model from the sidebar to start chatting with AI running locally on your machine.

      +
      +
      + + + + 100% Private +
      +
      + + + + + Low Latency +
      +
      + + + + + + Runs Locally +
      +
      +
      + `; +} + +function scrollToBottom() { + chatMessages.scrollTop = chatMessages.scrollHeight; +} + +// Toast Notifications +function showToast(message, type = 'info') { + const toast = document.createElement('div'); + toast.className = `toast ${type}`; + toast.innerHTML = ` + + ${type === 'success' ? '' : + type === 'error' ? '' : + type === 'warning' ? '' : + '' + } + + ${message} + `; + + toastContainer.appendChild(toast); + + setTimeout(() => { + toast.style.animation = 'slideIn 0.3s ease reverse'; + setTimeout(() => toast.remove(), 300); + }, 3000); +} diff --git a/samples/js/electron-chat-application/screenshots/electron-description-of-functions.png b/samples/js/electron-chat-application/screenshots/electron-description-of-functions.png new file mode 100644 index 00000000..ee46f8be Binary files /dev/null and b/samples/js/electron-chat-application/screenshots/electron-description-of-functions.png differ diff --git a/samples/js/electron-chat-application/screenshots/electron-transcription.png b/samples/js/electron-chat-application/screenshots/electron-transcription.png new file mode 100644 index 00000000..32295ac1 Binary files /dev/null and b/samples/js/electron-chat-application/screenshots/electron-transcription.png differ diff --git a/samples/js/electron-chat-application/styles.css b/samples/js/electron-chat-application/styles.css new file mode 100644 index 00000000..1f0e2fc2 --- /dev/null +++ b/samples/js/electron-chat-application/styles.css @@ -0,0 +1,1348 @@ +/* ===================================================== + Foundry Local Chat - Modern Chat Interface Styles + ===================================================== */ + +:root { + /* Color Palette - Dark Theme */ + --bg-primary: #0f0f1a; + --bg-secondary: #1a1a2e; + --bg-tertiary: #16213e; + --bg-hover: #1f2b4d; + --bg-active: #2a3a5f; + + --text-primary: #e8e8e8; + --text-secondary: #a0a0b0; + --text-muted: #6b6b80; + + --accent-primary: #6366f1; + --accent-secondary: #818cf8; + --accent-gradient: linear-gradient(135deg, #6366f1 0%, #8b5cf6 100%); + + --success: #22c55e; + --warning: #f59e0b; + --error: #ef4444; + + --border-color: #2a2a4a; + --border-subtle: rgba(255, 255, 255, 0.06); + + /* Sizing */ + --sidebar-width: 320px; + --sidebar-min-width: 240px; + --sidebar-max-width: 480px; + --header-height: 60px; + --input-height: 56px; + + /* Spacing */ + --space-xs: 4px; + --space-sm: 8px; + --space-md: 16px; + --space-lg: 24px; + --space-xl: 32px; + + /* Typography */ + --font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif; + --font-mono: 'SF Mono', 'Fira Code', 'Consolas', monospace; + + /* Transitions */ + --transition-fast: 150ms ease; + --transition-normal: 250ms ease; + + /* Shadows */ + --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.3); + --shadow-md: 0 4px 6px rgba(0, 0, 0, 0.3); + --shadow-lg: 0 10px 25px rgba(0, 0, 0, 0.4); + + /* Border Radius */ + --radius-sm: 6px; + --radius-md: 10px; + --radius-lg: 16px; + --radius-full: 9999px; +} + +/* ===================================================== + Reset & Base Styles + ===================================================== */ + +*, *::before, *::after { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +html, body { + height: 100%; + overflow: hidden; +} + +body { + font-family: var(--font-family); + font-size: 14px; + line-height: 1.5; + color: var(--text-primary); + background: var(--bg-primary); + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +/* ===================================================== + App Container + ===================================================== */ + +.app-container { + display: flex; + height: 100vh; + width: 100vw; + overflow: hidden; +} + +/* ===================================================== + Sidebar + ===================================================== */ + +.sidebar { + width: var(--sidebar-width); + min-width: var(--sidebar-min-width); + max-width: var(--sidebar-max-width); + height: 100%; + background: var(--bg-secondary); + border-right: 1px solid var(--border-color); + display: flex; + flex-direction: column; + transition: transform var(--transition-normal); + z-index: 100; + position: relative; +} + +.sidebar-resize-handle { + position: absolute; + top: 0; + right: 0; + width: 4px; + height: 100%; + cursor: col-resize; + background: transparent; + transition: background var(--transition-fast); + z-index: 10; +} + +.sidebar-resize-handle:hover, +.sidebar-resize-handle.dragging { + background: var(--accent-primary); +} + +.sidebar.collapsed { + width: 0; + min-width: 0; + transform: translateX(-100%); +} + +.sidebar-header { + padding: var(--space-md); + padding-top: 40px; /* Account for macOS title bar */ + display: flex; + align-items: center; + justify-content: space-between; + border-bottom: 1px solid var(--border-color); + -webkit-app-region: drag; +} + +.logo { + display: flex; + align-items: center; + gap: var(--space-sm); + font-weight: 600; + font-size: 16px; + color: var(--text-primary); +} + +.logo svg { + color: var(--accent-primary); +} + +.sidebar-toggle { + background: transparent; + border: none; + color: var(--text-secondary); + cursor: pointer; + padding: var(--space-xs); + border-radius: var(--radius-sm); + transition: all var(--transition-fast); + -webkit-app-region: no-drag; +} + +.sidebar-toggle:hover { + background: var(--bg-hover); + color: var(--text-primary); +} + +.sidebar-content { + flex: 1; + overflow-y: auto; + padding: var(--space-md); +} + +.section-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: var(--space-md); +} + +.section-header h3 { + font-size: 12px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--text-muted); +} + +.refresh-btn { + background: transparent; + border: none; + color: var(--text-muted); + cursor: pointer; + padding: var(--space-xs); + border-radius: var(--radius-sm); + transition: all var(--transition-fast); +} + +.refresh-btn:hover { + background: var(--bg-hover); + color: var(--text-primary); +} + +.refresh-btn.spinning svg { + animation: spin 1s linear infinite; +} + +@keyframes spin { + from { transform: rotate(0deg); } + to { transform: rotate(360deg); } +} + +/* Model List */ +.model-list { + display: flex; + flex-direction: column; + gap: var(--space-sm); +} + +.model-group { + margin-bottom: var(--space-md); +} + +.model-group-header { + display: flex; + align-items: center; + gap: var(--space-sm); + padding: var(--space-xs) 0; + margin-bottom: var(--space-xs); +} + +.model-group-header .status-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--text-muted); +} + +.model-group-header .status-dot.cached { + background: var(--success); +} + +.model-group-header .status-dot.loaded { + background: var(--success); +} + +.model-group-header span { + font-size: 11px; + font-weight: 500; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--text-muted); +} + +.model-item { + display: flex; + align-items: center; + padding: var(--space-sm) var(--space-md); + background: var(--bg-tertiary); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); + cursor: pointer; + transition: all var(--transition-fast); + gap: var(--space-sm); +} + +.model-item:hover { + background: var(--bg-hover); + border-color: var(--border-color); +} + +.model-item.active { + background: rgba(16, 185, 129, 0.15); + border-color: var(--success); +} + +.model-item.active .model-name, +.model-item.active .model-size { + color: var(--text-primary); +} + +.model-item.loading { + pointer-events: none; + opacity: 0.7; +} + +.model-icon { + width: 32px; + height: 32px; + border-radius: var(--radius-sm); + background: var(--bg-primary); + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; +} + +.model-icon svg { + width: 18px; + height: 18px; + color: var(--accent-secondary); +} + +.model-info { + flex: 1; + min-width: 0; +} + +.model-name { + font-weight: 500; + font-size: 13px; + color: var(--text-primary); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.model-size { + font-size: 11px; + color: var(--text-muted); +} + +.model-status { + display: flex; + align-items: center; + gap: var(--space-xs); +} + +.status-indicator { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--text-muted); +} + +.status-indicator.cached { + background: var(--warning); +} + +.status-indicator.loaded { + background: var(--success); +} + +.status-indicator.loading { + background: var(--warning); + animation: pulse 1.5s ease-in-out infinite; +} + +@keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.5; } +} + +.download-btn, +.load-btn { + padding: var(--space-xs) var(--space-sm); + font-size: 11px; + font-weight: 500; + background: var(--accent-primary); + color: white; + border: none; + border-radius: var(--radius-sm); + cursor: pointer; + transition: all var(--transition-fast); +} + +.download-btn:hover, +.load-btn:hover { + background: var(--accent-secondary); +} + +.unload-btn { + padding: var(--space-xs) var(--space-sm); + font-size: 11px; + font-weight: 500; + background: rgba(239, 68, 68, 0.15); + color: var(--error); + border: none; + border-radius: var(--radius-sm); + cursor: pointer; + transition: all var(--transition-fast); +} + +.unload-btn:hover { + background: var(--error); + color: white; +} + +.delete-model-btn { + width: 24px; + height: 24px; + font-size: 12px; + background: transparent; + color: var(--text-muted); + border: none; + border-radius: var(--radius-sm); + cursor: pointer; + transition: all var(--transition-fast); + display: flex; + align-items: center; + justify-content: center; + opacity: 0; +} + +.model-item:hover .delete-model-btn { + opacity: 1; +} + +.delete-model-btn:hover { + color: var(--error); +} + +/* Loading Spinner */ +.loading-spinner { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: var(--space-xl); + gap: var(--space-md); + color: var(--text-muted); +} + +.spinner { + width: 24px; + height: 24px; + border: 2px solid var(--border-color); + border-top-color: var(--accent-primary); + border-radius: 50%; + animation: spin 0.8s linear infinite; +} + +/* ===================================================== + Chat Area + ===================================================== */ + +.chat-area { + flex: 1; + display: flex; + flex-direction: column; + min-width: 0; + background: var(--bg-primary); +} + +.chat-header { + height: var(--header-height); + padding: 0 var(--space-lg); + padding-top: 20px; /* Account for macOS title bar */ + display: flex; + align-items: center; + justify-content: space-between; + border-bottom: 1px solid var(--border-color); + background: var(--bg-secondary); + -webkit-app-region: drag; +} + +.mobile-menu-btn { + display: none; + background: transparent; + border: none; + color: var(--text-secondary); + cursor: pointer; + padding: var(--space-xs); + border-radius: var(--radius-sm); + -webkit-app-region: no-drag; +} + +.chat-title { + display: flex; + align-items: center; + gap: var(--space-md); +} + +.chat-title h1 { + font-size: 18px; + font-weight: 600; +} + +.model-badge { + font-size: 12px; + padding: var(--space-xs) var(--space-sm); + background: var(--bg-tertiary); + border-radius: var(--radius-full); + color: var(--text-secondary); +} + +.new-chat-btn { + background: transparent; + border: none; + color: var(--text-secondary); + cursor: pointer; + padding: var(--space-sm); + border-radius: var(--radius-sm); + transition: all var(--transition-fast); + -webkit-app-region: no-drag; +} + +.new-chat-btn:hover { + background: var(--bg-hover); + color: var(--text-primary); +} + +/* Chat Messages */ +.chat-messages { + flex: 1; + overflow-y: auto; + padding: var(--space-lg); + display: flex; + flex-direction: column; + gap: var(--space-lg); +} + +/* Welcome Message */ +.welcome-message { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + text-align: center; + padding: var(--space-xl); + margin: auto; + max-width: 500px; +} + +.welcome-icon { + width: 80px; + height: 80px; + background: var(--accent-gradient); + border-radius: var(--radius-lg); + display: flex; + align-items: center; + justify-content: center; + margin-bottom: var(--space-lg); + color: white; +} + +.welcome-message h2 { + font-size: 24px; + font-weight: 600; + margin-bottom: var(--space-sm); +} + +.welcome-message p { + color: var(--text-secondary); + margin-bottom: var(--space-lg); +} + +.feature-highlights { + display: flex; + gap: var(--space-lg); + flex-wrap: wrap; + justify-content: center; +} + +.feature { + display: flex; + align-items: center; + gap: var(--space-sm); + color: var(--text-secondary); + font-size: 13px; +} + +.feature svg { + color: var(--accent-secondary); +} + +/* Message Bubbles */ +.message { + display: flex; + gap: var(--space-md); + max-width: 85%; + animation: fadeIn 0.3s ease; +} + +@keyframes fadeIn { + from { + opacity: 0; + transform: translateY(10px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +.message.user { + margin-left: auto; + flex-direction: row-reverse; +} + +.message-avatar { + width: 36px; + height: 36px; + border-radius: var(--radius-md); + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + font-weight: 600; + font-size: 14px; +} + +.message.user .message-avatar { + background: var(--accent-gradient); + color: white; +} + +.message.assistant .message-avatar { + background: var(--bg-tertiary); + color: var(--accent-secondary); +} + +.message-content { + display: flex; + flex-direction: column; + gap: var(--space-xs); +} + +.message-bubble { + padding: var(--space-md); + border-radius: var(--radius-lg); + line-height: 1.6; +} + +.message.user .message-bubble { + background: var(--accent-gradient); + color: white; + border-bottom-right-radius: var(--radius-sm); +} + +.message.assistant .message-bubble { + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-bottom-left-radius: var(--radius-sm); +} + +/* Message Stats */ +.message-stats { + display: flex; + gap: var(--space-md); + font-size: 11px; + color: var(--text-muted); + padding: 0 var(--space-sm); +} + +.stat-item { + display: flex; + align-items: center; + gap: var(--space-xs); +} + +/* Code Blocks */ +.message-bubble pre { + margin: var(--space-sm) 0; + padding: var(--space-md); + background: var(--bg-primary); + border-radius: var(--radius-md); + overflow-x: auto; + position: relative; +} + +.message-bubble code { + font-family: var(--font-mono); + font-size: 13px; +} + +.message-bubble pre code { + display: block; +} + +.message-bubble :not(pre) > code { + background: var(--bg-tertiary); + padding: 2px 6px; + border-radius: var(--radius-sm); +} + +.code-block-wrapper { + position: relative; + margin: var(--space-sm) 0; +} + +.code-block-wrapper pre { + margin: 0; + border-radius: var(--radius-md); +} + +.code-copy-btn { + position: absolute; + top: 8px; + right: 8px; + width: 28px; + height: 28px; + display: flex; + align-items: center; + justify-content: center; + background: rgba(30, 30, 50, 0.9); + border: 1px solid var(--border-color); + border-radius: var(--radius-sm); + color: var(--text-secondary); + cursor: pointer; + opacity: 0; + transition: all var(--transition-fast); + font-size: 14px; + line-height: 1; +} + +.code-block-wrapper:hover .code-copy-btn { + opacity: 1; +} + +.code-copy-btn:hover { + background: var(--bg-hover); + color: var(--text-primary); + border-color: var(--text-muted); +} + +.code-copy-btn .copy-icon { + display: inline; +} + +.code-copy-btn .check-icon { + display: none; +} + +.code-copy-btn.copied { + border-color: var(--success); + background: rgba(34, 197, 94, 0.2); + color: var(--success); +} + +.code-copy-btn.copied .copy-icon { + display: none; +} + +.code-copy-btn.copied .check-icon { + display: inline; +} + +/* Headings in messages */ +.message-bubble h2 { + font-size: 1.3em; + font-weight: 600; + margin: var(--space-md) 0 var(--space-sm) 0; + color: var(--text-primary); +} + +.message-bubble h3 { + font-size: 1.15em; + font-weight: 600; + margin: var(--space-md) 0 var(--space-sm) 0; + color: var(--text-primary); +} + +.message-bubble h4 { + font-size: 1.05em; + font-weight: 600; + margin: var(--space-sm) 0 var(--space-xs) 0; + color: var(--text-primary); +} + +.message-bubble h2:first-child, +.message-bubble h3:first-child, +.message-bubble h4:first-child { + margin-top: 0; +} + +/* Lists in messages */ +.message-bubble ul { + margin: var(--space-sm) 0; + padding-left: var(--space-lg); +} + +.message-bubble li { + margin: var(--space-xs) 0; +} + +/* Typing Indicator */ +.typing-indicator { + display: flex; + gap: 4px; + padding: var(--space-md); + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: var(--radius-lg); + border-bottom-left-radius: var(--radius-sm); + width: fit-content; +} + +.typing-indicator span { + width: 8px; + height: 8px; + background: var(--text-muted); + border-radius: 50%; + animation: typing 1.4s ease-in-out infinite; +} + +.typing-indicator span:nth-child(2) { + animation-delay: 0.2s; +} + +.typing-indicator span:nth-child(3) { + animation-delay: 0.4s; +} + +@keyframes typing { + 0%, 100% { transform: translateY(0); opacity: 0.5; } + 50% { transform: translateY(-4px); opacity: 1; } +} + +/* ===================================================== + Chat Input + ===================================================== */ + +.chat-input-container { + padding: var(--space-md) var(--space-lg) var(--space-lg); + background: var(--bg-primary); +} + +.chat-input-form { + max-width: 900px; + margin: 0 auto; +} + +.input-wrapper { + display: flex; + align-items: flex-end; + gap: var(--space-sm); + padding: var(--space-sm); + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: var(--radius-lg); + transition: all var(--transition-fast); +} + +.input-wrapper:focus-within { + border-color: var(--accent-primary); + box-shadow: 0 0 0 3px rgba(99, 102, 241, 0.1); +} + +.input-wrapper textarea { + flex: 1; + background: transparent; + border: none; + outline: none; + color: var(--text-primary); + font-family: inherit; + font-size: 14px; + line-height: 1.5; + resize: none; + max-height: 150px; + padding: var(--space-sm); +} + +.input-wrapper textarea::placeholder { + color: var(--text-muted); +} + +.input-wrapper textarea:disabled { + cursor: not-allowed; +} + +.send-btn { + width: 40px; + height: 40px; + background: var(--accent-gradient); + border: none; + border-radius: var(--radius-md); + color: white; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + transition: all var(--transition-fast); + flex-shrink: 0; +} + +.send-btn:hover:not(:disabled) { + transform: scale(1.05); +} + +.send-btn:active:not(:disabled) { + transform: scale(0.95); +} + +.send-btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +/* Record Button */ +.record-btn { + width: 40px; + height: 40px; + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + transition: all var(--transition-fast); + flex-shrink: 0; + color: var(--text-secondary); +} + +.record-btn:hover { + background: var(--bg-hover); + border-color: var(--text-muted); + color: var(--text-primary); +} + +.record-btn .stop-icon { + display: none; +} + +.record-btn.recording { + background: var(--error); + border-color: var(--error); + color: white; + animation: pulse-recording 1.5s ease-in-out infinite; +} + +.record-btn.recording .mic-icon { + display: none; +} + +.record-btn.recording .stop-icon { + display: block; +} + +@keyframes pulse-recording { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.7; } +} + +.record-btn.transcribing { + pointer-events: none; + opacity: 0.7; +} + +.input-hint { + text-align: center; + font-size: 11px; + color: var(--text-muted); + margin-top: var(--space-sm); + display: flex; + align-items: center; + justify-content: center; + gap: var(--space-xs); +} + +.hint-separator { + opacity: 0.5; +} + +.transcription-settings-link { + background: none; + border: none; + color: var(--text-muted); + font-size: 11px; + cursor: pointer; + padding: 0; + transition: color var(--transition-fast); +} + +.transcription-settings-link:hover { + color: var(--accent-secondary); + text-decoration: underline; +} + +/* Context Usage Indicator */ +.context-usage { + display: flex; + align-items: center; + justify-content: center; + gap: var(--space-sm); + margin-top: var(--space-xs); +} + +.context-label-text { + font-size: 11px; + color: var(--text-muted); +} + +.context-bar { + width: 100px; + height: 4px; + background: var(--bg-tertiary); + border-radius: var(--radius-full); + overflow: hidden; +} + +.context-fill { + height: 100%; + width: 0%; + background: var(--success); + border-radius: var(--radius-full); + transition: width 0.3s ease, background 0.3s ease; +} + +.context-fill.warning { + background: var(--warning); +} + +.context-fill.danger { + background: var(--error); +} + +.context-label { + font-size: 11px; + color: var(--text-muted); + min-width: 28px; +} + +.input-hint kbd { + padding: 2px 6px; + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: var(--radius-sm); + font-family: var(--font-mono); + font-size: 10px; +} + +/* ===================================================== + Toast Notifications + ===================================================== */ + +.toast-container { + position: fixed; + top: var(--space-lg); + right: var(--space-lg); + display: flex; + flex-direction: column; + gap: var(--space-sm); + z-index: 1000; +} + +.toast { + padding: var(--space-md) var(--space-lg); + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + box-shadow: var(--shadow-lg); + display: flex; + align-items: center; + gap: var(--space-sm); + animation: slideIn 0.3s ease; + max-width: 350px; +} + +@keyframes slideIn { + from { + opacity: 0; + transform: translateX(100%); + } + to { + opacity: 1; + transform: translateX(0); + } +} + +.toast.success { + border-left: 3px solid var(--success); +} + +.toast.error { + border-left: 3px solid var(--error); +} + +.toast.warning { + border-left: 3px solid var(--warning); +} + +/* ===================================================== + Scrollbar + ===================================================== */ + +::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +::-webkit-scrollbar-track { + background: transparent; +} + +::-webkit-scrollbar-thumb { + background: var(--border-color); + border-radius: var(--radius-full); +} + +::-webkit-scrollbar-thumb:hover { + background: var(--text-muted); +} + +/* ===================================================== + Responsive + ===================================================== */ + +@media (max-width: 768px) { + .sidebar { + position: fixed; + left: 0; + top: 0; + bottom: 0; + transform: translateX(-100%); + box-shadow: var(--shadow-lg); + } + + .sidebar.open { + transform: translateX(0); + } + + .mobile-menu-btn { + display: block; + } + + .message { + max-width: 95%; + } + + .feature-highlights { + flex-direction: column; + align-items: center; + } +} + +/* ===================================================== + Syntax Highlighting (Basic) + ===================================================== */ + +.hljs-keyword, +.hljs-selector-tag, +.hljs-built_in { + color: #c792ea; +} + +.hljs-string, +.hljs-attr { + color: #c3e88d; +} + +.hljs-number, +.hljs-literal { + color: #f78c6c; +} + +.hljs-comment { + color: #546e7a; + font-style: italic; +} + +.hljs-function, +.hljs-title { + color: #82aaff; +} + +.hljs-variable, +.hljs-params { + color: #f07178; +} + +/* ===================================================== + Modal + ===================================================== */ + +.modal-overlay { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(0, 0, 0, 0.7); + display: none; + align-items: center; + justify-content: center; + z-index: 2000; +} + +.modal-overlay.visible { + display: flex; +} + +.modal { + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: var(--radius-lg); + padding: var(--space-lg); + max-width: 400px; + width: 90%; + box-shadow: var(--shadow-lg); +} + +.modal h3 { + margin-bottom: var(--space-sm); + font-size: 18px; +} + +.modal p { + color: var(--text-secondary); + margin-bottom: var(--space-md); + font-size: 14px; +} + +.whisper-models { + display: flex; + flex-direction: column; + gap: var(--space-sm); + margin-bottom: var(--space-lg); + max-height: 200px; + overflow-y: auto; +} + +.whisper-model-item { + display: flex; + align-items: center; + justify-content: space-between; + padding: var(--space-sm) var(--space-md); + background: var(--bg-tertiary); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); +} + +.whisper-model-item.selected { + border-color: var(--success); + background: rgba(34, 197, 94, 0.1); +} + +.whisper-model-item .model-info { + display: flex; + flex-direction: column; +} + +.whisper-model-item .model-name { + font-weight: 500; + font-size: 13px; +} + +.whisper-model-item .model-size { + font-size: 11px; + color: var(--text-muted); +} + +.whisper-model-item .model-actions { + display: flex; + align-items: center; + gap: var(--space-xs); +} + +.whisper-model-item .download-btn, +.whisper-model-item .use-btn { + padding: var(--space-xs) var(--space-sm); + font-size: 11px; + font-weight: 500; + border: none; + border-radius: var(--radius-sm); + cursor: pointer; + transition: all var(--transition-fast); +} + +.whisper-model-item .download-btn { + background: var(--accent-primary); + color: white; +} + +.whisper-model-item .download-btn:hover { + background: var(--accent-secondary); +} + +.whisper-model-item .use-btn { + background: var(--success); + color: white; +} + +.whisper-model-item .use-btn:hover { + opacity: 0.9; +} + +.whisper-model-item .delete-btn { + padding: var(--space-xs); + font-size: 12px; + background: transparent; + color: var(--text-muted); + border: none; + border-radius: var(--radius-sm); + cursor: pointer; + transition: all var(--transition-fast); +} + +.whisper-model-item .delete-btn:hover { + color: var(--error); +} + +.current-whisper-model { + padding: var(--space-sm) var(--space-md); + background: var(--bg-tertiary); + border-radius: var(--radius-md); + margin-bottom: var(--space-md); + display: flex; + align-items: center; + gap: var(--space-sm); +} + +.current-whisper-model .label { + font-size: 12px; + color: var(--text-secondary); +} + +.current-whisper-model .model-name { + font-weight: 500; + color: var(--accent-secondary); +} + +.modal-actions { + display: flex; + justify-content: flex-end; + gap: var(--space-sm); +} + +.modal-btn { + padding: var(--space-sm) var(--space-md); + font-size: 13px; + font-weight: 500; + border: none; + border-radius: var(--radius-sm); + cursor: pointer; + transition: all var(--transition-fast); +} + +.modal-btn.secondary { + background: var(--bg-tertiary); + color: var(--text-secondary); +} + +.modal-btn.secondary:hover { + background: var(--bg-hover); + color: var(--text-primary); +} diff --git a/samples/js/hello-foundry-local/README.md b/samples/js/hello-foundry-local/README.md deleted file mode 100644 index 24e60d3e..00000000 --- a/samples/js/hello-foundry-local/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Sample: Hello Foundry Local! - -This is a simple example of how to use the Foundry Local SDK to run a model locally and make requests to it. The example demonstrates how to set up the SDK, initialize a model, and make a request to the model. - -Install the Foundry Local SDK and OpenAI packages using npm: - -```bash -npm install foundry-local-sdk openai -``` - -Run the application using Node.js: - -```bash -node src/app.js -``` diff --git a/samples/js/hello-foundry-local/src/app.js b/samples/js/hello-foundry-local/src/app.js deleted file mode 100644 index eb81e3e4..00000000 --- a/samples/js/hello-foundry-local/src/app.js +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -import { OpenAI } from "openai"; -import { FoundryLocalManager } from "foundry-local-sdk"; - -// By using an alias, the most suitable model will be downloaded -// to your end-user's device. -// TIP: You can find a list of available models by running the -// following command in your terminal: `foundry model list`. -const alias = "qwen2.5-coder-0.5b"; - -// Create a FoundryLocalManager instance. This will start the Foundry -// Local service if it is not already running. -const foundryLocalManager = new FoundryLocalManager() - -// Initialize the manager with a model. This will download the model -// if it is not already present on the user's device. -const modelInfo = await foundryLocalManager.init(alias) -console.log("Model Info:", modelInfo) - -const openai = new OpenAI({ - baseURL: foundryLocalManager.endpoint, - apiKey: foundryLocalManager.apiKey, -}); - -async function streamCompletion() { - const stream = await openai.chat.completions.create({ - model: modelInfo.id, - messages: [{ role: "user", content: "What is the golden ratio?" }], - stream: true, - }); - - for await (const chunk of stream) { - if (chunk.choices[0]?.delta?.content) { - process.stdout.write(chunk.choices[0].delta.content); - } - } -} - -streamCompletion(); diff --git a/samples/js/langchain-integration-example/.npmrc b/samples/js/langchain-integration-example/.npmrc new file mode 100644 index 00000000..b337e3f2 --- /dev/null +++ b/samples/js/langchain-integration-example/.npmrc @@ -0,0 +1 @@ +registry=https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/npm/registry/ diff --git a/samples/js/langchain-integration-example/README.md b/samples/js/langchain-integration-example/README.md new file mode 100644 index 00000000..3812d1e5 --- /dev/null +++ b/samples/js/langchain-integration-example/README.md @@ -0,0 +1,56 @@ +# LangChain integration example + +This sample demonstrates how to integrate the Foundry Local SDK with LangChain.js to create a simple application that uses local language models for text generation. + +## Prerequisites +- Ensure you have Node.js installed (version 20 or higher is recommended). + +## Setup project + +Navigate to the sample directory, setup the project, and install the Foundry Local and LangChain packages. + +1. Navigate to the sample directory and setup the project: + ```bash + cd samples/js/langchain-integration-example + npm init -y + npm pkg set type=module + ``` + +1. Install the Foundry Local and LangChain packages: + + **macOS / Linux:** + ```bash + npm install --foreground-scripts foundry-local-sdk@0.9.0-rc2 + npm install @langchain/openai @langchain/core --registry https://registry.npmjs.org + ``` + + **Windows:** + ```bash + npm install --foreground-scripts --winml foundry-local-sdk@0.9.0-rc2 + npm install @langchain/openai @langchain/core --registry https://registry.npmjs.org + ``` + +## Workaround for macOS / Linux + +> **Note:** There is a known issue where ONNX Runtime is not picked up on macOS / Linux. This will be fixed in ORT 1.24.3. In the meantime, add the ONNX Runtime native library to your library path before running the sample: +> +> **macOS:** +> ```bash +> export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$(pwd)/node_modules/@foundry-local-core/darwin-arm64 +> ``` +> +> **Linux:** +> ```bash +> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/node_modules/@foundry-local-core/linux-x64 +> ``` +> +> Run this from the sample directory after installing dependencies. The platform-specific path (e.g., `darwin-arm64`, `linux-x64`) will vary depending on your system architecture. + +## Run the sample + +Run the sample script using Node.js: + +```bash +cd samples/js/langchain-integration-example +node app.js +``` \ No newline at end of file diff --git a/samples/js/langchain-integration-example/app.js b/samples/js/langchain-integration-example/app.js new file mode 100644 index 00000000..0766740c --- /dev/null +++ b/samples/js/langchain-integration-example/app.js @@ -0,0 +1,85 @@ +import { ChatOpenAI } from "@langchain/openai"; +import { ChatPromptTemplate } from "@langchain/core/prompts"; +import { FoundryLocalManager } from 'foundry-local-sdk'; + +// Initialize the Foundry Local SDK +console.log('Initializing Foundry Local SDK...'); + +const endpointUrl = 'http://localhost:5764'; + +const manager = FoundryLocalManager.create({ + appName: 'foundry_local_samples', + logLevel: 'info', + webServiceUrls: endpointUrl +}); +console.log('āœ“ SDK initialized successfully'); + +// Get the model variant +const modelAlias = 'qwen2.5-0.5b'; +const model = await manager.catalog.getModel(modelAlias); +const variant = model.variants[0]; + +// Download the model +console.log(`\nDownloading model ${modelAlias}...`); +await variant.download((progress) => { + process.stdout.write(`\rDownloading... ${progress.toFixed(2)}%`); +}); +console.log('\nāœ“ Model downloaded'); + +// Load the model +console.log(`\nLoading model ${modelAlias}...`); +await variant.load(); +console.log('āœ“ Model loaded'); + +// Start the web service +console.log('\nStarting web service...'); +manager.startWebService(); +console.log('āœ“ Web service started'); + + +// Configure ChatOpenAI to use your locally-running model +const llm = new ChatOpenAI({ + model: variant.id, + configuration: { + baseURL: endpointUrl + '/v1', + apiKey: 'notneeded' + }, + temperature: 0.6, + streaming: false +}); + +// Create a translation prompt template +const prompt = ChatPromptTemplate.fromMessages([ + { + role: "system", + content: "You are a helpful assistant that translates {input_language} to {output_language}." + }, + { + role: "user", + content: "{input}" + } +]); + +// Build a simple chain by connecting the prompt to the language model +const chain = prompt.pipe(llm); + +const input = "I love to code."; +console.log(`Translating '${input}' to French...`); + +// Run the chain with your inputs +await chain.invoke({ + input_language: "English", + output_language: "French", + input: input +}).then(aiMsg => { + // Print the result content + console.log(`Response: ${aiMsg.content}`); +}).catch(err => { + console.error("Error:", err); +}); + +// Tidy up +console.log('Unloading model and stopping web service...'); +await variant.unload(); +manager.stopWebService(); +console.log(`āœ“ Model unloaded and web service stopped`); \ No newline at end of file diff --git a/samples/js/native-chat-completions/.npmrc b/samples/js/native-chat-completions/.npmrc new file mode 100644 index 00000000..b337e3f2 --- /dev/null +++ b/samples/js/native-chat-completions/.npmrc @@ -0,0 +1 @@ +registry=https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/npm/registry/ diff --git a/samples/js/native-chat-completions/README.md b/samples/js/native-chat-completions/README.md new file mode 100644 index 00000000..09539e8f --- /dev/null +++ b/samples/js/native-chat-completions/README.md @@ -0,0 +1,53 @@ +# Native chat completions with Foundry Local SDK + +This sample demonstrates how to use the Foundry Local SDK to perform native chat completions using a local model. It initializes the SDK, selects a model, and sends a chat completion request with a system prompt and user message. + +## Prerequisites +- Ensure you have Node.js installed (version 20 or higher is recommended). + +## Setup project + +Navigate to the sample directory, setup the project, and install the Foundry Local SDK package. + +1. Navigate to the sample directory and setup the project: + ```bash + cd samples/js/native-chat-completions + npm init -y + npm pkg set type=module + ``` + +1. Install the Foundry Local SDK package: + + **macOS / Linux:** + ```bash + npm install --foreground-scripts foundry-local-sdk@0.9.0-rc2 + ``` + + **Windows:** + ```bash + npm install --foreground-scripts --winml foundry-local-sdk@0.9.0-rc2 + ``` + +## Workaround for macOS / Linux + +> **Note:** There is a known issue where ONNX Runtime is not picked up on macOS / Linux. This will be fixed in ORT 1.24.3. In the meantime, add the ONNX Runtime native library to your library path before running the sample: +> +> **macOS:** > ```bash +> export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$(pwd)/node_modules/@foundry-local-core/darwin-arm64 +> ``` +> +> **Linux:** +> ```bash +> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/node_modules/@foundry-local-core/linux-x64 +> ``` +> +> Run this from the sample directory after installing dependencies. The platform-specific path (e.g., `darwin-arm64`, `linux-x64`) will vary depending on your system architecture. + +## Run the sample + +Run the sample script using Node.js: + +```bash +cd samples/js/native-chat-completions +node app.js +``` \ No newline at end of file diff --git a/samples/js/native-chat-completions/app.js b/samples/js/native-chat-completions/app.js new file mode 100644 index 00000000..535f06c2 --- /dev/null +++ b/samples/js/native-chat-completions/app.js @@ -0,0 +1,60 @@ +import { FoundryLocalManager } from 'foundry-local-sdk'; + +// Initialize the Foundry Local SDK +console.log('Initializing Foundry Local SDK...'); + +const manager = FoundryLocalManager.create({ + appName: 'foundry_local_samples', + logLevel: 'info' +}); +console.log('āœ“ SDK initialized successfully'); + +// Get the model variant +const modelAlias = 'qwen2.5-0.5b'; +const model = await manager.catalog.getModel(modelAlias); +const variant = model.variants[0]; + +// Download the model +console.log(`\nDownloading model ${modelAlias}...`); +await variant.download((progress) => { + process.stdout.write(`\rDownloading... ${progress.toFixed(2)}%`); +}); +console.log('\nāœ“ Model downloaded'); + +// Load the model +console.log(`\nLoading model ${modelAlias}...`); +await variant.load(); +console.log('āœ“ Model loaded'); + +// Create chat client +console.log('\nCreating chat client...'); +const chatClient = variant.createChatClient(); +console.log('āœ“ Chat client created'); + +// Example chat completion +console.log('\nTesting chat completion...'); +const completion = await chatClient.completeChat([ + { role: 'user', content: 'Why is the sky blue?' } +]); + +console.log('\nChat completion result:'); +console.log(completion.choices[0]?.message?.content); + +// Example streaming completion +console.log('\nTesting streaming completion...'); +await chatClient.completeStreamingChat( + [{ role: 'user', content: 'Write a short poem about programming.' }], + (chunk) => { + const content = chunk.choices?.[0]?.message?.content; + if (content) { + process.stdout.write(content); + } + } +); +console.log('\n'); + +// Unload the model +console.log('Unloading model...'); +await variant.unload(); +console.log(`āœ“ Model unloaded`); + \ No newline at end of file diff --git a/samples/js/web-server-example/.npmrc b/samples/js/web-server-example/.npmrc new file mode 100644 index 00000000..b337e3f2 --- /dev/null +++ b/samples/js/web-server-example/.npmrc @@ -0,0 +1 @@ +registry=https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/npm/registry/ diff --git a/samples/js/web-server-example/README.md b/samples/js/web-server-example/README.md new file mode 100644 index 00000000..c6c625bc --- /dev/null +++ b/samples/js/web-server-example/README.md @@ -0,0 +1,56 @@ +# Chat completions using an OpenAI-compatible web server + +This sample demonstrates how to use the Foundry Local SDK to perform chat completions using an OpenAI-compatible web server. It initializes the SDK with the server URL, selects a model, and sends a chat completion request with a system prompt and user message. + +## Prerequisites +- Ensure you have Node.js installed (version 20 or higher is recommended). + +## Setup project + +Navigate to the sample directory, setup the project, and install the required packages. + +1. Navigate to the sample directory and setup the project: + ```bash + cd samples/js/web-server-example + npm init -y + npm pkg set type=module + ``` + +1. Install the Foundry Local and OpenAI packages: + + **macOS / Linux:** + ```bash + npm install --foreground-scripts foundry-local-sdk@0.9.0-rc2 + npm install openai --registry https://registry.npmjs.org + ``` + + **Windows:** + ```bash + npm install --foreground-scripts --winml foundry-local-sdk@0.9.0-rc2 + npm install openai --registry https://registry.npmjs.org + ``` + +## Workaround for macOS / Linux + +> **Note:** There is a known issue where ONNX Runtime is not picked up on macOS / Linux. This will be fixed in ORT 1.24.3. In the meantime, add the ONNX Runtime native library to your library path before running the sample: +> +> **macOS:** +> ```bash +> export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$(pwd)/node_modules/@foundry-local-core/darwin-arm64 +> ``` +> +> **Linux:** +> ```bash +> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/node_modules/@foundry-local-core/linux-x64 +> ``` +> +> Run this from the sample directory after installing dependencies. The platform-specific path (e.g., `darwin-arm64`, `linux-x64`) will vary depending on your system architecture. + +## Run the sample + +Run the sample script using Node.js: + +```bash +cd samples/js/web-server-example +node app.js +``` \ No newline at end of file diff --git a/samples/js/web-server-example/app.js b/samples/js/web-server-example/app.js new file mode 100644 index 00000000..27f23ecd --- /dev/null +++ b/samples/js/web-server-example/app.js @@ -0,0 +1,61 @@ +import { FoundryLocalManager } from 'foundry-local-sdk'; +import { OpenAI } from 'openai'; + +// Initialize the Foundry Local SDK +console.log('Initializing Foundry Local SDK...'); + +const endpointUrl = 'http://localhost:5764'; + +const manager = FoundryLocalManager.create({ + appName: 'foundry_local_samples', + logLevel: 'info', + webServiceUrls: endpointUrl +}); +console.log('āœ“ SDK initialized successfully'); + +// Get the model variant +const modelAlias = 'qwen2.5-0.5b'; +const model = await manager.catalog.getModel(modelAlias); +const variant = model.variants[0]; + +// Download the model +console.log(`\nDownloading model ${modelAlias}...`); +await variant.download((progress) => { + process.stdout.write(`\rDownloading... ${progress.toFixed(2)}%`); +}); +console.log('\nāœ“ Model downloaded'); + +// Load the model +console.log(`\nLoading model ${modelAlias}...`); +await variant.load(); +console.log('āœ“ Model loaded'); + +// Start the web service +console.log('\nStarting web service...'); +manager.startWebService(); +console.log('āœ“ Web service started'); + +const openai = new OpenAI({ + baseURL: endpointUrl + '/v1', + apiKey: 'notneeded', +}); + +// Example chat completion +console.log('\nTesting chat completion with OpenAI client...'); +const response = await openai.chat.completions.create({ + model: variant.id, + messages: [ + { + role: "user", + content: "What is the golden ratio?", + }, + ], +}); + +console.log(response.choices[0].message.content); + +// Tidy up +console.log('Unloading model and stopping web service...'); +await variant.unload(); +manager.stopWebService(); +console.log(`āœ“ Model unloaded and web service stopped`); diff --git a/sdk_v2/cs/src/Catalog.cs b/sdk_v2/cs/src/Catalog.cs index eb9ba0d7..f57f57c0 100644 --- a/sdk_v2/cs/src/Catalog.cs +++ b/sdk_v2/cs/src/Catalog.cs @@ -197,4 +197,84 @@ public void Dispose() { _lock.Dispose(); } + + public async Task AddCatalogAsync(string name, Uri uri, string? clientId = null, + string? clientSecret = null, string? bearerToken = null, + string? tokenEndpoint = null, string? audience = null, + CancellationToken? ct = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(uri); + + await Utils.CallWithExceptionHandling(async () => + { + var request = new CoreInteropRequest + { + Params = new Dictionary + { + ["Name"] = name, + ["Uri"] = uri.ToString(), + ["ClientId"] = clientId ?? "", + ["ClientSecret"] = clientSecret ?? "", + ["BearerToken"] = bearerToken ?? "", + ["TokenEndpoint"] = tokenEndpoint ?? "", + ["Audience"] = audience ?? "" + } + }; + + var result = await _coreInterop.ExecuteCommandAsync("add_catalog", request, ct) + .ConfigureAwait(false); + if (result.Error != null) + { + throw new FoundryLocalException($"Error adding catalog '{name}': {result.Error}", _logger); + } + + // Force model list refresh to pick up new catalog's models + _lastFetch = DateTime.MinValue; + await UpdateModels(ct).ConfigureAwait(false); + }, $"Error adding catalog '{name}'.", _logger).ConfigureAwait(false); + } + + public async Task SelectCatalogAsync(string? catalogName, CancellationToken? ct = null) + { + await Utils.CallWithExceptionHandling(async () => + { + var request = new CoreInteropRequest + { + Params = new Dictionary + { + ["Name"] = catalogName ?? "" + } + }; + + var result = await _coreInterop.ExecuteCommandAsync("select_catalog", request, ct) + .ConfigureAwait(false); + if (result.Error != null) + { + throw new FoundryLocalException($"Error selecting catalog: {result.Error}", _logger); + } + + // Force model list refresh so the managed-side maps reflect the filter. + // The native core already has models cached; this just re-fetches the + // (now-filtered) list into _modelAliasToModel / _modelIdToModelVariant. + _lastFetch = DateTime.MinValue; + await UpdateModels(ct).ConfigureAwait(false); + }, "Error selecting catalog.", _logger).ConfigureAwait(false); + } + + public async Task> GetCatalogNamesAsync(CancellationToken? ct = null) + { + return await Utils.CallWithExceptionHandling(async () => + { + CoreInteropRequest? input = null; + var result = await _coreInterop.ExecuteCommandAsync("get_catalog_names", input, ct) + .ConfigureAwait(false); + if (result.Error != null) + { + throw new FoundryLocalException($"Error getting catalog names: {result.Error}", _logger); + } + + return JsonSerializer.Deserialize(result.Data!, JsonSerializationContext.Default.ListString) ?? []; + }, "Error getting catalog names.", _logger).ConfigureAwait(false); + } } diff --git a/sdk_v2/cs/src/Detail/JsonSerializationContext.cs b/sdk_v2/cs/src/Detail/JsonSerializationContext.cs index b9031426..de7962e5 100644 --- a/sdk_v2/cs/src/Detail/JsonSerializationContext.cs +++ b/sdk_v2/cs/src/Detail/JsonSerializationContext.cs @@ -21,6 +21,7 @@ namespace Microsoft.AI.Foundry.Local.Detail; [JsonSerializable(typeof(AudioCreateTranscriptionRequest))] [JsonSerializable(typeof(AudioCreateTranscriptionResponse))] [JsonSerializable(typeof(string[]))] // list loaded or cached models +[JsonSerializable(typeof(List))] // catalog names [JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, WriteIndented = false)] internal partial class JsonSerializationContext : JsonSerializerContext diff --git a/sdk_v2/cs/src/ICatalog.cs b/sdk_v2/cs/src/ICatalog.cs index 12347940..e2495e92 100644 --- a/sdk_v2/cs/src/ICatalog.cs +++ b/sdk_v2/cs/src/ICatalog.cs @@ -50,4 +50,35 @@ public interface ICatalog /// Optional CancellationToken. /// List of ModelVariant instances. Task> GetLoadedModelsAsync(CancellationToken? ct = null); + + /// + /// Add a private model catalog. Models from the new catalog become available + /// on the next ListModelsAsync or GetModelAsync call. + /// + /// Display name for the catalog (e.g. "my-private-catalog"). + /// Base URL of the private catalog service. + /// Optional OAuth2 client credentials ID. + /// Optional OAuth2 client credentials secret, or API key for legacy auth. + /// Optional pre-obtained bearer token (for testing/self-service auth). + /// Optional OAuth2 token endpoint URL (e.g. "https://idp.example.com/oauth/token"). + /// Optional OAuth2 audience parameter (e.g. "model-distribution-service"). + /// Optional CancellationToken. + Task AddCatalogAsync(string name, Uri uri, string? clientId = null, string? clientSecret = null, + string? bearerToken = null, string? tokenEndpoint = null, string? audience = null, + CancellationToken? ct = null); + + /// + /// Filter the catalog to only return models from the named catalog. + /// Pass null to reset and show models from all catalogs. + /// + /// Catalog name to filter to, or null to show all. + /// Optional CancellationToken. + Task SelectCatalogAsync(string? catalogName, CancellationToken? ct = null); + + /// + /// Get the names of all registered catalogs. + /// + /// Optional CancellationToken. + /// List of catalog name strings. + Task> GetCatalogNamesAsync(CancellationToken? ct = null); } diff --git a/sdk_v2/cs/test/FoundryLocal.Tests/CatalogManagementTests.cs b/sdk_v2/cs/test/FoundryLocal.Tests/CatalogManagementTests.cs new file mode 100644 index 00000000..81dc97f9 --- /dev/null +++ b/sdk_v2/cs/test/FoundryLocal.Tests/CatalogManagementTests.cs @@ -0,0 +1,61 @@ +// -------------------------------------------------------------------------------------------------------------------- +// +// Copyright (c) Microsoft. All rights reserved. +// +// -------------------------------------------------------------------------------------------------------------------- + +namespace Microsoft.AI.Foundry.Local.Tests; + +using System.Text.Json; +using Microsoft.AI.Foundry.Local.Detail; +using Moq; + +public class CatalogManagementTests +{ + private static async Task CreateCatalogWithIntercepts( + List extra) + { + var logger = Utils.CreateCapturingLoggerMock([]); + var lm = new Mock(); + lm.Setup(m => m.ListLoadedModelsAsync(It.IsAny())).ReturnsAsync(Array.Empty()); + + List intercepts = + [ + new() { CommandName = "get_catalog_name", ResponseData = "Test" }, + new() { CommandName = "get_model_list", + ResponseData = JsonSerializer.Serialize(Utils.TestCatalog.TestCatalog, + JsonSerializationContext.Default.ListModelInfo) }, + new() { CommandName = "get_cached_model_ids", ResponseData = "[]" }, + .. extra + ]; + + var ci = Utils.CreateCoreInteropWithIntercept(Utils.CoreInterop, intercepts); + return await Catalog.CreateAsync(lm.Object, ci.Object, logger.Object); + } + + [Test] + public async Task Test_AddAndSelectCatalog() + { + using var catalog = await CreateCatalogWithIntercepts( + [ + new() { CommandName = "add_catalog", ResponseData = "OK" }, + new() { CommandName = "select_catalog", ResponseData = "OK" } + ]); + + await catalog.AddCatalogAsync("priv", new Uri("https://mds.example.com"), "id", "secret"); + await catalog.SelectCatalogAsync("priv"); + await catalog.SelectCatalogAsync(null); + await Assert.That(catalog).IsNotNull(); + } + + [Test] + public async Task Test_GetCatalogNames() + { + using var catalog = await CreateCatalogWithIntercepts( + [new() { CommandName = "get_catalog_names", ResponseData = "[\"public\",\"private\"]" }]); + + var names = await catalog.GetCatalogNamesAsync(); + await Assert.That(names.Count).IsEqualTo(2); + await Assert.That(names).Contains("private"); + } +} diff --git a/sdk_v2/js/src/detail/coreInterop.ts b/sdk_v2/js/src/detail/coreInterop.ts index 167784e7..b5ddb57f 100644 --- a/sdk_v2/js/src/detail/coreInterop.ts +++ b/sdk_v2/js/src/detail/coreInterop.ts @@ -83,10 +83,11 @@ export class CoreInterop { const coreDir = path.dirname(corePath); const ext = CoreInterop._getLibraryExtension(); - // On Windows, explicitly load dependencies to work around DLL resolution challenges + // Explicitly preload native dependencies so the .NET runtime can resolve them + const libPrefix = process.platform === 'win32' ? '' : 'lib'; + koffi.load(path.join(coreDir, `${libPrefix}onnxruntime${ext}`)); + koffi.load(path.join(coreDir, `${libPrefix}onnxruntime-genai${ext}`)); if (process.platform === 'win32') { - koffi.load(path.join(coreDir, `onnxruntime${ext}`)); - koffi.load(path.join(coreDir, `onnxruntime-genai${ext}`)); process.env.PATH = `${coreDir};${process.env.PATH}`; } this.lib = koffi.load(corePath);