diff --git a/frontend/src/components/LLMSelection/ChutesAiOptions/index.jsx b/frontend/src/components/LLMSelection/ChutesAiOptions/index.jsx new file mode 100644 index 00000000000..3d3e51226f9 --- /dev/null +++ b/frontend/src/components/LLMSelection/ChutesAiOptions/index.jsx @@ -0,0 +1,114 @@ +import { useState, useEffect } from "react"; +import System from "@/models/system"; + +export default function ChutesAiOptions({ settings }) { + const [inputValue, setInputValue] = useState(settings?.ChutesApiKey); + const [apiKey, setApiKey] = useState(settings?.ChutesApiKey); + + return ( +
+
+ + setInputValue(e.target.value)} + onBlur={() => setApiKey(inputValue)} + /> +
+ + {!settings?.credentialsOnly && ( + + )} +
+ ); +} + +function ChutesAIModelSelection({ apiKey, settings }) { + const [customModels, setCustomModels] = useState([]); + const [loading, setLoading] = useState(true); + + useEffect(() => { + async function findCustomModels() { + if (!apiKey) { + setCustomModels([]); + setLoading(true); + return; + } + + try { + setLoading(true); + const { models } = await System.customModels("chutes", apiKey); + setCustomModels(models || []); + } catch (error) { + console.error("Failed to fetch custom models:", error); + setCustomModels([]); + } finally { + setLoading(false); + } + } + findCustomModels(); + }, [apiKey]); + + if (loading) { + return ( +
+ + +

+ Enter a valid API key to view all available models for your account. +

+
+ ); + } + + return ( +
+ + +

+ Select the Chutes AI model you want to use for your conversations. +

+
+ ); +} diff --git a/frontend/src/components/ProviderPrivacy/constants.js b/frontend/src/components/ProviderPrivacy/constants.js index ac01c3c33cd..d5bb26a4214 100644 --- a/frontend/src/components/ProviderPrivacy/constants.js +++ b/frontend/src/components/ProviderPrivacy/constants.js @@ -45,6 +45,7 @@ import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png"; import PrivateModeLogo from "@/media/llmprovider/privatemode.png"; import SambaNovaLogo from "@/media/llmprovider/sambanova.png"; import LemonadeLogo from "@/media/llmprovider/lemonade.png"; +import ChutesLogo from "@/media/llmprovider/chutes.png"; const LLM_PROVIDER_PRIVACY_MAP = { openai: { @@ -252,6 +253,11 @@ const LLM_PROVIDER_PRIVACY_MAP = { ], logo: LemonadeLogo, }, + chutes: { + name: "Chutes AI", + policyUrl: "https://chutes.ai/privacy", + logo: ChutesLogo, + }, }; const VECTOR_DB_PROVIDER_PRIVACY_MAP = { diff --git a/frontend/src/hooks/useGetProvidersModels.js b/frontend/src/hooks/useGetProvidersModels.js index b13a1c29519..97234929e7a 100644 --- a/frontend/src/hooks/useGetProvidersModels.js +++ b/frontend/src/hooks/useGetProvidersModels.js @@ -32,6 +32,7 @@ const PROVIDER_DEFAULT_MODELS = { "generic-openai": [], bedrock: [], xai: ["grok-beta"], + chutes: [], }; // For providers with large model lists (e.g. togetherAi) - we subgroup the options diff --git a/frontend/src/media/llmprovider/chutes.png b/frontend/src/media/llmprovider/chutes.png new file mode 100644 index 00000000000..302f5dbee0a Binary files /dev/null and b/frontend/src/media/llmprovider/chutes.png differ diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx index b6eda7fe2e3..96f6cf7eb6e 100644 --- a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx @@ -41,6 +41,7 @@ import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png"; import PrivateModeLogo from "@/media/llmprovider/privatemode.png"; import SambaNovaLogo from "@/media/llmprovider/sambanova.png"; import LemonadeLogo from "@/media/llmprovider/lemonade.png"; +import ChutesLogo from "@/media/llmprovider/chutes.png"; import PreLoader from "@/components/Preloader"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; @@ -79,6 +80,7 @@ import DockerModelRunnerOptions from "@/components/LLMSelection/DockerModelRunne import PrivateModeOptions from "@/components/LLMSelection/PrivateModeOptions"; import SambaNovaOptions from "@/components/LLMSelection/SambaNovaOptions"; import LemonadeOptions from "@/components/LLMSelection/LemonadeOptions"; +import ChutesAiOptions from "@/components/LLMSelection/ChutesAiOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react"; @@ -401,6 +403,15 @@ export const AVAILABLE_LLM_PROVIDERS = [ description: "Run GiteeAI's powerful LLMs.", requiredConfig: ["GiteeAIApiKey"], }, + { + name: "Chutes AI", + value: "chutes", + logo: ChutesLogo, + options: (settings) => , + description: + "Decentralized AI inference on Bittensor SN64, TEE-secured, OpenAI-compatible.", + requiredConfig: ["ChutesApiKey"], + }, { name: "Generic OpenAI", value: "generic-openai", diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx index 3e6f2a5e84d..2e0be7d378a 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx @@ -35,6 +35,7 @@ import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png"; import PrivateModeLogo from "@/media/llmprovider/privatemode.png"; import SambaNovaLogo from "@/media/llmprovider/sambanova.png"; import LemonadeLogo from "@/media/llmprovider/lemonade.png"; +import ChutesLogo from "@/media/llmprovider/chutes.png"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions"; @@ -71,6 +72,7 @@ import DockerModelRunnerOptions from "@/components/LLMSelection/DockerModelRunne import PrivateModeOptions from "@/components/LLMSelection/PrivateModeOptions"; import SambaNovaOptions from "@/components/LLMSelection/SambaNovaOptions"; import LemonadeOptions from "@/components/LLMSelection/LemonadeOptions"; +import ChutesAiOptions from "@/components/LLMSelection/ChutesAiOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import System from "@/models/system"; @@ -336,6 +338,14 @@ const LLMS = [ options: (settings) => , description: "Run GiteeAI's powerful LLMs.", }, + { + name: "Chutes AI", + value: "chutes", + logo: ChutesLogo, + options: (settings) => , + description: + "Decentralized AI inference on Bittensor SN64, TEE-secured, OpenAI-compatible.", + }, ]; export default function LLMPreference({ diff --git a/server/utils/AiProviders/chutes/index.js b/server/utils/AiProviders/chutes/index.js new file mode 100644 index 00000000000..d58ba3c0175 --- /dev/null +++ b/server/utils/AiProviders/chutes/index.js @@ -0,0 +1,186 @@ +const { NativeEmbedder } = require("../../EmbeddingEngines/native"); +const { + LLMPerformanceMonitor, +} = require("../../helpers/chat/LLMPerformanceMonitor"); +const { + handleDefaultStreamResponseV2, +} = require("../../helpers/chat/responses"); +const { MODEL_MAP } = require("../modelMap"); + +class ChutesLLM { + constructor(embedder = null, modelPreference = null) { + const { OpenAI: OpenAIApi } = require("openai"); + if (!process.env.CHUTES_API_KEY) + throw new Error("No Chutes API key was set."); + + this.className = "ChutesLLM"; + this.openai = new OpenAIApi({ + baseURL: "https://llm.chutes.ai/v1", + apiKey: process.env.CHUTES_API_KEY, + }); + this.model = + modelPreference || process.env.CHUTES_MODEL_PREF || "chutes/DeepSeek-V3.2-TEE"; + this.limits = { + history: this.promptWindowLimit() * 0.15, + system: this.promptWindowLimit() * 0.15, + user: this.promptWindowLimit() * 0.7, + }; + + this.embedder = embedder ?? new NativeEmbedder(); + this.defaultTemp = 0.7; + } + + #appendContext(contextTexts = []) { + if (!contextTexts || !contextTexts.length) return ""; + return ( + "\nContext:\n" + + contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("") + ); + } + + #log(text, ...args) { + console.log(`\x1b[32m[ChutesAI]\x1b[0m ${text}`, ...args); + } + + streamingEnabled() { + return "streamGetChatCompletion" in this; + } + + static promptWindowLimit(modelName) { + return MODEL_MAP.get("chutes", modelName) ?? 65536; + } + + promptWindowLimit() { + return MODEL_MAP.get("chutes", this.model) ?? 65536; + } + + async isValidChatCompletionModel(modelName = "") { + return !!modelName; + } + + /** + * Generates appropriate content array for a message + attachments. + */ + #generateContent({ userPrompt, attachments = [] }) { + if (!attachments.length) return userPrompt; + const content = [{ type: "text", text: userPrompt }]; + for (let attachment of attachments) { + content.push({ + type: "image_url", + image_url: { + url: attachment.contentString, + }, + }); + } + return content.flat(); + } + + constructPrompt({ + systemPrompt = "", + contextTexts = [], + chatHistory = [], + userPrompt = "", + attachments = [], + }) { + const prompt = [ + { + role: "system", + content: `${systemPrompt}${this.#appendContext(contextTexts)}`, + }, + ...chatHistory, + { + role: "user", + content: attachments.length + ? this.#generateContent({ userPrompt, attachments }) + : userPrompt, + }, + ]; + return prompt; + } + + async getChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `ChutesAI:chatCompletion: ${this.model} is not valid for chat completion!` + ); + + const result = await LLMPerformanceMonitor.measureAsyncFunction( + this.openai.chat.completions + .create({ + model: this.model, + messages, + temperature, + }) + .catch((e) => { + throw new Error(e.message); + }) + ); + + if ( + !result.output.hasOwnProperty("choices") || + result.output.choices.length === 0 + ) + return null; + + return { + textResponse: result.output.choices[0].message.content, + metrics: { + prompt_tokens: result.output.usage?.prompt_tokens || 0, + completion_tokens: result.output.usage?.completion_tokens || 0, + total_tokens: result.output.usage?.total_tokens || 0, + outputTps: result.output.usage?.completion_tokens / (result.output.usage?.completion_time || 1), + duration: result.output.usage?.total_time, + model: this.model, + provider: this.className, + timestamp: new Date(), + }, + }; + } + + async streamGetChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `ChutesAI:streamChatCompletion: ${this.model} is not valid for chat completion!` + ); + + const measuredStreamRequest = await LLMPerformanceMonitor.measureStream({ + func: this.openai.chat.completions.create({ + model: this.model, + stream: true, + messages, + temperature, + }), + messages, + runPromptTokenCalculation: false, + modelTag: this.model, + provider: this.className, + }); + return measuredStreamRequest; + } + + handleStream(response, stream, responseProps) { + return handleDefaultStreamResponseV2(response, stream, responseProps); + } + + async embedTextInput(textInput) { + return await this.embedder.embedTextInput(textInput); + } + + async embedChunks(textChunks = []) { + return await this.embedder.embedChunks(textChunks); + } + + async compressMessages(promptArgs = {}, rawHistory = []) { + const { messageArrayCompressor } = require("../../helpers/chat"); + const messageArray = this.constructPrompt(promptArgs); + return await messageArrayCompressor(this, messageArray, rawHistory); + } +} + +module.exports = { + ChutesLLM, +}; diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js index 2c122e825f4..3709d9da268 100644 --- a/server/utils/helpers/customModels.js +++ b/server/utils/helpers/customModels.js @@ -49,6 +49,7 @@ const SUPPORT_CUSTOM_MODELS = [ "privatemode", "sambanova", "lemonade", + "chutes", // Embedding Engines "native-embedder", "cohere-embedder", @@ -133,6 +134,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) { return await getLemonadeModels(basePath); case "lemonade-embedder": return await getLemonadeModels(basePath, "embedding"); + case "chutes": + return await getChutesModels(apiKey); default: return { models: [], error: "Invalid provider for custom models" }; } @@ -319,6 +322,29 @@ async function getGroqAiModels(_apiKey = null) { return { models, error: null }; } +async function getChutesModels(_apiKey = null) { + const { OpenAI: OpenAIApi } = require("openai"); + const apiKey = + _apiKey === true + ? process.env.CHUTES_API_KEY + : _apiKey || process.env.CHUTES_API_KEY || null; + const openai = new OpenAIApi({ + baseURL: "https://llm.chutes.ai/v1", + apiKey, + }); + const models = await openai.models + .list() + .then((results) => results.data) + .catch((e) => { + console.error(`ChutesAI:listModels`, e.message); + return []; + }); + + // Api Key was successful so lets save it for future uses + if (models.length > 0 && !!apiKey) process.env.CHUTES_API_KEY = apiKey; + return { models, error: null }; +} + async function liteLLMModels(basePath = null, apiKey = null) { const { OpenAI: OpenAIApi } = require("openai"); const openai = new OpenAIApi({ diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index 29b66fe8bc9..dc69ebc6642 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -175,6 +175,9 @@ function getLLMProvider({ provider = null, model = null } = {}) { case "groq": const { GroqLLM } = require("../AiProviders/groq"); return new GroqLLM(embedder, model); + case "chutes": + const { ChutesLLM } = require("../AiProviders/chutes"); + return new ChutesLLM(embedder, model); case "koboldcpp": const { KoboldCPPLLM } = require("../AiProviders/koboldCPP"); return new KoboldCPPLLM(embedder, model);