Docker Model Runner LLM/Agent Provider

README.md

@@ -104,6 +104,7 @@ AnythingLLM divides your documents into objects called `workspaces`. A Workspace
 - [Moonshot AI](https://www.moonshot.ai/)
 - [Microsoft Foundry Local](https://github.com/microsoft/Foundry-Local)
 - [CometAPI (chat models)](https://api.cometapi.com/)
+- [Docker Model Runner](https://docs.docker.com/ai/model-runner/)
 
 **Embedder models:**
 

docker/.env.example

@@ -149,6 +149,11 @@ GID='1000'
 # FOUNDRY_MODEL_PREF='phi-3.5-mini'
 # FOUNDRY_MODEL_TOKEN_LIMIT=4096
 
+# LLM_PROVIDER='docker-model-runner'
+# DOCKER_MODEL_RUNNER_BASE_PATH='http://127.0.0.1:12434'
+# DOCKER_MODEL_RUNNER_MODEL_PREF='phi-3.5-mini'
+# DOCKER_MODEL_RUNNER_MODEL_TOKEN_LIMIT=4096
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################

frontend/src/components/LLMSelection/DockerModelRunnerOptions/index.jsx

@@ -0,0 +1,152 @@
+import { useState, useEffect } from "react";
+import System from "@/models/system";
+import PreLoader from "@/components/Preloader";
+import { DOCKER_MODEL_RUNNER_COMMON_URLS } from "@/utils/constants";
+import useProviderEndpointAutoDiscovery from "@/hooks/useProviderEndpointAutoDiscovery";
+import { upperCaseFirst } from "text-case";
+
+export default function DockerModelRunnerOptions({ settings }) {
+  const {
+    autoDetecting: loading,
+    basePath,
+    basePathValue,
+    handleAutoDetectClick,
+  } = useProviderEndpointAutoDiscovery({
+    provider: "docker-model-runner",
+    initialBasePath: settings?.DockerModelRunnerBasePath,
+    ENDPOINTS: DOCKER_MODEL_RUNNER_COMMON_URLS,
+  });
+
+  const [maxTokens, setMaxTokens] = useState(
+    settings?.DockerModelRunnerModelTokenLimit || 4096
+  );
+
+  return (
+    <div className="w-full flex flex-col gap-y-7">
+      <div className="flex gap-[36px] mt-1.5 flex-wrap">
+        <div className="flex flex-col w-60">
+          <div className="flex justify-between items-center mb-2">
+            <label className="text-white text-sm font-semibold">Base URL</label>
+            {loading ? (
+              <PreLoader size="6" />
+            ) : (
+              <>
+                {!basePathValue.value && (
+                  <button
+                    onClick={handleAutoDetectClick}
+                    className="bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
+                  >
+                    Auto-Detect
+                  </button>
+                )}
+              </>
+            )}
+          </div>
+          <input
+            type="url"
+            name="DockerModelRunnerBasePath"
+            className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="http://localhost:12434/engines/llama.cpp/v1"
+            value={basePathValue.value}
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+            onChange={basePath.onChange}
+            onBlur={basePath.onBlur}
+          />
+        </div>
+        <DockerModelRunnerModelSelection
+          settings={settings}
+          basePath={basePathValue.value}
+        />
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            Token context window
+          </label>
+          <input
+            type="number"
+            name="DockerModelRunnerModelTokenLimit"
+            className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="4096"
+            min={1}
+            value={maxTokens}
+            onChange={(e) => setMaxTokens(Number(e.target.value))}
+            onScroll={(e) => e.target.blur()}
+            required={true}
+            autoComplete="off"
+          />
+        </div>
+      </div>
+    </div>
+  );
+}
+
+function DockerModelRunnerModelSelection({ settings, basePath = null }) {
+  const [customModels, setCustomModels] = useState([]);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    async function findCustomModels() {
+      if (!basePath) {
+        setCustomModels([]);
+        setLoading(false);
+        return;
+      }
+
+      setLoading(true);
+      const { models } = await System.customModels(
+        "docker-model-runner",
+        null,
+        basePath
+      );
+      setCustomModels(models || []);
+      setLoading(false);
+    }
+    findCustomModels();
+  }, [basePath]);
+
+  if (loading) {
+    return (
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-3">
+          Chat Model Selection
+        </label>
+        <PreLoader />
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex flex-col w-60">
+      <label className="text-white text-sm font-semibold block mb-3">
+        Chat Model Selection
+      </label>
+      <select
+        name="DockerModelRunnerModelPref"
+        required={true}
+        className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+      >
+        <option disabled={true} selected={true}>
+          -- select a model --
+        </option>
+        {customModels.length > 0 &&
+          customModels.map((model) => {
+            return (
+              <option
+                key={model.id}
+                value={model.id}
+                selected={settings?.DockerModelRunnerModelPref === model.id}
+              >
+                {upperCaseFirst(model.name)}
+              </option>
+            );
+          })}
+        {customModels.length === 0 && (
+          <option disabled={true} selected={true}>
+            -- no models found --
+          </option>
+        )}
+      </select>
+    </div>
+  );
+}

frontend/src/pages/GeneralSettings/LLMPreference/index.jsx

@@ -35,6 +35,7 @@ import DellProAiStudioLogo from "@/media/llmprovider/dpais.png";
 import MoonshotAiLogo from "@/media/llmprovider/moonshotai.png";
 import CometApiLogo from "@/media/llmprovider/cometapi.png";
 import FoundryLogo from "@/media/llmprovider/foundry-local.png";
+import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png";
 
 import PreLoader from "@/components/Preloader";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
@@ -67,6 +68,7 @@ import PPIOLLMOptions from "@/components/LLMSelection/PPIOLLMOptions";
 import DellProAiStudioOptions from "@/components/LLMSelection/DPAISOptions";
 import MoonshotAiOptions from "@/components/LLMSelection/MoonshotAiOptions";
 import FoundryOptions from "@/components/LLMSelection/FoundryOptions";
+import DockerModelRunnerOptions from "@/components/LLMSelection/DockerModelRunnerOptions";
 
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
@@ -327,6 +329,18 @@ export const AVAILABLE_LLM_PROVIDERS = [
       "FoundryModelTokenLimit",
     ],
   },
+  {
+    name: "Docker Model Runner",
+    value: "docker-model-runner",
+    logo: DockerModelRunnerLogo,
+    options: (settings) => <DockerModelRunnerOptions settings={settings} />,
+    description: "Run models locally using Docker Model Runner.",
+    requiredConfig: [
+      "DockerModelRunnerBasePath",
+      "DockerModelRunnerModelPref",
+      "DockerModelRunnerModelTokenLimit",
+    ],
+  },
   {
     name: "xAI",
     value: "xai",

frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx

@@ -41,6 +41,7 @@ import DPAISLogo from "@/media/llmprovider/dpais.png";
 import MoonshotAiLogo from "@/media/llmprovider/moonshotai.png";
 import CometApiLogo from "@/media/llmprovider/cometapi.png";
 import FoundryLogo from "@/media/llmprovider/foundry-local.png";
+import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png";
 
 import React, { useState, useEffect } from "react";
 import paths from "@/utils/paths";
@@ -269,6 +270,13 @@ export const LLM_SELECTION_PRIVACY = {
     ],
     logo: FoundryLogo,
   },
+  "docker-model-runner": {
+    name: "Docker Model Runner",
+    description: [
+      "Your model and chats are only accessible on the machine running Docker Model Runner",
+    ],
+    logo: DockerModelRunnerLogo,
+  },
 };
 
 export const VECTOR_DB_PRIVACY = {

frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx

@@ -29,6 +29,7 @@ import PPIOLogo from "@/media/llmprovider/ppio.png";
 import DellProAiStudioLogo from "@/media/llmprovider/dpais.png";
 import MoonshotAiLogo from "@/media/llmprovider/moonshotai.png";
 import CometApiLogo from "@/media/llmprovider/cometapi.png";
+import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png";
 
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
 import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
@@ -59,6 +60,7 @@ import PPIOLLMOptions from "@/components/LLMSelection/PPIOLLMOptions";
 import DellProAiStudioOptions from "@/components/LLMSelection/DPAISOptions";
 import MoonshotAiOptions from "@/components/LLMSelection/MoonshotAiOptions";
 import CometApiLLMOptions from "@/components/LLMSelection/CometApiLLMOptions";
+import DockerModelRunnerOptions from "@/components/LLMSelection/DockerModelRunnerOptions";
 
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import System from "@/models/system";
@@ -281,6 +283,13 @@ const LLMS = [
     options: (settings) => <CometApiLLMOptions settings={settings} />,
     description: "500+ AI Models all in one API.",
   },
+  {
+    name: "Docker Model Runner",
+    value: "docker-model-runner",
+    logo: DockerModelRunnerLogo,
+    options: (settings) => <DockerModelRunnerOptions settings={settings} />,
+    description: "Run models locally using Docker Model Runner.",
+  },
 ];
 
 export default function LLMPreference({

frontend/src/utils/constants.js

@@ -51,6 +51,14 @@ export const NVIDIA_NIM_COMMON_URLS = [
   "http://172.17.0.1:8000/v1/version",
 ];
 
+export const DOCKER_MODEL_RUNNER_COMMON_URLS = [
+  "http://localhost:12434/engines/llama.cpp/v1",
+  "http://127.0.0.1:12434/engines/llama.cpp/v1",
+  "http://model-runner.docker.internal/engines/llama.cpp/v1",
+  "http://host.docker.internal:12434/engines/llama.cpp/v1",
+  "http://172.17.0.1:12434/engines/llama.cpp/v1",
+];
+
 export function fullApiUrl() {
   if (API_BASE !== "/api") return API_BASE;
   return `${window.location.origin}/api`;

server/.env.example

@@ -148,6 +148,11 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long.
 # FOUNDRY_MODEL_PREF='phi-3.5-mini'
 # FOUNDRY_MODEL_TOKEN_LIMIT=4096
 
+# LLM_PROVIDER='docker-model-runner'
+# DOCKER_MODEL_RUNNER_BASE_PATH='http://127.0.0.1:12434'
+# DOCKER_MODEL_RUNNER_MODEL_PREF='phi-3.5-mini'
+# DOCKER_MODEL_RUNNER_MODEL_TOKEN_LIMIT=4096
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################

server/models/systemSettings.js

@@ -574,6 +574,12 @@ const SystemSettings = {
       FoundryModelPref: process.env.FOUNDRY_MODEL_PREF,
       FoundryModelTokenLimit: process.env.FOUNDRY_MODEL_TOKEN_LIMIT,
 
+      // Docker Model Runner Keys
+      DockerModelRunnerBasePath: process.env.DOCKER_MODEL_RUNNER_BASE_PATH,
+      DockerModelRunnerModelPref: process.env.DOCKER_MODEL_RUNNER_MODEL_PREF,
+      DockerModelRunnerModelTokenLimit:
+        process.env.DOCKER_MODEL_RUNNER_MODEL_TOKEN_LIMIT,
+
       AwsBedrockLLMConnectionMethod:
         process.env.AWS_BEDROCK_LLM_CONNECTION_METHOD || "iam",
       AwsBedrockLLMAccessKeyId: !!process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,