mirror of
https://github.com/RightNow-AI/openfang.git
synced 2026-08-14 08:52:02 +00:00
feat: add NVIDIA NIM to CLI provider selection wizards
Add NVIDIA NIM as a selectable provider in both the setup wizard and init wizard CLI screens, using NVIDIA_API_KEY env var and meta/llama-3.3-70b-instruct as the default model. Closes #787 Co-authored-by: ilteoood <6383527+ilteoood@users.noreply.github.com> Agent-Logs-Url: https://github.com/ilteoood/openfang/sessions/541bf2ad-f8d8-488c-84a2-4ca71e66730f
This commit is contained in:
co-authored by
ilteoood
parent
1c9d53df11
commit
5ae554ed51
@@ -32,9 +32,6 @@
|
||||
# Fireworks AI
|
||||
# FIREWORKS_API_KEY=...
|
||||
|
||||
# NVIDIA NIM
|
||||
# NVIDIA_API_KEY=nvapi-...
|
||||
|
||||
# ─── Local LLM Providers (no API key needed) ─────────────────────────
|
||||
|
||||
# Ollama (default: http://localhost:11434)
|
||||
|
||||
@@ -45,4 +45,3 @@ Thumbs.db
|
||||
*.swo
|
||||
*~
|
||||
.serena/
|
||||
crates/openfang-desktop/gen/schemas/linux-schema.json
|
||||
|
||||
Generated
+14
-14
@@ -3812,7 +3812,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-api"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -3852,7 +3852,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-channels"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"async-trait",
|
||||
@@ -3889,7 +3889,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-cli"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"clap_complete",
|
||||
@@ -3916,7 +3916,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-desktop"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"open",
|
||||
@@ -3942,7 +3942,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-extensions"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -3970,7 +3970,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-hands"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"dashmap",
|
||||
@@ -3987,7 +3987,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-kernel"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -4025,7 +4025,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-memory"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -4044,7 +4044,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-migrate"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"dirs 6.0.0",
|
||||
@@ -4063,7 +4063,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-runtime"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -4097,7 +4097,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-skills"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"hex",
|
||||
@@ -4120,7 +4120,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-types"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -4139,7 +4139,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openfang-wire"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -8818,7 +8818,7 @@ checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56"
|
||||
|
||||
[[package]]
|
||||
name = "xtask"
|
||||
version = "0.5.1"
|
||||
version = "0.4.9"
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
|
||||
@@ -188,6 +188,14 @@ const PROVIDERS: &[ProviderInfo] = &[
|
||||
needs_key: true,
|
||||
hint: "",
|
||||
},
|
||||
ProviderInfo {
|
||||
name: "nvidia",
|
||||
display: "NVIDIA NIM",
|
||||
env_var: "NVIDIA_API_KEY",
|
||||
default_model: "meta/llama-3.3-70b-instruct",
|
||||
needs_key: true,
|
||||
hint: "",
|
||||
},
|
||||
ProviderInfo {
|
||||
name: "claude-code",
|
||||
display: "Claude Code",
|
||||
|
||||
@@ -127,6 +127,12 @@ const PROVIDERS: &[ProviderInfo] = &[
|
||||
default_model: "codegeex-4",
|
||||
needs_key: true,
|
||||
},
|
||||
ProviderInfo {
|
||||
name: "nvidia",
|
||||
env_var: "NVIDIA_API_KEY",
|
||||
default_model: "meta/llama-3.3-70b-instruct",
|
||||
needs_key: true,
|
||||
},
|
||||
ProviderInfo {
|
||||
name: "claude-code",
|
||||
env_var: "",
|
||||
|
||||
@@ -869,9 +869,7 @@ fn builtin_aliases() -> HashMap<String, String> {
|
||||
("codex-4.1", "codex/gpt-4.1"),
|
||||
("codex-o4", "codex/o4-mini"),
|
||||
// NVIDIA NIM aliases
|
||||
("nemotron", "meta/llama-3.3-70b-instruct"),
|
||||
("nemotron-super", "nvidia/llama-3.3-nemotron-super-49b-v1.5"),
|
||||
("nemotron-ultra", "nvidia/llama-3.1-nemotron-ultra-253b-v1"),
|
||||
("nemotron", "nvidia/llama-3.1-nemotron-70b-instruct"),
|
||||
// Venice aliases
|
||||
("venice", "venice-uncensored"),
|
||||
// Claude Code aliases
|
||||
@@ -2152,50 +2150,8 @@ fn builtin_models() -> Vec<ModelCatalogEntry> {
|
||||
aliases: vec![],
|
||||
},
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// NVIDIA NIM (8)
|
||||
// NVIDIA NIM (5)
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
ModelCatalogEntry {
|
||||
id: "meta/llama-3.3-70b-instruct".into(),
|
||||
display_name: "Llama 3.3 70B Instruct (NVIDIA NIM)".into(),
|
||||
provider: "nvidia".into(),
|
||||
tier: ModelTier::Balanced,
|
||||
context_window: 128_000,
|
||||
max_output_tokens: 4_096,
|
||||
input_cost_per_m: 0.88,
|
||||
output_cost_per_m: 0.88,
|
||||
supports_tools: true,
|
||||
supports_vision: false,
|
||||
supports_streaming: true,
|
||||
aliases: vec![],
|
||||
},
|
||||
ModelCatalogEntry {
|
||||
id: "nvidia/llama-3.3-nemotron-super-49b-v1.5".into(),
|
||||
display_name: "Nemotron Super 49B v1.5 (NVIDIA NIM)".into(),
|
||||
provider: "nvidia".into(),
|
||||
tier: ModelTier::Smart,
|
||||
context_window: 128_000,
|
||||
max_output_tokens: 4_096,
|
||||
input_cost_per_m: 0.88,
|
||||
output_cost_per_m: 0.88,
|
||||
supports_tools: true,
|
||||
supports_vision: false,
|
||||
supports_streaming: true,
|
||||
aliases: vec!["nemotron-super".into()],
|
||||
},
|
||||
ModelCatalogEntry {
|
||||
id: "nvidia/llama-3.1-nemotron-ultra-253b-v1".into(),
|
||||
display_name: "Nemotron Ultra 253B (NVIDIA NIM)".into(),
|
||||
provider: "nvidia".into(),
|
||||
tier: ModelTier::Frontier,
|
||||
context_window: 128_000,
|
||||
max_output_tokens: 4_096,
|
||||
input_cost_per_m: 4.20,
|
||||
output_cost_per_m: 4.20,
|
||||
supports_tools: true,
|
||||
supports_vision: false,
|
||||
supports_streaming: true,
|
||||
aliases: vec!["nemotron-ultra".into()],
|
||||
},
|
||||
ModelCatalogEntry {
|
||||
id: "nvidia/llama-3.1-nemotron-70b-instruct".into(),
|
||||
display_name: "Nemotron 70B Instruct (NVIDIA NIM)".into(),
|
||||
|
||||
+3
-47
@@ -1,6 +1,6 @@
|
||||
# LLM Providers Guide
|
||||
|
||||
OpenFang ships with a comprehensive model catalog covering **3 native LLM drivers**, **21 providers**, **56 builtin models**, and **26 aliases**. Every provider uses one of three battle-tested drivers: the native **Anthropic** driver, the native **Gemini** driver, or the universal **OpenAI-compatible** driver. This guide is the single source of truth for configuring, selecting, and managing LLM providers in OpenFang.
|
||||
OpenFang ships with a comprehensive model catalog covering **3 native LLM drivers**, **20 providers**, **51 builtin models**, and **23 aliases**. Every provider uses one of three battle-tested drivers: the native **Anthropic** driver, the native **Gemini** driver, or the universal **OpenAI-compatible** driver. This guide is the single source of truth for configuring, selecting, and managing LLM providers in OpenFang.
|
||||
|
||||
---
|
||||
|
||||
@@ -549,41 +549,9 @@ For Gemini specifically, either `GEMINI_API_KEY` or `GOOGLE_API_KEY` will work.
|
||||
|
||||
---
|
||||
|
||||
### 21. NVIDIA NIM
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **Display Name** | NVIDIA NIM |
|
||||
| **Driver** | OpenAI-compatible |
|
||||
| **Env Var** | `NVIDIA_API_KEY` |
|
||||
| **Base URL** | `https://integrate.api.nvidia.com/v1` |
|
||||
| **Key Required** | Yes |
|
||||
| **Free Tier** | Yes (rate-limited) |
|
||||
| **Auth** | `Authorization: Bearer` header |
|
||||
| **Models** | 8 |
|
||||
|
||||
**Available Models:**
|
||||
- `meta/llama-3.3-70b-instruct` (Balanced) -- recommended default
|
||||
- `nvidia/llama-3.3-nemotron-super-49b-v1.5` (Smart) -- NVIDIA-tuned, 49B
|
||||
- `nvidia/llama-3.1-nemotron-ultra-253b-v1` (Frontier) -- max quality, 253B
|
||||
- `nvidia/llama-3.1-nemotron-70b-instruct` (Smart)
|
||||
- `meta/llama-3.1-405b-instruct` (Frontier)
|
||||
- `meta/llama-3.1-70b-instruct` (Balanced)
|
||||
- `mistralai/mistral-large-latest` (Smart)
|
||||
- `nvidia/nemotron-4-340b-instruct` (Frontier)
|
||||
|
||||
**Setup:**
|
||||
1. Sign up at [build.nvidia.com](https://build.nvidia.com)
|
||||
2. Generate an API key (starts with `nvapi-`)
|
||||
3. `export NVIDIA_API_KEY="nvapi-..."`
|
||||
|
||||
**Notes:** NVIDIA NIM provides hosted inference for open-source and NVIDIA-tuned models via an OpenAI-compatible API. Supports tool calling and streaming. Also known as `nvidia-nim`.
|
||||
|
||||
---
|
||||
|
||||
## Model Catalog
|
||||
|
||||
The complete catalog of all 56 builtin models, sorted by provider. Pricing is per million tokens.
|
||||
The complete catalog of all 51 builtin models, sorted by provider. Pricing is per million tokens.
|
||||
|
||||
| # | Model ID | Display Name | Provider | Tier | Context Window | Max Output | Input $/M | Output $/M | Tools | Vision |
|
||||
|---|----------|-------------|----------|------|---------------|------------|-----------|------------|-------|--------|
|
||||
@@ -640,14 +608,6 @@ The complete catalog of all 56 builtin models, sorted by provider. Pricing is pe
|
||||
| 51 | `grok-2-mini` | Grok 2 Mini | xai | Fast | 131,072 | 32,768 | $0.30 | $0.50 | Yes | No |
|
||||
| 52 | `hf/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B (HF) | huggingface | Balanced | 128,000 | 4,096 | $0.30 | $0.30 | No | No |
|
||||
| 53 | `replicate/meta-llama-3.3-70b-instruct` | Llama 3.3 70B (Replicate) | replicate | Balanced | 128,000 | 4,096 | $0.40 | $0.40 | No | No |
|
||||
| 54 | `meta/llama-3.3-70b-instruct` | Llama 3.3 70B Instruct (NVIDIA NIM) | nvidia | Balanced | 128,000 | 4,096 | $0.88 | $0.88 | Yes | No |
|
||||
| 55 | `nvidia/llama-3.3-nemotron-super-49b-v1.5` | Nemotron Super 49B v1.5 (NVIDIA NIM) | nvidia | Smart | 128,000 | 4,096 | $0.88 | $0.88 | Yes | No |
|
||||
| 56 | `nvidia/llama-3.1-nemotron-ultra-253b-v1` | Nemotron Ultra 253B (NVIDIA NIM) | nvidia | Frontier | 128,000 | 4,096 | $4.20 | $4.20 | Yes | No |
|
||||
| 57 | `nvidia/llama-3.1-nemotron-70b-instruct` | Nemotron 70B Instruct (NVIDIA NIM) | nvidia | Smart | 128,000 | 4,096 | $0.88 | $0.88 | Yes | No |
|
||||
| 58 | `meta/llama-3.1-405b-instruct` | Llama 3.1 405B Instruct (NVIDIA NIM) | nvidia | Frontier | 128,000 | 4,096 | $5.00 | $16.00 | Yes | No |
|
||||
| 59 | `meta/llama-3.1-70b-instruct` | Llama 3.1 70B Instruct (NVIDIA NIM) | nvidia | Balanced | 128,000 | 4,096 | $0.88 | $0.88 | Yes | No |
|
||||
| 60 | `mistralai/mistral-large-latest` | Mistral Large (NVIDIA NIM) | nvidia | Smart | 128,000 | 4,096 | $2.00 | $6.00 | Yes | No |
|
||||
| 61 | `nvidia/nemotron-4-340b-instruct` | Nemotron 4 340B Instruct (NVIDIA NIM) | nvidia | Frontier | 4,096 | 4,096 | $4.20 | $4.20 | Yes | No |
|
||||
|
||||
**Model Tiers:**
|
||||
|
||||
@@ -667,7 +627,7 @@ The complete catalog of all 56 builtin models, sorted by provider. Pricing is pe
|
||||
|
||||
## Model Aliases
|
||||
|
||||
All 26 aliases resolve to canonical model IDs. Aliases are case-insensitive.
|
||||
All 23 aliases resolve to canonical model IDs. Aliases are case-insensitive.
|
||||
|
||||
| Alias | Resolves To |
|
||||
|-------|------------|
|
||||
@@ -694,9 +654,6 @@ All 26 aliases resolve to canonical model IDs. Aliases are case-insensitive.
|
||||
| `sonar` | `sonar-pro` |
|
||||
| `jamba` | `jamba-1.5-large` |
|
||||
| `command-r` | `command-r-plus` |
|
||||
| `nemotron` | `meta/llama-3.3-70b-instruct` |
|
||||
| `nemotron-super` | `nvidia/llama-3.3-nemotron-super-49b-v1.5` |
|
||||
| `nemotron-ultra` | `nvidia/llama-3.1-nemotron-ultra-253b-v1` |
|
||||
|
||||
You can use aliases anywhere a model ID is accepted: in config files, REST API calls, chat commands, and the model routing configuration.
|
||||
|
||||
@@ -1090,7 +1047,6 @@ Quick reference for all provider environment variables:
|
||||
| Hugging Face | `HF_API_KEY` | Yes |
|
||||
| xAI | `XAI_API_KEY` | Yes |
|
||||
| Replicate | `REPLICATE_API_TOKEN` | Yes |
|
||||
| NVIDIA NIM | `NVIDIA_API_KEY` | Yes |
|
||||
|
||||
---
|
||||
|
||||
|
||||
Reference in New Issue
Block a user