Move declarative definitions (#10167)

Cargo.lock

@@ -5184,6 +5184,7 @@ dependencies = [
  "futures",
  "goose-local-inference",
  "goose-provider-types",
+ "include_dir",
  "pem",
  "pkcs1",
  "pkcs8",

crates/goose-providers/Cargo.toml

@@ -53,6 +53,7 @@ pkcs1 = { version = "0.7.5", default-features = false, features = ["pkcs8", "std
 # v0.10 matches the der/const-oid series used by sec1 v0.7 and pkcs1 v0.7; upgrading to v0.11 causes type mismatches across those crates.
 pkcs8 = { version = "0.10", default-features = false, features = ["alloc", "std"], optional = true }
 sec1 = { version = "0.7", default-features = false, features = ["der", "pkcs8", "std"], optional = true }
+include_dir = { workspace = true }
 
 [dev-dependencies]
 test-case = { workspace = true }

crates/goose-providers/examples/declarative.rs

@@ -19,15 +19,9 @@ async fn complete(provider: &dyn Provider, model: ModelConfig) -> Result<()> {
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    let deepseek = include_str!("deepseek.json");
-    let deepseek_model = ModelConfig::new("deepseek-v4-flash");
-    let zai = include_str!("zai.json");
-    let zai_model = ModelConfig::new("glm-4.5-flash");
+    let model = ModelConfig::new("deepseek-v4-flash");
+    let provider = goose_providers::deepseek::create(None, EnvKeyResolver {})?;
+    complete(provider.as_ref(), model).await?;
 
-    for (json, model) in [(deepseek, deepseek_model), (zai, zai_model)] {
-        let provider = goose_providers::declarative::from_json(json, None, EnvKeyResolver {})?;
-        println!("{}:", provider.get_name());
-        complete(provider.as_ref(), model).await?;
-    }
     Ok(())
 }

crates/goose-providers/examples/deepseek.json

@@ -1,30 +0,0 @@
-{
-  "name": "deepseek",
-  "engine": "openai",
-  "display_name": "DeepSeek",
-  "description": "Custom DeepSeek provider",
-  "api_key_env": "DEEPSEEK_API_KEY",
-  "base_url": "https://api.deepseek.com",
-  "models": [
-    {
-      "name": "deepseek-chat",
-      "context_limit": 128000,
-      "input_token_cost": null,
-      "output_token_cost": null,
-      "currency": null,
-      "supports_cache_control": null
-    },
-    {
-      "name": "deepseek-reasoner",
-      "context_limit": 128000,
-      "input_token_cost": null,
-      "output_token_cost": null,
-      "currency": null,
-      "supports_cache_control": null
-    }
-  ],
-  "headers": null,
-  "timeout_seconds": null,
-  "preserves_thinking": true,
-  "supports_streaming": true
-}

crates/goose-providers/examples/zai.json

@@ -1,25 +0,0 @@
-{
-  "name": "zai",
-  "engine": "anthropic",
-  "display_name": "Z.AI",
-  "description": "Z.AI GLM models via Anthropic-compatible API.",
-  "api_key_env": "ZHIPU_API_KEY",
-  "base_url": "https://api.z.ai/api/anthropic",
-  "catalog_provider_id": "zai",
-  "model_doc_link": "https://docs.z.ai/devpack/tool/goose",
-  "fast_model": "glm-4.5-air",
-  "preserves_thinking": true,
-  "models": [
-    { "name": "glm-5.1", "context_limit": 200000 },
-    { "name": "glm-5", "context_limit": 204800 },
-    { "name": "glm-5-turbo", "context_limit": 200000 },
-    { "name": "glm-4.7", "context_limit": 204800 },
-    { "name": "glm-4.7-flash", "context_limit": 200000 },
-    { "name": "glm-4.7-flashx", "context_limit": 200000 },
-    { "name": "glm-4.6", "context_limit": 204800 },
-    { "name": "glm-4.5", "context_limit": 131072 },
-    { "name": "glm-4.5-air", "context_limit": 131072 },
-    { "name": "glm-4.5-flash", "context_limit": 131072 }
-  ],
-  "supports_streaming": true
-}

crates/goose-providers/src/declarative.rs

@@ -1,9 +1,57 @@
-use std::{collections::HashMap, str::FromStr};
+#[macro_use]
+mod macros;
+
+use std::{collections::HashMap, path::Path, str::FromStr};
 
 use anyhow::Result;
+use include_dir::{include_dir, Dir};
 use serde::{Deserialize, Deserializer, Serialize};
 use utoipa::ToSchema;
 
+pub static FIXED_PROVIDERS: Dir = include_dir!("$CARGO_MANIFEST_DIR/src/declarative/definitions");
+
+pub(crate) mod declarative_providers {
+    use super::*;
+
+    expose_declarative_providers!(
+        alibaba,
+        atomic_chat,
+        cerebras,
+        deepseek,
+        empiriolabs,
+        fireworks,
+        futurmix,
+        groq,
+        iflytek,
+        iflytek_astron,
+        inception,
+        llama_swap,
+        lmstudio,
+        minimax,
+        mistral,
+        moonshot,
+        nearai,
+        novita,
+        nvidia,
+        ollama_cloud,
+        omlx,
+        opencode_go,
+        orcarouter,
+        ovhcloud,
+        perplexity,
+        routstr,
+        saladcloud,
+        scaleway,
+        tanzu,
+        tensorix,
+        together,
+        venice,
+        vercel_ai_gateway,
+        zai,
+        zhipu,
+    );
+}
+
 use crate::{
     anthropic,
     api_client::TlsConfig,
@@ -11,6 +59,14 @@ use crate::{
     ollama, openai,
 };
 
+pub fn fixed_provider_configs() -> anyhow::Result<Vec<DeclarativeProviderConfig>> {
+    declarative_providers::fixed_provider_configs()
+}
+
+pub fn fixed_provider_config_entries() -> Vec<(&'static str, &'static str)> {
+    declarative_providers::fixed_provider_config_entries()
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 pub struct EnvVarConfig {
     pub name: String,
@@ -94,7 +150,7 @@ fn default_requires_auth() -> bool {
     true
 }
 
-fn should_preserve_thinking_by_default(engine: &ProviderEngine) -> bool {
+pub fn should_preserve_thinking_by_default(engine: &ProviderEngine) -> bool {
     matches!(engine, ProviderEngine::OpenAI)
 }
 
@@ -195,7 +251,7 @@ fn resolve_config(config: &mut DeclarativeProviderConfig) -> Result<()> {
     Ok(())
 }
 
-fn config_from_json(json: &str) -> Result<DeclarativeProviderConfig> {
+pub fn deserialize_provider_config(json: &str) -> Result<DeclarativeProviderConfig> {
     let raw: serde_json::Value = serde_json::from_str(json)?;
     let preserves_thinking_was_set = raw.get("preserves_thinking").is_some();
     let mut config: DeclarativeProviderConfig = serde_json::from_value(raw)?;
@@ -204,10 +260,33 @@ fn config_from_json(json: &str) -> Result<DeclarativeProviderConfig> {
         config.preserves_thinking = should_preserve_thinking_by_default(&config.engine);
     }
 
+    Ok(config)
+}
+
+fn config_from_json(json: &str) -> Result<DeclarativeProviderConfig> {
+    let mut config = deserialize_provider_config(json)?;
     resolve_config(&mut config)?;
     Ok(config)
 }
 
+pub fn load_custom_providers(dir: &Path) -> Result<Vec<DeclarativeProviderConfig>> {
+    if !dir.exists() {
+        return Ok(Vec::new());
+    }
+
+    std::fs::read_dir(dir)?
+        .filter_map(|entry| {
+            let path = entry.ok()?.path();
+            (path.extension()? == "json").then_some(path)
+        })
+        .map(|path| {
+            let content = std::fs::read_to_string(&path)?;
+            deserialize_provider_config(&content)
+                .map_err(|e| anyhow::anyhow!("Failed to parse {}: {}", path.display(), e))
+        })
+        .collect()
+}
+
 pub fn from_json(
     json: &str,
     tls_config: Option<TlsConfig>,
@@ -231,6 +310,7 @@ pub fn from_json(
 mod tests {
     use super::*;
     use serde_json::json;
+    use std::collections::HashSet;
 
     fn model_json() -> serde_json::Value {
         json!({
@@ -277,6 +357,132 @@ mod tests {
         assert_eq!(ollama.engine, ProviderEngine::Ollama);
     }
 
+    #[test]
+    fn existing_json_files_still_deserialize_without_new_fields() {
+        let config =
+            deserialize_provider_config(crate::groq::JSON).expect("groq.json should parse");
+        assert!(config.env_vars.is_none());
+        assert!(config.dynamic_models.is_none());
+        assert!(config.model_doc_link.is_none());
+        assert!(config.setup_steps.is_empty());
+        assert!(config.preserves_thinking);
+    }
+
+    fn placeholder_var_names(template: &str) -> Vec<String> {
+        template
+            .split("${")
+            .skip(1)
+            .filter_map(|chunk| chunk.split_once('}'))
+            .map(|(name, _)| name.to_string())
+            .collect()
+    }
+
+    fn validate_provider_id(id: &str) -> Result<()> {
+        let mut chars = id.chars();
+        let Some(first) = chars.next() else {
+            anyhow::bail!("Invalid provider id: provider id cannot be empty");
+        };
+
+        if !(first.is_ascii_lowercase() || first.is_ascii_digit() || first == '_') {
+            anyhow::bail!("Invalid provider id: {id}");
+        }
+
+        if chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_' || ch == '-')
+        {
+            Ok(())
+        } else {
+            anyhow::bail!("Invalid provider id: {id}")
+        }
+    }
+
+    #[test]
+    fn expose_declarative_providers_enumerates_all_bundled_json_files() {
+        let enumerated: HashSet<_> = fixed_provider_config_entries()
+            .into_iter()
+            .map(|(path, _)| path.to_string())
+            .collect();
+        let bundled: HashSet<_> = FIXED_PROVIDERS
+            .files()
+            .filter(|file| file.path().extension().and_then(|s| s.to_str()) == Some("json"))
+            .map(|file| {
+                file.path()
+                    .file_name()
+                    .unwrap()
+                    .to_string_lossy()
+                    .into_owned()
+            })
+            .collect();
+
+        assert_eq!(enumerated, bundled);
+    }
+
+    #[test]
+    fn all_bundled_providers_are_valid() {
+        let mut seen_ids = HashSet::new();
+
+        for (path, json) in fixed_provider_config_entries() {
+            let config = deserialize_provider_config(json)
+                .unwrap_or_else(|e| panic!("{path} failed to parse: {e}"));
+
+            validate_provider_id(config.id())
+                .unwrap_or_else(|e| panic!("{path} has an invalid provider id: {e}"));
+            assert!(
+                seen_ids.insert(config.id().to_string()),
+                "{path} has a duplicate provider id: {}",
+                config.id()
+            );
+            assert!(!config.base_url.is_empty(), "{path} has an empty base_url");
+
+            if config.dynamic_models == Some(false) {
+                assert!(
+                    !config.models.is_empty(),
+                    "{path} disables dynamic_models but lists no static models"
+                );
+            }
+
+            let declared: HashSet<&str> = config
+                .env_vars
+                .iter()
+                .flatten()
+                .map(|v| v.name.as_str())
+                .collect();
+            let templates = std::iter::once(config.base_url.as_str())
+                .chain(config.base_path.as_deref())
+                .chain(
+                    config
+                        .headers
+                        .iter()
+                        .flat_map(|h| h.values())
+                        .map(String::as_str),
+                );
+            for template in templates {
+                for var in placeholder_var_names(template) {
+                    assert!(
+                        declared.contains(var.as_str()),
+                        "{path} references ${{{var}}} but declares no matching env_var"
+                    );
+                }
+            }
+        }
+
+        assert!(!seen_ids.is_empty(), "no bundled providers were found");
+    }
+
+    #[test]
+    fn fixed_provider_configs_are_unresolved() {
+        let configs = fixed_provider_configs().expect("bundled providers should load");
+        let config = configs
+            .iter()
+            .find(|config| config.env_vars.is_some())
+            .expect("at least one bundled provider should declare env_vars");
+
+        assert!(
+            config.base_url.contains("${"),
+            "{} should keep base_url placeholders unresolved",
+            config.id()
+        );
+    }
+
     #[test]
     fn from_json_defaults_openai_preserves_thinking_to_true() {
         let json = json!({

crates/goose-providers/src/declarative/macros.rs

@@ -0,0 +1,44 @@
+macro_rules! expose_declarative_provider {
+    ($module:ident, $definition:expr) => {
+        pub mod $module {
+            use anyhow::Result;
+
+            use crate::{
+                api_client::TlsConfig,
+                base::Provider,
+                declarative::{from_json, KeyResolver},
+            };
+
+            pub const JSON: &str = include_str!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/src/declarative/definitions/",
+                $definition,
+                ".json"
+            ));
+
+            pub fn create(
+                tls_config: Option<TlsConfig>,
+                key_resolver: impl KeyResolver,
+            ) -> Result<Box<dyn Provider>> {
+                from_json(JSON, tls_config, key_resolver)
+            }
+        }
+    };
+}
+
+macro_rules! expose_declarative_providers {
+    ($($module:ident),+ $(,)?) => {
+        $(expose_declarative_provider!($module, stringify!($module));)+
+
+        pub(crate) fn fixed_provider_configs() -> anyhow::Result<Vec<DeclarativeProviderConfig>> {
+            fixed_provider_config_entries()
+                .into_iter()
+                .map(|(_, json)| deserialize_provider_config(json))
+                .collect()
+        }
+
+        pub(crate) fn fixed_provider_config_entries() -> Vec<(&'static str, &'static str)> {
+            vec![$((concat!(stringify!($module), ".json"), $module::JSON)),+]
+        }
+    };
+}

crates/goose-providers/src/lib.rs

@@ -11,3 +11,5 @@ pub mod local_inference;
 pub mod ollama;
 pub mod openai;
 pub mod openai_compatible;
+
+pub use declarative::declarative_providers::*;

crates/goose/src/config/declarative_providers.rs

@@ -8,28 +8,21 @@ use crate::providers::inventory::declarative_inventory_identity;
 use crate::providers::ollama_def::OllamaProviderDef;
 use crate::providers::openai_def::OpenAiProviderDef;
 use anyhow::Result;
-use include_dir::{include_dir, Dir};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 use std::str::FromStr;
 
 use std::collections::HashMap;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::sync::Mutex;
 use utoipa::ToSchema;
 
 pub use goose_providers::declarative::*;
 
-static FIXED_PROVIDERS: Dir = include_dir!("$CARGO_MANIFEST_DIR/src/providers/declarative");
-
 pub fn custom_providers_dir() -> std::path::PathBuf {
     Paths::config_dir().join("custom_providers")
 }
 
-fn should_preserve_thinking_by_default(engine: &ProviderEngine) -> bool {
-    matches!(engine, ProviderEngine::OpenAI)
-}
-
 /// Expand `${VAR_NAME}` placeholders in a template string using the given env var configs.
 /// Resolves values via Config (secret if `secret`, param otherwise), falls back to `default`.
 /// Returns an error if a `required` var is missing.
@@ -341,92 +334,25 @@ pub fn load_provider(id: &str) -> Result<LoadedProvider> {
         });
     }
 
-    for file in FIXED_PROVIDERS.files() {
-        if file.path().extension().and_then(|s| s.to_str()) != Some("json") {
-            continue;
-        }
-
-        let content = file
-            .contents_utf8()
-            .ok_or_else(|| anyhow::anyhow!("Failed to read file as UTF-8: {:?}", file.path()))?;
-
-        let config: DeclarativeProviderConfig = match serde_json::from_str(content) {
-            Ok(config) => config,
-            Err(_) => continue,
-        };
-        if config.name == id {
-            return Ok(LoadedProvider {
-                config,
-                is_editable: false,
-            });
-        }
+    if let Some(config) = fixed_provider_configs()?
+        .into_iter()
+        .find(|config| config.name == id)
+    {
+        return Ok(LoadedProvider {
+            config,
+            is_editable: false,
+        });
     }
 
     Err(anyhow::anyhow!("Provider not found: {}", id))
 }
 
-pub fn load_custom_providers(dir: &Path) -> Result<Vec<DeclarativeProviderConfig>> {
-    if !dir.exists() {
-        return Ok(Vec::new());
-    }
-
-    std::fs::read_dir(dir)?
-        .filter_map(|entry| {
-            let path = entry.ok()?.path();
-            (path.extension()? == "json").then_some(path)
-        })
-        .map(|path| {
-            let content = std::fs::read_to_string(&path)?;
-            deserialize_provider_config(&content)
-                .map_err(|e| anyhow::anyhow!("Failed to parse {}: {}", path.display(), e))
-        })
-        .collect()
-}
-
-fn deserialize_provider_config(content: &str) -> Result<DeclarativeProviderConfig> {
-    let raw: serde_json::Value = serde_json::from_str(content)?;
-    let preserves_thinking_was_set = raw.get("preserves_thinking").is_some();
-    let mut config: DeclarativeProviderConfig = serde_json::from_value(raw)?;
-
-    if !preserves_thinking_was_set {
-        config.preserves_thinking = should_preserve_thinking_by_default(&config.engine);
-    }
-
-    Ok(config)
-}
-
-fn load_fixed_providers() -> Result<Vec<DeclarativeProviderConfig>> {
-    let mut res = Vec::new();
-    for file in FIXED_PROVIDERS.files() {
-        if file.path().extension().and_then(|s| s.to_str()) != Some("json") {
-            continue;
-        }
-
-        let content = file
-            .contents_utf8()
-            .ok_or_else(|| anyhow::anyhow!("Failed to read file as UTF-8: {:?}", file.path()))?;
-
-        match deserialize_provider_config(content) {
-            Ok(config) => res.push(config),
-            Err(e) => {
-                tracing::warn!(
-                    "Skipping invalid declarative provider {:?}: {}",
-                    file.path(),
-                    e
-                );
-            }
-        }
-    }
-
-    Ok(res)
-}
-
 pub fn register_declarative_providers(
     registry: &mut crate::providers::provider_registry::ProviderRegistry,
 ) -> Result<()> {
     let dir = custom_providers_dir();
     let custom_providers = load_custom_providers(&dir)?;
-    let fixed_providers = load_fixed_providers()?;
+    let fixed_providers = fixed_provider_configs()?;
     for config in fixed_providers {
         register_declarative_provider(registry, config, ProviderType::Declarative);
     }
@@ -677,89 +603,9 @@ mod tests {
         ));
     }
 
-    #[test]
-    fn test_existing_json_files_still_deserialize_without_new_fields() {
-        let json = include_str!("../providers/declarative/groq.json");
-        let config =
-            deserialize_provider_config(json).expect("groq.json should parse without env_vars");
-        assert!(config.env_vars.is_none());
-        assert!(config.dynamic_models.is_none());
-        assert!(config.model_doc_link.is_none());
-        assert!(config.setup_steps.is_empty());
-        assert!(config.preserves_thinking);
-    }
-
-    fn placeholder_var_names(template: &str) -> Vec<String> {
-        template
-            .split("${")
-            .skip(1)
-            .filter_map(|chunk| chunk.split_once('}'))
-            .map(|(name, _)| name.to_string())
-            .collect()
-    }
-
-    #[test]
-    fn test_all_bundled_providers_are_valid() {
-        let mut seen_ids = std::collections::HashSet::new();
-
-        for file in FIXED_PROVIDERS.files() {
-            if file.path().extension().and_then(|s| s.to_str()) != Some("json") {
-                continue;
-            }
-            let path = file.path().display().to_string();
-            let content = file
-                .contents_utf8()
-                .unwrap_or_else(|| panic!("{path} is not valid UTF-8"));
-            let config = deserialize_provider_config(content)
-                .unwrap_or_else(|e| panic!("{path} failed to parse: {e}"));
-
-            validate_provider_id(config.id())
-                .unwrap_or_else(|e| panic!("{path} has an invalid provider id: {e}"));
-            assert!(
-                seen_ids.insert(config.id().to_string()),
-                "{path} has a duplicate provider id: {}",
-                config.id()
-            );
-            assert!(!config.base_url.is_empty(), "{path} has an empty base_url");
-
-            if config.dynamic_models == Some(false) {
-                assert!(
-                    !config.models.is_empty(),
-                    "{path} disables dynamic_models but lists no static models"
-                );
-            }
-
-            let declared: std::collections::HashSet<&str> = config
-                .env_vars
-                .iter()
-                .flatten()
-                .map(|v| v.name.as_str())
-                .collect();
-            let templates = std::iter::once(config.base_url.as_str())
-                .chain(config.base_path.as_deref())
-                .chain(
-                    config
-                        .headers
-                        .iter()
-                        .flat_map(|h| h.values())
-                        .map(String::as_str),
-                );
-            for template in templates {
-                for var in placeholder_var_names(template) {
-                    assert!(
-                        declared.contains(var.as_str()),
-                        "{path} references ${{{var}}} but declares no matching env_var"
-                    );
-                }
-            }
-        }
-
-        assert!(!seen_ids.is_empty(), "no bundled providers were found");
-    }
-
     #[test]
     fn test_bundled_providers_wire_into_registry_metadata() {
-        let configs = load_fixed_providers().expect("bundled providers should load");
+        let configs = fixed_provider_configs().expect("bundled providers should load");
         assert!(!configs.is_empty(), "no bundled providers were found");
 
         for config in configs {