From 271c7506bd3ddfa9a4cfb05dc246807618fd0ec2 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Fri, 19 Jun 2026 15:28:38 -0700 Subject: [PATCH] fix(lfx): preserve custom static models through models.dev override (#13676) Custom models added to the bundled *_constants.py lists (e.g. openai_constants.py) were silently discarded once a models.dev snapshot was active. apply_models_dev_overrides() replaced a covered provider's entire static group with the models.dev rows and skipped any later same-provider group (the OpenAI embeddings group), so user-added LLM and embedding entries never reached get_unified_models_detailed() and the model/embedding pickers. Fold any static entry whose name models.dev does not cover into that provider's override list (mutating the same list object that is appended to the result, so a later embeddings group folds into it too). models.dev still wins for every name it does cover, preserving its fresher metadata. This is especially important for embeddings: the embedding dropdown has no free-text combobox, so a custom embedding must be present in the list to be selectable. Fixes #13556 Co-authored-by: Claude Opus 4.8 (1M context) --- .../src/lfx/base/models/models_dev_catalog.py | 34 +++++++- .../src/lfx/base/models/openai_constants.py | 9 ++ .../base/models/test_models_dev_catalog.py | 82 ++++++++++++++++--- 3 files changed, 110 insertions(+), 15 deletions(-) diff --git a/src/lfx/src/lfx/base/models/models_dev_catalog.py b/src/lfx/src/lfx/base/models/models_dev_catalog.py index 599c7e6f77..ae6754df45 100644 --- a/src/lfx/src/lfx/base/models/models_dev_catalog.py +++ b/src/lfx/src/lfx/base/models/models_dev_catalog.py @@ -345,6 +345,14 @@ def apply_models_dev_overrides( :data:`MODELS_DEV_PROVIDER_KEYS`) pass through unchanged. Override groups for covered providers that had no static group at all are appended. + User-added models survive the override: any entry in the bundled + ``*_constants.py`` lists whose name models.dev does not cover is carried + over (appended after the models.dev rows) rather than discarded. This + matters most for embedding models, whose component dropdown has no + free-text combobox, so a custom embedding must be present in the list to be + selectable. models.dev still wins for every model name it does cover, so + the fresher metadata (context windows, pricing, capabilities) is preserved. + models.dev exposes no ``deprecated`` field of its own, so this function preserves the static-list curation by name: any model that was already flagged deprecated in the bundled ``*_constants.py`` lists keeps that flag @@ -391,6 +399,22 @@ def apply_models_dev_overrides( if not overrides: return static_lists + def _fold_custom_static_entries(provider: str, group: list[dict[str, Any]]) -> None: + """Append static rows models.dev doesn't cover into the override list. + + models.dev only knows the models it ships, so any model a user added to + the bundled ``*_constants.py`` lists would otherwise be silently dropped + when its provider's group is replaced. Carrying those rows over (matched + by name) keeps user-added models selectable while models.dev still wins + for every model it does cover. The override list is mutated in place so a + later same-provider group (e.g. the OpenAI embeddings group) folds its + custom rows into the same list already appended to ``replaced``. + """ + override_names = {m.get("name") for m in overrides[provider] if isinstance(m, dict)} + custom_entries = [m for m in group if isinstance(m, dict) and m.get("name") not in override_names] + if custom_entries: + overrides[provider].extend(custom_entries) + replaced: list[list[dict[str, Any]]] = [] consumed_providers: set[str] = set() for group in static_lists: @@ -400,12 +424,16 @@ def apply_models_dev_overrides( if len(provider_names) == 1: (provider,) = provider_names if provider in overrides: + # Preserve any user-added static models models.dev doesn't know + # about before the group is replaced (or skipped as a duplicate). + _fold_custom_static_entries(provider, group) if provider in consumed_providers: # A second (or later) static group for the same provider # — e.g. OPENAI_EMBEDDING_MODELS_DETAILED appears after - # OPENAI_MODELS_DETAILED. The override already contains - # both LLMs and embeddings from that provider, so - # appending this static group would duplicate the rows. + # OPENAI_MODELS_DETAILED. The override already contains both + # LLMs and embeddings from that provider (plus any custom + # rows just folded in), so appending this static group would + # duplicate the rows. continue replaced.append(overrides[provider]) consumed_providers.add(provider) diff --git a/src/lfx/src/lfx/base/models/openai_constants.py b/src/lfx/src/lfx/base/models/openai_constants.py index 7492932ea1..bd61ef48f7 100644 --- a/src/lfx/src/lfx/base/models/openai_constants.py +++ b/src/lfx/src/lfx/base/models/openai_constants.py @@ -1,5 +1,14 @@ from .model_metadata import create_model_metadata +# Adding a custom model: append a ``create_model_metadata(...)`` entry below (for +# chat models) or a name to ``OPENAI_EMBEDDING_MODEL_NAMES`` (for embeddings). +# Custom entries survive the models.dev catalog override +# (``apply_models_dev_overrides``) — any name models.dev doesn't cover is carried +# over rather than discarded. The ``openai_api_base`` / ``OPENAI_API_BASE`` +# setting (for OpenAI-compatible servers) is configured separately per-component +# and is independent of this catalog. Embedding dropdowns have no free-text +# combobox, so a custom embedding must appear here to be selectable. + # Unified model metadata - single source of truth OPENAI_MODELS_DETAILED = [ # GPT-5.4 Series diff --git a/src/lfx/tests/unit/base/models/test_models_dev_catalog.py b/src/lfx/tests/unit/base/models/test_models_dev_catalog.py index 71822b33de..404505bd9f 100644 --- a/src/lfx/tests/unit/base/models/test_models_dev_catalog.py +++ b/src/lfx/tests/unit/base/models/test_models_dev_catalog.py @@ -196,10 +196,16 @@ def test_apply_overrides_replaces_covered_provider(): result = apply_models_dev_overrides([_anthropic_group(), _watsonx_group()], snapshot) - # Anthropic group must be replaced with 2 new entries; WatsonX preserved. + # Anthropic group is replaced with the 2 models.dev entries; the custom + # static "claude-old-1" (absent from the snapshot) is carried over so + # user-added models survive the override. WatsonX is preserved. assert len(result) == 2 anthropic_models = result[0] - assert {m["name"] for m in anthropic_models} == {"claude-opus-4-1-20250805", "claude-sonnet-4-5"} + assert {m["name"] for m in anthropic_models} == { + "claude-opus-4-1-20250805", + "claude-sonnet-4-5", + "claude-old-1", + } opus = next(m for m in anthropic_models if m["name"] == "claude-opus-4-1-20250805") assert opus["provider"] == "Anthropic" @@ -223,6 +229,48 @@ def test_apply_overrides_replaces_covered_provider(): assert result[1] == _watsonx_group() +def test_apply_overrides_preserves_custom_static_entries_for_covered_provider(): + """Custom static models survive the override; models.dev rows still win. + + A model added to a bundled ``*_constants.py`` list whose name models.dev + does not cover must be carried over (appended after the models.dev rows) + rather than discarded. Names models.dev *does* cover keep the fresher + models.dev translation. + """ + from lfx.base.models.models_dev_catalog import apply_models_dev_overrides + + static_anthropic = [ + # In the snapshot below → replaced by the models.dev translation. + {"provider": "Anthropic", "name": "claude-old-1", "tool_calling": False}, + # Not in the snapshot → must be preserved verbatim. + {"provider": "Anthropic", "name": "claude-custom", "tool_calling": True}, + ] + snapshot = { + "anthropic": { + "id": "anthropic", + "models": { + "claude-opus-4-1-20250805": {"id": "claude-opus-4-1-20250805", "tool_call": True}, + "claude-old-1": {"id": "claude-old-1", "tool_call": False}, + }, + }, + } + + result = apply_models_dev_overrides([static_anthropic], snapshot) + + anthropic_models = result[0] + # models.dev rows first (in snapshot order), then any custom carry-overs. + assert [m["name"] for m in anthropic_models] == [ + "claude-opus-4-1-20250805", + "claude-old-1", + "claude-custom", + ] + # The custom entry is preserved exactly as authored (not re-translated). + assert anthropic_models[-1] == {"provider": "Anthropic", "name": "claude-custom", "tool_calling": True} + # The name models.dev covers keeps the models.dev translation, not the static row. + old_one = next(m for m in anthropic_models if m["name"] == "claude-old-1") + assert old_one.get("created") is not None # came through _translate_model_entry + + def test_apply_overrides_handles_missing_or_invalid_release_date(): """release_date is best-effort: ISO YYYY-MM-DD parses, anything else → 0. @@ -534,6 +582,9 @@ def test_apply_overrides_drops_subsequent_static_groups_for_overridden_provider( static_openai_embeddings = [ {"provider": "OpenAI", "name": "text-embedding-3-large", "model_type": "embeddings"}, {"provider": "OpenAI", "name": "text-embedding-3-small", "model_type": "embeddings"}, + # Custom embedding absent from models.dev — must be carried over, not + # dropped with the rest of this (already-consumed) trailing group. + {"provider": "OpenAI", "name": "text-embedding-custom", "model_type": "embeddings"}, ] snapshot = { "openai": { @@ -566,6 +617,8 @@ def test_apply_overrides_drops_subsequent_static_groups_for_overridden_provider( "embedding row must not be duplicated by the trailing static embedding group" ) assert seen.get("text-embedding-3-small") == 1 + # Custom embedding survives exactly once despite living in the dropped group. + assert seen.get("text-embedding-custom") == 1 def test_apply_overrides_appends_new_provider_when_no_static_group(): @@ -643,15 +696,18 @@ def test_get_unified_models_detailed_sorts_provider_lists(): anthropic = next(p for p in unified if p["provider"] == "Anthropic") names = [m["model_name"] for m in anthropic["models"]] - # Non-deprecated dated rows first (newest first), then undated rows - # in their original order, then deprecated last. - assert names == [ - "claude-newest", - "claude-mid", - "claude-undated-a", - "claude-undated-b", - "claude-old-deprecated", - ] + # The real bundled Anthropic constants are now carried over alongside + # the synthetic snapshot rows (custom static entries survive the + # override), so assert on the synthetic rows' presence and their + # relative ordering rather than an exact, exhaustive list. + synthetic = {"claude-newest", "claude-mid", "claude-undated-a", "claude-undated-b", "claude-old-deprecated"} + assert synthetic.issubset(set(names)), f"missing synthetic models: {synthetic - set(names)}" + + # Non-deprecated dated rows first (newest first), then undated rows in + # their original order, then the deprecated row last. + idx = {n: names.index(n) for n in synthetic} + assert idx["claude-newest"] < idx["claude-mid"] < idx["claude-old-deprecated"] + assert idx["claude-undated-a"] < idx["claude-undated-b"] < idx["claude-old-deprecated"] finally: models_dev_catalog.set_active_snapshot(prior) models_dev_catalog.invalidate_catalog_cache() @@ -684,7 +740,9 @@ def test_install_snapshot_invalidates_get_models_detailed_cache(): anthropic_groups = [g for g in live_view if any(m.get("provider") == "Anthropic" for m in g)] assert anthropic_groups, "Anthropic group missing after override install" names = {m["name"] for m in anthropic_groups[0]} - assert names == {"synthetic-claude"} + # The snapshot row is present; the real bundled Anthropic constants are + # also carried over now that custom static entries survive the override. + assert "synthetic-claude" in names, "synthetic-claude missing after cache invalidation" finally: # Restore prior state so subsequent tests aren't polluted. models_dev_catalog.set_active_snapshot(prior)