Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 47 additions & 1 deletion core/config/backend_capabilities.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,8 @@ type BackendCapability struct {
// contract. Model variants that share a backend may narrow this further;
// use VoiceCloningForModel for UI/API decisions.
VoiceCloning *VoiceCloningCapability
// TTSVoices lists named voices built into the backend.
TTSVoices []TTSVoice
// Description is a human-readable summary of the backend.
Description string
}
Expand All @@ -263,6 +265,22 @@ func referenceVoiceCloning() *VoiceCloningCapability {
}
}

// TTSVoicesForModel returns model-specific metadata or the backend's built-in
// catalog. The returned slice is safe for callers to modify.
func TTSVoicesForModel(cfg *ModelConfig) []TTSVoice {
if cfg == nil {
return nil
}
if len(cfg.TTSConfig.Voices) > 0 {
return slices.Clone(cfg.TTSConfig.Voices)
}
capability := GetBackendCapability(cfg.Backend)
if capability == nil {
return nil
}
return slices.Clone(capability.TTSVoices)
}

// BackendCapabilities maps each backend name (as used in model configs and gallery
// entries) to its verified capabilities. This is the single source of truth for
// what each backend supports.
Expand Down Expand Up @@ -580,7 +598,35 @@ var BackendCapabilities = map[string]BackendCapability{
PossibleUsecases: []string{UsecaseTTS},
DefaultUsecases: []string{UsecaseTTS},
VoiceCloning: referenceVoiceCloning(),
Description: "Pocket TTS — lightweight text-to-speech",
TTSVoices: []TTSVoice{
{Name: "juergen", Language: "de_DE", Gender: "male"},
{Name: "alba", Language: "en_US", Gender: "female"},
{Name: "bill_boerst", Language: "en_US", Gender: "male"},
{Name: "charles", Language: "en_US", Gender: "male"},
{Name: "george", Language: "en_US", Gender: "male"},
{Name: "javert", Language: "en_US", Gender: "male"},
{Name: "jean", Language: "en_US", Gender: "male"},
{Name: "marius", Language: "en_US", Gender: "male"},
{Name: "michael", Language: "en_US", Gender: "male"},
{Name: "paul", Language: "en_US", Gender: "male"},
{Name: "peter_yearsley", Language: "en_US", Gender: "male"},
{Name: "stuart_bell", Language: "en_US", Gender: "male"},
{Name: "anna", Language: "en_US", Gender: "female"},
{Name: "azelma", Language: "en_US", Gender: "female"},
{Name: "caro_davy", Language: "en_US", Gender: "female"},
{Name: "cosette", Language: "en_US", Gender: "female"},
{Name: "eponine", Language: "en_US", Gender: "female"},
{Name: "eve", Language: "en_US", Gender: "female"},
{Name: "fantine", Language: "en_US", Gender: "female"},
{Name: "jane", Language: "en_US", Gender: "female"},
{Name: "mary", Language: "en_US", Gender: "female"},
{Name: "vera", Language: "en_US", Gender: "female"},
{Name: "lola", Language: "es_ES", Gender: "female"},
{Name: "estelle", Language: "fr_FR", Gender: "female"},
{Name: "giovanni", Language: "it_IT", Gender: "male"},
{Name: "rafael", Language: "pt_PT", Gender: "male"},
},
Description: "Pocket TTS — lightweight text-to-speech",
},
"qwen-tts": {
GRPCMethods: []GRPCMethod{MethodTTS},
Expand Down
22 changes: 22 additions & 0 deletions core/config/backend_capabilities_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,28 @@ var _ = Describe("VoiceCloningForModel", func() {
)
})

var _ = Describe("TTSVoicesForModel", func() {
It("returns the built-in Pocket TTS voice catalog", func() {
voices := TTSVoicesForModel(&ModelConfig{Name: "pocket", Backend: "pocket-tts"})
Expect(voices).To(ContainElement(TTSVoice{Name: "alba", Language: "en_US", Gender: "female"}))
Expect(voices).To(ContainElement(TTSVoice{Name: "giovanni", Language: "it_IT", Gender: "male"}))
})

It("resolves the catalog for pinned backend variants", func() {
voices := TTSVoicesForModel(&ModelConfig{Name: "pocket", Backend: "cuda12-pocket-tts"})
Expect(voices).To(ContainElement(TTSVoice{Name: "alba", Language: "en_US", Gender: "female"}))
})

It("prefers model-specific voice metadata", func() {
configured := []TTSVoice{{Name: "custom", Language: "en_GB"}}
voices := TTSVoicesForModel(&ModelConfig{
Backend: "pocket-tts",
TTSConfig: TTSConfig{Voices: configured},
})
Expect(voices).To(Equal(configured))
})
})

// llama.cpp serves Qwen3-TTS as well as the text LLMs it is known for, so the
// backend has to advertise TTS. That advertisement is what makes narrowing
// mandatory: the per-backend switch in VoiceCloningForModel ends in a
Expand Down
7 changes: 7 additions & 0 deletions core/config/meta/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -878,6 +878,13 @@ func DefaultRegistry() map[string]FieldMetaOverride {
Component: "input",
Order: 91,
},
"tts.voices": {
Section: "tts",
Label: "Named Voices",
Description: "Named voices that this model accepts. Each entry requires a name and can include language and gender metadata.",
Component: "json-editor",
Order: 92,
},

// --- Diffusers ---
"diffusers.pipeline_type": {
Expand Down
11 changes: 11 additions & 0 deletions core/config/model_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,17 @@ type TTSConfig struct {
// A pointer preserves the distinction between an explicit false and the
// default automatic behavior.
VoiceCloning *bool `yaml:"voice_cloning,omitempty" json:"voice_cloning,omitempty"`

// Voices describes named voices accepted by this model. Backends with a
// built-in catalog supply defaults when this list is empty.
Voices []TTSVoice `yaml:"voices,omitempty" json:"voices,omitempty"`
}

// TTSVoice describes one named voice accepted by a text-to-speech model.
type TTSVoice struct {
Name string `yaml:"name" json:"name"`
Language string `yaml:"language,omitempty" json:"language,omitempty"`
Gender string `yaml:"gender,omitempty" json:"gender,omitempty"`
}

// @Description ModelConfig represents a model configuration
Expand Down
2 changes: 2 additions & 0 deletions core/http/auth/features.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ var RouteFeatureRegistry = []RouteFeature{
{"POST", "/v1/audio/speech", FeatureAudioSpeech},
{"POST", "/audio/speech", FeatureAudioSpeech},
{"POST", "/tts", FeatureAudioSpeech},
{"GET", "/v1/audio/voices", FeatureAudioSpeech},
{"GET", "/audio/voices", FeatureAudioSpeech},
{"POST", "/v1/text-to-speech/:voice-id", FeatureAudioSpeech},
{"GET", "/api/voice-profiles", FeatureAudioSpeech},
{"GET", "/api/voice-profiles/:id/audio", FeatureAudioSpeech},
Expand Down
2 changes: 1 addition & 1 deletion core/http/endpoints/localai/api_instructions.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ var instructionDefs = []instructionDef{
Name: "audio",
Description: "Text-to-speech, voice activity detection, transcription, speaker diarization, sound classification, and sound generation",
Tags: []string{"audio"},
Intro: "Diarization (/v1/audio/diarization) returns speaker-labelled time segments. Backends with native ASR-diarization (vibevoice-cpp) can also emit per-segment text via include_text=true; backends with a dedicated pipeline (sherpa-onnx + pyannote) emit segmentation only. Response formats: json (default), verbose_json (adds speakers summary + text), rttm (NIST format). Sound classification (/v1/audio/classification) returns scored AudioSet sound-event tags (audio tagging via the ced backend); top_k and threshold control the returned set.",
Intro: "GET /v1/audio/voices lists named voices for installed TTS models and accepts an optional model filter. Diarization (/v1/audio/diarization) returns speaker-labelled time segments. Backends with native ASR-diarization (vibevoice-cpp) can also emit per-segment text via include_text=true; backends with a dedicated pipeline (sherpa-onnx + pyannote) emit segmentation only. Response formats: json (default), verbose_json (adds speakers summary + text), rttm (NIST format). Sound classification (/v1/audio/classification) returns scored AudioSet sound-event tags (audio tagging via the ced backend); top_k and threshold control the returned set.",
},
{
Name: "voice-library",
Expand Down
99 changes: 99 additions & 0 deletions core/http/endpoints/localai/tts_voices.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package localai

import (
"net/http"

"github.com/labstack/echo/v4"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/http/auth"
"github.com/mudler/LocalAI/core/schema"
"gorm.io/gorm"
)

// TTSModelVoices groups named voices by installed model.
type TTSModelVoices struct {
Model string `json:"model"`
Voices []config.TTSVoice `json:"voices"`
}

// TTSVoicesResponse is returned by the TTS voice discovery endpoint.
type TTSVoicesResponse struct {
Data []TTSModelVoices `json:"data"`
}

// TTSVoicesEndpoint lists named voices advertised by installed model configs.
//
// @Summary List text-to-speech voices
// @Description List named voices and their language and gender metadata. Use the optional model query parameter to filter the response.
// @Tags audio
// @Produce json
// @Param model query string false "Installed model name"
// @Success 200 {object} TTSVoicesResponse
// @Failure 404 {object} schema.ErrorResponse
// @Router /v1/audio/voices [get]
func TTSVoicesEndpoint(loader *config.ModelConfigLoader, databases ...*gorm.DB) echo.HandlerFunc {
var authDB *gorm.DB
if len(databases) > 0 {
authDB = databases[0]
}
return func(c echo.Context) error {
allowed, err := ttsVoiceModelAllowlist(c, authDB)
if err != nil {
return c.JSON(http.StatusInternalServerError, schema.ErrorResponse{Error: &schema.APIError{
Code: http.StatusInternalServerError, Message: "failed to check permissions", Type: "server_error",
}})
}
modelName := c.QueryParam("model")
if modelName != "" {
cfg, ok := loader.GetModelConfig(modelName)
if !ok || (allowed != nil && !allowed[modelName]) {
return c.JSON(http.StatusNotFound, schema.ErrorResponse{Error: &schema.APIError{
Code: http.StatusNotFound, Message: "model not found", Type: "not_found",
}})
}
return c.JSON(http.StatusOK, TTSVoicesResponse{Data: []TTSModelVoices{{
Model: cfg.Name, Voices: ttsVoicesForConfig(loader, &cfg),
}}})
}

response := TTSVoicesResponse{Data: []TTSModelVoices{}}
for _, cfg := range loader.GetAllModelsConfigs() {
if allowed != nil && !allowed[cfg.Name] {
continue
}
voices := ttsVoicesForConfig(loader, &cfg)
if len(voices) == 0 {
continue
}
response.Data = append(response.Data, TTSModelVoices{Model: cfg.Name, Voices: voices})
}
return c.JSON(http.StatusOK, response)
}
}

func ttsVoicesForConfig(loader *config.ModelConfigLoader, cfg *config.ModelConfig) []config.TTSVoice {
resolved, isAlias, err := loader.ResolveAlias(cfg)
if err == nil && isAlias {
return config.TTSVoicesForModel(resolved)
}
return config.TTSVoicesForModel(cfg)
}

func ttsVoiceModelAllowlist(c echo.Context, db *gorm.DB) (map[string]bool, error) {
if db == nil {
return nil, nil
}
user := auth.GetUser(c)
if user == nil || user.Role == auth.RoleAdmin {
return nil, nil
}
permissions, err := auth.GetCachedUserPermissions(c, db, user.ID)
if err != nil || !permissions.AllowedModels.Enabled {
return nil, err
}
allowed := make(map[string]bool, len(permissions.AllowedModels.Models))
for _, model := range permissions.AllowedModels.Models {
allowed[model] = true
}
return allowed, nil
}
66 changes: 66 additions & 0 deletions core/http/endpoints/localai/tts_voices_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package localai_test

import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"

"github.com/labstack/echo/v4"
"github.com/mudler/LocalAI/core/config"
. "github.com/mudler/LocalAI/core/http/endpoints/localai"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("TTSVoicesEndpoint", func() {
var loader *config.ModelConfigLoader

BeforeEach(func() {
dir, err := os.MkdirTemp("", "localai-tts-voices-test")
Expect(err).NotTo(HaveOccurred())
DeferCleanup(os.RemoveAll, dir)
Expect(os.WriteFile(filepath.Join(dir, "pocket.yaml"), []byte("name: pocket\nbackend: pocket-tts\n"), 0o600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(dir, "custom.yaml"), []byte("name: custom\nbackend: custom\nknown_usecases: [tts]\ntts:\n voices:\n - name: narrator\n language: en_GB\n"), 0o600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(dir, "pocket-alias.yaml"), []byte("name: pocket-alias\nalias: pocket\n"), 0o600)).To(Succeed())
loader = config.NewModelConfigLoader(dir)
Expect(loader.LoadModelConfigsFromPath(dir)).To(Succeed())
})

It("returns the target catalog under an alias name", func() {
e := echo.New()
e.GET("/v1/audio/voices", TTSVoicesEndpoint(loader))
rec := httptest.NewRecorder()
e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/audio/voices?model=pocket-alias", nil))
Expect(rec.Code).To(Equal(http.StatusOK))
Expect(rec.Body.String()).To(ContainSubstring(`"model":"pocket-alias"`))
Expect(rec.Body.String()).To(ContainSubstring(`"name":"alba"`))
})

It("lists voice metadata for installed TTS models", func() {
e := echo.New()
e.GET("/v1/audio/voices", TTSVoicesEndpoint(loader))
rec := httptest.NewRecorder()
e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/audio/voices", nil))
Expect(rec.Code).To(Equal(http.StatusOK))
Expect(rec.Body.String()).To(ContainSubstring(`"model":"custom"`))
Expect(rec.Body.String()).To(ContainSubstring(`"name":"narrator"`))
Expect(rec.Body.String()).To(ContainSubstring(`"model":"pocket"`))
Expect(rec.Body.String()).To(ContainSubstring(`"name":"alba"`))
})

It("filters by model and rejects an unknown model", func() {
e := echo.New()
e.GET("/v1/audio/voices", TTSVoicesEndpoint(loader))

found := httptest.NewRecorder()
e.ServeHTTP(found, httptest.NewRequest(http.MethodGet, "/v1/audio/voices?model=pocket", nil))
Expect(found.Code).To(Equal(http.StatusOK))
Expect(found.Body.String()).To(ContainSubstring(`"model":"pocket"`))
Expect(found.Body.String()).NotTo(ContainSubstring(`"model":"custom"`))

missing := httptest.NewRecorder()
e.ServeHTTP(missing, httptest.NewRequest(http.MethodGet, "/v1/audio/voices?model=missing", nil))
Expect(missing.Code).To(Equal(http.StatusNotFound))
})
})
4 changes: 4 additions & 0 deletions core/http/routes/localai.go
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
"vram_estimate": "/api/models/vram-estimate",
"model_load_status": "/api/models/:id/load-status",
"tts": "/tts",
"tts_voices": "/v1/audio/voices",
"voice_profiles": "/api/voice-profiles",
"transcription": "/v1/audio/transcriptions",
"image_generation": "/v1/images/generations",
Expand All @@ -345,6 +346,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
"transcription": "/v1/audio/transcriptions",
"diarization": "/v1/audio/diarization",
"sound_classification": "/v1/audio/classification",
"tts_voices": "/v1/audio/voices",
"image_generation": "/v1/images/generations",
},
"config_management": map[string]string{
Expand All @@ -366,6 +368,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
},
"ai_functions": map[string]string{
"tts": "/tts",
"tts_voices": "/v1/audio/voices",
"voice_profiles": "/api/voice-profiles",
"vad": "/vad",
"video": "/video",
Expand Down Expand Up @@ -414,6 +417,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
"p2p": appConfig.P2PToken != "",
"tracing": true,
"voice_profiles": true,
"tts_voices": true,
},
})
})
Expand Down
2 changes: 2 additions & 0 deletions core/http/routes/openai.go
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,8 @@ func RegisterOpenAIRoutes(app *echo.Echo,

app.POST("/v1/audio/speech", audioSpeechHandler, audioSpeechMiddleware...)
app.POST("/audio/speech", audioSpeechHandler, audioSpeechMiddleware...)
app.GET("/v1/audio/voices", localai.TTSVoicesEndpoint(application.ModelConfigLoader(), application.AuthDB()))
app.GET("/audio/voices", localai.TTSVoicesEndpoint(application.ModelConfigLoader(), application.AuthDB()))

// images
imageHandler := openai.ImageEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig())
Expand Down
31 changes: 31 additions & 0 deletions docs/content/features/text-to-audio.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,37 @@ curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{

Returns an `audio/wav` file.

## List available voices

Use `GET /v1/audio/voices` to list named voices for installed TTS models:

```bash
curl http://localhost:8080/v1/audio/voices
```

Add the `model` query parameter to return one installed model:

```bash
curl 'http://localhost:8080/v1/audio/voices?model=pocket-tts'
```

Each voice can include `language` and `gender` metadata. LocalAI supplies the
built-in Pocket TTS catalog. Other models can declare their catalog in YAML:

```yaml
name: custom-tts
backend: custom
known_usecases: [tts]
tts:
voices:
- name: narrator
language: en_GB
gender: female
```

LocalAI returns `404` when the requested model is not installed. Models without
voice metadata do not appear in the unfiltered response.

## Voice Library

Administrators can manage reusable voice-cloning references from **Operate → Voice Library** in the LocalAI WebUI. The library replaces per-model filesystem and YAML setup for supported cloning backends:
Expand Down
Loading
Loading