feat: use native MLX backend when --model-path is set on Apple Silicon
Build-tagged backend selection: MLX on darwin/arm64/mlx, HTTP elsewhere. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
f4303fada2
commit
4d3e54c81a
3 changed files with 37 additions and 3 deletions
|
|
@ -79,9 +79,12 @@ type usageInfo struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func runServe(cmd *cli.Command, args []string) error {
|
func runServe(cmd *cli.Command, args []string) error {
|
||||||
// Create a backend — use HTTP backend pointing to configured API URL.
|
// Try native MLX backend first (macOS arm64 with mlx tag + model-path set),
|
||||||
// On macOS with MLX build tag, this will use the native MLX backend instead.
|
// fall back to HTTP proxy backend.
|
||||||
backend := ml.NewHTTPBackend(apiURL, modelName)
|
backend, err := createServeBackend()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
|
|
|
||||||
9
internal/cmd/ml/serve_backend_default.go
Normal file
9
internal/cmd/ml/serve_backend_default.go
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
//go:build !(darwin && arm64 && mlx)
|
||||||
|
|
||||||
|
package ml
|
||||||
|
|
||||||
|
import "forge.lthn.ai/core/cli/pkg/ml"
|
||||||
|
|
||||||
|
func createServeBackend() (ml.Backend, error) {
|
||||||
|
return ml.NewHTTPBackend(apiURL, modelName), nil
|
||||||
|
}
|
||||||
22
internal/cmd/ml/serve_backend_mlx.go
Normal file
22
internal/cmd/ml/serve_backend_mlx.go
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
//go:build darwin && arm64 && mlx
|
||||||
|
|
||||||
|
package ml
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
|
"forge.lthn.ai/core/cli/pkg/ml"
|
||||||
|
)
|
||||||
|
|
||||||
|
func createServeBackend() (ml.Backend, error) {
|
||||||
|
if serveModelPath != "" {
|
||||||
|
slog.Info("ml serve: loading native MLX backend", "path", serveModelPath)
|
||||||
|
b, err := ml.NewMLXBackend(serveModelPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("mlx backend: %w", err)
|
||||||
|
}
|
||||||
|
return b, nil
|
||||||
|
}
|
||||||
|
return ml.NewHTTPBackend(apiURL, modelName), nil
|
||||||
|
}
|
||||||
Loading…
Add table
Reference in a new issue