Download
97 lines 2.6 KiBGoModified 2026-10-31 raw
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697
package llm

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"slices"
	"strings"
	"unicode"
)

// Models lists the models the server offers, by name.
func (c *Client) Models(ctx context.Context) ([]string, error) {
	var list struct {
		Data []struct {
			ID string `json:"id"`
		} `json:"data"`
	}
	if err := c.getJSON(ctx, c.BaseURL+"/models", &list); err != nil {
		return nil, fmt.Errorf("listing models: %w", err)
	}
	var names []string
	for _, m := range list.Data {
		// A name is drawn on screen and saved as the model to use: one
		// with terminal controls in it is not a name to offer.
		if m.ID != "" && !strings.ContainsFunc(m.ID, func(r rune) bool { return unicode.IsControl(r) || unicode.Is(unicode.Cf, r) }) {
			names = append(names, m.ID)
		}
	}
	slices.Sort(names)
	return names, nil
}

// maxContext bounds the context sizes believed from a server.
const maxContext = 100_000_000

// ContextSize asks the server how many tokens the model can take in,
// prompt and reply together: vLLM and SGLang list it with the model, and
// llama.cpp gives it in /props, beside the API. It returns 0 if the server
// does not say.
func (c *Client) ContextSize(ctx context.Context) (int, error) {
	var list struct {
		Data []struct {
			ID          string `json:"id"`
			MaxModelLen int    `json:"max_model_len"`
		} `json:"data"`
	}
	if err := c.getJSON(ctx, c.BaseURL+"/models", &list); err != nil {
		return 0, err
	}
	for _, m := range list.Data {
		if m.ID == c.Model && m.MaxModelLen > 0 && m.MaxModelLen <= maxContext {
			return m.MaxModelLen, nil
		}
	}
	var props struct {
		Settings struct {
			NCtx int `json:"n_ctx"`
		} `json:"default_generation_settings"`
	}
	if err := c.getJSON(ctx, strings.TrimSuffix(c.BaseURL, "/v1")+"/props", &props); err != nil {
		return 0, nil // not llama.cpp
	}
	if n := props.Settings.NCtx; n > 0 && n <= maxContext {
		return n, nil
	}
	return 0, nil
}

// getJSON reads a JSON document from the server.
func (c *Client) getJSON(ctx context.Context, url string, v any) error {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return err
	}
	if c.APIKey != "" {
		req.Header.Set("Authorization", "Bearer "+c.APIKey)
	}
	resp, err := c.HTTP.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := io.ReadAll(io.LimitReader(resp.Body, 10<<20))
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("%s: status %d: %.200s", url, resp.StatusCode, body)
	}
	if err := json.Unmarshal(body, v); err != nil {
		return fmt.Errorf("%s is not the JSON expected: %w", url, err)
	}
	return nil
}