Download
201 lines 7 KiBGoModified 2026-10-26 raw
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201
package llm

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
)

// sse serves the chunks as a streaming completion.
func sse(t *testing.T, check func(req map[string]any), chunks ...string) *httptest.Server {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		body, _ := io.ReadAll(r.Body)
		var req map[string]any
		if err := json.Unmarshal(body, &req); err != nil {
			t.Errorf("bad request body: %v", err)
		}
		if check != nil {
			check(req)
		}
		w.Header().Set("Content-Type", "text/event-stream")
		for _, c := range chunks {
			fmt.Fprintf(w, "data: %s\n\n", c)
		}
		fmt.Fprint(w, "data: [DONE]\n\n")
	}))
	t.Cleanup(srv.Close)
	return srv
}

func delta(d string) string { return `{"choices":[{"delta":` + d + `}]}` }

func TestStreamToolCalls(t *testing.T) {
	srv := sse(t, func(req map[string]any) {
		if req["stream"] != true || req["tool_choice"] != "auto" || fmt.Sprint(req["stream_options"]) != "map[include_usage:true]" {
			t.Errorf("request = %v", req)
		}
		tools, _ := req["tools"].([]any)
		if len(tools) != 1 {
			t.Errorf("tools = %v", req["tools"])
		}
	},
		delta(`{"reasoning_content":"let me "}`),
		delta(`{"reasoning_content":"look"}`),
		delta(`{"content":"Reading."}`),
		delta(`{"tool_calls":[{"index":0,"id":"c1","function":{"name":"Read","arguments":"{\"pa"}}]}`),
		delta(`{"tool_calls":[{"index":0,"function":{"arguments":"th\":\"a.go\"}"}}]}`),
		delta(`{"tool_calls":[{"index":1,"id":"c2","function":{"name":"Safe_Bash","arguments":"{}"}}]}`),
		`{"choices":[],"usage":{"prompt_tokens":1200,"completion_tokens":34}}`,
	)
	c := New(srv.URL, "", "m")
	var reply, reasoning strings.Builder
	r, err := c.Complete(context.Background(), []Message{{Role: "user", Content: "hi"}},
		[]Tool{{Name: "Read", Parameters: map[string]any{"type": "object"}}},
		func(text string, isReasoning bool) {
			if isReasoning {
				reasoning.WriteString(text)
			} else {
				reply.WriteString(text)
			}
		})
	if err != nil {
		t.Fatal(err)
	}
	if r.Reply != "Reading." || r.Reasoning != "let me look" || reply.String() != "Reading." || reasoning.String() != "let me look" {
		t.Errorf("reply %q reasoning %q streamed %q / %q", r.Reply, r.Reasoning, reply.String(), reasoning.String())
	}
	if len(r.Calls) != 2 || r.Calls[0].Function.Arguments != `{"path":"a.go"}` || r.Calls[1].ID != "c2" {
		t.Errorf("calls = %+v", r.Calls)
	}
	if r.Usage != (Usage{PromptTokens: 1200, CompletionTokens: 34}) {
		t.Errorf("usage = %+v", r.Usage)
	}
}

func TestInlineThinkingSplitAcrossChunks(t *testing.T) {
	srv := sse(t, nil,
		delta(`{"content":"<thi"}`),
		delta(`{"content":"nk>hmm, </th"}`),
		delta(`{"content":"ink>\n\nAnswer"}`),
	)
	var reply, reasoning strings.Builder
	r, err := New(srv.URL, "", "m").Complete(context.Background(), nil, nil, func(text string, isReasoning bool) {
		if isReasoning {
			reasoning.WriteString(text)
		} else {
			reply.WriteString(text)
		}
	})
	if err != nil {
		t.Fatal(err)
	}
	if r.Reply != "Answer" || reply.String() != "Answer" || strings.TrimSpace(reasoning.String()) != "hmm," {
		t.Errorf("reply %q streamed %q reasoning %q", r.Reply, reply.String(), reasoning.String())
	}
}

func TestNonStreamingFallbackAndErrors(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Authorization") != "Bearer k" {
			w.WriteHeader(http.StatusUnauthorized)
			fmt.Fprint(w, `{"error":{"message":"bad key"}}`)
			return
		}
		fmt.Fprint(w, `{"choices":[{"message":{"role":"assistant","content":"plain"}}]}`)
	}))
	defer srv.Close()
	r, err := New(srv.URL, "k", "m").Complete(context.Background(), nil, nil, nil)
	if err != nil || r.Reply != "plain" {
		t.Errorf("got %q, %v", r.Reply, err)
	}
	_, err = New(srv.URL, "wrong", "m").Complete(context.Background(), nil, nil, nil)
	if err == nil || !strings.Contains(err.Error(), "bad key") {
		t.Errorf("err = %v", err)
	}
}

func TestRedact(t *testing.T) {
	if got := string(redact([]byte("Authorization: Bearer sk-123"), "sk-123")); strings.Contains(got, "sk-123") {
		t.Error(got)
	}
}

func TestContextSize(t *testing.T) {
	routes := map[string]string{
		"/vllm/v1/models":  `{"data":[{"id":"other","max_model_len":8192},{"id":"m","max_model_len":32768}]}`,
		"/llama/v1/models": `{"data":[{"id":"m"}]}`,
		"/llama/props":     `{"default_generation_settings":{"n_ctx":16384}}`,
		"/other/v1/models": `{"data":[{"id":"m","max_model_len":-1}]}`,
	}
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		body, ok := routes[r.URL.Path]
		if !ok {
			w.WriteHeader(http.StatusNotFound)
		}
		fmt.Fprint(w, body)
	}))
	defer srv.Close()
	for server, want := range map[string]int{"vllm": 32768, "llama": 16384, "other": 0} {
		got, err := New(srv.URL+"/"+server+"/v1", "", "m").ContextSize(context.Background())
		if err != nil || got != want {
			t.Errorf("%s: ContextSize = %d, %v; want %d", server, got, err, want)
		}
	}
}

// A redirect, even to the same host name on another port, never gets the
// key: Go would forward the Authorization header there.
func TestRedirectKeepsKey(t *testing.T) {
	got := make(chan string, 4)
	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		got <- r.Header.Get("Authorization")
	}))
	t.Cleanup(other.Close)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, other.URL+r.URL.Path, http.StatusTemporaryRedirect)
	}))
	t.Cleanup(srv.Close)
	c := New(srv.URL, "sk-SECRET", "m")
	if _, err := c.Complete(context.Background(), []Message{{Role: "user", Content: "hi"}}, nil, nil); !errors.Is(err, ErrRedirect) {
		t.Errorf("Complete: %v", err)
	}
	if _, err := c.Models(context.Background()); err == nil {
		t.Error("Models followed the redirect")
	}
	close(got)
	for auth := range got {
		t.Errorf("the redirect target got %q", auth)
	}
}

// A stream is bounded whatever MaxTokens says, tool calls' names
// included, and one that stops without saying it is done is an error.
func TestStreamLimits(t *testing.T) {
	name := strings.Repeat("x", 1<<20)
	var chunks []string
	for range 100 {
		chunks = append(chunks, delta(`{"tool_calls":[{"index":0,"function":{"name":"`+name+`"}}]}`))
	}
	c := New(sse(t, nil, chunks...).URL, "", "m")
	c.MaxTokens = 100
	if _, err := c.Complete(context.Background(), []Message{{Role: "user", Content: "hi"}}, nil, nil); !errors.Is(err, ErrTruncated) {
		t.Errorf("a 100 MB tool name: %v", err)
	}

	cut := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		fmt.Fprintf(w, "data: %s\n\n", delta(`{"content":"Running."}`))
		fmt.Fprintf(w, "data: %s\n\n", delta(`{"tool_calls":[{"index":0,"id":"c1","function":{"name":"Safe_Bash","arguments":"{\"script\":\"ls\"}"}}]}`))
	}))
	t.Cleanup(cut.Close)
	r, err := New(cut.URL, "", "m").Complete(context.Background(), []Message{{Role: "user", Content: "hi"}}, nil, nil)
	if err == nil || len(r.Calls) != 0 {
		t.Errorf("a stream cut short: %+v, %v", r, err)
	}
}