Skip to content
1        10        20        30        40        50        60        70        80        90        100       110       120       130       140       150       1601//! Minimal client for the Anthropic Messages API, with tool use.23const std = @import("std");45const config = @import("config.zig");67const log = std.log.scoped(.llm);89const api_version = "2023-06-01";10const max_output_tokens = 2048;1112pub const ToolUse = struct {13    id: []const u8,14    name: []const u8,15    input: std.json.Value,16};1718pub const Reply = struct {19    /// The raw content blocks. They are replayed into the conversation as-is.20    content: std.json.Value,21    text: []const u8,22    tool_uses: []const ToolUse,23};2425pub const Client = struct {26    api_key: []const u8,27    model: []const u8,28    /// Base URL without a trailing slash, e.g. https://api.anthropic.com.29    base_url: []const u8,30    auth: config.Auth,3132    /// `messages` holds complete JSON message objects, already serialized.33    /// `tools_json` is a JSON array of tool definitions.34    pub fn complete(35        self: *Client,36        allocator: std.mem.Allocator,37        system: []const u8,38        tools_json: []const u8,39        messages: []const []const u8,40    ) !Reply {41        const body = try buildBody(allocator, self.model, system, tools_json, messages);42        const url = try std.fmt.allocPrint(allocator, "{s}/v1/messages", .{self.base_url});4344        var http = std.http.Client{ .allocator = allocator };45        defer http.deinit();4647        const auth_value = switch (self.auth) {48            .x_api_key => self.api_key,49            .bearer => try std.fmt.allocPrint(allocator, "Bearer {s}", .{self.api_key}),50        };51        const auth_name = switch (self.auth) {52            .x_api_key => "x-api-key",53            .bearer => "authorization",54        };55        const headers = [_]std.http.Header{56            .{ .name = auth_name, .value = auth_value },57            .{ .name = "anthropic-version", .value = api_version },58        };5960        var response = std.ArrayList(u8).init(allocator);61        const fetched = try http.fetch(.{62            .location = .{ .url = url },63            .method = .POST,64            .payload = body,65            .headers = .{ .content_type = .{ .override = "application/json" } },66            .extra_headers = &headers,67            .response_storage = .{ .dynamic = &response },68        });69        if (fetched.status != .ok) {70            log.err("messages API returned {d}: {s}", .{ @intFromEnum(fetched.status), response.items });71            return error.ModelRequestFailed;72        }73        return parseReply(allocator, response.items);74    }75};7677fn buildBody(78    allocator: std.mem.Allocator,79    model: []const u8,80    system: []const u8,81    tools_json: []const u8,82    messages: []const []const u8,83) ![]u8 {84    var out = std.ArrayList(u8).init(allocator);85    const writer = out.writer();86    try writer.writeAll("{\"model\":");87    try std.json.stringify(model, .{}, writer);88    try writer.print(",\"max_tokens\":{d},\"system\":", .{max_output_tokens});89    try std.json.stringify(system, .{}, writer);90    try writer.writeAll(",\"tools\":");91    try writer.writeAll(tools_json);92    try writer.writeAll(",\"messages\":[");93    for (messages, 0..) |message, index| {94        if (index > 0) try writer.writeByte(',');95        try writer.writeAll(message);96    }97    try writer.writeAll("]}");98    return out.items;99}100101fn parseReply(allocator: std.mem.Allocator, bytes: []const u8) !Reply {102    const root = try std.json.parseFromSliceLeaky(std.json.Value, allocator, bytes, .{});103    const root_object = switch (root) {104        .object => |object| object,105        else => return error.ModelRequestFailed,106    };107    const content = root_object.get("content") orelse return error.ModelRequestFailed;108    const blocks = switch (content) {109        .array => |array| array.items,110        else => return error.ModelRequestFailed,111    };112113    var text = std.ArrayList(u8).init(allocator);114    var tool_uses = std.ArrayList(ToolUse).init(allocator);115    for (blocks) |block| {116        const block_object = switch (block) {117            .object => |object| object,118            else => continue,119        };120        const block_type = stringField(block_object, "type") orelse continue;121        if (std.mem.eql(u8, block_type, "text")) {122            try text.appendSlice(stringField(block_object, "text") orelse "");123        } else if (std.mem.eql(u8, block_type, "tool_use")) {124            try tool_uses.append(.{125                .id = stringField(block_object, "id") orelse return error.ModelRequestFailed,126                .name = stringField(block_object, "name") orelse return error.ModelRequestFailed,127                .input = block_object.get("input") orelse .null,128            });129        }130    }131    return .{ .content = content, .text = text.items, .tool_uses = tool_uses.items };132}133134fn stringField(object: std.json.ObjectMap, key: []const u8) ?[]const u8 {135    return switch (object.get(key) orelse return null) {136        .string => |value| value,137        else => null,138    };139}140141const testing = std.testing;142143test "buildBody sends the model, system prompt, tools and messages" {144    var arena = std.heap.ArenaAllocator.init(testing.allocator);145    defer arena.deinit();146    const a = arena.allocator();147148    const body = try buildBody(149        a,150        "claude-test",151        "You are \"Totenschloss\".\nBe brief.",152        "[{\"name\":\"run_sql\"}]",153        &.{154            "{\"role\":\"user\",\"content\":\"first\"}",155            "{\"role\":\"assistant\",\"content\":\"second\"}",156        },157    );158159    const root = try std.json.parseFromSliceLeaky(std.json.Value, a, body, .{});160    const object = root.object;161    try testing.expectEqualStrings("claude-test", object.get("model").?.string);162    try testing.expectEqual(@as(i64, max_output_tokens), object.get("max_tokens").?.integer);163    try testing.expectEqualStrings("You are \"Totenschloss\".\nBe brief.", object.get("system").?.string);164165    const tools = object.get("tools").?.array.items;166    try testing.expectEqual(@as(usize, 1), tools.len);167    try testing.expectEqualStrings("run_sql", tools[0].object.get("name").?.string);168169    const messages = object.get("messages").?.array.items;170    try testing.expectEqual(@as(usize, 2), messages.len);171    try testing.expectEqualStrings("first", messages[0].object.get("content").?.string);172    try testing.expectEqualStrings("second", messages[1].object.get("content").?.string);173}174175test "parseReply joins text blocks and reports tool uses" {176    var arena = std.heap.ArenaAllocator.init(testing.allocator);177    defer arena.deinit();178    const a = arena.allocator();179180    const body =181        \\{"content":[182        \\  {"type":"text","text":"There are "},183        \\  {"type":"text","text":"42 players."},184        \\  {"type":"thinking","thinking":"ignored"},185        \\  {"type":"tool_use","id":"toolu_1","name":"run_sql","input":{"sql":"SELECT 42"}}186        \\]}187    ;188    const reply = try parseReply(a, body);189    try testing.expectEqualStrings("There are 42 players.", reply.text);190    try testing.expectEqual(@as(usize, 1), reply.tool_uses.len);191    try testing.expectEqualStrings("toolu_1", reply.tool_uses[0].id);192    try testing.expectEqualStrings("run_sql", reply.tool_uses[0].name);193    try testing.expectEqualStrings("SELECT 42", reply.tool_uses[0].input.object.get("sql").?.string);194}195196test "parseReply accepts a text-only reply" {197    var arena = std.heap.ArenaAllocator.init(testing.allocator);198    defer arena.deinit();199    const reply = try parseReply(arena.allocator(), "{\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}");200    try testing.expectEqualStrings("Hi", reply.text);201    try testing.expectEqual(@as(usize, 0), reply.tool_uses.len);202}203204test "parseReply rejects malformed replies" {205    var arena = std.heap.ArenaAllocator.init(testing.allocator);206    defer arena.deinit();207    const a = arena.allocator();208209    try testing.expectError(error.ModelRequestFailed, parseReply(a, "{}"));210    try testing.expectError(error.ModelRequestFailed, parseReply(a, "[]"));211    try testing.expectError(error.ModelRequestFailed, parseReply(a, "{\"content\":{}}"));212    try testing.expectError(213        error.ModelRequestFailed,214        parseReply(a, "{\"content\":[{\"type\":\"tool_use\",\"name\":\"run_sql\",\"input\":{}}]}"),215    );216    try testing.expectError(217        error.ModelRequestFailed,218        parseReply(a, "{\"content\":[{\"type\":\"tool_use\",\"id\":\"x\",\"input\":{}}]}"),219    );220}221222test "non-string tool fields are ignored instead of crashing" {223    var arena = std.heap.ArenaAllocator.init(testing.allocator);224    defer arena.deinit();225    const reply = try parseReply(arena.allocator(), "{\"content\":[{\"type\":42,\"text\":7}]}");226    try testing.expectEqualStrings("", reply.text);227    try testing.expectEqual(@as(usize, 0), reply.tool_uses.len);228}229230// ---- HTTP tests -------------------------------------------------------------231232/// Serves exactly one request and records it, so `complete` can be tested end to end.233const TestHttp = struct {234    listener: std.net.Server,235    port: u16,236    status_line: []const u8,237    response: []const u8,238    request: std.ArrayList(u8),239    thread: ?std.Thread = null,240241    fn start(status_line: []const u8, response: []const u8) !TestHttp {242        const address = try std.net.Address.parseIp4("127.0.0.1", 0);243        var listener = try address.listen(.{ .reuse_address = true });244        errdefer listener.deinit();245        return .{246            .listener = listener,247            .port = listener.listen_address.getPort(),248            .status_line = status_line,249            .response = response,250            .request = std.ArrayList(u8).init(std.heap.page_allocator),251        };252    }253254    fn run(self: *TestHttp) !void {255        self.thread = try std.Thread.spawn(.{}, serveOnce, .{self});256    }257258    fn deinit(self: *TestHttp) void {259        if (self.thread) |thread| thread.join();260        self.listener.deinit();261        self.request.deinit();262    }263};264265fn serveOnce(self: *TestHttp) void {266    var connection = self.listener.accept() catch return;267    defer connection.stream.close();268    var reader = connection.stream.reader();269270    var head = std.ArrayList(u8).init(std.heap.page_allocator);271    defer head.deinit();272    var byte: [1]u8 = undefined;273    while (head.items.len < 64 * 1024) {274        const count = reader.read(&byte) catch break;275        if (count == 0) break;276        head.append(byte[0]) catch break;277        if (std.mem.endsWith(u8, head.items, "\r\n\r\n")) break;278    }279280    const length = contentLength(head.items) orelse 0;281    const body = std.heap.page_allocator.alloc(u8, length) catch return;282    defer std.heap.page_allocator.free(body);283    reader.readNoEof(body) catch {};284285    self.request.appendSlice(head.items) catch return;286    self.request.appendSlice(body) catch return;287288    var response_head = std.ArrayList(u8).init(std.heap.page_allocator);289    defer response_head.deinit();290    response_head.writer().print(291        "HTTP/1.1 {s}\r\ncontent-type: application/json\r\ncontent-length: {d}\r\nconnection: close\r\n\r\n",292        .{ self.status_line, self.response.len },293    ) catch return;294    connection.stream.writeAll(response_head.items) catch return;295    connection.stream.writeAll(self.response) catch return;296}297298fn contentLength(head: []const u8) ?usize {299    var lines = std.mem.splitSequence(u8, head, "\r\n");300    _ = lines.next();301    while (lines.next()) |line| {302        const colon = std.mem.indexOfScalar(u8, line, ':') orelse continue;303        if (std.ascii.eqlIgnoreCase(line[0..colon], "content-length")) {304            return std.fmt.parseInt(usize, std.mem.trim(u8, line[colon + 1 ..], " \t"), 10) catch null;305        }306    }307    return null;308}309310fn indexOfIgnoreCase(haystack: []const u8, needle: []const u8) ?usize {311    if (needle.len == 0 or haystack.len < needle.len) return null;312    var index: usize = 0;313    while (index + needle.len <= haystack.len) : (index += 1) {314        if (std.ascii.eqlIgnoreCase(haystack[index .. index + needle.len], needle)) return index;315    }316    return null;317}318319test "complete sends an x-api-key request and parses the reply" {320    var arena = std.heap.ArenaAllocator.init(testing.allocator);321    defer arena.deinit();322    const a = arena.allocator();323324    var server = try TestHttp.start(325        "200 OK",326        "{\"content\":[{\"type\":\"text\",\"text\":\"42 players\"},{\"type\":\"tool_use\",\"id\":\"t1\",\"name\":\"run_sql\",\"input\":{\"sql\":\"SELECT 42\"}}]}",327    );328    try server.run();329    defer server.deinit();330331    var client = Client{332        .api_key = "secret-key",333        .model = "claude-test",334        .base_url = try std.fmt.allocPrint(a, "http://127.0.0.1:{d}", .{server.port}),335        .auth = .x_api_key,336    };337    const reply = try client.complete(338        a,339        "Be brief.\nNo SQL.",340        "[{\"name\":\"run_sql\"}]",341        &.{ "{\"role\":\"user\",\"content\":\"top players?\"}", "{\"role\":\"assistant\",\"content\":\"one moment\"}" },342    );343    try testing.expectEqualStrings("42 players", reply.text);344    try testing.expectEqual(@as(usize, 1), reply.tool_uses.len);345    try testing.expectEqualStrings("run_sql", reply.tool_uses[0].name);346347    const request = server.request.items;348    try testing.expect(std.mem.startsWith(u8, request, "POST /v1/messages HTTP/1.1\r\n"));349    try testing.expect(indexOfIgnoreCase(request, "x-api-key: secret-key") != null);350    try testing.expect(indexOfIgnoreCase(request, "authorization:") == null);351    try testing.expect(indexOfIgnoreCase(request, "anthropic-version: 2023-06-01") != null);352    try testing.expect(indexOfIgnoreCase(request, "content-type: application/json") != null);353354    const body = request[std.mem.indexOf(u8, request, "\r\n\r\n").? + 4 ..];355    const sent = try std.json.parseFromSliceLeaky(std.json.Value, a, body, .{});356    try testing.expectEqualStrings("claude-test", sent.object.get("model").?.string);357    try testing.expectEqualStrings("Be brief.\nNo SQL.", sent.object.get("system").?.string);358    try testing.expectEqual(@as(usize, 2), sent.object.get("messages").?.array.items.len);359    try testing.expectEqualStrings(360        "top players?",361        sent.object.get("messages").?.array.items[0].object.get("content").?.string,362    );363}364365test "complete sends a bearer token for the inference provider" {366    var arena = std.heap.ArenaAllocator.init(testing.allocator);367    defer arena.deinit();368    const a = arena.allocator();369370    var server = try TestHttp.start("200 OK", "{\"content\":[{\"type\":\"text\",\"text\":\"ok\"}]}");371    try server.run();372    defer server.deinit();373374    var client = Client{375        .api_key = "inference-key",376        .model = "some-model",377        .base_url = try std.fmt.allocPrint(a, "http://127.0.0.1:{d}/anthropic", .{server.port}),378        .auth = .bearer,379    };380    const reply = try client.complete(a, "", "[]", &.{"{\"role\":\"user\",\"content\":\"hi\"}"});381    try testing.expectEqualStrings("ok", reply.text);382383    const request = server.request.items;384    try testing.expect(std.mem.startsWith(u8, request, "POST /anthropic/v1/messages HTTP/1.1\r\n"));385    try testing.expect(indexOfIgnoreCase(request, "authorization: Bearer inference-key") != null);386    try testing.expect(indexOfIgnoreCase(request, "x-api-key:") == null);387}