Skip to content
1        10        20        30        40        50        60        70        80        90        100       110       1201//! One request's view of the lake: a fresh in-memory DuckDB with the DuckLake2//! catalog attached read-only. Nothing is shared between requests, so a bad query3//! or a stuck connection cannot leak into the next one.4//!5//! The only process-wide piece is `Watchdog`: one long-lived thread that6//! interrupts queries whose deadline passed. Requests register a stack-allocated7//! entry with it instead of spawning a thread of their own.89const std = @import("std");10const duckdb = @import("duckdb.zig");11const sql_guard = @import("sql_guard.zig");12const Config = @import("config.zig").Config;1314const c = duckdb.c;15const log = std.log.scoped(.session);1617/// Rows kept per query for the UI. The model gets a smaller slice of these.18pub const max_rows_returned = 200;1920pub const QueryOutput = struct {21    sql: []const u8,22    columns: []const []const u8 = &.{},23    rows: []const []const ?[]const u8 = &.{},24    total_rows: usize = 0,25    error_message: ?[]const u8 = null,26};2728/// Downloads the DuckLake and httpfs extensions into the extension cache once at29/// startup, so per-request LOAD calls never touch the network for them. The30/// extensions DuckDB normally bundles (core_functions, json, parquet, icu) are31/// linked into the binary, so only these two are fetched.32pub fn installExtensions(cfg: *const Config) !void {33    var database = try duckdb.Database.openMemory(cfg.extension_dir, cfg.temp_dir);34    defer database.close();35    var connection = try duckdb.Connection.connect(&database);36    defer connection.disconnect();37    try execute(&connection, "INSTALL ducklake");38    try execute(&connection, "INSTALL httpfs");39}4041pub const Session = struct {42    database: duckdb.Database,43    connection: duckdb.Connection,44    timeout_ms: u64,4546    pub fn open(allocator: std.mem.Allocator, cfg: *const Config) !Session {47        var database = try duckdb.Database.openMemory(cfg.extension_dir, cfg.temp_dir);48        errdefer database.close();49        var connection = try duckdb.Connection.connect(&database);50        errdefer connection.disconnect();5152        const attach = try std.fmt.allocPrintZ(allocator, "ATTACH 'ducklake:{s}' AS lake (READ_ONLY)", .{cfg.lake_catalog});53        try execute(&connection, "LOAD ducklake");54        try execute(&connection, "LOAD httpfs");55        try execute(&connection, attach);56        try execute(&connection, "SET memory_limit = '1GB'");57        // Grouped queries over a few days of match data otherwise try to buffer the58        // whole result in insertion order and hit the 1 GB cap; no reply depends on59        // the incidental order of an un-ordered result.60        try execute(&connection, "SET preserve_insertion_order = false");61        // The lake's data files are HTTPS, so httpfs stays on. Local files are off,62        // and the lock below means the model cannot switch this back.63        try execute(&connection, "SET disabled_filesystems = 'LocalFileSystem'");64        try execute(&connection, "SET lock_configuration = true");6566        return .{ .database = database, .connection = connection, .timeout_ms = cfg.query_timeout_ms };67    }6869    pub fn deinit(self: *Session) void {70        self.connection.disconnect();71        self.database.close();72    }7374    /// Runs one user- or model-written query after the guard and the statement-kind check.75    /// Problems come back in `error_message`; only a broken watchdog or an allocation76    /// failure is returned as an error.77    pub fn runQuery(self: *Session, allocator: std.mem.Allocator, sql: []const u8) !QueryOutput {78        if (try sql_guard.check(allocator, &self.connection, sql)) |reason| {79            return .{ .sql = sql, .error_message = reason };80        }8182        const sql_z = try allocator.dupeZ(u8, sql);83        var extracted = self.connection.extract(sql_z);84        defer extracted.deinit();85        if (extracted.count != 1) return .{ .sql = sql, .error_message = "exactly one SQL statement is allowed" };8687        var statement = extracted.prepare(0);88        defer statement.deinit();89        if (!statement.ok) {90            return .{ .sql = sql, .error_message = try allocator.dupe(u8, statement.errorMessage()) };91        }92        if (statement.kind() != c.DUCKDB_STATEMENT_TYPE_SELECT) {93            return .{ .sql = sql, .error_message = "only SELECT queries are allowed" };94        }9596        // The watchdog interrupts this query if it overruns, so the request never97        // blocks forever. Registration is dropped before the session can be closed.98        try Watchdog.ensureRunning();99        var deadline: Watchdog.Entry = undefined;100        try Watchdog.register(&deadline, &self.connection, self.timeout_ms);101        defer Watchdog.unregister(&deadline);102103        var result = statement.execute();104        defer result.deinit();105106        if (!result.ok) {107            return .{ .sql = sql, .error_message = try allocator.dupe(u8, result.errorMessage()) };108        }109        return collect(allocator, sql, &result);110    }111112    /// Lists lake tables and columns as plain text for the model.113    pub fn describe(self: *Session, allocator: std.mem.Allocator) ![]const u8 {114        var result = self.connection.query(115            "SELECT table_name, column_name, data_type FROM duckdb_columns() " ++116                "WHERE database_name = 'lake' ORDER BY table_name, column_index",117        );118        defer result.deinit();119        if (!result.ok) return allocator.dupe(u8, result.errorMessage());120121        var out = std.ArrayList(u8).init(allocator);122        const writer = out.writer();123        var current_table: ?[]u8 = null;124        var row: usize = 0;125        while (row < result.rowCount()) : (row += 1) {126            const table = (try result.valueText(allocator, 0, row)) orelse continue;127            const column = (try result.valueText(allocator, 1, row)) orelse continue;128            const data_type = (try result.valueText(allocator, 2, row)) orelse continue;129            if (current_table == null or !std.mem.eql(u8, current_table.?, table)) {130                if (current_table != null) try writer.writeAll(")\n");131                try writer.print("lake.{s} (", .{table});132                current_table = table;133            } else {134                try writer.writeAll(", ");135            }136            try writer.print("{s} {s}", .{ column, data_type });137        }138        if (current_table != null) try writer.writeAll(")\n");139        if (out.items.len == 0) return allocator.dupe(u8, "no tables are visible in the lake");140        return out.items;141    }142};143144/// Interrupts queries that run past their timeout. One supervisor thread watches145/// every active query: `runQuery` registers a deadline before it calls DuckDB and146/// unregisters afterwards.147///148/// It used to spawn a short-lived thread per query and join it, which made a149/// failed `pthread_join` abort the whole server through std's `unreachable`.150/// Registering a deadline instead keeps all thread lifecycle out of the request151/// path: there is nothing to join, and a query cannot outlive its session152/// because `unregister` blocks until an in-flight interrupt has finished.153pub const Watchdog = struct {154    pub const Entry = struct {155        connection: *duckdb.Connection,156        timeout_ns: u64,157        start: std.time.Instant,158        fired: bool = false,159        next: ?*Entry = null,160    };161162    var mutex: std.Thread.Mutex = .{};163    var cond: std.Thread.Condition = .{};164    var entries: ?*Entry = null;165    var start_once = std.once(startSupervisor);166    var start_error: ?anyerror = null;167168    /// Starts the supervisor on first use. Its thread never exits and is never169    /// joined; the process owns it for its whole lifetime.170    pub fn ensureRunning() !void {171        start_once.call();172        if (start_error) |err| return err;173    }174175    fn startSupervisor() void {176        _ = std.Thread.spawn(.{}, supervise, .{}) catch |err| {177            start_error = err;178        };179    }180181    /// Marks `entry` as running now with the given deadline. The entry must stay182    /// alive, and in this list, until `unregister` is called.183    pub fn register(entry: *Entry, connection: *duckdb.Connection, timeout_ms: u64) !void {184        const now = try std.time.Instant.now();185        mutex.lock();186        defer mutex.unlock();187        entry.* = .{188            .connection = connection,189            .timeout_ns = timeout_ms * std.time.ns_per_ms,190            .start = now,191            .next = entries,192        };193        entries = entry;194        cond.signal();195    }196197    /// Removes `entry` from the watch list. Returns only once the supervisor can198    /// no longer touch the connection, so callers may close the session after it.199    pub fn unregister(entry: *Entry) void {200        mutex.lock();201        defer mutex.unlock();202        var slot = &entries;203        while (slot.*) |current| {204            if (current == entry) {205                slot.* = current.next;206                return;207            }208            slot = &current.next;209        }210    }211212    fn supervise() void {213        mutex.lock();214        defer mutex.unlock();215        while (true) {216            if (entries == null) {217                cond.wait(&mutex);218                continue;219            }220221            const now = std.time.Instant.now() catch return;222            var next_wakeup: u64 = std.math.maxInt(u64);223            var cursor = entries;224            while (cursor) |entry| : (cursor = entry.next) {225                const elapsed = now.since(entry.start);226                if (elapsed >= entry.timeout_ns) {227                    if (!entry.fired) {228                        entry.fired = true;229                        log.warn("query exceeded {d} ms, interrupting", .{entry.timeout_ns / std.time.ns_per_ms});230                        entry.connection.interrupt();231                    }232                } else {233                    next_wakeup = @min(next_wakeup, entry.timeout_ns - elapsed);234                }235            }236237            if (next_wakeup == std.math.maxInt(u64)) {238                // Everything here already timed out; wait for the next registration.239                cond.wait(&mutex);240            } else {241                cond.timedWait(&mutex, next_wakeup) catch {};242            }243        }244    }245};246247fn execute(connection: *duckdb.Connection, sql: [:0]const u8) !void {248    var result = connection.query(sql);249    defer result.deinit();250    if (!result.ok) {251        log.err("{s} failed: {s}", .{ sql, result.errorMessage() });252        return error.DuckDbFailed;253    }254}255256fn collect(allocator: std.mem.Allocator, sql: []const u8, result: *duckdb.Result) !QueryOutput {257    const column_count = result.columnCount();258    const columns = try allocator.alloc([]const u8, column_count);259    for (columns, 0..) |*name, col| {260        name.* = try allocator.dupe(u8, result.columnName(col));261    }262263    const total = result.rowCount();264    const shown = @min(total, max_rows_returned);265    const rows = try allocator.alloc([]const ?[]const u8, shown);266    for (rows, 0..) |*row, index| {267        const cells = try allocator.alloc(?[]const u8, column_count);268        for (cells, 0..) |*cell, col| {269            cell.* = try result.valueText(allocator, col, index);270        }271        row.* = cells;272    }273274    return .{ .sql = sql, .columns = columns, .rows = rows, .total_rows = total };275}276277test "watchdog interrupts a query that passes its deadline" {278    var database = try duckdb.Database.openMemory("/tmp", "/tmp");279    defer database.close();280    var connection = try duckdb.Connection.connect(&database);281    defer connection.disconnect();282283    try Watchdog.ensureRunning();284    var deadline: Watchdog.Entry = undefined;285    try Watchdog.register(&deadline, &connection, 20);286    defer Watchdog.unregister(&deadline);287288    // Long enough that the interrupt lands while it is still running.289    var result = connection.query(290        "SELECT sum(a.range * b.range) FROM range(20000000) a, range(40) b",291    );292    defer result.deinit();293    try std.testing.expect(!result.ok);294}295296test "collect copies columns and rows, including NULL" {297    var database = try duckdb.Database.openMemory("/tmp", "/tmp");298    defer database.close();299    var connection = try duckdb.Connection.connect(&database);300    defer connection.disconnect();301302    var result = connection.query("SELECT 42 AS number, NULL::VARCHAR AS missing, 'text' AS label");303    defer result.deinit();304    try std.testing.expect(result.ok);305306    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);307    defer arena.deinit();308    const output = try collect(arena.allocator(), "SELECT ...", &result);309    try std.testing.expectEqual(@as(usize, 3), output.columns.len);310    try std.testing.expectEqualStrings("number", output.columns[0]);311    try std.testing.expectEqualStrings("missing", output.columns[1]);312    try std.testing.expectEqualStrings("label", output.columns[2]);313    try std.testing.expectEqual(@as(usize, 1), output.rows.len);314    try std.testing.expectEqualStrings("42", output.rows[0][0].?);315    try std.testing.expect(output.rows[0][1] == null);316    try std.testing.expectEqualStrings("text", output.rows[0][2].?);317    try std.testing.expectEqual(@as(usize, 1), output.total_rows);318}319320test "collect caps returned rows at max_rows_returned" {321    var database = try duckdb.Database.openMemory("/tmp", "/tmp");322    defer database.close();323    var connection = try duckdb.Connection.connect(&database);324    defer connection.disconnect();325326    var result = connection.query("SELECT range AS n FROM range(300)");327    defer result.deinit();328    try std.testing.expect(result.ok);329330    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);331    defer arena.deinit();332    const output = try collect(arena.allocator(), "SELECT range AS n FROM range(300)", &result);333    try std.testing.expectEqual(@as(usize, 300), output.total_rows);334    try std.testing.expectEqual(max_rows_returned, output.rows.len);335    try std.testing.expectEqualStrings("0", output.rows[0][0].?);336    try std.testing.expectEqualStrings("199", output.rows[max_rows_returned - 1][0].?);337}338339fn testSession(database: *duckdb.Database, connection: *duckdb.Connection) Session {340    return .{ .database = database.*, .connection = connection.*, .timeout_ms = 5_000 };341}342343test "runQuery rejects guarded statements and returns rows for selects" {344    var database = try duckdb.Database.openMemory("/tmp", "/tmp");345    var connection = try duckdb.Connection.connect(&database);346    var sess = testSession(&database, &connection);347    defer sess.deinit();348349    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);350    defer arena.deinit();351    const a = arena.allocator();352353    const rejected = try sess.runQuery(a, "DROP TABLE lake.leaderboard");354    try std.testing.expect(std.mem.indexOf(u8, rejected.error_message.?, "SELECT") != null);355    try std.testing.expectEqual(@as(usize, 0), rejected.rows.len);356357    const multi = try sess.runQuery(a, "SELECT 1; SELECT 2");358    try std.testing.expectEqualStrings("exactly one SQL statement is allowed", multi.error_message.?);359360    const missing = try sess.runQuery(a, "SELECT * FROM missing_table");361    try std.testing.expect(std.mem.indexOf(u8, missing.error_message.?, "missing_table") != null);362363    const ok = try sess.runQuery(a, "SELECT 42 AS answer, NULL::VARCHAR AS note");364    try std.testing.expect(ok.error_message == null);365    try std.testing.expectEqualStrings("answer", ok.columns[0]);366    try std.testing.expectEqualStrings("42", ok.rows[0][0].?);367    try std.testing.expect(ok.rows[0][1] == null);368    try std.testing.expectEqual(@as(usize, 1), ok.total_rows);369}370371test "describe lists lake tables in a compact form" {372    var database = try duckdb.Database.openMemory("/tmp", "/tmp");373    var connection = try duckdb.Connection.connect(&database);374    var sess = testSession(&database, &connection);375    defer sess.deinit();376377    try execute(&connection, "ATTACH ':memory:' AS lake");378    try execute(&connection, "CREATE TABLE lake.heroes (id INTEGER, name VARCHAR)");379    try execute(&connection, "CREATE TABLE lake.matches (id BIGINT)");380381    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);382    defer arena.deinit();383    const text = try sess.describe(arena.allocator());384    try std.testing.expectEqualStrings(385        "lake.heroes (id INTEGER, name VARCHAR)\nlake.matches (id BIGINT)\n",386        text,387    );388}389390test "describe reports an empty lake" {391    var database = try duckdb.Database.openMemory("/tmp", "/tmp");392    var connection = try duckdb.Connection.connect(&database);393    var sess = testSession(&database, &connection);394    defer sess.deinit();395396    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);397    defer arena.deinit();398    try std.testing.expectEqualStrings(399        "no tables are visible in the lake",400        try sess.describe(arena.allocator()),401    );402}