LINE1 10 20 30 40 50 60 70 80 90 100 110 1201//! One request's view of the lake: a fresh in-memory DuckDB with the DuckLake2//! catalog attached read-only. Nothing is shared between requests, so a bad query3//! or a stuck connection cannot leak into the next one.4//!5//! The only process-wide piece is `Watchdog`: one long-lived thread that6//! interrupts queries whose deadline passed. Requests register a stack-allocated7//! entry with it instead of spawning a thread of their own.89const std = @import("std");10const duckdb = @import("duckdb.zig");11const sql_guard = @import("sql_guard.zig");12const Config = @import("config.zig").Config;1314const c = duckdb.c;15const log = std.log.scoped(.session);1617/// Rows kept per query for the UI. The model gets a smaller slice of these.18pub const max_rows_returned = 200;1920pub const QueryOutput = struct {21 sql: []const u8,22 columns: []const []const u8 = &.{},23 rows: []const []const ?[]const u8 = &.{},24 total_rows: usize = 0,25 error_message: ?[]const u8 = null,26};2728/// Downloads the DuckLake and httpfs extensions into the extension cache once at29/// startup, so per-request LOAD calls never touch the network for them. The30/// extensions DuckDB normally bundles (core_functions, json, parquet, icu) are31/// linked into the binary, so only these two are fetched.32pub fn installExtensions(cfg: *const Config) !void {33 var database = try duckdb.Database.openMemory(cfg.extension_dir, cfg.temp_dir);34 defer database.close();35 var connection = try duckdb.Connection.connect(&database);36 defer connection.disconnect();37 try execute(&connection, "INSTALL ducklake");38 try execute(&connection, "INSTALL httpfs");39}4041pub const Session = struct {42 database: duckdb.Database,43 connection: duckdb.Connection,44 timeout_ms: u64,4546 pub fn open(allocator: std.mem.Allocator, cfg: *const Config) !Session {47 var database = try duckdb.Database.openMemory(cfg.extension_dir, cfg.temp_dir);48 errdefer database.close();49 var connection = try duckdb.Connection.connect(&database);50 errdefer connection.disconnect();5152 const attach = try std.fmt.allocPrintZ(allocator, "ATTACH 'ducklake:{s}' AS lake (READ_ONLY)", .{cfg.lake_catalog});53 try execute(&connection, "LOAD ducklake");54 try execute(&connection, "LOAD httpfs");55 try execute(&connection, attach);56 try execute(&connection, "SET memory_limit = '1GB'");57 // Grouped queries over a few days of match data otherwise try to buffer the58 // whole result in insertion order and hit the 1 GB cap; no reply depends on59 // the incidental order of an un-ordered result.60 try execute(&connection, "SET preserve_insertion_order = false");61 // The lake's data files are HTTPS, so httpfs stays on. Local files are off,62 // and the lock below means the model cannot switch this back.63 try execute(&connection, "SET disabled_filesystems = 'LocalFileSystem'");64 try execute(&connection, "SET lock_configuration = true");6566 return .{ .database = database, .connection = connection, .timeout_ms = cfg.query_timeout_ms };67 }6869 pub fn deinit(self: *Session) void {70 self.connection.disconnect();71 self.database.close();72 }7374 /// Runs one user- or model-written query after the guard and the statement-kind check.75 /// Problems come back in `error_message`; only a broken watchdog or an allocation76 /// failure is returned as an error.77 pub fn runQuery(self: *Session, allocator: std.mem.Allocator, sql: []const u8) !QueryOutput {78 if (try sql_guard.check(allocator, &self.connection, sql)) |reason| {79 return .{ .sql = sql, .error_message = reason };80 }8182 const sql_z = try allocator.dupeZ(u8, sql);83 var extracted = self.connection.extract(sql_z);84 defer extracted.deinit();85 if (extracted.count != 1) return .{ .sql = sql, .error_message = "exactly one SQL statement is allowed" };8687 var statement = extracted.prepare(0);88 defer statement.deinit();89 if (!statement.ok) {90 return .{ .sql = sql, .error_message = try allocator.dupe(u8, statement.errorMessage()) };91 }92 if (statement.kind() != c.DUCKDB_STATEMENT_TYPE_SELECT) {93 return .{ .sql = sql, .error_message = "only SELECT queries are allowed" };94 }9596 // The watchdog interrupts this query if it overruns, so the request never97 // blocks forever. Registration is dropped before the session can be closed.98 try Watchdog.ensureRunning();99 var deadline: Watchdog.Entry = undefined;100 try Watchdog.register(&deadline, &self.connection, self.timeout_ms);101 defer Watchdog.unregister(&deadline);102103 var result = statement.execute();104 defer result.deinit();105106 if (!result.ok) {107 return .{ .sql = sql, .error_message = try allocator.dupe(u8, result.errorMessage()) };108 }109 return collect(allocator, sql, &result);110 }111112 /// Lists lake tables and columns as plain text for the model.113 pub fn describe(self: *Session, allocator: std.mem.Allocator) ![]const u8 {114 var result = self.connection.query(115 "SELECT table_name, column_name, data_type FROM duckdb_columns() " ++116 "WHERE database_name = 'lake' ORDER BY table_name, column_index",117 );118 defer result.deinit();119 if (!result.ok) return allocator.dupe(u8, result.errorMessage());120121 var out = std.ArrayList(u8).init(allocator);122 const writer = out.writer();123 var current_table: ?[]u8 = null;124 var row: usize = 0;125 while (row < result.rowCount()) : (row += 1) {126 const table = (try result.valueText(allocator, 0, row)) orelse continue;127 const column = (try result.valueText(allocator, 1, row)) orelse continue;128 const data_type = (try result.valueText(allocator, 2, row)) orelse continue;129 if (current_table == null or !std.mem.eql(u8, current_table.?, table)) {130 if (current_table != null) try writer.writeAll(")\n");131 try writer.print("lake.{s} (", .{table});132 current_table = table;133 } else {134 try writer.writeAll(", ");135 }136 try writer.print("{s} {s}", .{ column, data_type });137 }138 if (current_table != null) try writer.writeAll(")\n");139 if (out.items.len == 0) return allocator.dupe(u8, "no tables are visible in the lake");140 return out.items;141 }142};143144/// Interrupts queries that run past their timeout. One supervisor thread watches145/// every active query: `runQuery` registers a deadline before it calls DuckDB and146/// unregisters afterwards.147///148/// It used to spawn a short-lived thread per query and join it, which made a149/// failed `pthread_join` abort the whole server through std's `unreachable`.150/// Registering a deadline instead keeps all thread lifecycle out of the request151/// path: there is nothing to join, and a query cannot outlive its session152/// because `unregister` blocks until an in-flight interrupt has finished.153pub const Watchdog = struct {154 pub const Entry = struct {155 connection: *duckdb.Connection,156 timeout_ns: u64,157 start: std.time.Instant,158 fired: bool = false,159 next: ?*Entry = null,160 };161162 var mutex: std.Thread.Mutex = .{};163 var cond: std.Thread.Condition = .{};164 var entries: ?*Entry = null;165 var start_once = std.once(startSupervisor);166 var start_error: ?anyerror = null;167168 /// Starts the supervisor on first use. Its thread never exits and is never169 /// joined; the process owns it for its whole lifetime.170 pub fn ensureRunning() !void {171 start_once.call();172 if (start_error) |err| return err;173 }174175 fn startSupervisor() void {176 _ = std.Thread.spawn(.{}, supervise, .{}) catch |err| {177 start_error = err;178 };179 }180181 /// Marks `entry` as running now with the given deadline. The entry must stay182 /// alive, and in this list, until `unregister` is called.183 pub fn register(entry: *Entry, connection: *duckdb.Connection, timeout_ms: u64) !void {184 const now = try std.time.Instant.now();185 mutex.lock();186 defer mutex.unlock();187 entry.* = .{188 .connection = connection,189 .timeout_ns = timeout_ms * std.time.ns_per_ms,190 .start = now,191 .next = entries,192 };193 entries = entry;194 cond.signal();195 }196197 /// Removes `entry` from the watch list. Returns only once the supervisor can198 /// no longer touch the connection, so callers may close the session after it.199 pub fn unregister(entry: *Entry) void {200 mutex.lock();201 defer mutex.unlock();202 var slot = &entries;203 while (slot.*) |current| {204 if (current == entry) {205 slot.* = current.next;206 return;207 }208 slot = ¤t.next;209 }210 }211212 fn supervise() void {213 mutex.lock();214 defer mutex.unlock();215 while (true) {216 if (entries == null) {217 cond.wait(&mutex);218 continue;219 }220221 const now = std.time.Instant.now() catch return;222 var next_wakeup: u64 = std.math.maxInt(u64);223 var cursor = entries;224 while (cursor) |entry| : (cursor = entry.next) {225 const elapsed = now.since(entry.start);226 if (elapsed >= entry.timeout_ns) {227 if (!entry.fired) {228 entry.fired = true;229 log.warn("query exceeded {d} ms, interrupting", .{entry.timeout_ns / std.time.ns_per_ms});230 entry.connection.interrupt();231 }232 } else {233 next_wakeup = @min(next_wakeup, entry.timeout_ns - elapsed);234 }235 }236237 if (next_wakeup == std.math.maxInt(u64)) {238 // Everything here already timed out; wait for the next registration.239 cond.wait(&mutex);240 } else {241 cond.timedWait(&mutex, next_wakeup) catch {};242 }243 }244 }245};246247fn execute(connection: *duckdb.Connection, sql: [:0]const u8) !void {248 var result = connection.query(sql);249 defer result.deinit();250 if (!result.ok) {251 log.err("{s} failed: {s}", .{ sql, result.errorMessage() });252 return error.DuckDbFailed;253 }254}255256fn collect(allocator: std.mem.Allocator, sql: []const u8, result: *duckdb.Result) !QueryOutput {257 const column_count = result.columnCount();258 const columns = try allocator.alloc([]const u8, column_count);259 for (columns, 0..) |*name, col| {260 name.* = try allocator.dupe(u8, result.columnName(col));261 }262263 const total = result.rowCount();264 const shown = @min(total, max_rows_returned);265 const rows = try allocator.alloc([]const ?[]const u8, shown);266 for (rows, 0..) |*row, index| {267 const cells = try allocator.alloc(?[]const u8, column_count);268 for (cells, 0..) |*cell, col| {269 cell.* = try result.valueText(allocator, col, index);270 }271 row.* = cells;272 }273274 return .{ .sql = sql, .columns = columns, .rows = rows, .total_rows = total };275}276277test "watchdog interrupts a query that passes its deadline" {278 var database = try duckdb.Database.openMemory("/tmp", "/tmp");279 defer database.close();280 var connection = try duckdb.Connection.connect(&database);281 defer connection.disconnect();282283 try Watchdog.ensureRunning();284 var deadline: Watchdog.Entry = undefined;285 try Watchdog.register(&deadline, &connection, 20);286 defer Watchdog.unregister(&deadline);287288 // Long enough that the interrupt lands while it is still running.289 var result = connection.query(290 "SELECT sum(a.range * b.range) FROM range(20000000) a, range(40) b",291 );292 defer result.deinit();293 try std.testing.expect(!result.ok);294}295296test "collect copies columns and rows, including NULL" {297 var database = try duckdb.Database.openMemory("/tmp", "/tmp");298 defer database.close();299 var connection = try duckdb.Connection.connect(&database);300 defer connection.disconnect();301302 var result = connection.query("SELECT 42 AS number, NULL::VARCHAR AS missing, 'text' AS label");303 defer result.deinit();304 try std.testing.expect(result.ok);305306 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);307 defer arena.deinit();308 const output = try collect(arena.allocator(), "SELECT ...", &result);309 try std.testing.expectEqual(@as(usize, 3), output.columns.len);310 try std.testing.expectEqualStrings("number", output.columns[0]);311 try std.testing.expectEqualStrings("missing", output.columns[1]);312 try std.testing.expectEqualStrings("label", output.columns[2]);313 try std.testing.expectEqual(@as(usize, 1), output.rows.len);314 try std.testing.expectEqualStrings("42", output.rows[0][0].?);315 try std.testing.expect(output.rows[0][1] == null);316 try std.testing.expectEqualStrings("text", output.rows[0][2].?);317 try std.testing.expectEqual(@as(usize, 1), output.total_rows);318}319320test "collect caps returned rows at max_rows_returned" {321 var database = try duckdb.Database.openMemory("/tmp", "/tmp");322 defer database.close();323 var connection = try duckdb.Connection.connect(&database);324 defer connection.disconnect();325326 var result = connection.query("SELECT range AS n FROM range(300)");327 defer result.deinit();328 try std.testing.expect(result.ok);329330 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);331 defer arena.deinit();332 const output = try collect(arena.allocator(), "SELECT range AS n FROM range(300)", &result);333 try std.testing.expectEqual(@as(usize, 300), output.total_rows);334 try std.testing.expectEqual(max_rows_returned, output.rows.len);335 try std.testing.expectEqualStrings("0", output.rows[0][0].?);336 try std.testing.expectEqualStrings("199", output.rows[max_rows_returned - 1][0].?);337}338339fn testSession(database: *duckdb.Database, connection: *duckdb.Connection) Session {340 return .{ .database = database.*, .connection = connection.*, .timeout_ms = 5_000 };341}342343test "runQuery rejects guarded statements and returns rows for selects" {344 var database = try duckdb.Database.openMemory("/tmp", "/tmp");345 var connection = try duckdb.Connection.connect(&database);346 var sess = testSession(&database, &connection);347 defer sess.deinit();348349 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);350 defer arena.deinit();351 const a = arena.allocator();352353 const rejected = try sess.runQuery(a, "DROP TABLE lake.leaderboard");354 try std.testing.expect(std.mem.indexOf(u8, rejected.error_message.?, "SELECT") != null);355 try std.testing.expectEqual(@as(usize, 0), rejected.rows.len);356357 const multi = try sess.runQuery(a, "SELECT 1; SELECT 2");358 try std.testing.expectEqualStrings("exactly one SQL statement is allowed", multi.error_message.?);359360 const missing = try sess.runQuery(a, "SELECT * FROM missing_table");361 try std.testing.expect(std.mem.indexOf(u8, missing.error_message.?, "missing_table") != null);362363 const ok = try sess.runQuery(a, "SELECT 42 AS answer, NULL::VARCHAR AS note");364 try std.testing.expect(ok.error_message == null);365 try std.testing.expectEqualStrings("answer", ok.columns[0]);366 try std.testing.expectEqualStrings("42", ok.rows[0][0].?);367 try std.testing.expect(ok.rows[0][1] == null);368 try std.testing.expectEqual(@as(usize, 1), ok.total_rows);369}370371test "describe lists lake tables in a compact form" {372 var database = try duckdb.Database.openMemory("/tmp", "/tmp");373 var connection = try duckdb.Connection.connect(&database);374 var sess = testSession(&database, &connection);375 defer sess.deinit();376377 try execute(&connection, "ATTACH ':memory:' AS lake");378 try execute(&connection, "CREATE TABLE lake.heroes (id INTEGER, name VARCHAR)");379 try execute(&connection, "CREATE TABLE lake.matches (id BIGINT)");380381 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);382 defer arena.deinit();383 const text = try sess.describe(arena.allocator());384 try std.testing.expectEqualStrings(385 "lake.heroes (id INTEGER, name VARCHAR)\nlake.matches (id BIGINT)\n",386 text,387 );388}389390test "describe reports an empty lake" {391 var database = try duckdb.Database.openMemory("/tmp", "/tmp");392 var connection = try duckdb.Connection.connect(&database);393 var sess = testSession(&database, &connection);394 defer sess.deinit();395396 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);397 defer arena.deinit();398 try std.testing.expectEqualStrings(399 "no tables are visible in the lake",400 try sess.describe(arena.allocator()),401 );402}