tickets

63.0 KB

1. // mpackdb DB engine — Zig port of mpackdb v1.0.7 (MPackDB.js / IndexManager.js)
2. // semantics. File-level compatible with the JS implementation (D12):
3. //
4. // <base>.mpack append-only records: [4-byte LE total size][msgpack map]
5. // <base>.meta.json { nextId?, deleted:[offsets], version, schema? }
6. // <base>.<field>.txt sorted index lines "key,offset,length\n"
7. // <base>.idxstate.json { coveredBytes }
8. // <base>.lock lock file containing the holder's pid (O_EXCL protocol,
9. // stale takeover after staleLockTimeout ms)
10. //
11. // Divergences from the JS implementation (documented in the 066 report):
12. // - the in-memory index is the COMPLETE entry set (JS keeps on-disk + delta);
13. // persist() writes the full set instead of read-merge-write. Sequential
14. // cross-process use (close one side, open the other) behaves identically.
15. // - string index keys sort bytewise (JS: String.localeCompare, ICU collation).
16. // Equality lookups are unaffected; range ordering can differ for
17. // mixed-case/accented keys.
18. // - null field values are not indexed (JS indexes them under the key "null").
19. // - external data-file replacement while open (inode change) is not detected.
20. // - a fresh `*id` table's first id is 1 (JS: 0) — ticket #4, see FIRST_ID.
22. const std = @import("std");
23. const linux = std.os.linux;
24. const msgpack = @import("msgpack.zig");
26. pub const PkType = enum(u8) { number = 0, uuid = 1, string = 2 };
28. /// A `*id` table's first auto-increment id (ticket #4, the creator's ruling:
29. /// "start at 1"). Only a table WITHOUT a stored counter starts here: a
30. /// .meta.json that carries `nextId` — every existing table, including one the
31. /// JS mpackdb wrote — keeps counting from it. The file format is unchanged;
32. /// the JS mpackdb 1.0.7 starts a fresh table at 0, which is the one divergence.
33. pub const FIRST_ID: f64 = 1;
34. pub const IdxType = enum(u8) { lexical = 0, numeric = 1 };
36. pub const Error = error{
37. NoDbFile,
38. NoPrimaryKey,
39. /// mission 077 (074 GAP 10): a query named a field with no index. There is
40. /// no index to binary-search and mpackdb does not silently full-scan, so
41. /// the answer used to be an empty result — indistinguishable from "no such
42. /// record". It is a mistake, and it says so now.
43. NoSuchIndex,
44. DuplicateKey,
45. /// ticket #29: update()'s record must carry the primary key. Without it
46. /// the old record was deleted, the new one written without a key, and the
47. /// table left corrupt (fetch null, a unique index still finding it).
48. RecordLacksPrimaryKey,
49. LockTimeout,
50. CorruptRecord,
51. IoError,
52. OutOfMemory,
53. InvalidFormat,
54. Truncated,
55. MapTooLarge,
56. };
58. pub const Key = union(enum) {
59. num: f64,
60. str: []const u8,
61. };
63. pub const Entry = struct {
64. key: Key,
65. off: u64,
66. len: u64,
67. };
69. const FieldIndex = struct {
70. typ: IdxType,
71. entries: std.ArrayList(Entry) = .empty, // sorted by (key, off)
72. };
74. pub const Loc = struct { off: u64, len: u64 };
76. pub const Record = struct {
77. value: msgpack.Value,
78. off: u64,
79. len: u64,
80. };
82. // =========================================================================
83. // Low-level file IO (raw linux syscalls — plugins avoid std.Io plumbing)
84. // =========================================================================
86. const fio = struct {
87. fn openz(alloc: std.mem.Allocator, path: []const u8, flags: linux.O, mode: u32) ?i32 {
88. const path_z = alloc.dupeZ(u8, path) catch return null;
89. defer alloc.free(path_z);
90. const rc = linux.open(path_z, flags, mode);
91. const fd: i32 = @bitCast(@as(u32, @truncate(rc)));
92. if (fd < 0) return null;
93. return fd;
94. }
96. fn openRead(alloc: std.mem.Allocator, path: []const u8) ?i32 {
97. return openz(alloc, path, .{}, 0);
98. }
100. fn openAppend(alloc: std.mem.Allocator, path: []const u8) ?i32 {
101. return openz(alloc, path, .{ .ACCMODE = .WRONLY, .CREAT = true, .APPEND = true }, 0o644);
102. }
104. fn openTrunc(alloc: std.mem.Allocator, path: []const u8) ?i32 {
105. return openz(alloc, path, .{ .ACCMODE = .WRONLY, .CREAT = true, .TRUNC = true }, 0o644);
106. }
108. /// O_CREAT|O_EXCL — returns null when the file already exists.
109. fn openExcl(alloc: std.mem.Allocator, path: []const u8) ?i32 {
110. return openz(alloc, path, .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true }, 0o644);
111. }
113. fn close(fd: i32) void {
114. _ = linux.close(fd);
115. }
117. fn writeAll(fd: i32, bytes: []const u8) bool {
118. var written: usize = 0;
119. while (written < bytes.len) {
120. const rc = linux.write(fd, bytes.ptr + written, bytes.len - written);
121. if (@as(isize, @bitCast(rc)) <= 0) return false;
122. written += rc;
123. }
124. return true;
125. }
127. fn pread(fd: i32, buf: []u8, offset: u64) ?usize {
128. var got: usize = 0;
129. while (got < buf.len) {
130. const rc = linux.pread(fd, buf.ptr + got, buf.len - got, @intCast(offset + got));
131. const n: isize = @bitCast(rc);
132. if (n < 0) return null;
133. if (n == 0) break;
134. got += @intCast(n);
135. }
136. return got;
137. }
139. fn readAll(alloc: std.mem.Allocator, path: []const u8) ?[]u8 {
140. const fd = openRead(alloc, path) orelse return null;
141. defer close(fd);
142. var content: std.ArrayList(u8) = .empty;
143. var buf: [65536]u8 = undefined;
144. while (true) {
145. const rc = linux.read(fd, &buf, buf.len);
146. if (@as(isize, @bitCast(rc)) <= 0) break;
147. content.appendSlice(alloc, buf[0..rc]) catch return null;
148. }
149. return content.toOwnedSlice(alloc) catch null;
150. }
152. fn fileSize(alloc: std.mem.Allocator, path: []const u8) ?u64 {
153. const path_z = alloc.dupeZ(u8, path) catch return null;
154. defer alloc.free(path_z);
155. var stx: linux.Statx = undefined;
156. const rc = linux.statx(linux.AT.FDCWD, path_z, 0, linux.STATX.BASIC_STATS, &stx);
157. if (rc != 0) return null;
158. return stx.size;
159. }
161. fn mtimeMs(alloc: std.mem.Allocator, path: []const u8) ?i64 {
162. const path_z = alloc.dupeZ(u8, path) catch return null;
163. defer alloc.free(path_z);
164. var stx: linux.Statx = undefined;
165. const rc = linux.statx(linux.AT.FDCWD, path_z, 0, linux.STATX.BASIC_STATS, &stx);
166. if (rc != 0) return null;
167. return @as(i64, stx.mtime.sec) * 1000 + @divTrunc(@as(i64, stx.mtime.nsec), 1_000_000);
168. }
170. fn rename(alloc: std.mem.Allocator, old_path: []const u8, new_path: []const u8) bool {
171. const old_z = alloc.dupeZ(u8, old_path) catch return false;
172. defer alloc.free(old_z);
173. const new_z = alloc.dupeZ(u8, new_path) catch return false;
174. defer alloc.free(new_z);
175. return linux.renameat(linux.AT.FDCWD, old_z, linux.AT.FDCWD, new_z) == 0;
176. }
178. fn unlink(alloc: std.mem.Allocator, path: []const u8) bool {
179. const path_z = alloc.dupeZ(u8, path) catch return false;
180. defer alloc.free(path_z);
181. return linux.unlinkat(linux.AT.FDCWD, path_z, 0) == 0;
182. }
184. fn mkdirAll(alloc: std.mem.Allocator, dir_path: []const u8) void {
185. if (dir_path.len == 0 or std.mem.eql(u8, dir_path, ".")) return;
186. var i: usize = 1;
187. while (i <= dir_path.len) : (i += 1) {
188. if (i == dir_path.len or dir_path[i] == '/') {
189. const part = alloc.dupeZ(u8, dir_path[0..i]) catch return;
190. defer alloc.free(part);
191. _ = linux.mkdirat(linux.AT.FDCWD, part, 0o755);
192. }
193. }
194. }
196. fn sleepMs(ms: u64) void {
197. var req: linux.timespec = .{ .sec = @intCast(ms / 1000), .nsec = @intCast((ms % 1000) * 1_000_000) };
198. var rem: linux.timespec = undefined;
199. _ = linux.nanosleep(&req, &rem);
200. }
202. fn nowMs() i64 {
203. var ts: linux.timespec = undefined;
204. _ = linux.clock_gettime(linux.CLOCK.REALTIME, &ts);
205. return @as(i64, ts.sec) * 1000 + @divTrunc(@as(i64, ts.nsec), 1_000_000);
206. }
207. };
209. // =========================================================================
210. // JS-compatible helpers
211. // =========================================================================
213. /// Format an f64 the way JS String(n) does for the values mpackdb produces:
214. /// integral values print without a decimal point. (Exotic floats may diverge
215. /// from V8's shortest-round-trip formatting; index keys are ints/strings in
216. /// practice.)
217. pub fn jsNumFmt(buf: []u8, v: f64) []const u8 {
218. if (v == @floor(v) and @abs(v) <= 9007199254740992.0) {
219. const i: i64 = @intFromFloat(v);
220. return std.fmt.bufPrint(buf, "{d}", .{i}) catch buf[0..0];
221. }
222. return std.fmt.bufPrint(buf, "{d}", .{v}) catch buf[0..0];
223. }
225. /// JS parseInt(s, 10) semantics: optional sign, leading digits, ignore rest.
226. /// Returns null for NaN.
227. fn jsParseInt(s: []const u8) ?f64 {
228. var i: usize = 0;
229. while (i < s.len and (s[i] == ' ' or s[i] == '\t')) i += 1;
230. var sign: f64 = 1;
231. if (i < s.len and (s[i] == '+' or s[i] == '-')) {
232. if (s[i] == '-') sign = -1;
233. i += 1;
234. }
235. var got = false;
236. var v: f64 = 0;
237. while (i < s.len and s[i] >= '0' and s[i] <= '9') : (i += 1) {
238. v = v * 10 + @as(f64, @floatFromInt(s[i] - '0'));
239. got = true;
240. }
241. if (!got) return null;
242. return sign * v;
243. }
245. /// _compareKeys: numbers numerically; otherwise stringified byte compare
246. /// (JS uses localeCompare — see divergence note in the header).
247. pub fn cmpKeys(a: Key, b: Key) i32 {
248. if (a == .num and b == .num) {
249. if (a.num < b.num) return -1;
250. if (a.num > b.num) return 1;
251. return 0;
252. }
253. var buf_a: [32]u8 = undefined;
254. var buf_b: [32]u8 = undefined;
255. const sa = if (a == .str) a.str else jsNumFmt(&buf_a, a.num);
256. const sb = if (b == .str) b.str else jsNumFmt(&buf_b, b.num);
257. return switch (std.mem.order(u8, sa, sb)) {
258. .lt => -1,
259. .eq => 0,
260. .gt => 1,
261. };
262. }
264. fn entryLess(_: void, a: Entry, b: Entry) bool {
265. const c = cmpKeys(a.key, b.key);
266. if (c != 0) return c < 0;
267. return a.off < b.off;
268. }
270. /// mpack.js uuid(): 9-char base36 ms timestamp (padStart '0') + 3 base36 chars.
271. pub fn genUuid(buf: *[12]u8) []const u8 {
272. const digits = "0123456789abcdefghijklmnopqrstuvwxyz";
273. const t: u64 = @intCast(fio.nowMs());
274. var tmp: [16]u8 = undefined;
275. var n: usize = 0;
276. var v = t;
277. while (v > 0) : (v /= 36) {
278. tmp[n] = digits[@intCast(v % 36)];
279. n += 1;
280. }
281. var i: usize = 0;
282. while (i < 9) : (i += 1) {
283. buf[8 - i] = if (i < n) tmp[i] else '0';
284. }
285. if (!rng_init) {
286. var ts: linux.timespec = undefined;
287. _ = linux.clock_gettime(linux.CLOCK.MONOTONIC, &ts);
288. rng = std.Random.DefaultPrng.init(@bitCast(@as(i64, ts.sec) *% 1_000_000_000 +% ts.nsec));
289. rng_init = true;
290. }
291. var r = rng.random();
292. for (0..3) |j| {
293. buf[9 + j] = digits[r.uintLessThan(u8, 36)];
294. }
295. return buf[0..12];
296. }
298. var rng: std.Random.DefaultPrng = .{ .s = undefined };
299. var rng_init: bool = false;
301. // =========================================================================
302. // The database
303. // =========================================================================
305. pub const OpenOptions = struct {
306. primary_key: ?[]const u8 = null, // with */@/! prefixes, like the JS API
307. indexes: []const []const u8 = &.{}, // with prefixes
308. compact: bool = true,
309. stale_lock_timeout_ms: i64 = 30000,
310. debug: bool = false,
311. };
313. /// The identity of a table on disk (ticket #110): dirname + basename with any
314. /// extension stripped, exactly the computation `Db.open` uses to turn a given
315. /// path into `<base>.mpack` — so two spellings of the same file (`x.db`, `x`)
316. /// share one key. The plugin ABI layer (mpackdb.zig) keys its process-wide
317. /// handle registry on this, so a table opened twice in one process — from any
318. /// module instance or realm — shares the one live `Db` instead of each open
319. /// racing the other's file state.
320. pub fn tableKey(alloc: std.mem.Allocator, db_file: []const u8) Error![]u8 {
321. const dir = std.fs.path.dirname(db_file) orelse ".";
322. var base = std.fs.path.basename(db_file);
323. if (std.mem.lastIndexOfScalar(u8, base, '.')) |dot| {
324. if (dot > 0) base = base[0..dot];
325. }
326. return std.fmt.allocPrint(alloc, "{s}/{s}", .{ dir, base }) catch Error.OutOfMemory;
327. }
329. pub const Db = struct {
330. arena_state: std.heap.ArenaAllocator,
331. alloc: std.mem.Allocator, // arena — freed wholesale on close
332. // THE BUFFERS OF ONE OPERATION ARE NOT THE HANDLE'S (hybriel#126): the meta read
333. // back and rendered anew, the index files, the paths. On the arena they stayed for
334. // the handle's life, and the meta grows with every tombstone, so each update leaked
335. // more than the last (notes: 2000 updates of one note, +100 MB). They are freed now.
336. tmp: std.mem.Allocator,
338. // paths
339. data_path: []const u8,
340. meta_path: []const u8,
341. lock_path: []const u8,
342. idxstate_path: []const u8,
344. // schema
345. pk: ?[]const u8 = null,
346. pk_type: PkType = .string,
347. index_fields: std.ArrayList([]const u8) = .empty, // pk first (when set), like JS
348. unique_fields: std.ArrayList([]const u8) = .empty,
349. idx: std.StringArrayHashMapUnmanaged(FieldIndex) = .empty,
351. // meta
352. next_id: f64 = FIRST_ID,
353. deleted: std.ArrayList(u64) = .empty,
354. version: u64 = 0,
356. covered_bytes: u64 = 0,
357. compact_on_open: bool = true,
358. stale_lock_timeout_ms: i64 = 30000,
359. debug: bool = false,
361. // last DuplicateKey detail for the plugin surface
362. last_error_buf: [256]u8 = undefined,
363. last_error: []const u8 = "",
365. pub fn open(gpa: std.mem.Allocator, db_file: []const u8, opts: OpenOptions) Error!*Db {
366. if (db_file.len == 0) return Error.NoDbFile;
368. const self = gpa.create(Db) catch return Error.OutOfMemory;
369. self.* = .{
370. .arena_state = std.heap.ArenaAllocator.init(gpa),
371. .alloc = undefined,
372. .tmp = gpa,
373. .data_path = undefined,
374. .meta_path = undefined,
375. .lock_path = undefined,
376. .idxstate_path = undefined,
377. };
378. self.alloc = self.arena_state.allocator();
379. errdefer {
380. self.arena_state.deinit();
381. gpa.destroy(self);
382. }
383. const a = self.alloc;
385. self.compact_on_open = opts.compact;
386. self.stale_lock_timeout_ms = opts.stale_lock_timeout_ms;
387. self.debug = opts.debug;
389. // dirname / basename (extension stripped, like JS basename(f, extname(f)))
390. const dir = std.fs.path.dirname(db_file) orelse ".";
391. var base = std.fs.path.basename(db_file);
392. if (std.mem.lastIndexOfScalar(u8, base, '.')) |dot| {
393. if (dot > 0) base = base[0..dot];
394. }
395. fio.mkdirAll(a, dir);
397. self.data_path = std.fmt.allocPrint(a, "{s}/{s}.mpack", .{ dir, base }) catch return Error.OutOfMemory;
398. self.meta_path = std.fmt.allocPrint(a, "{s}/{s}.meta.json", .{ dir, base }) catch return Error.OutOfMemory;
399. self.lock_path = std.fmt.allocPrint(a, "{s}/{s}.lock", .{ dir, base }) catch return Error.OutOfMemory;
400. self.idxstate_path = std.fmt.allocPrint(a, "{s}/{s}.idxstate.json", .{ dir, base }) catch return Error.OutOfMemory;
402. // ---- parse primary key ----
403. if (opts.primary_key) |pk_raw| {
404. if (pk_raw.len > 0) {
405. if (pk_raw[0] == '*') {
406. self.pk = a.dupe(u8, pk_raw[1..]) catch return Error.OutOfMemory;
407. self.pk_type = .number;
408. } else if (pk_raw[0] == '@') {
409. self.pk = a.dupe(u8, pk_raw[1..]) catch return Error.OutOfMemory;
410. self.pk_type = .uuid;
411. } else {
412. self.pk = a.dupe(u8, pk_raw) catch return Error.OutOfMemory;
413. self.pk_type = .string;
414. }
415. }
416. }
417. if (self.pk) |p| self.unique_fields.append(a, p) catch return Error.OutOfMemory;
419. // ---- parse indexes (pk first, then configured; dedupe) ----
420. if (self.pk) |p| {
421. self.index_fields.append(a, p) catch return Error.OutOfMemory;
422. const t: IdxType = if (self.pk_type == .number) .numeric else .lexical;
423. self.idx.put(a, p, .{ .typ = t }) catch return Error.OutOfMemory;
424. }
425. for (opts.indexes) |raw| {
426. var clean = raw;
427. var is_unique = false;
428. if (clean.len > 0 and clean[0] == '!') {
429. is_unique = true;
430. clean = clean[1..];
431. }
432. var typ: IdxType = .lexical;
433. if (clean.len > 0 and clean[0] == '*') {
434. typ = .numeric;
435. clean = clean[1..];
436. } else if (clean.len > 0 and clean[0] == '@') {
437. clean = clean[1..];
438. }
439. if (self.idx.contains(clean)) continue;
440. const owned = a.dupe(u8, clean) catch return Error.OutOfMemory;
441. self.index_fields.append(a, owned) catch return Error.OutOfMemory;
442. self.idx.put(a, owned, .{ .typ = typ }) catch return Error.OutOfMemory;
443. if (is_unique) self.unique_fields.append(a, owned) catch return Error.OutOfMemory;
444. }
446. // ---- load meta ----
447. self.loadMeta();
449. // ---- compact on init (JS default; also creates an empty data file) ----
450. // AN OPEN WITH NOTHING TO CHANGE WRITES NOTHING (ticket #21): a store
451. // with no tombstones has nothing to compact, and rewriting it anyway
452. // made opening a backup change it.
453. var did_compact = false;
454. const nothing_to_compact = self.deleted.items.len == 0 and
455. fio.fileSize(self.tmp, self.data_path) != null;
456. if (self.compact_on_open and !nothing_to_compact) {
457. try self.acquireLock();
458. const cr = self.compactLocked();
459. self.releaseLock();
460. try cr;
461. did_compact = true;
462. }
464. // ---- persist schema into meta (JS: init-schema under lock).
465. // compactLocked already wrote meta (with schema); skip the extra bump.
466. if (!did_compact and (self.pk != null or self.index_fields.items.len > 0) and
467. !self.metaOnDiskIsCurrent())
468. {
469. try self.acquireLock();
470. const mr = self.persistMetaLocked();
471. self.releaseLock();
472. try mr;
473. }
475. // ---- indexes (JS: (re)build under the lock; force after compaction) ----
476. if (self.index_fields.items.len > 0) {
477. try self.acquireLock();
478. const ir = self.initIndexes(did_compact);
479. self.releaseLock();
480. try ir;
481. }
483. return self;
484. }
486. pub fn close(self: *Db) void {
487. // persist indexes + idxstate (JS: IndexManager.close → persist under lock)
488. if (self.index_fields.items.len > 0) {
489. if (self.acquireLock()) {
490. self.persistIndexesLocked() catch {};
491. self.releaseLock();
492. } else |_| {}
493. }
494. const gpa = self.arena_state.child_allocator;
495. self.arena_state.deinit();
496. gpa.destroy(self);
497. }
499. fn dbg(self: *Db, comptime fmt: []const u8, args: anytype) void {
500. if (self.debug) std.debug.print("[mpackdb] " ++ fmt ++ "\n", args);
501. }
503. // =====================================================================
504. // Locking (JS _acquireFileLock protocol)
505. // =====================================================================
507. fn acquireLock(self: *Db) Error!void {
508. var retries: u32 = 0;
509. while (true) {
510. if (fio.openExcl(self.tmp, self.lock_path)) |fd| {
511. var pid_buf: [16]u8 = undefined;
512. const pid_s = std.fmt.bufPrint(&pid_buf, "{d}", .{linux.getpid()}) catch "0";
513. _ = fio.writeAll(fd, pid_s);
514. fio.close(fd);
515. return;
516. }
517. // stale lock takeover
518. if (self.stale_lock_timeout_ms > 0) {
519. if (fio.mtimeMs(self.tmp, self.lock_path)) |mt| {
520. if (fio.nowMs() - mt > self.stale_lock_timeout_ms) {
521. _ = fio.unlink(self.tmp, self.lock_path);
522. continue;
523. }
524. }
525. }
526. if (retries > 480) return Error.LockTimeout; // ~12s at 25ms
527. fio.sleepMs(25);
528. retries += 1;
529. }
530. }
532. fn releaseLock(self: *Db) void {
533. _ = fio.unlink(self.tmp, self.lock_path);
534. }
536. // =====================================================================
537. // Meta (.meta.json)
538. // =====================================================================
540. fn appendFmt(self: *Db, list: *std.ArrayList(u8), comptime fmt: []const u8, args: anytype) Error!void {
541. const s = std.fmt.allocPrint(self.tmp, fmt, args) catch return Error.OutOfMemory;
542. defer self.tmp.free(s);
543. list.appendSlice(self.tmp, s) catch return Error.OutOfMemory;
544. }
546. fn loadMeta(self: *Db) void {
547. const content = fio.readAll(self.tmp, self.meta_path) orelse return;
548. defer self.tmp.free(content);
549. self.applyMetaJson(content);
550. }
552. fn applyMetaJson(self: *Db, content: []const u8) void {
553. const parsed = std.json.parseFromSlice(std.json.Value, self.tmp, content, .{}) catch return;
554. defer parsed.deinit();
555. if (parsed.value != .object) return;
556. const obj = parsed.value.object;
557. self.next_id = FIRST_ID;
558. self.deleted.clearRetainingCapacity();
559. self.version = 0;
560. if (obj.get("nextId")) |v| {
561. self.next_id = switch (v) {
562. .integer => |i| @floatFromInt(i),
563. .float => |f| f,
564. else => FIRST_ID,
565. };
566. }
567. if (obj.get("version")) |v| {
568. if (v == .integer) self.version = @intCast(@max(v.integer, 0));
569. }
570. if (obj.get("deleted")) |v| {
571. if (v == .array) {
572. for (v.array.items) |it| {
573. const off: u64 = switch (it) {
574. .integer => |i| @intCast(@max(i, 0)),
575. .float => |f| @intFromFloat(@max(f, 0)),
576. else => continue,
577. };
578. self.deleted.append(self.alloc, off) catch {};
579. }
580. }
581. }
582. }
584. /// Pick up other processes' persisted state (JS refresh()): adopt disk meta
585. /// when its version is newer; index appended tail records.
586. pub fn refresh(self: *Db) void {
587. if (fio.readAll(self.tmp, self.meta_path)) |content| {
588. defer self.tmp.free(content);
589. const parsed = std.json.parseFromSlice(std.json.Value, self.tmp, content, .{}) catch return;
590. defer parsed.deinit();
591. if (parsed.value == .object) {
592. var disk_version: u64 = 0;
593. if (parsed.value.object.get("version")) |v| {
594. if (v == .integer) disk_version = @intCast(@max(v.integer, 0));
595. }
596. if (disk_version > self.version) self.applyMetaJson(content);
597. }
598. }
599. // catchUp: index records another process appended
600. if (self.index_fields.items.len > 0) {
601. const size = fio.fileSize(self.tmp, self.data_path) orelse 0;
602. if (size > self.covered_bytes) self.catchUp(size);
603. }
604. }
606. fn persistMetaLocked(self: *Db) Error!void {
607. self.version += 1;
608. var out: std.ArrayList(u8) = .empty;
609. defer out.deinit(self.tmp);
610. try self.renderMeta(&out);
612. // atomic tmp + rename (JS: `${metaPath}.${pid}.tmp`)
613. var tmp_buf: [512]u8 = undefined;
614. const tmp_path = std.fmt.bufPrint(&tmp_buf, "{s}.{d}.tmp", .{ self.meta_path, linux.getpid() }) catch return Error.IoError;
615. const fd = fio.openTrunc(self.tmp, tmp_path) orelse return Error.IoError;
616. const ok = fio.writeAll(fd, out.items);
617. fio.close(fd);
618. if (!ok) return Error.IoError;
619. if (!fio.rename(self.tmp, tmp_path, self.meta_path)) return Error.IoError;
620. }
622. /// Does .meta.json already say, byte for byte, what this handle would write
623. /// at its current version? Then an open has nothing to persist (ticket #21).
624. fn metaOnDiskIsCurrent(self: *Db) bool {
625. const disk = fio.readAll(self.tmp, self.meta_path) orelse return false;
626. defer self.tmp.free(disk);
627. var out: std.ArrayList(u8) = .empty;
628. defer out.deinit(self.tmp);
629. self.renderMeta(&out) catch return false;
630. return std.mem.eql(u8, disk, out.items);
631. }
633. fn renderMeta(self: *Db, out: *std.ArrayList(u8)) Error!void {
634. try self.appendFmt(out, "{{", .{});
635. if (self.pk != null and self.pk_type == .number) {
636. var nbuf: [32]u8 = undefined;
637. try self.appendFmt(out, "\"nextId\":{s},", .{jsNumFmt(&nbuf, self.next_id)});
638. }
639. try self.appendFmt(out, "\"deleted\":[", .{});
640. for (self.deleted.items, 0..) |off, i| {
641. if (i > 0) try self.appendFmt(out, ",", .{});
642. try self.appendFmt(out, "{d}", .{off});
643. }
644. try self.appendFmt(out, "],\"version\":{d}", .{self.version});
645. if (self.pk != null or self.index_fields.items.len > 0) {
646. try self.appendFmt(out, ",\"schema\":{{", .{});
647. var first = true;
648. if (self.pk) |p| {
649. const prefix: []const u8 = switch (self.pk_type) {
650. .number => "*",
651. .uuid => "@",
652. .string => "",
653. };
654. try self.appendFmt(out, "\"primaryKey\":\"{s}{s}\"", .{ prefix, p });
655. first = false;
656. }
657. if (!first) try self.appendFmt(out, ",", .{});
658. try self.appendFmt(out, "\"indexes\":[", .{});
659. var n: usize = 0;
660. for (self.index_fields.items) |field| {
661. if (self.pk != null and std.mem.eql(u8, field, self.pk.?)) continue;
662. if (n > 0) try self.appendFmt(out, ",", .{});
663. const uni = for (self.unique_fields.items) |u| {
664. if (std.mem.eql(u8, u, field)) break true;
665. } else false;
666. const numeric = (self.idx.get(field) orelse FieldIndex{ .typ = .lexical }).typ == .numeric;
667. try self.appendFmt(out, "\"{s}{s}{s}\"", .{
668. if (uni) "!" else "",
669. if (numeric) "*" else "",
670. field,
671. });
672. n += 1;
673. }
674. try self.appendFmt(out, "]}}", .{});
675. }
676. try self.appendFmt(out, "}}", .{});
677. }
679. // =====================================================================
680. // Data file scanning
681. // =====================================================================
683. fn deletedSet(self: *Db, alloc: std.mem.Allocator) std.AutoHashMapUnmanaged(u64, void) {
684. var set: std.AutoHashMapUnmanaged(u64, void) = .empty;
685. for (self.deleted.items) |off| set.put(alloc, off, {}) catch {};
686. return set;
687. }
689. /// Sequential scan yielding all live records. Caller supplies an arena for
690. /// decoded values. Used for rebuilds, compaction and unindexed finds.
691. pub const Scanner = struct {
692. db: *Db,
693. fd: i32 = -1,
694. offset: u64 = 0,
695. size: u64 = 0,
696. skip_deleted: bool = true,
697. deleted_set: std.AutoHashMapUnmanaged(u64, void) = .empty,
698. scratch: std.mem.Allocator,
700. pub fn init(db: *Db, scratch: std.mem.Allocator, skip_deleted: bool) Scanner {
701. var s = Scanner{ .db = db, .scratch = scratch, .skip_deleted = skip_deleted };
702. s.size = fio.fileSize(scratch, db.data_path) orelse 0;
703. if (s.size > 0) {
704. s.fd = fio.openRead(scratch, db.data_path) orelse -1;
705. }
706. if (skip_deleted) s.deleted_set = db.deletedSet(scratch);
707. return s;
708. }
710. pub fn deinit(self: *Scanner) void {
711. if (self.fd >= 0) fio.close(self.fd);
712. self.fd = -1;
713. }
715. /// Returns the next record (decoded into `arena`) or null at EOF.
716. pub fn next(self: *Scanner, arena: std.mem.Allocator) Error!?Record {
717. while (true) {
718. if (self.fd < 0 or self.offset + 4 > self.size) return null;
719. var hdr: [4]u8 = undefined;
720. const got = fio.pread(self.fd, &hdr, self.offset) orelse return Error.IoError;
721. if (got < 4) return null;
722. const rec_size = std.mem.readInt(u32, &hdr, .little);
723. if (rec_size <= 4) return Error.CorruptRecord;
724. if (self.offset + rec_size > self.size) return null; // incomplete tail
725. const off = self.offset;
726. self.offset += rec_size;
727. if (self.skip_deleted and self.deleted_set.contains(off)) continue;
728. const buf = arena.alloc(u8, rec_size - 4) catch return Error.OutOfMemory;
729. const got2 = fio.pread(self.fd, buf, off + 4) orelse return Error.IoError;
730. if (got2 < rec_size - 4) return Error.Truncated;
731. const d = msgpack.decode(arena, buf) catch return Error.CorruptRecord;
732. return Record{ .value = d.value, .off = off, .len = rec_size };
733. }
734. }
736. /// Like next() but without decoding — yields the raw framed bytes.
737. pub fn nextRaw(self: *Scanner, arena: std.mem.Allocator) Error!?struct { bytes: []u8, off: u64 } {
738. while (true) {
739. if (self.fd < 0 or self.offset + 4 > self.size) return null;
740. var hdr: [4]u8 = undefined;
741. const got = fio.pread(self.fd, &hdr, self.offset) orelse return Error.IoError;
742. if (got < 4) return null;
743. const rec_size = std.mem.readInt(u32, &hdr, .little);
744. if (rec_size <= 4) return Error.CorruptRecord;
745. if (self.offset + rec_size > self.size) return null;
746. const off = self.offset;
747. self.offset += rec_size;
748. if (self.skip_deleted and self.deleted_set.contains(off)) continue;
749. const buf = arena.alloc(u8, rec_size) catch return Error.OutOfMemory;
750. const got2 = fio.pread(self.fd, buf, off) orelse return Error.IoError;
751. if (got2 < rec_size) return Error.Truncated;
752. return .{ .bytes = buf, .off = off };
753. }
754. }
755. };
757. /// Read + decode one record by location.
758. pub fn readAt(self: *Db, arena: std.mem.Allocator, loc: Loc) Error!msgpack.Value {
759. const fd = fio.openRead(self.tmp, self.data_path) orelse return Error.IoError;
760. defer fio.close(fd);
761. if (loc.len <= 4) return Error.CorruptRecord;
762. const buf = arena.alloc(u8, loc.len - 4) catch return Error.OutOfMemory;
763. const got = fio.pread(fd, buf, loc.off + 4) orelse return Error.IoError;
764. if (got < buf.len) return Error.Truncated;
765. const d = msgpack.decode(arena, buf) catch return Error.CorruptRecord;
766. return d.value;
767. }
769. // =====================================================================
770. // Index management
771. // =====================================================================
773. fn keyFromValue(v: msgpack.Value) ?Key {
774. return switch (v) {
775. .number => |n| Key{ .num = n },
776. .str => |s| Key{ .str = s },
777. else => null, // null/bool/objects are not indexed (see header note)
778. };
779. }
781. /// Duplicate a key's string into the db arena so it outlives the op arena.
782. fn ownKey(self: *Db, k: Key) Error!Key {
783. return switch (k) {
784. .num => k,
785. .str => |s| Key{ .str = self.alloc.dupe(u8, s) catch return Error.OutOfMemory },
786. };
787. }
789. fn lowerBound(entries: []const Entry, key: Key) usize {
790. var lo: usize = 0;
791. var hi: usize = entries.len;
792. while (lo < hi) {
793. const mid = lo + (hi - lo) / 2;
794. if (cmpKeys(entries[mid].key, key) < 0) {
795. lo = mid + 1;
796. } else {
797. hi = mid;
798. }
799. }
800. return lo;
801. }
803. /// Binary search: all entries whose key equals `key` (appended to `out`).
804. pub fn indexGet(self: *Db, field: []const u8, key: Key, alloc: std.mem.Allocator, out: *std.ArrayList(Entry)) Error!void {
805. const fi = self.idx.getPtr(field) orelse return Error.NoSuchIndex;
806. const parsed = self.parseKeyForField(field, key);
807. const entries = fi.entries.items;
808. var i = lowerBound(entries, parsed);
809. while (i < entries.len and cmpKeys(entries[i].key, parsed) == 0) : (i += 1) {
810. out.append(alloc, entries[i]) catch return Error.OutOfMemory;
811. }
812. }
814. /// Range scan [from, to] (either side optional), ascending.
815. pub fn indexRange(self: *Db, field: []const u8, from: ?Key, to: ?Key, alloc: std.mem.Allocator, out: *std.ArrayList(Entry)) Error!void {
816. const fi = self.idx.getPtr(field) orelse return Error.NoSuchIndex;
817. const entries = fi.entries.items;
818. var i: usize = if (from) |f| lowerBound(entries, self.parseKeyForField(field, f)) else 0;
819. const to_key: ?Key = if (to) |t| self.parseKeyForField(field, t) else null;
820. while (i < entries.len) : (i += 1) {
821. if (to_key) |t| {
822. if (cmpKeys(entries[i].key, t) > 0) break;
823. }
824. out.append(alloc, entries[i]) catch return Error.OutOfMemory;
825. }
826. }
828. /// _parseKey semantics: numeric index fields parse string keys via
829. /// parseInt; on NaN the key stays a string.
830. fn parseKeyForField(self: *Db, field: []const u8, key: Key) Key {
831. const fi = self.idx.get(field) orelse return key;
832. if (fi.typ == .numeric and key == .str) {
833. if (jsParseInt(key.str)) |n| return Key{ .num = n };
834. }
835. return key;
836. }
838. fn indexInsertEntry(self: *Db, field: []const u8, key: Key, loc: Loc) Error!void {
839. const fi = self.idx.getPtr(field) orelse return;
840. const owned = try self.ownKey(key);
841. const e = Entry{ .key = owned, .off = loc.off, .len = loc.len };
842. // insert at sorted position
843. const pos = blk: {
844. var lo: usize = 0;
845. var hi: usize = fi.entries.items.len;
846. while (lo < hi) {
847. const mid = lo + (hi - lo) / 2;
848. if (entryLess({}, fi.entries.items[mid], e)) {
849. lo = mid + 1;
850. } else {
851. hi = mid;
852. }
853. }
854. break :blk lo;
855. };
856. fi.entries.insert(self.alloc, pos, e) catch return Error.OutOfMemory;
857. }
859. /// Index a record's fields at loc.
860. fn indexInsertRecord(self: *Db, rec: msgpack.Value, loc: Loc) Error!void {
861. for (self.index_fields.items) |field| {
862. const v = rec.get(field) orelse continue;
863. const key = keyFromValue(v) orelse continue;
864. try self.indexInsertEntry(field, key, loc);
865. }
866. self.covered_bytes = @max(self.covered_bytes, loc.off + loc.len);
867. }
869. /// Remove all entries pointing at offset `off` (all fields).
870. fn indexRemoveOffset(self: *Db, off: u64) void {
871. var it = self.idx.iterator();
872. while (it.next()) |kv| {
873. const list = &kv.value_ptr.entries;
874. var i: usize = 0;
875. while (i < list.items.len) {
876. if (list.items[i].off == off) {
877. _ = list.orderedRemove(i);
878. } else {
879. i += 1;
880. }
881. }
882. }
883. }
885. fn indexPath(self: *Db, buf: []u8, field: []const u8) []const u8 {
886. // <dir>/<base>.<field>.txt — derive from idxstate path (…/base.idxstate.json)
887. const prefix = self.idxstate_path[0 .. self.idxstate_path.len - "idxstate.json".len];
888. return std.fmt.bufPrint(buf, "{s}{s}.txt", .{ prefix, field }) catch buf[0..0];
889. }
891. fn initIndexes(self: *Db, force_rebuild: bool) Error!void {
892. var need_rebuild = force_rebuild;
893. const data_size = fio.fileSize(self.tmp, self.data_path) orelse 0;
895. if (!need_rebuild) {
896. for (self.index_fields.items) |field| {
897. var pbuf: [512]u8 = undefined;
898. const p = self.indexPath(&pbuf, field);
899. const isize_ = fio.fileSize(self.tmp, p);
900. if (isize_ == null or isize_.? == 0) {
901. if (data_size > 0) {
902. need_rebuild = true;
903. break;
904. }
905. }
906. }
907. }
908. if (!need_rebuild) {
909. // coveredBytes from idxstate — no state file means rebuild (JS)
910. if (self.readIdxState()) |cb| {
911. self.covered_bytes = cb;
912. } else {
913. need_rebuild = true;
914. }
915. }
917. if (need_rebuild) {
918. try self.rebuildIndexes();
919. return;
920. }
922. // Load index files into memory
923. for (self.index_fields.items) |field| {
924. var pbuf: [512]u8 = undefined;
925. const p = self.indexPath(&pbuf, field);
926. const content = fio.readAll(self.tmp, p) orelse continue;
927. defer self.tmp.free(content);
928. self.loadIndexLines(field, content);
929. }
930. // sort (files are sorted by JS localeCompare; re-sort under our order)
931. var it = self.idx.iterator();
932. while (it.next()) |kv| {
933. std.sort.pdq(Entry, kv.value_ptr.entries.items, {}, entryLess);
934. }
935. if (data_size > self.covered_bytes) self.catchUp(data_size);
936. }
938. fn loadIndexLines(self: *Db, field: []const u8, content: []const u8) void {
939. const fi = self.idx.getPtr(field) orelse return;
940. var lines = std.mem.splitScalar(u8, content, '\n');
941. while (lines.next()) |line| {
942. if (line.len == 0) continue;
943. // key = up to FIRST comma (JS split(',')[0]); then offset, length
944. const c1 = std.mem.indexOfScalar(u8, line, ',') orelse continue;
945. const rest = line[c1 + 1 ..];
946. const c2 = std.mem.indexOfScalar(u8, rest, ',') orelse continue;
947. const off_s = rest[0..c2];
948. const len_s = rest[c2 + 1 ..];
949. const off = std.fmt.parseInt(u64, off_s, 10) catch continue;
950. const len = std.fmt.parseInt(u64, len_s, 10) catch continue;
951. if (len == 0) continue;
952. const key_s = line[0..c1];
953. var key: Key = undefined;
954. if (fi.typ == .numeric) {
955. key = if (jsParseInt(key_s)) |n| Key{ .num = n } else Key{ .str = self.alloc.dupe(u8, key_s) catch return };
956. } else {
957. key = Key{ .str = self.alloc.dupe(u8, key_s) catch return };
958. }
959. fi.entries.append(self.alloc, .{ .key = key, .off = off, .len = len }) catch return;
960. }
961. }
963. fn rebuildIndexes(self: *Db) Error!void {
964. var it0 = self.idx.iterator();
965. while (it0.next()) |kv| kv.value_ptr.entries.clearRetainingCapacity();
967. var scratch_state = std.heap.ArenaAllocator.init(self.arena_state.child_allocator);
968. defer scratch_state.deinit();
969. const scratch = scratch_state.allocator();
971. // JS rebuild scans ALL records (no tombstone filter — deleted offsets
972. // simply get filtered at read time). Mirror that.
973. var scanner = Scanner.init(self, scratch, false);
974. defer scanner.deinit();
975. while (try scanner.next(scratch)) |rec| {
976. try self.indexInsertRecord(rec.value, .{ .off = rec.off, .len = rec.len });
977. }
978. self.covered_bytes = fio.fileSize(self.tmp, self.data_path) orelse 0;
979. try self.writeIndexFiles();
980. try self.writeIdxState();
981. }
983. fn catchUp(self: *Db, file_size: u64) void {
984. var scratch_state = std.heap.ArenaAllocator.init(self.arena_state.child_allocator);
985. defer scratch_state.deinit();
986. const scratch = scratch_state.allocator();
988. var scanner = Scanner.init(self, scratch, false);
989. defer scanner.deinit();
990. scanner.offset = self.covered_bytes;
991. while (true) {
992. const maybe = scanner.next(scratch) catch break;
993. const rec = maybe orelse break;
994. self.indexInsertRecord(rec.value, .{ .off = rec.off, .len = rec.len }) catch break;
995. }
996. self.covered_bytes = @max(self.covered_bytes, file_size);
997. }
999. /// A file that already holds these exact bytes is left alone — its mtime
1000. /// included — so closing a handle that changed nothing writes nothing.
1001. fn fileHolds(alloc: std.mem.Allocator, path: []const u8, bytes: []const u8) bool {
1002. const disk = fio.readAll(alloc, path) orelse return false;
1003. defer alloc.free(disk);
1004. return std.mem.eql(u8, disk, bytes);
1005. }
1007. fn writeIndexFiles(self: *Db) Error!void {
1008. for (self.index_fields.items) |field| {
1009. const fi = self.idx.getPtr(field) orelse continue;
1010. var out: std.ArrayList(u8) = .empty;
1011. defer out.deinit(self.tmp);
1012. for (fi.entries.items) |e| {
1013. var kbuf: [32]u8 = undefined;
1014. const ks = switch (e.key) {
1015. .num => |n| jsNumFmt(&kbuf, n),
1016. .str => |s| s,
1017. };
1018. try self.appendFmt(&out, "{s},{d},{d}\n", .{ ks, e.off, e.len });
1019. }
1020. var pbuf: [512]u8 = undefined;
1021. const p = self.indexPath(&pbuf, field);
1022. if (fileHolds(self.tmp, p, out.items)) continue;
1023. var tbuf: [512]u8 = undefined;
1024. const tmp = std.fmt.bufPrint(&tbuf, "{s}.{d}.tmp", .{ p, linux.getpid() }) catch return Error.IoError;
1025. const fd = fio.openTrunc(self.tmp, tmp) orelse return Error.IoError;
1026. const ok = fio.writeAll(fd, out.items);
1027. fio.close(fd);
1028. if (!ok) return Error.IoError;
1029. if (!fio.rename(self.tmp, tmp, p)) return Error.IoError;
1030. }
1031. }
1033. fn readIdxState(self: *Db) ?u64 {
1034. const content = fio.readAll(self.tmp, self.idxstate_path) orelse return null;
1035. defer self.tmp.free(content);
1036. const parsed = std.json.parseFromSlice(std.json.Value, self.tmp, content, .{}) catch return null;
1037. defer parsed.deinit();
1038. if (parsed.value != .object) return null;
1039. const v = parsed.value.object.get("coveredBytes") orelse return null;
1040. return switch (v) {
1041. .integer => |i| @intCast(@max(i, 0)),
1042. .float => |f| @intFromFloat(@max(f, 0)),
1043. else => null,
1044. };
1045. }
1047. fn writeIdxState(self: *Db) Error!void {
1048. var buf: [128]u8 = undefined;
1049. const json = std.fmt.bufPrint(&buf, "{{\"coveredBytes\":{d}}}", .{self.covered_bytes}) catch return Error.IoError;
1050. if (fileHolds(self.tmp, self.idxstate_path, json)) return;
1051. var tbuf: [512]u8 = undefined;
1052. const tmp = std.fmt.bufPrint(&tbuf, "{s}.{d}.tmp", .{ self.idxstate_path, linux.getpid() }) catch return Error.IoError;
1053. const fd = fio.openTrunc(self.tmp, tmp) orelse return Error.IoError;
1054. const ok = fio.writeAll(fd, json);
1055. fio.close(fd);
1056. if (!ok) return Error.IoError;
1057. if (!fio.rename(self.tmp, tmp, self.idxstate_path)) return Error.IoError;
1058. }
1060. fn persistIndexesLocked(self: *Db) Error!void {
1061. try self.writeIndexFiles();
1062. // coverage can only grow (JS: max of ours and on-disk state)
1063. if (self.readIdxState()) |cb| self.covered_bytes = @max(self.covered_bytes, cb);
1064. try self.writeIdxState();
1065. }
1067. // =====================================================================
1068. // Operations
1069. // =====================================================================
1071. /// A mutable record under construction (op-arena entries).
1072. pub const MutableRecord = struct {
1073. entries: std.ArrayList(msgpack.Entry) = .empty,
1075. pub fn get(self: *const MutableRecord, key: []const u8) ?msgpack.Value {
1076. for (self.entries.items) |e| {
1077. if (e.key == .str and std.mem.eql(u8, e.key.str, key)) return e.value;
1078. }
1079. return null;
1080. }
1082. pub fn set(self: *MutableRecord, alloc: std.mem.Allocator, key: []const u8, v: msgpack.Value) Error!void {
1083. for (self.entries.items) |*e| {
1084. if (e.key == .str and std.mem.eql(u8, e.key.str, key)) {
1085. e.value = v;
1086. return;
1087. }
1088. }
1089. self.entries.append(alloc, .{ .key = .{ .str = key }, .value = v }) catch return Error.OutOfMemory;
1090. }
1092. pub fn toValue(self: *const MutableRecord) msgpack.Value {
1093. return .{ .map = self.entries.items };
1094. }
1095. };
1097. pub const InsertResult = union(enum) {
1098. pk_num: f64,
1099. pk_str: []const u8, // op-arena
1100. record: msgpack.Value,
1101. };
1103. /// insert() — auto primary key, unique checks, append, index.
1104. /// `record` must be a map value; `arena` is the op arena (record memory).
1105. pub fn insert(self: *Db, arena: std.mem.Allocator, record: msgpack.Value, skip_pk: bool) Error!InsertResult {
1106. if (record != .map) return Error.CorruptRecord;
1107. try self.acquireLock();
1108. defer self.releaseLock();
1109. self.refresh();
1110. return self.insertLocked(arena, record, skip_pk);
1111. }
1113. fn insertLocked(self: *Db, arena: std.mem.Allocator, record: msgpack.Value, skip_pk: bool) Error!InsertResult {
1114. // copy into mutable form
1115. var rec = MutableRecord{};
1116. for (record.map) |e| rec.entries.append(arena, e) catch return Error.OutOfMemory;
1118. // _hasPrimaryKeyValue: present and not null/undefined
1119. var has_pk_value = false;
1120. if (self.pk) |p| {
1121. if (rec.get(p)) |v| {
1122. has_pk_value = (v != .nil and v != .undef);
1123. }
1124. }
1126. var auto_gen = false;
1127. if (self.pk != null and !skip_pk and !has_pk_value) {
1128. if (self.pk_type == .number) {
1129. rec.set(arena, self.pk.?, .{ .number = self.next_id }) catch return Error.OutOfMemory;
1130. self.next_id += 1;
1131. auto_gen = true;
1132. } else if (self.pk_type == .uuid) {
1133. // A GENERATED ID IS CHECKED, NOT TRUSTED (ticket #113): the format is a
1134. // millisecond stamp + 3 base36 chars, so a bulk put collides within a
1135. // millisecond (two of 7800 in a measured run) and the unique check below
1136. // skips generated keys — two live rows then shared one pk and the next
1137. // update() deleted both and re-inserted one. Draw again until it is free.
1138. var ubuf: [12]u8 = undefined;
1139. var tries: usize = 0;
1140. while (true) : (tries += 1) {
1141. const u = genUuid(&ubuf);
1142. var taken: std.ArrayList(Loc) = .empty;
1143. defer taken.deinit(arena);
1144. try self.findPkLocs(arena, .{ .str = u }, &taken);
1145. if (taken.items.len == 0 or tries >= 64) break;
1146. }
1147. const owned = arena.dupe(u8, ubuf[0..12]) catch return Error.OutOfMemory;
1148. rec.set(arena, self.pk.?, .{ .str = owned }) catch return Error.OutOfMemory;
1149. auto_gen = true;
1150. }
1151. }
1153. // unique constraints (skip auto-generated pk)
1154. if (self.index_fields.items.len > 0 and self.unique_fields.items.len > 0) {
1155. var del_set = self.deletedSet(arena);
1156. for (self.unique_fields.items) |field| {
1157. if (auto_gen and self.pk != null and std.mem.eql(u8, field, self.pk.?)) continue;
1158. const v = rec.get(field) orelse continue;
1159. if (v == .undef) continue;
1160. const key = keyFromValue(v) orelse continue;
1161. var hits: std.ArrayList(Entry) = .empty;
1162. defer hits.deinit(arena);
1163. try self.indexGet(field, key, arena, &hits);
1164. var live: usize = 0;
1165. for (hits.items) |h| {
1166. if (!del_set.contains(h.off)) live += 1;
1167. }
1168. if (live > 0) {
1169. var kbuf: [32]u8 = undefined;
1170. const ks = switch (key) {
1171. .num => |n| jsNumFmt(&kbuf, n),
1172. .str => |s| s,
1173. };
1174. self.last_error = std.fmt.bufPrint(&self.last_error_buf, "Duplicate key: {s}={s}", .{ field, ks }) catch "Duplicate key";
1175. return Error.DuplicateKey;
1176. }
1177. }
1178. }
1180. // serialize + append
1181. const framed = msgpack.serialize(arena, rec.toValue()) catch return Error.OutOfMemory;
1182. const offset = fio.fileSize(self.tmp, self.data_path) orelse 0;
1183. const fd = fio.openAppend(self.tmp, self.data_path) orelse return Error.IoError;
1184. const ok = fio.writeAll(fd, framed);
1185. fio.close(fd);
1186. if (!ok) return Error.IoError;
1188. const loc = Loc{ .off = offset, .len = framed.len };
1189. if (self.index_fields.items.len > 0) {
1190. try self.indexInsertRecord(rec.toValue(), loc);
1191. }
1193. if (self.pk != null and self.pk_type == .number and !skip_pk and !has_pk_value) {
1194. try self.persistMetaLocked();
1195. }
1197. if (self.pk) |p| {
1198. const v = rec.get(p) orelse return Error.CorruptRecord;
1199. return switch (v) {
1200. .number => |n| InsertResult{ .pk_num = n },
1201. .str => |s| InsertResult{ .pk_str = s },
1202. else => InsertResult{ .record = rec.toValue() },
1203. };
1204. }
1205. return InsertResult{ .record = rec.toValue() };
1206. }
1208. /// Locate live records by primary key (index-backed when available).
1209. pub fn findPkLocs(self: *Db, arena: std.mem.Allocator, key: Key, out: *std.ArrayList(Loc)) Error!void {
1210. if (self.pk == null) return Error.NoPrimaryKey;
1211. self.refresh();
1212. var del_set = self.deletedSet(arena);
1213. if (self.index_fields.items.len > 0) {
1214. var hits: std.ArrayList(Entry) = .empty;
1215. defer hits.deinit(arena);
1216. try self.indexGet(self.pk.?, key, arena, &hits);
1217. for (hits.items) |h| {
1218. if (del_set.contains(h.off)) continue;
1219. out.append(arena, .{ .off = h.off, .len = h.len }) catch return Error.OutOfMemory;
1220. }
1221. return;
1222. }
1223. // no indexes: full scan comparing the pk field
1224. var scanner = Scanner.init(self, arena, true);
1225. defer scanner.deinit();
1226. while (try scanner.next(arena)) |rec| {
1227. const v = rec.value.get(self.pk.?) orelse continue;
1228. const k = keyFromValue(v) orelse continue;
1229. if (cmpKeys(self.parseKeyForField(self.pk.?, k), self.parseKeyForField(self.pk.?, key)) == 0) {
1230. out.append(arena, .{ .off = rec.off, .len = rec.len }) catch return Error.OutOfMemory;
1231. }
1232. }
1233. }
1235. /// Locate live records by secondary index equality.
1236. pub fn findIndexLocs(self: *Db, arena: std.mem.Allocator, field: []const u8, key: Key, out: *std.ArrayList(Loc)) Error!void {
1237. self.refresh();
1238. var del_set = self.deletedSet(arena);
1239. var hits: std.ArrayList(Entry) = .empty;
1240. defer hits.deinit(arena);
1241. try self.indexGet(field, key, arena, &hits);
1242. for (hits.items) |h| {
1243. if (del_set.contains(h.off)) continue;
1244. out.append(arena, .{ .off = h.off, .len = h.len }) catch return Error.OutOfMemory;
1245. }
1246. }
1248. /// Locate live records by index range [from, to] ascending.
1249. pub fn findRangeLocs(self: *Db, arena: std.mem.Allocator, field: []const u8, from: ?Key, to: ?Key, out: *std.ArrayList(Loc)) Error!void {
1250. self.refresh();
1251. var del_set = self.deletedSet(arena);
1252. var hits: std.ArrayList(Entry) = .empty;
1253. defer hits.deinit(arena);
1254. try self.indexRange(field, from, to, arena, &hits);
1255. var seen: std.AutoHashMapUnmanaged(u64, void) = .empty;
1256. for (hits.items) |h| {
1257. if (del_set.contains(h.off)) continue;
1258. if (seen.contains(h.off)) continue;
1259. seen.put(arena, h.off, {}) catch {};
1260. out.append(arena, .{ .off = h.off, .len = h.len }) catch return Error.OutOfMemory;
1261. }
1262. }
1264. /// delete by primary key. Returns the deleted records (decoded into arena).
1265. pub fn deletePk(self: *Db, arena: std.mem.Allocator, key: Key, out_records: *std.ArrayList(msgpack.Value)) Error!usize {
1266. try self.acquireLock();
1267. defer self.releaseLock();
1268. self.refresh();
1269. return self.deletePkLocked(arena, key, out_records);
1270. }
1272. fn deletePkLocked(self: *Db, arena: std.mem.Allocator, key: Key, out_records: *std.ArrayList(msgpack.Value)) Error!usize {
1273. var locs: std.ArrayList(Loc) = .empty;
1274. defer locs.deinit(arena);
1275. try self.findPkLocs(arena, key, &locs);
1276. for (locs.items) |loc| {
1277. const rec = try self.readAt(arena, loc);
1278. out_records.append(arena, rec) catch return Error.OutOfMemory;
1279. self.deleted.append(self.alloc, loc.off) catch return Error.OutOfMemory;
1280. self.indexRemoveOffset(loc.off);
1281. }
1282. try self.persistMetaLocked();
1283. return locs.items.len;
1284. }
1286. /// update by primary key: delete + insert(skip_pk) — JS update() semantics.
1287. /// `new_record` must already carry the primary key (the JS callback
1288. /// contract: the record keeps its pk unless the caller removes it).
1289. pub fn updatePk(self: *Db, arena: std.mem.Allocator, key: Key, new_record: msgpack.Value) Error!usize {
1290. if (self.pk) |p| {
1291. const carried = for (new_record.map) |e| {
1292. if (e.key == .str and std.mem.eql(u8, e.key.str, p)) break e.value != .nil and e.value != .undef;
1293. } else false;
1294. if (!carried) return Error.RecordLacksPrimaryKey;
1295. }
1296. try self.acquireLock();
1297. defer self.releaseLock();
1298. self.refresh();
1299. var old: std.ArrayList(msgpack.Value) = .empty;
1300. defer old.deinit(arena);
1301. const n = try self.deletePkLocked(arena, key, &old);
1302. if (n == 0) return 0;
1303. var i: usize = 0;
1304. while (i < n) : (i += 1) {
1305. _ = try self.insertLocked(arena, new_record, true);
1306. }
1307. return n;
1308. }
1310. /// compact() — rewrite the data file without tombstones, rebuild indexes.
1311. pub fn compact(self: *Db) Error!void {
1312. try self.acquireLock();
1313. defer self.releaseLock();
1314. self.refresh();
1315. try self.compactLocked();
1316. if (self.index_fields.items.len > 0) {
1317. try self.rebuildIndexes();
1318. }
1319. }
1321. fn compactLocked(self: *Db) Error!void {
1322. var scratch_state = std.heap.ArenaAllocator.init(self.arena_state.child_allocator);
1323. defer scratch_state.deinit();
1324. const scratch = scratch_state.allocator();
1326. var tbuf: [512]u8 = undefined;
1327. const tmp = std.fmt.bufPrint(&tbuf, "{s}.tmp", .{self.data_path}) catch return Error.IoError;
1328. const out_fd = fio.openTrunc(self.tmp, tmp) orelse return Error.IoError;
1330. var scanner = Scanner.init(self, scratch, true);
1331. var ok = true;
1332. while (true) {
1333. const maybe = scanner.nextRaw(scratch) catch {
1334. ok = false;
1335. break;
1336. };
1337. const raw = maybe orelse break;
1338. if (!fio.writeAll(out_fd, raw.bytes)) {
1339. ok = false;
1340. break;
1341. }
1342. }
1343. scanner.deinit();
1344. fio.close(out_fd);
1345. if (!ok) return Error.IoError;
1346. if (!fio.rename(self.tmp, tmp, self.data_path)) return Error.IoError;
1348. self.deleted.clearRetainingCapacity();
1349. try self.persistMetaLocked();
1350. }
1352. pub fn persistNow(self: *Db) Error!void {
1353. try self.acquireLock();
1354. defer self.releaseLock();
1355. self.refresh();
1356. try self.persistIndexesLocked();
1357. }
1358. };
1360. // =========================================================================
1361. // Tests
1362. // =========================================================================
1364. const testing = std.testing;
1366. fn tmpBase(buf: []u8, comptime name: []const u8) []const u8 {
1367. return std.fmt.bufPrint(buf, "/tmp/mpackdb-zigtest-{d}-" ++ name, .{linux.getpid()}) catch unreachable;
1368. }
1370. fn cleanup(alloc: std.mem.Allocator, base: []const u8) void {
1371. var buf: [512]u8 = undefined;
1372. const suffixes = [_][]const u8{ ".mpack", ".meta.json", ".idxstate.json", ".lock", ".id.txt", ".email.txt", ".age.txt", ".uuid.txt" };
1373. for (suffixes) |suffix| {
1374. const p = std.fmt.bufPrint(&buf, "{s}{s}", .{ base, suffix }) catch continue;
1375. _ = fio.unlink(alloc, p);
1376. }
1377. }
1379. fn strKey(s: []const u8) Key {
1380. return .{ .str = s };
1381. }
1383. fn numKey(n: f64) Key {
1384. return .{ .num = n };
1385. }
1387. fn makeUser(arena: std.mem.Allocator, name: []const u8, email: []const u8, age: f64) !msgpack.Value {
1388. const entries = try arena.alloc(msgpack.Entry, 3);
1389. entries[0] = .{ .key = .{ .str = "name" }, .value = .{ .str = name } };
1390. entries[1] = .{ .key = .{ .str = "email" }, .value = .{ .str = email } };
1391. entries[2] = .{ .key = .{ .str = "age" }, .value = .{ .number = age } };
1392. return .{ .map = entries };
1393. }
1395. test "engine: insert/find/delete/update with numeric pk + indexes" {
1396. var base_buf: [128]u8 = undefined;
1397. const base = tmpBase(&base_buf, "crud");
1398. cleanup(testing.allocator, base);
1399. defer cleanup(testing.allocator, base);
1401. var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
1402. defer arena_state.deinit();
1403. const arena = arena_state.allocator();
1405. const db = try Db.open(testing.allocator, base, .{
1406. .primary_key = "*id",
1407. .indexes = &.{ "email", "*age" },
1408. });
1410. // insert three — a fresh table counts from 1 (ticket #4)
1411. const r1 = try db.insert(arena, try makeUser(arena, "Alice", "[email protected]", 30), false);
1412. try testing.expectEqual(@as(f64, 1), r1.pk_num);
1413. const r2 = try db.insert(arena, try makeUser(arena, "Bob", "[email protected]", 25), false);
1414. try testing.expectEqual(@as(f64, 2), r2.pk_num);
1415. _ = try db.insert(arena, try makeUser(arena, "Carol", "[email protected]", 35), false);
1417. // find by pk
1418. var locs: std.ArrayList(Loc) = .empty;
1419. try db.findPkLocs(arena, numKey(2), &locs);
1420. try testing.expectEqual(@as(usize, 1), locs.items.len);
1421. const bob = try db.readAt(arena, locs.items[0]);
1422. try testing.expectEqualStrings("Bob", bob.get("name").?.str);
1424. // find by secondary index
1425. var locs2: std.ArrayList(Loc) = .empty;
1426. try db.findIndexLocs(arena, "email", strKey("[email protected]"), &locs2);
1427. try testing.expectEqual(@as(usize, 1), locs2.items.len);
1429. // range on numeric index: age 26..40 → Alice(30), Carol(35)
1430. var locs3: std.ArrayList(Loc) = .empty;
1431. try db.findRangeLocs(arena, "age", numKey(26), numKey(40), &locs3);
1432. try testing.expectEqual(@as(usize, 2), locs3.items.len);
1434. // update Bob's age
1435. var bob_new = Db.MutableRecord{};
1436. for (bob.map) |e| try bob_new.entries.append(arena, e);
1437. try bob_new.set(arena, "age", .{ .number = 26 });
1438. const updated = try db.updatePk(arena, numKey(2), bob_new.toValue());
1439. try testing.expectEqual(@as(usize, 1), updated);
1441. var locs4: std.ArrayList(Loc) = .empty;
1442. try db.findPkLocs(arena, numKey(2), &locs4);
1443. try testing.expectEqual(@as(usize, 1), locs4.items.len);
1444. const bob2 = try db.readAt(arena, locs4.items[0]);
1445. try testing.expectEqual(@as(f64, 26), bob2.get("age").?.number);
1447. // delete Alice
1448. var deleted_recs: std.ArrayList(msgpack.Value) = .empty;
1449. const dn = try db.deletePk(arena, numKey(1), &deleted_recs);
1450. try testing.expectEqual(@as(usize, 1), dn);
1451. try testing.expectEqualStrings("Alice", deleted_recs.items[0].get("name").?.str);
1453. var locs5: std.ArrayList(Loc) = .empty;
1454. try db.findPkLocs(arena, numKey(1), &locs5);
1455. try testing.expectEqual(@as(usize, 0), locs5.items.len);
1457. db.close();
1459. // reopen (compaction drops the tombstone) and verify persistence
1460. const db2 = try Db.open(testing.allocator, base, .{
1461. .primary_key = "*id",
1462. .indexes = &.{ "email", "*age" },
1463. });
1464. defer db2.close();
1465. var locs6: std.ArrayList(Loc) = .empty;
1466. try db2.findPkLocs(arena, numKey(2), &locs6);
1467. try testing.expectEqual(@as(usize, 1), locs6.items.len);
1468. const bob3 = try db2.readAt(arena, locs6.items[0]);
1469. try testing.expectEqual(@as(f64, 26), bob3.get("age").?.number);
1470. // nextId continues after reopen
1471. const r4 = try db2.insert(arena, try makeUser(arena, "Dan", "[email protected]", 40), false);
1472. try testing.expectEqual(@as(f64, 4), r4.pk_num);
1473. }
1475. test "engine: a stored counter wins over FIRST_ID (ticket #4)" {
1476. var base_buf: [128]u8 = undefined;
1477. const base = tmpBase(&base_buf, "counter");
1478. cleanup(testing.allocator, base);
1479. defer cleanup(testing.allocator, base);
1481. var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
1482. defer arena_state.deinit();
1483. const arena = arena_state.allocator();
1485. // A table the JS mpackdb created and never wrote to: its meta says 0, and
1486. // an existing table keeps its own counter.
1487. var mbuf: [160]u8 = undefined;
1488. const meta_path = try std.fmt.bufPrint(&mbuf, "{s}.meta.json", .{base});
1489. const fd = fio.openTrunc(testing.allocator, meta_path).?;
1490. try testing.expect(fio.writeAll(fd, "{\"nextId\":0,\"deleted\":[],\"version\":1}"));
1491. fio.close(fd);
1493. const db = try Db.open(testing.allocator, base, .{ .primary_key = "*id" });
1494. defer db.close();
1495. const r = try db.insert(arena, try makeUser(arena, "Alice", "[email protected]", 1), false);
1496. try testing.expectEqual(@as(f64, 0), r.pk_num);
1497. }
1499. test "engine: unique index rejects duplicates" {
1500. var base_buf: [128]u8 = undefined;
1501. const base = tmpBase(&base_buf, "uniq");
1502. cleanup(testing.allocator, base);
1503. defer cleanup(testing.allocator, base);
1505. var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
1506. defer arena_state.deinit();
1507. const arena = arena_state.allocator();
1509. const db = try Db.open(testing.allocator, base, .{
1510. .primary_key = "*id",
1511. .indexes = &.{"!email"},
1512. });
1513. defer db.close();
1515. _ = try db.insert(arena, try makeUser(arena, "Alice", "[email protected]", 30), false);
1516. const dup = db.insert(arena, try makeUser(arena, "Evil", "[email protected]", 31), false);
1517. try testing.expectError(Error.DuplicateKey, dup);
1518. try testing.expect(std.mem.indexOf(u8, db.last_error, "[email protected]") != null);
1519. }
1521. test "engine: uuid primary key" {
1522. var base_buf: [128]u8 = undefined;
1523. const base = tmpBase(&base_buf, "uuid");
1524. cleanup(testing.allocator, base);
1525. defer cleanup(testing.allocator, base);
1527. var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
1528. defer arena_state.deinit();
1529. const arena = arena_state.allocator();
1531. const db = try Db.open(testing.allocator, base, .{ .primary_key = "@uuid" });
1532. defer db.close();
1534. const r = try db.insert(arena, try makeUser(arena, "Alice", "[email protected]", 1), false);
1535. try testing.expectEqual(@as(usize, 12), r.pk_str.len);
1536. var locs: std.ArrayList(Loc) = .empty;
1537. try db.findPkLocs(arena, strKey(r.pk_str), &locs);
1538. try testing.expectEqual(@as(usize, 1), locs.items.len);
1539. }
1541. test "jsNumFmt / jsParseInt / uuid shape" {
1542. var buf: [32]u8 = undefined;
1543. try testing.expectEqualStrings("42", jsNumFmt(&buf, 42));
1544. try testing.expectEqualStrings("-7", jsNumFmt(&buf, -7));
1545. try testing.expectEqualStrings("1.5", jsNumFmt(&buf, 1.5));
1546. try testing.expectEqual(@as(f64, 1), jsParseInt("1.5").?);
1547. try testing.expectEqual(@as(f64, -12), jsParseInt("-12abc").?);
1548. try testing.expect(jsParseInt("abc") == null);
1549. var ubuf: [12]u8 = undefined;
1550. const u = genUuid(&ubuf);
1551. try testing.expectEqual(@as(usize, 12), u.len);
1552. for (u) |ch| try testing.expect((ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'z'));
1553. }