Fixes #97: SQLite is now async, optimized, tests

deno.jsonc

@@ -18,7 +18,7 @@
   },
   "test": {
     "files": {
-      "exclude": ["plugos/forked"]
+      "exclude": ["plugos/forked", "plugos/sqlite/deno-sqlite"]
     }
   },
   "fmt": {

plugos/environments/worker_bundle.json

@@ -1,3 +1,3 @@
 {
-  "worker.js": "data:application/javascript;base64,KCgpID0+IHsgdmFyIG1vZD0oKCk9PntmdW5jdGlvbiBjKHIpe3IoKS5jYXRjaChlPT57Y29uc29sZS5lcnJvcigiQ2F1Z2h0IGVycm9yIixlLm1lc3NhZ2UpfSl9dmFyIGE9Y2xhc3N7Y29uc3RydWN0b3IoZSxuPSEwKXt0aGlzLnByaW50PW4sdGhpcy5jYWxsYmFjaz1lfWxvZyguLi5lKXt0aGlzLnB1c2goImxvZyIsZSl9d2FybiguLi5lKXt0aGlzLnB1c2goIndhcm4iLGUpfWVycm9yKC4uLmUpe3RoaXMucHVzaCgiZXJyb3IiLGUpfWluZm8oLi4uZSl7dGhpcy5wdXNoKCJpbmZvIixlKX1wdXNoKGUsbil7dGhpcy5jYWxsYmFjayhlLHRoaXMubG9nTWVzc2FnZShuKSksdGhpcy5wcmludCYmY29uc29sZVtlXSguLi5uKX1sb2dNZXNzYWdlKGUpe2xldCBuPVtdO2ZvcihsZXQgdCBvZiBlKXN3aXRjaCh0eXBlb2YgdCl7Y2FzZSJzdHJpbmciOmNhc2UibnVtYmVyIjpuLnB1c2goIiIrdCk7YnJlYWs7Y2FzZSJ1bmRlZmluZWQiOm4ucHVzaCgidW5kZWZpbmVkIik7YnJlYWs7ZGVmYXVsdDp0cnl7bGV0IG89SlNPTi5zdHJpbmdpZnkodCxudWxsLDIpO28ubGVuZ3RoPjUwMCYmKG89by5zdWJzdHJpbmcoMCw1MDApKyIuLi4iKSxuLnB1c2gobyl9Y2F0Y2h7bi5wdXNoKCJbY2lyY3VsYXIgb2JqZWN0XSIpfX1yZXR1cm4gbi5qb2luKCIgIil9fTt0eXBlb2YgRGVubz4idSImJihzZWxmLkRlbm89e2FyZ3M6W10sYnVpbGQ6e2FyY2g6Ing4Nl82NCJ9LGVudjp7Z2V0KCl7fX19KTt2YXIgZD1uZXcgTWFwLGk9bmV3IE1hcDtmdW5jdGlvbiBzKHIpe3R5cGVvZiB3aW5kb3c8InUiJiZ3aW5kb3cucGFyZW50IT09d2luZG93P3dpbmRvdy5wYXJlbnQucG9zdE1lc3NhZ2UociwiKiIpOnNlbGYucG9zdE1lc3NhZ2Uocil9dmFyIGw9MDtzZWxmLnN5c2NhbGw9YXN5bmMociwuLi5lKT0+YXdhaXQgbmV3IFByb21pc2UoKG4sdCk9PntsKyssaS5zZXQobCx7cmVzb2x2ZTpuLHJlamVjdDp0fSkscyh7dHlwZToic3lzY2FsbCIsaWQ6bCxuYW1lOnIsYXJnczplfSl9KTt2YXIgdT1uZXcgTWFwO3NlbGYucmVxdWlyZT1yPT57bGV0IGU9dS5nZXQocik7aWYoIWUpdGhyb3cgbmV3IEVycm9yKGBEeW5hbWljYWxseSBpbXBvcnRpbmcgbm9uLXByZWxvYWRlZCBsaWJyYXJ5ICR7cn1gKTtyZXR1cm4gZX07c2VsZi5jb25zb2xlPW5ldyBhKChyLGUpPT57cyh7dHlwZToibG9nIixsZXZlbDpyLG1lc3NhZ2U6ZX0pfSwhMSk7ZnVuY3Rpb24gZyhyKXtyZXR1cm5gcmV0dXJuICgke3J9KVsiZGVmYXVsdCJdYH1zZWxmLmFkZEV2ZW50TGlzdGVuZXIoIm1lc3NhZ2UiLHI9PntjKGFzeW5jKCk9PntsZXQgZT1yLmRhdGE7c3dpdGNoKGUudHlwZSl7Y2FzZSJsb2FkIjp7bGV0IG49bmV3IEZ1bmN0aW9uKGcoZS5jb2RlKSk7ZC5zZXQoZS5uYW1lLG4oKSkscyh7dHlwZToiaW5pdGVkIixuYW1lOmUubmFtZX0pfWJyZWFrO2Nhc2UibG9hZC1kZXBlbmRlbmN5Ijp7bGV0IHQ9bmV3IEZ1bmN0aW9uKGByZXR1cm4gJHtlLmNvZGV9YCkoKTt1LnNldChlLm5hbWUsdCkscyh7dHlwZToiZGVwZW5kZW5jeS1pbml0ZWQiLG5hbWU6ZS5uYW1lfSl9YnJlYWs7Y2FzZSJpbnZva2UiOntsZXQgbj1kLmdldChlLm5hbWUpO2lmKCFuKXRocm93IG5ldyBFcnJvcihgRnVuY3Rpb24gbm90IGxvYWRlZDogJHtlLm5hbWV9YCk7dHJ5e2xldCB0PWF3YWl0IFByb21pc2UucmVzb2x2ZShuKC4uLmUuYXJnc3x8W10pKTtzKHt0eXBlOiJyZXN1bHQiLGlkOmUuaWQscmVzdWx0OnR9KX1jYXRjaCh0KXtzKHt0eXBlOiJyZXN1bHQiLGlkOmUuaWQsZXJyb3I6dC5tZXNzYWdlLHN0YWNrOnQuc3RhY2t9KX19YnJlYWs7Y2FzZSJzeXNjYWxsLXJlc3BvbnNlIjp7bGV0IG49ZS5pZCx0PWkuZ2V0KG4pO2lmKCF0KXRocm93IGNvbnNvbGUubG9nKCJDdXJyZW50IG91dHN0YW5kaW5nIHJlcXVlc3RzIixpLCJsb29raW5nIHVwIixuKSxFcnJvcigiSW52YWxpZCByZXF1ZXN0IGlkIik7aS5kZWxldGUobiksZS5lcnJvcj90LnJlamVjdChuZXcgRXJyb3IoZS5lcnJvcikpOnQucmVzb2x2ZShlLnJlc3VsdCl9YnJlYWt9fSl9KTt9KSgpOwogcmV0dXJuIG1vZDt9KSgp"
+  "worker.js": "data:application/javascript;base64,KCgpID0+IHsgdmFyIG1vZD0oKCk9PntmdW5jdGlvbiBjKHMpe3MoKS5jYXRjaChlPT57Y29uc29sZS5lcnJvcigiQ2F1Z2h0IGVycm9yIixlLm1lc3NhZ2UpfSl9dmFyIGE9Y2xhc3N7Y29uc3RydWN0b3IoZSxuPSEwKXt0aGlzLnByaW50PW4sdGhpcy5jYWxsYmFjaz1lfWxvZyguLi5lKXt0aGlzLnB1c2goImxvZyIsZSl9d2FybiguLi5lKXt0aGlzLnB1c2goIndhcm4iLGUpfWVycm9yKC4uLmUpe3RoaXMucHVzaCgiZXJyb3IiLGUpfWluZm8oLi4uZSl7dGhpcy5wdXNoKCJpbmZvIixlKX1wdXNoKGUsbil7dGhpcy5jYWxsYmFjayhlLHRoaXMubG9nTWVzc2FnZShuKSksdGhpcy5wcmludCYmY29uc29sZVtlXSguLi5uKX1sb2dNZXNzYWdlKGUpe2xldCBuPVtdO2ZvcihsZXQgciBvZiBlKXN3aXRjaCh0eXBlb2Ygcil7Y2FzZSJzdHJpbmciOmNhc2UibnVtYmVyIjpuLnB1c2goIiIrcik7YnJlYWs7Y2FzZSJ1bmRlZmluZWQiOm4ucHVzaCgidW5kZWZpbmVkIik7YnJlYWs7ZGVmYXVsdDp0cnl7bGV0IG89SlNPTi5zdHJpbmdpZnkocixudWxsLDIpO28ubGVuZ3RoPjUwMCYmKG89by5zdWJzdHJpbmcoMCw1MDApKyIuLi4iKSxuLnB1c2gobyl9Y2F0Y2h7bi5wdXNoKCJbY2lyY3VsYXIgb2JqZWN0XSIpfX1yZXR1cm4gbi5qb2luKCIgIil9fTt0eXBlb2YgRGVubz4idSImJihzZWxmLkRlbm89e2FyZ3M6W10sYnVpbGQ6e2FyY2g6Ing4Nl82NCJ9LGVudjp7Z2V0KCl7fX19KTt2YXIgZD1uZXcgTWFwLGk9bmV3IE1hcDtmdW5jdGlvbiB0KHMpe3R5cGVvZiB3aW5kb3c8InUiJiZ3aW5kb3cucGFyZW50IT09d2luZG93P3dpbmRvdy5wYXJlbnQucG9zdE1lc3NhZ2UocywiKiIpOnNlbGYucG9zdE1lc3NhZ2Uocyl9dmFyIGw9MDtzZWxmLnN5c2NhbGw9YXN5bmMocywuLi5lKT0+YXdhaXQgbmV3IFByb21pc2UoKG4scik9PntsKyssaS5zZXQobCx7cmVzb2x2ZTpuLHJlamVjdDpyfSksdCh7dHlwZToic3lzY2FsbCIsaWQ6bCxuYW1lOnMsYXJnczplfSl9KTt2YXIgdT1uZXcgTWFwO3NlbGYucmVxdWlyZT1zPT57bGV0IGU9dS5nZXQocyk7aWYoIWUpdGhyb3cgbmV3IEVycm9yKGBEeW5hbWljYWxseSBpbXBvcnRpbmcgbm9uLXByZWxvYWRlZCBsaWJyYXJ5ICR7c31gKTtyZXR1cm4gZX07c2VsZi5jb25zb2xlPW5ldyBhKChzLGUpPT57dCh7dHlwZToibG9nIixsZXZlbDpzLG1lc3NhZ2U6ZX0pfSwhMSk7ZnVuY3Rpb24gZyhzKXtyZXR1cm5gcmV0dXJuICgke3N9KVsiZGVmYXVsdCJdYH1zZWxmLmFkZEV2ZW50TGlzdGVuZXIoIm1lc3NhZ2UiLHM9PntjKGFzeW5jKCk9PntsZXQgZT1zLmRhdGE7c3dpdGNoKGUudHlwZSl7Y2FzZSJsb2FkIjp7bGV0IG49bmV3IEZ1bmN0aW9uKGcoZS5jb2RlKSk7ZC5zZXQoZS5uYW1lLG4oKSksdCh7dHlwZToiaW5pdGVkIixuYW1lOmUubmFtZX0pfWJyZWFrO2Nhc2UibG9hZC1kZXBlbmRlbmN5Ijp7bGV0IHI9bmV3IEZ1bmN0aW9uKGByZXR1cm4gJHtlLmNvZGV9YCkoKTt1LnNldChlLm5hbWUsciksdCh7dHlwZToiZGVwZW5kZW5jeS1pbml0ZWQiLG5hbWU6ZS5uYW1lfSl9YnJlYWs7Y2FzZSJpbnZva2UiOntsZXQgbj1kLmdldChlLm5hbWUpO2lmKCFuKXRocm93IG5ldyBFcnJvcihgRnVuY3Rpb24gbm90IGxvYWRlZDogJHtlLm5hbWV9YCk7dHJ5e2xldCByPWF3YWl0IFByb21pc2UucmVzb2x2ZShuKC4uLmUuYXJnc3x8W10pKTt0KHt0eXBlOiJyZXN1bHQiLGlkOmUuaWQscmVzdWx0OnJ9KX1jYXRjaChyKXt0KHt0eXBlOiJyZXN1bHQiLGlkOmUuaWQsZXJyb3I6ci5tZXNzYWdlLHN0YWNrOnIuc3RhY2t9KX19YnJlYWs7Y2FzZSJzeXNjYWxsLXJlc3BvbnNlIjp7bGV0IG49ZS5pZCxyPWkuZ2V0KG4pO2lmKCFyKXRocm93IGNvbnNvbGUubG9nKCJDdXJyZW50IG91dHN0YW5kaW5nIHJlcXVlc3RzIixpLCJsb29raW5nIHVwIixuKSxFcnJvcigiSW52YWxpZCByZXF1ZXN0IGlkIik7aS5kZWxldGUobiksZS5lcnJvcj9yLnJlamVjdChuZXcgRXJyb3IoZS5lcnJvcikpOnIucmVzb2x2ZShlLnJlc3VsdCl9YnJlYWt9fSl9KTt9KSgpOwogcmV0dXJuIG1vZDt9KSgp"
 }

plugos/gen.ts

@@ -1,5 +1,7 @@
 import { AssetBundle } from "./asset_bundle/bundle.ts";
 import { compile } from "./compile.ts";
+
+console.log("Generating sandbox worker...");
 const bundlePath =
   new URL("./environments/worker_bundle.json", import.meta.url).pathname;
 const workerPath =
@@ -15,4 +17,21 @@ Deno.writeTextFile(
 );
 
 console.log(`Wrote updated bundle to ${bundlePath}`);
+
+console.log("Now generating SQLite worker...");
+const sqliteBundlePath =
+  new URL("./sqlite/worker_bundle.json", import.meta.url).pathname;
+const sqliteWorkerPath =
+  new URL("./sqlite/worker.ts", import.meta.url).pathname;
+
+const sqliteWorkerCode = await compile(sqliteWorkerPath);
+
+const sqliteAssetBundle = new AssetBundle();
+sqliteAssetBundle.writeTextFileSync("worker.js", sqliteWorkerCode);
+Deno.writeTextFile(
+  sqliteBundlePath,
+  JSON.stringify(sqliteAssetBundle.toJSON(), null, 2),
+);
+
+console.log(`Wrote updated bundle to ${sqliteBundlePath}`);
 Deno.exit(0);

plugos/sqlite/async_sqlite.test.ts

@@ -0,0 +1,16 @@
+import { AsyncSQLite } from "./async_sqlite.ts";
+import { assertEquals } from "../../test_deps.ts";
+
+Deno.test("Async SQLite test", async () => {
+  const db = new AsyncSQLite(":memory:");
+  await db.init();
+  await db.execute("CREATE TABLE test (name TEXT)");
+  await db.execute("INSERT INTO test (name) VALUES (?)", "test");
+  await db.execute("INSERT INTO test (name) VALUES (?)", "test 2");
+  assertEquals(await db.query("SELECT * FROM test ORDER BY name"), [{
+    name: "test",
+  }, {
+    name: "test 2",
+  }]);
+  db.stop();
+});

plugos/sqlite/async_sqlite.ts

@@ -0,0 +1,70 @@
+import { AssetBundle } from "../asset_bundle/bundle.ts";
+import workerBundleJson from "./worker_bundle.json" assert { type: "json" };
+
+const workerBundle = new AssetBundle(workerBundleJson);
+
+export class AsyncSQLite {
+  worker: Worker;
+  requestId = 0;
+  outstandingRequests = new Map<
+    number,
+    { resolve: (val: any) => void; reject: (error: Error) => void }
+  >();
+
+  constructor(readonly dbPath: string) {
+    const workerHref = URL.createObjectURL(
+      new Blob([
+        workerBundle.readFileSync("worker.js"),
+      ], {
+        type: "application/javascript",
+      }),
+    );
+    this.worker = new Worker(
+      workerHref,
+      {
+        type: "module",
+      },
+    );
+    this.worker.addEventListener("message", (event: MessageEvent) => {
+      const { data } = event;
+      // console.log("Got data back", data);
+      const { id, result, error } = data;
+      const req = this.outstandingRequests.get(id);
+      if (!req) {
+        console.error("Invalid request id", id);
+        return;
+      }
+      if (result !== undefined) {
+        req.resolve(result);
+      } else if (error) {
+        req.reject(new Error(error));
+      }
+      this.outstandingRequests.delete(id);
+    });
+  }
+
+  private request(message: Record<string, any>): Promise<any> {
+    this.requestId++;
+    return new Promise((resolve, reject) => {
+      this.outstandingRequests.set(this.requestId, { resolve, reject });
+      // console.log("Sending request", message);
+      this.worker.postMessage({ ...message, id: this.requestId });
+    });
+  }
+
+  init(): Promise<void> {
+    return this.request({ type: "init", dbPath: this.dbPath });
+  }
+
+  execute(query: string, ...params: any[]): Promise<number> {
+    return this.request({ type: "execute", query, params });
+  }
+
+  query(query: string, ...params: any[]): Promise<any[]> {
+    return this.request({ type: "query", query, params });
+  }
+
+  stop() {
+    this.worker.terminate();
+  }
+}

plugos/sqlite/deno-sqlite/mod.ts

@@ -11,6 +11,3 @@ export type {
   Row,
   RowObject,
 } from "./src/query.ts";
-
-import { compile } from "./build/sqlite.js";
-await compile();

plugos/sqlite/worker.ts

@@ -0,0 +1,61 @@
+// This file is never loaded directly, it's loaded via a bundle. Run `deno task generate` to update.
+import { DB } from "./deno-sqlite/mod.ts";
+
+let db: DB | undefined;
+
+import { compile } from "./deno-sqlite/build/sqlite.js";
+const ready = compile();
+
+globalThis.addEventListener("message", (event: MessageEvent) => {
+  const { data } = event;
+  // console.log("Got message", data);
+  ready.then(() => {
+    switch (data.type) {
+      case "init": {
+        try {
+          db = new DB(data.dbPath);
+        } catch (e: any) {
+          // console.error("Error!!!", e, data);
+          respondError(data.id, e);
+          break;
+        }
+        respond(data.id, true);
+        break;
+      }
+      case "execute": {
+        if (!db) {
+          respondError(data.id, new Error("Not initialized"));
+          break;
+        }
+        try {
+          db.query(data.query, data.params);
+          respond(data.id, db.changes);
+        } catch (e: any) {
+          respondError(data.id, e);
+        }
+        break;
+      }
+      case "query": {
+        if (!db) {
+          respondError(data.id, new Error("Not initialized"));
+          break;
+        }
+        try {
+          const result = db.queryEntries(data.query, data.params);
+          respond(data.id, result);
+        } catch (e: any) {
+          respondError(data.id, e);
+        }
+        break;
+      }
+    }
+  }).catch(console.error);
+});
+
+function respond(id: number, result: any) {
+  globalThis.postMessage({ id, result });
+}
+
+function respondError(id: number, error: Error) {
+  globalThis.postMessage({ id, error: error.message });
+}

plugos/syscalls/fulltext.sqlite.ts

@@ -1,48 +1,43 @@
-import { SQLite } from "../../server/deps.ts";
+import { AsyncSQLite } from "../../plugos/sqlite/async_sqlite.ts";
 import { SysCallMapping } from "../system.ts";
-import { asyncExecute, asyncQuery } from "./store.deno.ts";
 
-export function ensureFTSTable(
-  db: SQLite,
+export async function ensureFTSTable(
+  db: AsyncSQLite,
   tableName: string,
 ) {
-  const result = db.query(
+  const result = await db.query(
     `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
-    [tableName],
+    tableName,
   );
   if (result.length === 0) {
-    asyncExecute(
-      db,
+    await db.execute(
       `CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`,
     );
 
     console.log(`Created fts5 table ${tableName}`);
   }
-  return Promise.resolve();
 }
 
 export function fullTextSearchSyscalls(
-  db: SQLite,
+  db: AsyncSQLite,
   tableName: string,
 ): SysCallMapping {
   return {
     "fulltext.index": async (_ctx, key: string, value: string) => {
-      await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
-      await asyncExecute(
-        db,
+      await db.execute(`DELETE FROM ${tableName} WHERE key = ?`, key);
+      await db.execute(
         `INSERT INTO ${tableName} (key, value) VALUES (?, ?)`,
         key,
         value,
       );
     },
     "fulltext.delete": async (_ctx, key: string) => {
-      await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
+      await db.execute(`DELETE FROM ${tableName} WHERE key = ?`, key);
     },
     "fulltext.search": async (_ctx, phrase: string, limit: number) => {
       console.log("Got search query", phrase);
       return (
-        await asyncQuery<any>(
-          db,
+        await db.query(
           `SELECT key, rank FROM ${tableName} WHERE value MATCH ? ORDER BY key, rank LIMIT ?`,
           phrase,
           limit,

plugos/syscalls/store.deno.test.ts

@@ -1,11 +1,12 @@
 import { assertEquals } from "../../test_deps.ts";
-import { SQLite } from "../../server/deps.ts";
 import { createSandbox } from "../environments/deno_sandbox.ts";
 import { System } from "../system.ts";
 import { ensureTable, storeSyscalls } from "./store.deno.ts";
+import { AsyncSQLite } from "../sqlite/async_sqlite.ts";
 
 Deno.test("Test store", async () => {
-  const db = new SQLite(":memory:");
+  const db = new AsyncSQLite(":memory:");
+  await db.init();
   await ensureTable(db, "test_table");
   const system = new System("server");
   const syscalls = storeSyscalls(db, "test_table");
@@ -100,7 +101,8 @@ Deno.test("Test store", async () => {
   });
 
   allRoberts = await syscalls["store.query"](dummyCtx, {});
+  // console.log("All Roberts", allRoberts);
   assertEquals(allRoberts.length, 2);
 
-  db.close();
+  db.stop();
 });

plugos/syscalls/store.deno.ts

@@ -1,4 +1,4 @@
-import { SQLite } from "../../server/deps.ts";
+import { AsyncSQLite } from "../sqlite/async_sqlite.ts";
 import { SysCallMapping } from "../system.ts";
 
 export type Item = {
@@ -12,18 +12,17 @@ export type KV = {
   value: any;
 };
 
-export function ensureTable(db: SQLite, tableName: string) {
-  const result = db.query(
+export async function ensureTable(db: AsyncSQLite, tableName: string) {
+  const result = await db.query(
     `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
-    [tableName],
+    tableName,
   );
   if (result.length === 0) {
-    db.execute(
+    await db.execute(
       `CREATE TABLE ${tableName} (key STRING PRIMARY KEY, value TEXT);`,
     );
     console.log(`Created table ${tableName}`);
   }
-  return Promise.resolve();
 }
 
 export type Query = {
@@ -72,68 +71,53 @@ export function queryToSql(
   };
 }
 
-export function asyncQuery<T extends Record<string, unknown>>(
-  db: SQLite,
-  query: string,
-  ...params: any[]
-): Promise<T[]> {
-  // console.log("Querying", query, params);
-  return Promise.resolve(db.queryEntries(query, params));
-}
-
-export function asyncExecute(
-  db: SQLite,
-  query: string,
-  ...params: any[]
-): Promise<number> {
-  // console.log("Exdecting", query, params);
-  db.query(query, params);
-  return Promise.resolve(db.changes);
-}
-
 export function storeSyscalls(
-  db: SQLite,
+  db: AsyncSQLite,
   tableName: string,
 ): SysCallMapping {
   const apiObj: SysCallMapping = {
     "store.delete": async (_ctx, key: string) => {
-      await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
+      await db.execute(`DELETE FROM ${tableName} WHERE key = ?`, key);
     },
     "store.deletePrefix": async (_ctx, prefix: string) => {
-      await asyncExecute(
-        db,
+      await db.execute(
         `DELETE FROM ${tableName} WHERE key LIKE ?`,
         `${prefix}%`,
       );
     },
     "store.deleteQuery": async (_ctx, query: Query) => {
       const { sql, params } = queryToSql(query);
-      await asyncExecute(db, `DELETE FROM ${tableName} ${sql}`, ...params);
+      await db.execute(`DELETE FROM ${tableName} ${sql}`, ...params);
     },
     "store.deleteAll": async () => {
-      await asyncExecute(db, `DELETE FROM ${tableName}`);
+      await db.execute(`DELETE FROM ${tableName}`);
     },
     "store.set": async (_ctx, key: string, value: any) => {
-      await asyncExecute(
-        db,
-        `UPDATE ${tableName} SET value = ? WHERE key = ?`,
-        JSON.stringify(value),
+      await db.execute(
+        `INSERT INTO ${tableName}
+            (key, value)
+          VALUES (?, ?)
+          ON CONFLICT(key)
+          DO UPDATE SET value=excluded.value`,
         key,
+        JSON.stringify(value),
       );
-      if (db.changes === 0) {
-        await asyncExecute(
-          db,
-          `INSERT INTO ${tableName} (key, value) VALUES (?, ?)`,
-          key,
-          JSON.stringify(value),
-        );
-      }
     },
-    // TODO: Optimize
-    "store.batchSet": async (ctx, kvs: KV[]) => {
-      for (const { key, value } of kvs) {
-        await apiObj["store.set"](ctx, key, value);
+    "store.batchSet": async (_ctx, kvs: KV[]) => {
+      if (kvs.length === 0) {
+        return;
       }
+      const values = kvs.flatMap((
+        kv,
+      ) => [kv.key, JSON.stringify(kv.value)]);
+      await db.execute(
+        `INSERT INTO ${tableName}
+            (key, value)
+          VALUES ${kvs.map((_) => "(?, ?)").join(",")}
+          ON CONFLICT(key)
+          DO UPDATE SET value=excluded.value`,
+        ...values,
+      );
     },
     "store.batchDelete": async (ctx, keys: string[]) => {
       for (const key of keys) {
@@ -141,8 +125,7 @@ export function storeSyscalls(
       }
     },
     "store.get": async (_ctx, key: string): Promise<any | null> => {
-      const result = await asyncQuery<Item>(
-        db,
+      const result = await db.query(
         `SELECT value FROM ${tableName} WHERE key = ?`,
         key,
       );
@@ -154,8 +137,7 @@ export function storeSyscalls(
     },
     "store.queryPrefix": async (_ctx, prefix: string) => {
       return (
-        await asyncQuery<Item>(
-          db,
+        await db.query(
           `SELECT key, value FROM ${tableName} WHERE key LIKE ?`,
           `${prefix}%`,
         )
@@ -167,8 +149,7 @@ export function storeSyscalls(
     "store.query": async (_ctx, query: Query) => {
       const { sql, params } = queryToSql(query);
       return (
-        await asyncQuery<Item>(
-          db,
+        await db.query(
           `SELECT key, value FROM ${tableName} ${sql}`,
           ...params,
         )

plugs/core/anchor.ts

@@ -17,7 +17,7 @@ export async function indexAnchors({ name: pageName, tree }: IndexTreeEvent) {
       value: "" + n.from,
     });
   });
-  console.log("Found", anchors.length, "anchors(s)");
+  // console.log("Found", anchors.length, "anchors(s)");
   await index.batchSet(pageName, anchors);
 }
 

plugs/core/item.ts

@@ -17,7 +17,7 @@ export async function indexItems({ name, tree }: IndexTreeEvent) {
   const items: { key: string; value: Item }[] = [];
   removeQueries(tree);
 
-  console.log("Indexing items", name);
+  // console.log("Indexing items", name);
 
   const coll = collectNodesOfType(tree, "ListItem");
 
@@ -59,7 +59,7 @@ export async function indexItems({ name, tree }: IndexTreeEvent) {
       value: item,
     });
   });
-  console.log("Found", items.length, "item(s)");
+  // console.log("Found", items.length, "item(s)");
   await index.batchSet(name, items);
 }
 

plugs/core/page.ts

@@ -30,10 +30,10 @@ import { extractMeta } from "../query/data.ts";
 export async function indexLinks({ name, tree }: IndexTreeEvent) {
   const backLinks: { key: string; value: string }[] = [];
   // [[Style Links]]
-  console.log("Now indexing", name);
+  // console.log("Now indexing", name);
   const pageMeta = extractMeta(tree);
   if (Object.keys(pageMeta).length > 0) {
-    console.log("Extracted page meta data", pageMeta);
+    // console.log("Extracted page meta data", pageMeta);
     // Don't index meta data starting with $
     for (const key in pageMeta) {
       if (key.startsWith("$")) {
@@ -53,7 +53,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
       value: name,
     });
   });
-  console.log("Found", backLinks.length, "wiki link(s)");
+  // console.log("Found", backLinks.length, "wiki link(s)");
   await index.batchSet(name, backLinks);
 }
 
@@ -212,8 +212,11 @@ export async function reindexSpace() {
   await index.clearPageIndex();
   console.log("Listing all pages");
   const pages = await space.listPages();
+  let counter = 0;
   for (const { name } of pages) {
-    console.log("Indexing", name);
+    counter++;
+
+    console.log(`Indexing page ${counter}/${pages.length}: ${name}`);
     const text = await space.readPage(name);
     const parsed = await markdown.parseMarkdown(text);
     await events.dispatchEvent("page:index", {

plugs/core/search.ts

@@ -8,11 +8,7 @@ import {
   FileData,
   FileEncoding,
 } from "../../common/spaces/space_primitives.ts";
-import {
-  base64DecodeDataUrl,
-  base64Encode,
-  base64EncodedDataUrl,
-} from "../../plugos/asset_bundle/base64.ts";
+import { base64EncodedDataUrl } from "../../plugos/asset_bundle/base64.ts";
 
 const searchPrefix = "🔍 ";
 

plugs/query/data.ts

@@ -53,7 +53,7 @@ export async function indexData({ name, tree }: IndexTreeEvent) {
       return;
     }
   });
-  console.log("Found", dataObjects.length, "data objects");
+  // console.log("Found", dataObjects.length, "data objects");
   await index.batchSet(name, dataObjects);
 }
 

plugs/tasks/task.ts

@@ -80,7 +80,7 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) {
     // console.log("Task", task);
   });
 
-  console.log("Found", tasks.length, "task(s)");
+  // console.log("Found", tasks.length, "task(s)");
   await index.batchSet(name, tasks);
 }
 

server/deps.ts

@@ -1,4 +1,3 @@
 export * from "../common/deps.ts";
-export { DB as SQLite } from "../plugos/forked/deno-sqlite/mod.ts";
 export { Application, Router } from "https://deno.land/x/oak@v11.1.0/mod.ts";
 export * as etag from "https://deno.land/x/oak@v11.1.0/etag.ts";

server/http_server.ts

@@ -1,4 +1,4 @@
-import { Application, etag, path, Router, SQLite } from "./deps.ts";
+import { Application, etag, path, Router } from "./deps.ts";
 import { Manifest, SilverBulletHooks } from "../common/manifest.ts";
 import { loadMarkdownExtensions } from "../common/markdown_ext.ts";
 import buildMarkdown from "../common/parser.ts";
@@ -37,6 +37,7 @@ import { systemSyscalls } from "./syscalls/system.ts";
 import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
 import assetSyscalls from "../plugos/syscalls/asset.ts";
 import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
+import { AsyncSQLite } from "../plugos/sqlite/async_sqlite.ts";
 
 export type ServerOptions = {
   port: number;
@@ -53,7 +54,7 @@ export class HttpServer {
   system: System<SilverBulletHooks>;
   private space: Space;
   private eventHook: EventHook;
-  private db: SQLite;
+  private db: AsyncSQLite;
   private port: number;
   password?: string;
   settings: { [key: string]: any } = {};
@@ -97,7 +98,10 @@ export class HttpServer {
     this.space = new Space(this.spacePrimitives);
 
     // The database used for persistence (SQLite)
-    this.db = new SQLite(path.join(options.pagesPath, "data.db"));
+    this.db = new AsyncSQLite(path.join(options.pagesPath, "data.db"));
+    this.db.init().catch((e) => {
+      console.error("Error initializing database", e);
+    });
 
     // The cron hook
     this.system.addHook(new DenoCronHook());

server/syscalls/index.test.ts

@@ -0,0 +1,51 @@
+import { assertEquals } from "https://deno.land/std@0.152.0/testing/asserts.ts";
+import { AsyncSQLite } from "../../plugos/sqlite/async_sqlite.ts";
+import { ensureTable, pageIndexSyscalls } from "./index.ts";
+
+const fakeContext = {} as any;
+
+Deno.test("Page index", async () => {
+  const db = new AsyncSQLite(":memory:");
+  await db.init();
+  await ensureTable(db);
+  const syscalls = pageIndexSyscalls(db);
+  await syscalls["index.set"](fakeContext, "page1", "key1", "value1");
+  assertEquals(
+    "value1",
+    await syscalls["index.get"](fakeContext, "page1", "key1"),
+  );
+  await syscalls["index.set"](fakeContext, "page1", "key1", "value2");
+  assertEquals(
+    "value2",
+    await syscalls["index.get"](fakeContext, "page1", "key1"),
+  );
+  await syscalls["index.set"](fakeContext, "page1", "key2", "value1");
+  assertEquals(
+    [
+      { key: "key1", page: "page1", value: "value2" },
+      { key: "key2", page: "page1", value: "value1" },
+    ],
+    await syscalls["index.queryPrefix"](fakeContext, ""),
+  );
+  await syscalls["index.delete"](fakeContext, "page1", "key1");
+  assertEquals(
+    [
+      { key: "key2", page: "page1", value: "value1" },
+    ],
+    await syscalls["index.queryPrefix"](fakeContext, ""),
+  );
+  await syscalls["index.batchSet"](fakeContext, "page1", [
+    { key: "key1", value: "value1" },
+    { key: "key2", value: "value2" },
+    { key: "key3", value: "value3" },
+  ]);
+  assertEquals(
+    [
+      { key: "key1", page: "page1", value: "value1" },
+      { key: "key2", page: "page1", value: "value2" },
+      { key: "key3", page: "page1", value: "value3" },
+    ],
+    await syscalls["index.queryPrefix"](fakeContext, ""),
+  );
+  db.stop();
+});

server/syscalls/index.ts

@@ -1,12 +1,7 @@
 // import { Knex } from "knex";
 import { SysCallMapping } from "../../plugos/system.ts";
-import {
-  asyncExecute,
-  asyncQuery,
-  Query,
-  queryToSql,
-} from "../../plugos/syscalls/store.deno.ts";
-import { SQLite } from "../deps.ts";
+import { Query, queryToSql } from "../../plugos/syscalls/store.deno.ts";
+import { AsyncSQLite } from "../../plugos/sqlite/async_sqlite.ts";
 
 type Item = {
   page: string;
@@ -21,59 +16,61 @@ export type KV = {
 
 const tableName = "page_index";
 
-export function ensureTable(db: SQLite): Promise<void> {
-  const result = db.query(
+export async function ensureTable(db: AsyncSQLite): Promise<void> {
+  const result = await db.query(
     `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
-    [tableName],
+    tableName,
   );
   if (result.length === 0) {
-    db.execute(
+    await db.execute(
       `CREATE TABLE ${tableName} (key STRING, page STRING, value TEXT, PRIMARY KEY (page, key));`,
     );
-    db.execute(
+    await db.execute(
       `CREATE INDEX ${tableName}_idx ON ${tableName}(key);`,
     );
     console.log(`Created table ${tableName}`);
   }
-  return Promise.resolve();
 }
 
-export function pageIndexSyscalls(db: SQLite): SysCallMapping {
+export function pageIndexSyscalls(db: AsyncSQLite): SysCallMapping {
   const apiObj: SysCallMapping = {
     "index.set": async (_ctx, page: string, key: string, value: any) => {
-      await asyncExecute(
-        db,
-        `UPDATE ${tableName} SET value = ? WHERE key = ? AND page = ?`,
-        JSON.stringify(value),
-        key,
+      await db.execute(
+        `INSERT INTO ${tableName}
+            (page, key, value)
+          VALUES (?, ?, ?)
+          ON CONFLICT(page, key)
+          DO UPDATE SET value=excluded.value`,
         page,
+        key,
+        JSON.stringify(value),
       );
-      if (db.changes === 0) {
-        await asyncExecute(
-          db,
-          `INSERT INTO ${tableName} (key, page, value) VALUES (?, ?, ?)`,
-          key,
-          page,
-          JSON.stringify(value),
-        );
-      }
     },
-    "index.batchSet": async (ctx, page: string, kvs: KV[]) => {
-      for (const { key, value } of kvs) {
-        await apiObj["index.set"](ctx, page, key, value);
+    "index.batchSet": async (_ctx, page: string, kvs: KV[]) => {
+      if (kvs.length === 0) {
+        return;
       }
+      const values = kvs.flatMap((
+        kv,
+      ) => [page, kv.key, JSON.stringify(kv.value)]);
+      await db.execute(
+        `INSERT INTO ${tableName}
+            (page, key, value)
+          VALUES ${kvs.map((_) => "(?, ?, ?)").join(",")}
+          ON CONFLICT(key, page)
+          DO UPDATE SET value=excluded.value`,
+        ...values,
+      );
     },
     "index.delete": async (_ctx, page: string, key: string) => {
-      await asyncExecute(
-        db,
+      await db.execute(
         `DELETE FROM ${tableName} WHERE key = ? AND page = ?`,
         key,
         page,
       );
     },
     "index.get": async (_ctx, page: string, key: string) => {
-      const result = await asyncQuery<Item>(
-        db,
+      const result = await db.query(
         `SELECT value FROM ${tableName} WHERE key = ? AND page = ?`,
         key,
         page,
@@ -86,9 +83,8 @@ export function pageIndexSyscalls(db: SQLite): SysCallMapping {
     },
     "index.queryPrefix": async (_ctx, prefix: string) => {
       return (
-        await asyncQuery<Item>(
-          db,
-          `SELECT key, page, value FROM ${tableName} WHERE key LIKE ?`,
+        await db.query(
+          `SELECT key, page, value FROM ${tableName} WHERE key LIKE ? ORDER BY key, page ASC`,
           `${prefix}%`,
         )
       ).map(({ key, value, page }) => ({
@@ -100,11 +96,7 @@ export function pageIndexSyscalls(db: SQLite): SysCallMapping {
     "index.query": async (_ctx, query: Query) => {
       const { sql, params } = queryToSql(query);
       return (
-        await asyncQuery<Item>(
-          db,
-          `SELECT key, value FROM ${tableName} ${sql}`,
-          ...params,
-        )
+        await db.query(`SELECT key, value FROM ${tableName} ${sql}`, ...params)
       ).map(({ key, value, page }: any) => ({
         key,
         page,
@@ -115,16 +107,14 @@ export function pageIndexSyscalls(db: SQLite): SysCallMapping {
       await apiObj["index.deletePrefixForPage"](ctx, page, "");
     },
     "index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => {
-      await asyncExecute(
-        db,
+      await db.execute(
         `DELETE FROM ${tableName} WHERE key LIKE ? AND page = ?`,
         `${prefix}%`,
         page,
       );
     },
     "index.clearPageIndex": async () => {
-      await asyncExecute(
-        db,
+      await db.execute(
         `DELETE FROM ${tableName}`,
       );
     },