neil/silverbullet
1
0
Fork 0
Code Activity

Fixes #97: SQLite is now async, optimized, tests

Browse Source
This commit is contained in:
Zef Hemel
2022-10-21 10:00:43 +02:00
parent b9da9b7965
commit c1a78e0105
65 changed files with 329 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/deps.ts
-1
View File
@@ -1,4 +1,3 @@
export * from "../common/deps.ts";
export { DB as SQLite } from "../plugos/forked/deno-sqlite/mod.ts";
export { Application, Router } from "https://deno.land/x/oak@v11.1.0/mod.ts";
export * as etag from "https://deno.land/x/oak@v11.1.0/etag.ts";
server/http_server.ts
+7 -3
View File
@@ -1,4 +1,4 @@
import { Application, etag, path, Router, SQLite } from "./deps.ts";
import { Application, etag, path, Router } from "./deps.ts";
import { Manifest, SilverBulletHooks } from "../common/manifest.ts";
import { loadMarkdownExtensions } from "../common/markdown_ext.ts";
import buildMarkdown from "../common/parser.ts";
@@ -37,6 +37,7 @@ import { systemSyscalls } from "./syscalls/system.ts";
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
import assetSyscalls from "../plugos/syscalls/asset.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { AsyncSQLite } from "../plugos/sqlite/async_sqlite.ts";
export type ServerOptions = {
port: number;
@@ -53,7 +54,7 @@ export class HttpServer {
system: System<SilverBulletHooks>;
private space: Space;
private eventHook: EventHook;
private db: SQLite;
private db: AsyncSQLite;
private port: number;
password?: string;
settings: { [key: string]: any } = {};
@@ -97,7 +98,10 @@ export class HttpServer {
this.space = new Space(this.spacePrimitives);
// The database used for persistence (SQLite)
this.db = new SQLite(path.join(options.pagesPath, "data.db"));
this.db = new AsyncSQLite(path.join(options.pagesPath, "data.db"));
this.db.init().catch((e) => {
console.error("Error initializing database", e);
});
// The cron hook
this.system.addHook(new DenoCronHook());
server/syscalls/index.test.ts
+51
View File
@@ -0,0 +1,51 @@
import { assertEquals } from "https://deno.land/std@0.152.0/testing/asserts.ts";
import { AsyncSQLite } from "../../plugos/sqlite/async_sqlite.ts";
import { ensureTable, pageIndexSyscalls } from "./index.ts";
const fakeContext = {} as any;
Deno.test("Page index", async () => {
const db = new AsyncSQLite(":memory:");
await db.init();
await ensureTable(db);
const syscalls = pageIndexSyscalls(db);
await syscalls["index.set"](fakeContext, "page1", "key1", "value1");
assertEquals(
"value1",
await syscalls["index.get"](fakeContext, "page1", "key1"),
);
await syscalls["index.set"](fakeContext, "page1", "key1", "value2");
assertEquals(
"value2",
await syscalls["index.get"](fakeContext, "page1", "key1"),
);
await syscalls["index.set"](fakeContext, "page1", "key2", "value1");
assertEquals(
[
{ key: "key1", page: "page1", value: "value2" },
{ key: "key2", page: "page1", value: "value1" },
],
await syscalls["index.queryPrefix"](fakeContext, ""),
);
await syscalls["index.delete"](fakeContext, "page1", "key1");
assertEquals(
[
{ key: "key2", page: "page1", value: "value1" },
],
await syscalls["index.queryPrefix"](fakeContext, ""),
);
await syscalls["index.batchSet"](fakeContext, "page1", [
{ key: "key1", value: "value1" },
{ key: "key2", value: "value2" },
{ key: "key3", value: "value3" },
]);
assertEquals(
[
{ key: "key1", page: "page1", value: "value1" },
{ key: "key2", page: "page1", value: "value2" },
{ key: "key3", page: "page1", value: "value3" },
],
await syscalls["index.queryPrefix"](fakeContext, ""),
);
db.stop();
});
server/syscalls/index.ts
+37 -47
View File
@@ -1,12 +1,7 @@
// import { Knex } from "knex";
import { SysCallMapping } from "../../plugos/system.ts";
import {
asyncExecute,
asyncQuery,
Query,
queryToSql,
} from "../../plugos/syscalls/store.deno.ts";
import { SQLite } from "../deps.ts";
import { Query, queryToSql } from "../../plugos/syscalls/store.deno.ts";
import { AsyncSQLite } from "../../plugos/sqlite/async_sqlite.ts";
type Item = {
page: string;
@@ -21,59 +16,61 @@ export type KV = {
const tableName = "page_index";
export function ensureTable(db: SQLite): Promise<void> {
const result = db.query(
export async function ensureTable(db: AsyncSQLite): Promise<void> {
const result = await db.query(
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
[tableName],
tableName,
);
if (result.length === 0) {
db.execute(
await db.execute(
`CREATE TABLE ${tableName} (key STRING, page STRING, value TEXT, PRIMARY KEY (page, key));`,
);
db.execute(
await db.execute(
`CREATE INDEX ${tableName}_idx ON ${tableName}(key);`,
);
console.log(`Created table ${tableName}`);
}
return Promise.resolve();
}
export function pageIndexSyscalls(db: SQLite): SysCallMapping {
export function pageIndexSyscalls(db: AsyncSQLite): SysCallMapping {
const apiObj: SysCallMapping = {
"index.set": async (_ctx, page: string, key: string, value: any) => {
await asyncExecute(
db,
`UPDATE ${tableName} SET value = ? WHERE key = ? AND page = ?`,
JSON.stringify(value),
key,
await db.execute(
`INSERT INTO ${tableName}
(page, key, value)
VALUES (?, ?, ?)
ON CONFLICT(page, key)
DO UPDATE SET value=excluded.value`,
page,
key,
JSON.stringify(value),
);
if (db.changes === 0) {
await asyncExecute(
db,
`INSERT INTO ${tableName} (key, page, value) VALUES (?, ?, ?)`,
key,
page,
JSON.stringify(value),
);
}
},
"index.batchSet": async (ctx, page: string, kvs: KV[]) => {
for (const { key, value } of kvs) {
await apiObj["index.set"](ctx, page, key, value);
"index.batchSet": async (_ctx, page: string, kvs: KV[]) => {
if (kvs.length === 0) {
return;
}
const values = kvs.flatMap((
kv,
) => [page, kv.key, JSON.stringify(kv.value)]);
await db.execute(
`INSERT INTO ${tableName}
(page, key, value)
VALUES ${kvs.map((_) => "(?, ?, ?)").join(",")}
ON CONFLICT(key, page)
DO UPDATE SET value=excluded.value`,
...values,
);
},
"index.delete": async (_ctx, page: string, key: string) => {
await asyncExecute(
db,
await db.execute(
`DELETE FROM ${tableName} WHERE key = ? AND page = ?`,
key,
page,
);
},
"index.get": async (_ctx, page: string, key: string) => {
const result = await asyncQuery<Item>(
db,
const result = await db.query(
`SELECT value FROM ${tableName} WHERE key = ? AND page = ?`,
key,
page,
@@ -86,9 +83,8 @@ export function pageIndexSyscalls(db: SQLite): SysCallMapping {
},
"index.queryPrefix": async (_ctx, prefix: string) => {
return (
await asyncQuery<Item>(
db,
`SELECT key, page, value FROM ${tableName} WHERE key LIKE ?`,
await db.query(
`SELECT key, page, value FROM ${tableName} WHERE key LIKE ? ORDER BY key, page ASC`,
`${prefix}%`,
)
).map(({ key, value, page }) => ({
@@ -100,11 +96,7 @@ export function pageIndexSyscalls(db: SQLite): SysCallMapping {
"index.query": async (_ctx, query: Query) => {
const { sql, params } = queryToSql(query);
return (
await asyncQuery<Item>(
db,
`SELECT key, value FROM ${tableName} ${sql}`,
...params,
)
await db.query(`SELECT key, value FROM ${tableName} ${sql}`, ...params)
).map(({ key, value, page }: any) => ({
key,
page,
@@ -115,16 +107,14 @@ export function pageIndexSyscalls(db: SQLite): SysCallMapping {
await apiObj["index.deletePrefixForPage"](ctx, page, "");
},
"index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => {
await asyncExecute(
db,
await db.execute(
`DELETE FROM ${tableName} WHERE key LIKE ? AND page = ?`,
`${prefix}%`,
page,
);
},
"index.clearPageIndex": async () => {
await asyncExecute(
db,
await db.execute(
`DELETE FROM ${tableName}`,
);
},