SilverBullet pivot to become an offline-first PWA (#403)
This commit is contained in:
Zef Hemel
@@ -6,7 +6,7 @@ export default function assetSyscalls(system: System<any>): SysCallMapping {
|
||||
ctx,
|
||||
name: string,
|
||||
): string => {
|
||||
return system.loadedPlugs.get(ctx.plug.name)!.assets!.readFileAsDataUrl(
|
||||
return system.loadedPlugs.get(ctx.plug.name!)!.assets!.readFileAsDataUrl(
|
||||
name,
|
||||
);
|
||||
},
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
import { sandboxCompile, sandboxCompileModule } from "../compile.ts";
|
||||
import { SysCallMapping } from "../system.ts";
|
||||
import { Manifest } from "../types.ts";
|
||||
|
||||
import importMap from "../../import_map.json" assert { type: "json" };
|
||||
import { base64EncodedDataUrl } from "../asset_bundle/base64.ts";
|
||||
|
||||
export function esbuildSyscalls(
|
||||
imports: Manifest<any>[],
|
||||
): SysCallMapping {
|
||||
return {
|
||||
"esbuild.compile": async (
|
||||
_ctx,
|
||||
filename: string,
|
||||
code: string,
|
||||
functionName?: string,
|
||||
): Promise<string> => {
|
||||
// Override this to point to a URL
|
||||
importMap.imports["$sb/"] = "https://deno.land/x/silverbullet/plug-api/";
|
||||
const importUrl = new URL(
|
||||
base64EncodedDataUrl(
|
||||
"application/json",
|
||||
new TextEncoder().encode(JSON.stringify(importMap)),
|
||||
),
|
||||
);
|
||||
return await sandboxCompile(
|
||||
filename,
|
||||
code,
|
||||
functionName,
|
||||
{
|
||||
debug: true,
|
||||
imports,
|
||||
importMap: importUrl,
|
||||
},
|
||||
);
|
||||
},
|
||||
"esbuild.compileModule": async (
|
||||
_ctx,
|
||||
moduleName: string,
|
||||
): Promise<string> => {
|
||||
return await sandboxCompileModule(moduleName, {
|
||||
imports,
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
import type {
|
||||
SandboxFetchRequest,
|
||||
SandboxFetchResponse,
|
||||
} from "../../plug-api/plugos-syscall/fetch.ts";
|
||||
import { base64Encode } from "../asset_bundle/base64.ts";
|
||||
import { SysCallMapping } from "../system.ts";
|
||||
|
||||
export async function sandboxFetch(
|
||||
url: string,
|
||||
req?: SandboxFetchRequest,
|
||||
): Promise<SandboxFetchResponse> {
|
||||
const result = await fetch(
|
||||
url,
|
||||
req && {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: req.body,
|
||||
},
|
||||
);
|
||||
const body = await (await result.blob()).arrayBuffer();
|
||||
return {
|
||||
ok: result.ok,
|
||||
status: result.status,
|
||||
headers: Object.fromEntries(result.headers.entries()),
|
||||
base64Body: base64Encode(new Uint8Array(body)),
|
||||
};
|
||||
}
|
||||
|
||||
export function sandboxFetchSyscalls(): SysCallMapping {
|
||||
return {
|
||||
"sandboxFetch.fetch": (
|
||||
_ctx,
|
||||
url: string,
|
||||
options?: SandboxFetchRequest,
|
||||
): Promise<SandboxFetchResponse> => {
|
||||
return sandboxFetch(url, options);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -2,13 +2,12 @@ import { assert } from "../../test_deps.ts";
|
||||
import { FileMeta } from "../../common/types.ts";
|
||||
import { path } from "../deps.ts";
|
||||
import fileSystemSyscalls from "./fs.deno.ts";
|
||||
import { urlToPathname } from "../util.ts";
|
||||
|
||||
const fakeCtx = {} as any;
|
||||
|
||||
Deno.test("Test FS operations", async () => {
|
||||
const thisFolder = path.resolve(
|
||||
path.dirname(urlToPathname(new URL(import.meta.url))),
|
||||
path.dirname(new URL(import.meta.url).pathname),
|
||||
);
|
||||
const syscalls = fileSystemSyscalls(thisFolder);
|
||||
const allFiles: FileMeta[] = await syscalls["fs.listFiles"](
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
import { FullTextSearchOptions } from "../../plug-api/plugos-syscall/fulltext.ts";
|
||||
import { ISQLite } from "../sqlite/sqlite_interface.ts";
|
||||
import { SysCallMapping } from "../system.ts";
|
||||
|
||||
export async function ensureFTSTable(
|
||||
db: ISQLite,
|
||||
tableName: string,
|
||||
) {
|
||||
const result = await db.query(
|
||||
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
|
||||
tableName,
|
||||
);
|
||||
if (result.length === 0) {
|
||||
await db.execute(
|
||||
`CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`,
|
||||
);
|
||||
|
||||
// console.log(`Created fts5 table ${tableName}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function fullTextSearchSyscalls(
|
||||
db: ISQLite,
|
||||
tableName: string,
|
||||
): SysCallMapping {
|
||||
return {
|
||||
"fulltext.index": async (_ctx, key: string, value: string) => {
|
||||
await db.execute(`DELETE FROM ${tableName} WHERE key = ?`, key);
|
||||
await db.execute(
|
||||
`INSERT INTO ${tableName} (key, value) VALUES (?, ?)`,
|
||||
key,
|
||||
value,
|
||||
);
|
||||
},
|
||||
"fulltext.delete": async (_ctx, key: string) => {
|
||||
await db.execute(`DELETE FROM ${tableName} WHERE key = ?`, key);
|
||||
},
|
||||
"fulltext.search": async (
|
||||
_ctx,
|
||||
phrase: string,
|
||||
options: FullTextSearchOptions,
|
||||
) => {
|
||||
return (
|
||||
await db.query(
|
||||
`SELECT key, bm25(fts) AS score, snippet(fts, 1, ?, ?, ?, ?) as snippet
|
||||
FROM ${tableName}
|
||||
WHERE value
|
||||
MATCH ?
|
||||
ORDER BY score LIMIT ?`,
|
||||
options.highlightPrefix || "",
|
||||
options.highlightPostfix || "",
|
||||
options.highlightEllipsis || "...",
|
||||
options.summaryMaxLength || 50,
|
||||
phrase,
|
||||
options.limit || 20,
|
||||
)
|
||||
).map((item) => ({
|
||||
name: item.key,
|
||||
score: item.score,
|
||||
snippet: item.snippet,
|
||||
}));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
import type { LogEntry } from "../sandbox.ts";
|
||||
import type { SysCallMapping, System } from "../system.ts";
|
||||
|
||||
export default function sandboxSyscalls(system: System<any>): SysCallMapping {
|
||||
return {
|
||||
"sandbox.getLogs": (): LogEntry[] => {
|
||||
let allLogs: LogEntry[] = [];
|
||||
for (const plug of system.loadedPlugs.values()) {
|
||||
if (plug.sandbox) {
|
||||
allLogs = allLogs.concat(plug.sandbox.logBuffer);
|
||||
}
|
||||
}
|
||||
allLogs = allLogs.sort((a, b) => a.date - b.date);
|
||||
return allLogs;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
import { assertEquals } from "../../test_deps.ts";
|
||||
import { createSandbox } from "../environments/deno_sandbox.ts";
|
||||
import { System } from "../system.ts";
|
||||
import { ensureTable, storeSyscalls } from "./store.sqlite.ts";
|
||||
import { AsyncSQLite } from "../sqlite/async_sqlite.ts";
|
||||
|
||||
Deno.test("Test store", async () => {
|
||||
const db = new AsyncSQLite(":memory:");
|
||||
await db.init();
|
||||
await ensureTable(db, "test_table");
|
||||
const system = new System("server");
|
||||
const syscalls = storeSyscalls(db, "test_table");
|
||||
system.registerSyscalls([], syscalls);
|
||||
const plug = await system.load(
|
||||
{
|
||||
name: "test",
|
||||
functions: {
|
||||
test1: {
|
||||
code: `(() => {
|
||||
return {
|
||||
default: async () => {
|
||||
await self.syscall("store.set", "name", "Pete");
|
||||
return await self.syscall("store.get", "name");
|
||||
}
|
||||
};
|
||||
})()`,
|
||||
},
|
||||
},
|
||||
},
|
||||
createSandbox,
|
||||
);
|
||||
assertEquals(await plug.invoke("test1", []), "Pete");
|
||||
await system.unloadAll();
|
||||
|
||||
const dummyCtx: any = {};
|
||||
|
||||
await syscalls["store.deleteAll"](dummyCtx);
|
||||
await syscalls["store.batchSet"](dummyCtx, [
|
||||
{
|
||||
key: "pete",
|
||||
value: {
|
||||
age: 20,
|
||||
firstName: "Pete",
|
||||
lastName: "Roberts",
|
||||
},
|
||||
},
|
||||
{
|
||||
key: "petejr",
|
||||
value: {
|
||||
age: 8,
|
||||
firstName: "Pete Jr",
|
||||
lastName: "Roberts",
|
||||
},
|
||||
},
|
||||
{
|
||||
key: "petesr",
|
||||
value: {
|
||||
age: 78,
|
||||
firstName: "Pete Sr",
|
||||
lastName: "Roberts",
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
let allRoberts = await syscalls["store.query"](dummyCtx, {
|
||||
filter: [{ op: "=", prop: "lastName", value: "Roberts" }],
|
||||
orderBy: "age",
|
||||
orderDesc: true,
|
||||
});
|
||||
|
||||
assertEquals(allRoberts.length, 3);
|
||||
assertEquals(allRoberts[0].key, "petesr");
|
||||
|
||||
allRoberts = await syscalls["store.query"](dummyCtx, {
|
||||
filter: [{ op: "=", prop: "lastName", value: "Roberts" }],
|
||||
orderBy: "age",
|
||||
limit: 1,
|
||||
});
|
||||
|
||||
assertEquals(allRoberts.length, 1);
|
||||
assertEquals(allRoberts[0].key, "petejr");
|
||||
|
||||
allRoberts = await syscalls["store.query"](dummyCtx, {
|
||||
filter: [
|
||||
{ op: ">", prop: "age", value: 10 },
|
||||
{ op: "<", prop: "age", value: 30 },
|
||||
],
|
||||
orderBy: "age",
|
||||
});
|
||||
|
||||
assertEquals(allRoberts.length, 1);
|
||||
assertEquals(allRoberts[0].key, "pete");
|
||||
|
||||
// Delete the middle one
|
||||
|
||||
await syscalls["store.deleteQuery"](dummyCtx, {
|
||||
filter: [
|
||||
{ op: ">", prop: "age", value: 10 },
|
||||
{ op: "<", prop: "age", value: 30 },
|
||||
],
|
||||
});
|
||||
|
||||
allRoberts = await syscalls["store.query"](dummyCtx, {});
|
||||
// console.log("All Roberts", allRoberts);
|
||||
assertEquals(allRoberts.length, 2);
|
||||
|
||||
db.stop();
|
||||
});
|
||||
@@ -1,66 +1,55 @@
|
||||
import Dexie from "https://esm.sh/dexie@3.2.2";
|
||||
import { SysCallMapping } from "../system.ts";
|
||||
|
||||
export type KV = {
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
import { DexieKVStore } from "../lib/kv_store.dexie.ts";
|
||||
import { KV } from "../lib/kv_store.ts";
|
||||
|
||||
export function storeSyscalls(
|
||||
dbName: string,
|
||||
tableName: string,
|
||||
db: DexieKVStore,
|
||||
): SysCallMapping {
|
||||
const db = new Dexie(dbName);
|
||||
db.version(1).stores({
|
||||
[tableName]: "key",
|
||||
});
|
||||
const items = db.table(tableName);
|
||||
|
||||
return {
|
||||
"store.delete": async (_ctx, key: string) => {
|
||||
await items.delete(key);
|
||||
"store.delete": (_ctx, key: string) => {
|
||||
return db.del(key);
|
||||
},
|
||||
|
||||
"store.deletePrefix": async (_ctx, prefix: string) => {
|
||||
await items.where("key").startsWith(prefix).delete();
|
||||
"store.deletePrefix": (_ctx, prefix: string) => {
|
||||
return db.deletePrefix(prefix);
|
||||
},
|
||||
|
||||
"store.deleteAll": async () => {
|
||||
await items.clear();
|
||||
"store.deleteAll": () => {
|
||||
return db.deleteAll();
|
||||
},
|
||||
|
||||
"store.set": async (_ctx, key: string, value: any) => {
|
||||
await items.put({
|
||||
key,
|
||||
value,
|
||||
});
|
||||
"store.set": (_ctx, key: string, value: any) => {
|
||||
return db.set(key, value);
|
||||
},
|
||||
|
||||
"store.batchSet": async (_ctx, kvs: KV[]) => {
|
||||
await items.bulkPut(
|
||||
kvs.map(({ key, value }) => ({
|
||||
key,
|
||||
value,
|
||||
})),
|
||||
);
|
||||
"store.batchSet": (_ctx, kvs: KV[]) => {
|
||||
return db.batchSet(kvs);
|
||||
},
|
||||
|
||||
"store.get": async (_ctx, key: string): Promise<any | null> => {
|
||||
const result = await items.get({
|
||||
key,
|
||||
});
|
||||
return result ? result.value : null;
|
||||
"store.batchDelete": (_ctx, keys: string[]) => {
|
||||
return db.batchDelete(keys);
|
||||
},
|
||||
|
||||
"store.queryPrefix": async (
|
||||
"store.batchGet": (
|
||||
_ctx,
|
||||
keys: string[],
|
||||
): Promise<(any | undefined)[]> => {
|
||||
return db.batchGet(keys);
|
||||
},
|
||||
|
||||
"store.get": (_ctx, key: string): Promise<any | null> => {
|
||||
return db.get(key);
|
||||
},
|
||||
|
||||
"store.has": (_ctx, key: string): Promise<boolean> => {
|
||||
return db.has(key);
|
||||
},
|
||||
|
||||
"store.queryPrefix": (
|
||||
_ctx,
|
||||
keyPrefix: string,
|
||||
): Promise<{ key: string; value: any }[]> => {
|
||||
const results = await items.where("key").startsWith(keyPrefix).toArray();
|
||||
return results.map((result) => ({
|
||||
key: result.key,
|
||||
value: result.value,
|
||||
}));
|
||||
return db.queryPrefix(keyPrefix);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,170 +0,0 @@
|
||||
import { ISQLite } from "../sqlite/sqlite_interface.ts";
|
||||
import { SysCallMapping } from "../system.ts";
|
||||
|
||||
export type Item = {
|
||||
page: string;
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
export type KV = {
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
export async function ensureTable(db: ISQLite, tableName: string) {
|
||||
const result = await db.query(
|
||||
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
|
||||
tableName,
|
||||
);
|
||||
if (result.length === 0) {
|
||||
await db.execute(
|
||||
`CREATE TABLE ${tableName} (key STRING PRIMARY KEY, value TEXT);`,
|
||||
);
|
||||
// console.log(`Created table ${tableName}`);
|
||||
}
|
||||
}
|
||||
|
||||
export type Query = {
|
||||
filter?: Filter[];
|
||||
orderBy?: string;
|
||||
orderDesc?: boolean;
|
||||
limit?: number;
|
||||
select?: string[];
|
||||
};
|
||||
|
||||
export type Filter = {
|
||||
op: string;
|
||||
prop: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
export function queryToSql(
|
||||
query: Query,
|
||||
): { sql: string; params: any[] } {
|
||||
const whereClauses: string[] = [];
|
||||
const clauses: string[] = [];
|
||||
const params: any[] = [];
|
||||
if (query.filter) {
|
||||
for (const filter of query.filter) {
|
||||
whereClauses.push(
|
||||
`json_extract(value, '$.${filter.prop}') ${filter.op} ?`,
|
||||
);
|
||||
params.push(filter.value);
|
||||
}
|
||||
}
|
||||
if (query.orderBy) {
|
||||
clauses.push(
|
||||
`ORDER BY json_extract(value, '$.${query.orderBy}') ${
|
||||
query.orderDesc ? "desc" : "asc"
|
||||
}`,
|
||||
);
|
||||
}
|
||||
if (query.limit) {
|
||||
clauses.push(`LIMIT ${query.limit}`);
|
||||
}
|
||||
return {
|
||||
sql: whereClauses.length > 0
|
||||
? `WHERE ${whereClauses.join(" AND ")} ${clauses.join(" ")}`
|
||||
: clauses.join(" "),
|
||||
params,
|
||||
};
|
||||
}
|
||||
|
||||
export function storeSyscalls(
|
||||
db: ISQLite,
|
||||
tableName: string,
|
||||
): SysCallMapping {
|
||||
const apiObj: SysCallMapping = {
|
||||
"store.delete": async (_ctx, key: string) => {
|
||||
await db.execute(`DELETE FROM ${tableName} WHERE key = ?`, key);
|
||||
},
|
||||
"store.deletePrefix": async (_ctx, prefix: string) => {
|
||||
await db.execute(
|
||||
`DELETE FROM ${tableName} WHERE key LIKE ?`,
|
||||
`${prefix}%`,
|
||||
);
|
||||
},
|
||||
"store.deleteQuery": async (_ctx, query: Query) => {
|
||||
const { sql, params } = queryToSql(query);
|
||||
await db.execute(`DELETE FROM ${tableName} ${sql}`, ...params);
|
||||
},
|
||||
"store.deleteAll": async () => {
|
||||
await db.execute(`DELETE FROM ${tableName}`);
|
||||
},
|
||||
"store.set": async (_ctx, key: string, value: any) => {
|
||||
await db.execute(
|
||||
`INSERT INTO ${tableName}
|
||||
(key, value)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT(key)
|
||||
DO UPDATE SET value=excluded.value`,
|
||||
key,
|
||||
JSON.stringify(value),
|
||||
);
|
||||
},
|
||||
"store.batchSet": async (_ctx, kvs: KV[]) => {
|
||||
if (kvs.length === 0) {
|
||||
return;
|
||||
}
|
||||
const values = kvs.flatMap((
|
||||
kv,
|
||||
) => [kv.key, JSON.stringify(kv.value)]);
|
||||
await db.execute(
|
||||
`INSERT INTO ${tableName}
|
||||
(key, value)
|
||||
VALUES ${kvs.map((_) => "(?, ?)").join(",")}
|
||||
ON CONFLICT(key)
|
||||
DO UPDATE SET value=excluded.value`,
|
||||
...values,
|
||||
);
|
||||
},
|
||||
"store.batchDelete": async (ctx, keys: string[]) => {
|
||||
for (const key of keys) {
|
||||
await apiObj["store.delete"](ctx, key);
|
||||
}
|
||||
},
|
||||
"store.get": async (_ctx, key: string): Promise<any | null> => {
|
||||
const result = await db.query(
|
||||
`SELECT value FROM ${tableName} WHERE key = ?`,
|
||||
key,
|
||||
);
|
||||
if (result.length) {
|
||||
return JSON.parse(result[0].value);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
"store.has": async (_ctx, key: string): Promise<boolean> => {
|
||||
const result = await db.query(
|
||||
`SELECT count(value) as cnt FROM ${tableName} WHERE key = ?`,
|
||||
key,
|
||||
);
|
||||
return result[0].cnt === 1;
|
||||
},
|
||||
"store.queryPrefix": async (_ctx, prefix: string) => {
|
||||
return (
|
||||
await db.query(
|
||||
`SELECT key, value FROM ${tableName} WHERE key LIKE ?`,
|
||||
`${prefix}%`,
|
||||
)
|
||||
).map(({ key, value }) => ({
|
||||
key,
|
||||
value: JSON.parse(value),
|
||||
}));
|
||||
},
|
||||
"store.query": async (_ctx, query: Query) => {
|
||||
const { sql, params } = queryToSql(query);
|
||||
return (
|
||||
await db.query(
|
||||
`SELECT key, value FROM ${tableName} ${sql}`,
|
||||
...params,
|
||||
)
|
||||
).map(({ key, value }: { key: string; value: string }) => ({
|
||||
key,
|
||||
value: JSON.parse(value),
|
||||
}));
|
||||
},
|
||||
};
|
||||
return apiObj;
|
||||
}
|
||||
Reference in New Issue
Block a user