neil/silverbullet
1
0
Fork 0
Code

PlugOS refactor and other tweaks (#631)

Browse Source 
* Prep for in-process plug loading (e.g. for CF workers, Deno Deploy)
* Prototype of fixed in-process loading plugs
* Fix: buttons not to scroll with content
* Better positioning of modal especially on mobile
* Move query caching outside query
* Fix annoying mouse behavior when filter box appears
* Page navigator search tweaks
This commit is contained in:
Zef Hemel 2024-01-15 16:43:12 +01:00 committed by GitHub
parent a9eb252658
commit a2dbf7b3db
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
65 changed files with 591 additions and 617 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cli/plug_run.test.ts
View File

@ -2,9 +2,7 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { compileManifest } from "../plugos/compile.ts";
import { esbuild } from "../plugos/deps.ts";
import { runPlug } from "./plug_run.ts";
import assets from "../dist/plug_asset_bundle.json" assert {
type: "json",
};
import assets from "../dist/plug_asset_bundle.json" with { type: "json" };
import { assertEquals } from "../test_deps.ts";
import { path } from "../common/deps.ts";

2
cmd/plug_run.ts
View File

@ -1,6 +1,6 @@
import { runPlug } from "../cli/plug_run.ts";
import { path } from "../common/deps.ts";
import assets from "../dist/plug_asset_bundle.json" assert {
import assets from "../dist/plug_asset_bundle.json" with {
type: "json",
};
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";

4
cmd/server.ts
View File

@ -1,8 +1,8 @@
import { HttpServer } from "../server/http_server.ts";
import clientAssetBundle from "../dist/client_asset_bundle.json" assert {
import clientAssetBundle from "../dist/client_asset_bundle.json" with {
type: "json",
};
import plugAssetBundle from "../dist/plug_asset_bundle.json" assert {
import plugAssetBundle from "../dist/plug_asset_bundle.json" with {
type: "json",
};
import { AssetBundle, AssetJson } from "../plugos/asset_bundle/bundle.ts";

2
common/space_index.ts
View File

@ -4,7 +4,7 @@ import { System } from "../plugos/system.ts";
const indexVersionKey = ["$indexVersion"];
// Bump this one every time a full reinxex is needed
const desiredIndexVersion = 2;
const desiredIndexVersion = 3;
let indexOngoing = false;

4
common/syscalls/language.ts
View File

@ -16,9 +16,7 @@ export function languageSyscalls(): SysCallMapping {
}
return parse(lang, code);
},
"language.listLanguages": (
_ctx,
): string[] => {
"language.listLanguages": (): string[] => {
return Object.keys(builtinLanguages);
},
};

14
plug-api/lib/async.ts
View File

@ -37,24 +37,24 @@ export class PromiseQueue {
resolve: (value: any) => void;
reject: (error: any) => void;
}[] = [];
private running = false;
private processing = false;
runInQueue(fn: () => Promise<any>): Promise<any> {
return new Promise((resolve, reject) => {
this.queue.push({ fn, resolve, reject });
if (!this.running) {
this.run();
if (!this.processing) {
this.process();
}
});
}
private async run(): Promise<void> {
private async process(): Promise<void> {
if (this.queue.length === 0) {
this.running = false;
this.processing = false;
return;
}
this.running = true;
this.processing = true;
const { fn, resolve, reject } = this.queue.shift()!;
try {
@ -64,7 +64,7 @@ export class PromiseQueue {
reject(error);
}
this.run(); // Continue processing the next promise in the queue
this.process(); // Continue processing the next promise in the queue
}
}

3
plug-api/lib/frontmatter.ts
View File

@ -7,6 +7,7 @@ import {
replaceNodesMatchingAsync,
traverseTreeAsync,
} from "$sb/lib/tree.ts";
import { expandPropertyNames } from "$sb/lib/json.ts";
export type FrontMatter = { tags?: string[] } & Record<string, any>;
@ -116,6 +117,8 @@ export async function extractFrontmatter(
data.tags = [...new Set([...tags.map((t) => t.replace(/^#/, ""))])];
// console.log("Extracted tags", data.tags);
// Expand property names (e.g. "foo.bar" => { foo: { bar: true } })
data = expandPropertyNames(data);
return data;
}

2
common/limited_map.test.ts → plug-api/lib/limited_map.test.ts
View File

@ -1,5 +1,5 @@
import { sleep } from "$sb/lib/async.ts";
import { assertEquals } from "../test_deps.ts";
import { assertEquals } from "../../test_deps.ts";
import { LimitedMap } from "./limited_map.ts";
Deno.test("limited map", async () => {

15
common/limited_map.ts → plug-api/lib/limited_map.ts
View File

@ -1,4 +1,8 @@
type LimitedMapRecord<V> = { value: V; la: number };
type LimitedMapRecord<V> = {
value: V;
la: number;
expTimer?: number;
};
export class LimitedMap<V> {
private map: Map<string, LimitedMapRecord<V>>;
@ -16,8 +20,13 @@ export class LimitedMap<V> {
* @param ttl time to live (in ms)
*/
set(key: string, value: V, ttl?: number) {
const entry: LimitedMapRecord<V> = { value, la: Date.now() };
if (ttl) {
setTimeout(() => {
const existingEntry = this.map.get(key);
if (existingEntry?.expTimer) {
clearTimeout(existingEntry.expTimer);
}
entry.expTimer = setTimeout(() => {
this.map.delete(key);
}, ttl);
}
@ -26,7 +35,7 @@ export class LimitedMap<V> {
const oldestKey = this.getOldestKey();
this.map.delete(oldestKey!);
}
this.map.set(key, { value, la: Date.now() });
this.map.set(key, entry);
}
get(key: string): V | undefined {

17
plug-api/lib/memory_cache.test.ts Normal file
View File

@ -0,0 +1,17 @@
import { sleep } from "$sb/lib/async.ts";
import { ttlCache } from "$sb/lib/memory_cache.ts";
import { assertEquals } from "../../test_deps.ts";
Deno.test("Memory cache", async () => {
let calls = 0;
async function expensiveFunction(key: string) {
calls++;
await sleep(1);
return key;
}
assertEquals("key", await ttlCache("key", expensiveFunction, 0.01));
assertEquals(1, calls);
assertEquals("key", await ttlCache("key", expensiveFunction, 0.01));
assertEquals(1, calls);
await sleep(10);
});

21
plug-api/lib/memory_cache.ts Normal file
View File

@ -0,0 +1,21 @@
import { LimitedMap } from "$sb/lib/limited_map.ts";
const cache = new LimitedMap<any>(50);
export async function ttlCache<K, V>(
key: K,
fn: (key: K) => Promise<V>,
ttlSecs?: number,
): Promise<V> {
if (!ttlSecs) {
return fn(key);
}
const serializedKey = JSON.stringify(key);
const cached = cache.get(serializedKey);
if (cached) {
return cached;
}
const result = await fn(key);
cache.set(serializedKey, result, ttlSecs * 1000);
return result;
}

3
plug-api/plugos-syscall/asset.ts
View File

@ -2,10 +2,11 @@ import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts";
import { syscall } from "./syscall.ts";
export async function readAsset(
plugName: string,
name: string,
encoding: "utf8" | "dataurl" = "utf8",
): Promise<string> {
const dataUrl = await syscall("asset.readAsset", name) as string;
const dataUrl = await syscall("asset.readAsset", plugName, name) as string;
switch (encoding) {
case "utf8":
return new TextDecoder().decode(base64DecodeDataUrl(dataUrl));

5
plug-api/types.ts
View File

@ -73,11 +73,6 @@ export type Query = {
render?: string;
renderAll?: boolean;
distinct?: boolean;
/**
* When set, the DS implementation _may_ cache the result for the given number of seconds.
*/
cacheSecs?: number;
};
export type KvQuery = Omit<Query, "querySource"> & {

7
plugos/hooks/endpoint.test.ts
View File

@ -16,12 +16,7 @@ Deno.test("Run a plugos endpoint server", async () => {
tempDir,
);
await system.load(
new URL(`file://${workerPath}`),
"test",
0,
createSandbox,
);
await system.load("test", createSandbox(new URL(`file://${workerPath}`)));
const app = new Hono();
const port = 3123;

2
plugos/hooks/event.ts
View File

@ -70,7 +70,7 @@ export class EventHook implements Hook<EventHookT> {
}
} catch (e: any) {
console.error(
`Error dispatching event ${eventName} to plug ${plug.name}: ${e.message}`,
`Error dispatching event ${eventName} to ${plug.name}.${name}: ${e.message}`,
);
}
})());

3
plugos/hooks/mq.ts
View File

@ -1,6 +1,5 @@
import { Hook, Manifest } from "../types.ts";
import { System } from "../system.ts";
import { fullQueueName } from "../lib/mq_util.ts";
import { MQMessage } from "$sb/types.ts";
import { MessageQueue } from "../lib/mq.ts";
import { throttle } from "$sb/lib/async.ts";
@ -61,7 +60,7 @@ export class MQHook implements Hook<MQHookT> {
}
const subscriptions = functionDef.mqSubscriptions;
for (const subscriptionDef of subscriptions) {
const queue = fullQueueName(plug.name!, subscriptionDef.queue);
const queue = subscriptionDef.queue;
// console.log("Subscribing to queue", queue);
this.subscriptions.push(
this.mq.subscribe(

2
plugos/lib/datastore.test.ts
View File

@ -7,7 +7,7 @@ import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts";
import { PrefixedKvPrimitives } from "./prefixed_kv_primitives.ts";
async function test(db: KvPrimitives) {
const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), false, {
const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), {
count: (arr: any[]) => arr.length,
});
await datastore.set(["user", "peter"], { name: "Peter" });

27
plugos/lib/datastore.ts
View File

@ -2,18 +2,13 @@ import { applyQueryNoFilterKV, evalQueryExpression } from "$sb/lib/query.ts";
import { FunctionMap, KV, KvKey, KvQuery } from "$sb/types.ts";
import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts";
import { KvPrimitives } from "./kv_primitives.ts";
import { LimitedMap } from "../../common/limited_map.ts";
/**
* This is the data store class you'll actually want to use, wrapping the primitives
* in a more user-friendly way
*/
export class DataStore {
private cache = new LimitedMap<any>(20);
constructor(
readonly kv: KvPrimitives,
private enableCache = false,
private functionMap: FunctionMap = builtinFunctions,
) {
}
@ -63,21 +58,6 @@ export class DataStore {
}
async query<T = any>(query: KvQuery): Promise<KV<T>[]> {
let cacheKey: string | undefined;
const cacheSecs = query.cacheSecs;
// Should we do caching?
if (cacheSecs && this.enableCache) {
// Remove the cacheSecs from the query
query = { ...query, cacheSecs: undefined };
console.log("Going to cache query", query);
cacheKey = JSON.stringify(query);
const cachedResult = this.cache.get(cacheKey);
if (cachedResult) {
// Let's use the cached result
return cachedResult;
}
}
const results: KV<T>[] = [];
let itemCount = 0;
// Accumulate results
@ -104,12 +84,7 @@ export class DataStore {
}
}
// Apply order by, limit, and select
const finalResult = applyQueryNoFilterKV(query, results, this.functionMap);
if (cacheKey) {
// Store in the cache
this.cache.set(cacheKey, finalResult, cacheSecs! * 1000);
}
return finalResult;
return applyQueryNoFilterKV(query, results, this.functionMap);
}
async queryDelete(query: KvQuery): Promise<void> {

7
plugos/lib/mq_util.ts
View File

@ -1,7 +0,0 @@
// Adds a plug name to a queue name if it doesn't already have one.
export function fullQueueName(plugName: string, queueName: string) {
if (queueName.includes(".")) {
return queueName;
}
return plugName + "." + queueName;
}

3
plugos/plug.ts
View File

@ -21,7 +21,6 @@ export class Plug<HookT> {
constructor(
private system: System<HookT>,
public workerUrl: URL | undefined,
readonly name: string,
private hash: number,
private sandboxFactory: SandboxFactory<HookT>,
@ -44,7 +43,7 @@ export class Plug<HookT> {
// Invoke a syscall
syscall(name: string, args: any[]): Promise<any> {
return this.system.syscallWithContext({ plug: this }, name, args);
return this.system.syscall({ plug: this.name }, name, args);
}
/**

44
plugos/runtime.test.ts
View File

@ -1,20 +1,25 @@
import { createSandbox } from "./sandboxes/deno_worker_sandbox.ts";
import { System } from "./system.ts";
import { assertEquals } from "../test_deps.ts";
import { assert, assertEquals } from "../test_deps.ts";
import { compileManifest } from "./compile.ts";
import { esbuild } from "./deps.ts";
import {
createSandbox as createNoSandbox,
runWithSystemLock,
} from "./sandboxes/no_sandbox.ts";
import { sleep } from "$sb/lib/async.ts";
import { SysCallMapping } from "./system.ts";
Deno.test("Run a deno sandbox", async () => {
const system = new System("server");
system.registerSyscalls([], {
addNumbers: (_ctx, a, b) => {
console.log("This is the context", _ctx.plug.name);
return a + b;
},
failingSyscall: () => {
throw new Error("#fail");
},
});
} as SysCallMapping);
system.registerSyscalls(["restricted"], {
restrictedSyscall: () => {
return "restricted";
@ -34,10 +39,8 @@ Deno.test("Run a deno sandbox", async () => {
);
const plug = await system.load(
new URL(`file://${workerPath}`),
"test",
0,
createSandbox,
createSandbox(new URL(`file://${workerPath}`)),
);
assertEquals({
@ -52,12 +55,31 @@ Deno.test("Run a deno sandbox", async () => {
`file://${workerPath}`
);
const plug2 = await system.loadNoSandbox("test", plugExport);
const plug2 = await system.load("test", createNoSandbox(plugExport));
assertEquals({
addedNumbers: 3,
yamlMessage: "hello: world\n",
}, await plug2.invoke("boot", []));
let running = false;
await Promise.all([
runWithSystemLock(system, async () => {
console.log("Starting first run");
running = true;
await sleep(5);
assertEquals({
addedNumbers: 3,
yamlMessage: "hello: world\n",
}, await plug2.invoke("boot", []));
console.log("Done first run");
running = false;
}),
runWithSystemLock(system, async () => {
assert(!running);
console.log("Starting second run");
assertEquals({
addedNumbers: 3,
yamlMessage: "hello: world\n",
}, await plug2.invoke("boot", []));
console.log("Done second run");
}),
]);
await system.unloadAll();

42
plugos/sandboxes/deno_worker_sandbox.ts
View File

@ -1,26 +1,26 @@
import { WorkerSandbox } from "./worker_sandbox.ts";
import { Plug } from "../plug.ts";
import { Sandbox } from "./sandbox.ts";
import type { SandboxFactory } from "./sandbox.ts";
// Uses Deno's permissions to lock the worker down significantly
export function createSandbox<HookT>(plug: Plug<HookT>): Sandbox<HookT> {
return new WorkerSandbox(plug, {
deno: {
permissions: {
// Allow network access
net: true,
// This is required for console logging to work, apparently?
env: true,
// No talking to native code
ffi: false,
// No invocation of shell commands
run: false,
// No read access to the file system
read: false,
// No write access to the file system
write: false,
export function createSandbox<HookT>(workerUrl: URL): SandboxFactory<HookT> {
return (plug) =>
new WorkerSandbox(plug, workerUrl, {
deno: {
permissions: {
// Allow network access
net: true,
// This is required for console logging to work, apparently?
env: true,
// No talking to native code
ffi: false,
// No invocation of shell commands
run: false,
// No read access to the file system
read: false,
// No write access to the file system
write: false,
},
},
},
// Have to do this because the "deno" option is not standard and doesn't typecheck yet
} as any);
// Have to do this because the "deno" option is not standard and doesn't typecheck yet
});
}

114
plugos/sandboxes/no_sandbox.ts
View File

@ -2,6 +2,38 @@ import { PromiseQueue } from "$sb/lib/async.ts";
import { Plug } from "../plug.ts";
import { Sandbox } from "./sandbox.ts";
import { Manifest } from "../types.ts";
import { System } from "../system.ts";
import { SandboxFactory } from "./sandbox.ts";
/**
* This implements a "no sandbox" sandbox that actually runs code the main thread, without any isolation.
* This is useful for (often serverless) environments like CloudFlare workers and Deno Deploy that do not support workers.
* Since these environments often also don't allow dynamic loading (or even eval'ing) of code, plug code needs to be
* imported as a regular ESM module (which is possible).
*
* To make this work, a global `syscall` function needs to be injected into the global scope.
* Since a syscall relies on a System, we need to track the active System in a global variable.
* The issue with this is that it means that only a single System can be active at a given time per JS process.
* To enforce this, we have a runWithSystemLock function that can be used to run code in a System-locked context, effectively queuing the execution of tasks sequentially.
* This isn't great, but it's the best we can do.
*
* Luckily, in the only contexts in which you need to run plugs this way are serverless, where code will be
* run in a bunch of isolates with hopefully low parallelism of requests per isolate.
*/
/**
* A type representing the `plug` export of a plug, used via e.g. `import { plug } from "./some.plug.js`
* Values of this type are passed into the `noSandboxFactory` function when called on a system.load
*/
export type PlugExport = {
manifest: Manifest<any>;
functionMapping: Record<string, (...args: any[]) => any>;
};
// The global variable tracking the currently active system (if any)
let activeSystem:
| System<any>
| undefined;
// We need to hard inject the syscall function into the global scope
declare global {
@ -9,60 +41,76 @@ declare global {
syscall(name: string, ...args: any[]): Promise<any>;
}
}
export type PlugExport<HookT> = {
manifest: Manifest<HookT>;
functionMapping: Record<string, Function>;
};
const functionQueue = new PromiseQueue();
let activePlug: Plug<any> | undefined;
// @ts-ignore: globalThis
globalThis.syscall = (name: string, ...args: any[]): Promise<any> => {
if (!activePlug) {
throw new Error("No active plug");
if (!activeSystem) {
throw new Error(`No currently active system, can't invoke syscall ${name}`);
}
console.log("Calling syscall", name, args);
return activePlug.syscall(name, args);
// Invoke syscall with no active plug set (because we don't know which plug is invoking the syscall)
return activeSystem.syscall({}, name, args);
};
// Global sequential task queue for running tasks in a System-locked context
const taskQueue = new PromiseQueue();
/**
* Schedules a task to run in a System-locked context
* in effect this will ensure only one such context is active at a given time allowing for no parallelism
* @param system to activate while running the task
* @param task callback to run
* @returns the result of the task once it completes
*/
export function runWithSystemLock(
system: System<any>,
task: () => Promise<any>,
): Promise<any> {
return taskQueue.runInQueue(async () => {
// Set the global active system, which is used by the syscall function
activeSystem = system;
try {
// Run the logic, note putting the await here is crucial to make sure the `finally` block runs at the right time
return await task();
} finally {
// And then reset the global active system whether the thing blew up or not
activeSystem = undefined;
}
});
}
/**
* Implements a no-sandbox sandbox that runs code in the main thread
*/
export class NoSandbox<HookT> implements Sandbox<HookT> {
manifest?: Manifest<HookT> | undefined;
manifest: Manifest<HookT>;
constructor(
private plug: Plug<HookT>,
private plugExport: PlugExport<HookT>,
readonly plug: Plug<HookT>,
readonly plugExport: PlugExport,
) {
this.manifest = plugExport.manifest;
plug.manifest = this.manifest;
}
init(): Promise<void> {
// Nothing to do
return Promise.resolve();
}
invoke(name: string, args: any[]): Promise<any> {
activePlug = this.plug;
return functionQueue.runInQueue(async () => {
try {
const fn = this.plugExport.functionMapping[name];
if (!fn) {
throw new Error(`Function not loaded: ${name}`);
}
return await fn(...args);
} finally {
activePlug = undefined;
}
});
const fn = this.plugExport.functionMapping[name];
if (!fn) {
throw new Error(`Function not defined: ${name}`);
}
return Promise.resolve(fn(...args));
}
stop() {
// Nothing to do
}
}
export function noSandboxFactory<HookT>(
plugExport: PlugExport<HookT>,
): (plug: Plug<HookT>) => Sandbox<HookT> {
return (plug: Plug<HookT>) => new NoSandbox(plug, plugExport);
export function createSandbox<HookT>(
plugExport: PlugExport,
): SandboxFactory<HookT> {
return (plug: Plug<any>) => new NoSandbox(plug, plugExport);
}

6
plugos/sandboxes/web_worker_sandbox.ts
View File

@ -1,7 +1,7 @@
import { WorkerSandbox } from "./worker_sandbox.ts";
import type { Plug } from "../plug.ts";
import { Sandbox } from "./sandbox.ts";
import type { SandboxFactory } from "./sandbox.ts";
export function createSandbox<HookT>(plug: Plug<HookT>): Sandbox<HookT> {
return new WorkerSandbox(plug);
export function createSandbox<HookT>(workerUrl: URL): SandboxFactory<HookT> {
return (plug: Plug<HookT>) => new WorkerSandbox(plug, workerUrl);
}

3
plugos/sandboxes/worker_sandbox.ts
View File

@ -21,6 +21,7 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
constructor(
readonly plug: Plug<HookT>,
public workerUrl: URL,
private workerOptions = {},
) {
}
@ -35,7 +36,7 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
console.warn("Double init of sandbox, ignoring");
return Promise.resolve();
}
this.worker = new Worker(this.plug.workerUrl!, {
this.worker = new Worker(this.workerUrl, {
...this.workerOptions,
type: "module",
});

7
plugos/syscalls/asset.ts
View File

@ -2,11 +2,8 @@ import { SysCallMapping, System } from "../system.ts";
export default function assetSyscalls(system: System<any>): SysCallMapping {
return {
"asset.readAsset": (
ctx,
name: string,
): string => {
return system.loadedPlugs.get(ctx.plug.name!)!.assets!.readFileAsDataUrl(
"asset.readAsset": (_ctx, plugName: string, name: string): string => {
return system.loadedPlugs.get(plugName)!.assets!.readFileAsDataUrl(
name,
);
},

64
plugos/syscalls/datastore.ts
View File

@ -1,75 +1,47 @@
import { KV, KvKey, KvQuery } from "$sb/types.ts";
import type { DataStore } from "../lib/datastore.ts";
import type { SyscallContext, SysCallMapping } from "../system.ts";
import type { SysCallMapping } from "../system.ts";
/**
* Exposes the datastore API to plugs, but scoping everything to a prefix based on the plug's name
* @param ds the datastore to wrap
* @param prefix prefix to scope all keys to to which the plug name will be appended
*/
export function dataStoreSyscalls(
ds: DataStore,
prefix: KvKey = ["ds"],
): SysCallMapping {
export function dataStoreSyscalls(ds: DataStore): SysCallMapping {
return {
"datastore.delete": (ctx, key: KvKey) => {
return ds.delete(applyPrefix(ctx, key));
"datastore.delete": (_ctx, key: KvKey) => {
return ds.delete(key);
},
"datastore.set": (ctx, key: KvKey, value: any) => {
return ds.set(applyPrefix(ctx, key), value);
"datastore.set": (_ctx, key: KvKey, value: any) => {
return ds.set(key, value);
},
"datastore.batchSet": (ctx, kvs: KV[]) => {
return ds.batchSet(
kvs.map((kv) => ({ key: applyPrefix(ctx, kv.key), value: kv.value })),
);
"datastore.batchSet": (_ctx, kvs: KV[]) => {
return ds.batchSet(kvs);
},
"datastore.batchDelete": (ctx, keys: KvKey[]) => {
return ds.batchDelete(keys.map((k) => applyPrefix(ctx, k)));
"datastore.batchDelete": (_ctx, keys: KvKey[]) => {
return ds.batchDelete(keys);
},
"datastore.batchGet": (
ctx,
_ctx,
keys: KvKey[],
): Promise<(any | undefined)[]> => {
return ds.batchGet(keys.map((k) => applyPrefix(ctx, k)));
return ds.batchGet(keys);
},
"datastore.get": (ctx, key: KvKey): Promise<any | null> => {
return ds.get(applyPrefix(ctx, key));
"datastore.get": (_ctx, key: KvKey): Promise<any | null> => {
return ds.get(key);
},
"datastore.query": async (
ctx,
query: KvQuery,
): Promise<KV[]> => {
return (await ds.query({
...query,
prefix: applyPrefix(ctx, query.prefix),
})).map((kv) => ({
key: stripPrefix(kv.key),
value: kv.value,
}));
"datastore.query": async (_ctx, query: KvQuery): Promise<KV[]> => {
return (await ds.query(query));
},
"datastore.queryDelete": (
ctx,
query: KvQuery,
): Promise<void> => {
return ds.queryDelete({
...query,
prefix: applyPrefix(ctx, query.prefix),
});
"datastore.queryDelete": (_ctx, query: KvQuery): Promise<void> => {
return ds.queryDelete(query);
},
};
function applyPrefix(ctx: SyscallContext, key?: KvKey): KvKey {
return [...prefix, ctx.plug.name!, ...(key ? key : [])];
}
function stripPrefix(key: KvKey): KvKey {
return key.slice(prefix.length + 1);
}
}

4
plugos/syscalls/fs.deno.test.ts
View File

@ -3,15 +3,13 @@ import { assert } from "../../test_deps.ts";
import { path } from "../deps.ts";
import fileSystemSyscalls from "./fs.deno.ts";
const fakeCtx = {} as any;
Deno.test("Test FS operations", async () => {
const thisFolder = path.resolve(
path.dirname(new URL(import.meta.url).pathname),
);
const syscalls = fileSystemSyscalls(thisFolder);
const allFiles: FileMeta[] = await syscalls["fs.listFiles"](
fakeCtx,
{},
thisFolder,
true,
);

21
plugos/syscalls/mq.ts
View File

@ -1,25 +1,24 @@
import { SysCallMapping } from "../system.ts";
import { fullQueueName } from "../lib/mq_util.ts";
import { MessageQueue } from "../lib/mq.ts";
export function mqSyscalls(
mq: MessageQueue,
): SysCallMapping {
return {
"mq.send": (ctx, queue: string, body: any) => {
return mq.send(fullQueueName(ctx.plug.name!, queue), body);
"mq.send": (_ctx, queue: string, body: any) => {
return mq.send(queue, body);
},
"mq.batchSend": (ctx, queue: string, bodies: any[]) => {
return mq.batchSend(fullQueueName(ctx.plug.name!, queue), bodies);
"mq.batchSend": (_ctx, queue: string, bodies: any[]) => {
return mq.batchSend(queue, bodies);
},
"mq.ack": (ctx, queue: string, id: string) => {
return mq.ack(fullQueueName(ctx.plug.name!, queue), id);
"mq.ack": (_ctx, queue: string, id: string) => {
return mq.ack(queue, id);
},
"mq.batchAck": (ctx, queue: string, ids: string[]) => {
return mq.batchAck(fullQueueName(ctx.plug.name!, queue), ids);
"mq.batchAck": (_ctx, queue: string, ids: string[]) => {
return mq.batchAck(queue, ids);
},
"mq.getQueueStats": (ctx, queue: string) => {
return mq.getQueueStats(fullQueueName(ctx.plug.name!, queue));
"mq.getQueueStats": (_ctx, queue: string) => {
return mq.getQueueStats(queue);
},
};
}

20
plugos/syscalls/transport.ts
View File

@ -1,20 +0,0 @@
import { SyscallContext, SysCallMapping } from "../system.ts";
export function proxySyscalls(
names: string[],
transportCall: (
ctx: SyscallContext,
name: string,
...args: any[]
) => Promise<any>,
): SysCallMapping {
const syscalls: SysCallMapping = {};
for (const name of names) {
syscalls[name] = (ctx, ...args: any[]) => {
return transportCall(ctx, name, ...args);
};
}
return syscalls;
}

83
plugos/system.ts
View File

@ -3,7 +3,6 @@ import { EventEmitter } from "./event.ts";
import type { SandboxFactory } from "./sandboxes/sandbox.ts";
import { Plug } from "./plug.ts";
import { InMemoryManifestCache, ManifestCache } from "./manifest_cache.ts";
import { noSandboxFactory, PlugExport } from "./sandboxes/no_sandbox.ts";
export interface SysCallMapping {
[key: string]: (ctx: SyscallContext, ...args: any) => Promise<any> | any;
@ -16,11 +15,12 @@ export type SystemEvents<HookT> = {
// Passed to every syscall, allows to pass in additional context that the syscall may use
export type SyscallContext = {
plug: Plug<any>;
// This is the plug that is invoking the syscall,
// which may be undefined where this cannot be determined (e.g. when running in a NoSandbox)
plug?: string;
};
type SyscallSignature = (
ctx: SyscallContext,
...args: any[]
) => Promise<any> | any;
@ -75,7 +75,11 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
}
}
syscallWithContext(
localSyscall(name: string, args: any): Promise<any> {
return this.syscall({}, name, args);
}
syscall(
ctx: SyscallContext,
name: string,
args: any[],
@ -84,36 +88,29 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
if (!syscall) {
throw Error(`Unregistered syscall ${name}`);
}
for (const permission of syscall.requiredPermissions) {
if (!ctx.plug) {
throw Error(`Syscall ${name} requires permission and no plug is set`);
if (ctx.plug) {
// Only when running in a plug context do we check permissions
const plug = this.loadedPlugs.get(ctx.plug!);
if (!plug) {
throw new Error(
`Plug ${ctx.plug} not found while attempting to invoke ${name}}`,
);
}
if (!ctx.plug.grantedPermissions.includes(permission)) {
throw Error(`Missing permission '${permission}' for syscall ${name}`);
for (const permission of syscall.requiredPermissions) {
if (!plug.grantedPermissions.includes(permission)) {
throw Error(`Missing permission '${permission}' for syscall ${name}`);
}
}
}
return Promise.resolve(syscall.callback(ctx, ...args));
}
localSyscall(
contextPlugName: string,
syscallName: string,
args: any[],
): Promise<any> {
return this.syscallWithContext(
{ plug: this.plugs.get(contextPlugName)! },
syscallName,
args,
);
}
async load(
workerUrl: URL,
name: string,
hash: number,
sandboxFactory: SandboxFactory<HookT>,
hash = -1,
): Promise<Plug<HookT>> {
const plug = new Plug(this, workerUrl, name, hash, sandboxFactory);
const plug = new Plug(this, name, hash, sandboxFactory);
// Wait for worker to boot, and pass back its manifest
await plug.ready;
@ -139,44 +136,6 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
return plug;
}
/**
* Loads a plug without a sandbox, which means it will run in the same context as the caller
* @param name
* @param plugExport extracted via e.g. `import { plug } from "./some.plug.js`
* @returns Plug instance
*/
async loadNoSandbox(
name: string,
plugExport: PlugExport<HookT>,
): Promise<Plug<HookT>> {
const plug = new Plug(
this,
undefined,
name,
-1,
noSandboxFactory(plugExport),
);
const manifest = plugExport.manifest;
// Validate the manifest
let errors: string[] = [];
for (const feature of this.enabledHooks) {
errors = [...errors, ...feature.validateManifest(plug.manifest!)];
}
if (errors.length > 0) {
throw new Error(`Invalid manifest: ${errors.join(", ")}`);
}
if (this.plugs.has(manifest.name)) {
this.unload(manifest.name);
}
console.log("Activated plug without sandbox", manifest.name);
this.plugs.set(manifest.name, plug);
await this.emit("plugLoaded", plug);
return plug;
}
unload(name: string) {
const plug = this.plugs.get(name);
if (!plug) {

28
plugos/worker_runtime.ts
View File

@ -9,6 +9,20 @@ declare global {
function syscall(name: string, ...args: any[]): Promise<any>;
}
// Are we running in a (web) worker?
// Determines if we're running in a web worker environment (Deno or browser)
// - in a browser's main threads, typeof window is "object"
// - in a browser's worker threads, typeof window === "undefined"
// - in Deno's main thread typeof window === "object"
// - in Deno's workers typeof window === "undefined
// - in Cloudflare workers typeof window === "undefined", but typeof globalThis.WebSocketPair is defined
const runningAsWebWorker = typeof window === "undefined" &&
// @ts-ignore: globalThis
typeof globalThis.WebSocketPair === "undefined";
// console.log("Running as web worker:", runningAsWebWorker);
if (typeof Deno === "undefined") {
// @ts-ignore: Deno hack
self.Deno = {
@ -35,13 +49,11 @@ const pendingRequests = new Map<
let syscallReqId = 0;
const workerMode = typeof window === "undefined";
function workerPostMessage(msg: ControllerMessage) {
self.postMessage(msg);
}
if (workerMode) {
if (runningAsWebWorker) {
globalThis.syscall = async (name: string, ...args: any[]) => {
return await new Promise((resolve, reject) => {
syscallReqId++;
@ -61,7 +73,7 @@ export function setupMessageListener(
functionMapping: Record<string, Function>,
manifest: any,
) {
if (!workerMode) {
if (!runningAsWebWorker) {
// Don't do any of this stuff if this is not a web worker
// This caters to the NoSandbox run mode
return;
@ -163,10 +175,10 @@ export async function sandboxFetch(
return syscall("sandboxFetch.fetch", reqInfo, options);
}
// @ts-ignore: monkey patching fetch
globalThis.nativeFetch = globalThis.fetch;
// Monkey patch fetch()
export function monkeyPatchFetch() {
globalThis.nativeFetch = globalThis.fetch;
// @ts-ignore: monkey patching fetch
globalThis.fetch = async function (
reqInfo: RequestInfo,
@ -192,4 +204,6 @@ export function monkeyPatchFetch() {
};
}
monkeyPatchFetch();
if (runningAsWebWorker) {
monkeyPatchFetch();
}

4
plugs/editor/complete.ts
View File

@ -18,9 +18,7 @@ export async function pageComplete(completeEvent: CompleteEvent) {
completeEvent.linePrefix,
);
const tagToQuery = isInTemplateContext ? "template" : "page";
let allPages: PageMeta[] = await queryObjects<PageMeta>(tagToQuery, {
cacheSecs: 5,
});
let allPages: PageMeta[] = await queryObjects<PageMeta>(tagToQuery, {}, 5);
const prefix = match[1];
if (prefix.startsWith("!")) {
// Federation prefix, let's first see if we're matching anything from federation that is locally synced

2
plugs/editor/editor.ts
View File

@ -31,6 +31,6 @@ export async function moveToPosCommand() {
await editor.moveCursor(pos);
}
export async function customFlashMessage(_ctx: any, message: string) {
export async function customFlashMessage(_def: any, message: string) {
await editor.flashNotification(message);
}

5
plugs/index/anchor.ts
View File

@ -43,10 +43,7 @@ export async function anchorComplete(completeEvent: CompleteEvent) {
// "bare" anchor, match any page for completion purposes
filter = undefined;
}
const allAnchors = await queryObjects<AnchorObject>("anchor", {
filter,
cacheSecs: 5,
});
const allAnchors = await queryObjects<AnchorObject>("anchor", { filter }, 5);
return {
from: completeEvent.pos - match[1].length,
options: allAnchors.map((a) => ({

16
plugs/index/api.ts
View File

@ -3,6 +3,7 @@ import { KV, KvKey, KvQuery, ObjectQuery, ObjectValue } from "$sb/types.ts";
import { QueryProviderEvent } from "$sb/app_event.ts";
import { builtins } from "./builtins.ts";
import { AttributeObject, determineType } from "./attributes.ts";
import { ttlCache } from "$sb/lib/memory_cache.ts";
const indexKey = "idx";
const pageKey = "ridx";
@ -159,15 +160,18 @@ function cleanKey(ref: string, page: string) {
}
}
export async function queryObjects<T>(
export function queryObjects<T>(
tag: string,
query: ObjectQuery,
ttlSecs?: number,
): Promise<ObjectValue<T>[]> {
return (await datastore.query({
...query,
prefix: [indexKey, tag],
distinct: true,
})).map(({ value }) => value);
return ttlCache(query, async () => {
return (await datastore.query({
...query,
prefix: [indexKey, tag],
distinct: true,
})).map(({ value }) => value);
}, ttlSecs);
}
export async function query(

3
plugs/index/attributes.ts
View File

@ -52,8 +52,7 @@ export async function objectAttributeCompleter(
select: [{ name: "name" }, { name: "attributeType" }, { name: "tag" }, {
name: "readOnly",
}],
cacheSecs: 5,
});
}, 5);
return allAttributes.map((value) => {
return {
name: value.name,

2
plugs/index/command.ts
View File

@ -6,7 +6,7 @@ import { isTemplate } from "$sb/lib/cheap_yaml.ts";
export async function reindexCommand() {
await editor.flashNotification("Performing full page reindex...");
await system.invokeFunction("reindexSpace");
await system.invokeFunction("index.reindexSpace");
await editor.flashNotification("Done with page index!");
}

8
plugs/index/plug_api.ts
View File

@ -1,5 +1,6 @@
import { KV, KvQuery, ObjectQuery, ObjectValue } from "$sb/types.ts";
import { invokeFunction } from "$sb/silverbullet-syscall/system.ts";
import { ttlCache } from "$sb/lib/memory_cache.ts";
export function indexObjects<T>(
page: string,
@ -21,8 +22,13 @@ export function query(
export function queryObjects<T>(
tag: string,
query: ObjectQuery,
ttlSecs?: number,
): Promise<ObjectValue<T>[]> {
return invokeFunction("index.queryObjects", tag, query);
return ttlCache(
query,
() => invokeFunction("index.queryObjects", tag, query),
ttlSecs, // no-op when undefined
);
}
export function getObjectByRef<T>(

3
plugs/index/tags.ts
View File

@ -73,8 +73,7 @@ export async function tagComplete(completeEvent: CompleteEvent) {
filter: ["=", ["attr", "parent"], ["string", parent]],
select: [{ name: "name" }],
distinct: true,
cacheSecs: 5,
});
}, 5);
if (parent === "page") {
// Also add template, even though that would otherwise not appear because has "builtin" as a parent

12
plugs/markdown/markdown_render.test.ts
View File

@ -9,16 +9,16 @@ import { renderMarkdownToHtml } from "./markdown_render.ts";
Deno.test("Markdown render", async () => {
const system = new System<any>("server");
await system.load(
new URL("../../dist_plug_bundle/_plug/editor.plug.js", import.meta.url),
"editor",
0,
createSandbox,
createSandbox(
new URL("../../dist_plug_bundle/_plug/editor.plug.js", import.meta.url),
),
);
await system.load(
new URL("../../dist_plug_bundle/_plug/tasks.plug.js", import.meta.url),
"tasks",
0,
createSandbox,
createSandbox(
new URL("../../dist_plug_bundle/_plug/tasks.plug.js", import.meta.url),
),
);
const lang = buildMarkdown(loadMarkdownExtensions(system));
const testFile = Deno.readTextFileSync(

4
plugs/markdown/preview.ts
View File

@ -11,8 +11,8 @@ export async function updateMarkdownPreview() {
const text = await editor.getText();
const mdTree = await markdown.parseMarkdown(text);
// const cleanMd = await cleanMarkdown(text);
const css = await asset.readAsset("assets/preview.css");
const js = await asset.readAsset("assets/preview.js");
const css = await asset.readAsset("markdown", "assets/preview.css");
const js = await asset.readAsset("markdown", "assets/preview.js");
await expandCodeWidgets(mdTree, currentPage);
const html = renderMarkdownToHtml(mdTree, {

4
plugs/tasks/complete.ts
View File

@ -9,9 +9,7 @@ export async function completeTaskState(completeEvent: CompleteEvent) {
if (!taskMatch) {
return null;
}
const allStates = await queryObjects<TaskStateObject>("taskstate", {
cacheSecs: 5,
});
const allStates = await queryObjects<TaskStateObject>("taskstate", {}, 5);
const states = [...new Set(allStates.map((s) => s.state))];
return {

3
plugs/template/complete.ts
View File

@ -56,8 +56,7 @@ export async function templateSlashComplete(
"boolean",
false,
]]],
cacheSecs: 5,
});
}, 5);
return allTemplates.map((template) => ({
label: template.trigger!,
detail: "template",

4
server/crypto.ts
View File

@ -17,6 +17,7 @@ export class JWTIssuer {
async init(authString: string) {
const [secret] = await this.kv.batchGet([[jwtSecretKey]]);
if (!secret) {
console.log("Generating new JWT secret key");
return this.generateNewKey();
} else {
this.key = await crypto.subtle.importKey(
@ -34,6 +35,9 @@ export class JWTIssuer {
]]);
const newAuthHash = await this.hashSHA256(authString);
if (currentAuthHash && currentAuthHash !== newAuthHash) {
console.log(
"Authentication has changed since last run, so invalidating all existing tokens",
);
// It has, so we need to generate a new key to invalidate all existing tokens
await this.generateNewKey();
}

24
server/http_server.ts
View File

@ -344,6 +344,16 @@ export class HttpServer {
}),
);
// For when the day comes...
// this.app.use("*", async (c, next) => {
// // if (["POST", "PUT", "DELETE"].includes(c.req.method)) {
// const spaceServer = await this.ensureSpaceServer(c.req);
// return runWithSystemLock(spaceServer.system!, async () => {
// await next();
// });
// // }
// });
// File list
this.app.get(
"/index.json",
@ -382,10 +392,10 @@ export class HttpServer {
});
// RPC syscall
this.app.post("/.rpc/:plug/:syscall", async (c) => {
this.app.post("/.rpc/:plugName/:syscall", async (c) => {
const req = c.req;
const plugName = req.param("plug")!;
const syscall = req.param("syscall")!;
const plugName = req.param("plugName")!;
const spaceServer = await this.ensureSpaceServer(req);
const body = await req.json();
try {
@ -394,11 +404,11 @@ export class HttpServer {
}
const args: string[] = body;
try {
const plug = spaceServer.system!.loadedPlugs.get(plugName);
if (!plug) {
throw new Error(`Plug ${plugName} not found`);
}
const result = await plug.syscall(syscall, args);
const result = await spaceServer.system!.syscall(
{ plug: plugName },
syscall,
args,
);
return c.json({
result: result,
});

67
server/server_system.ts
View File

@ -35,6 +35,20 @@ import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
import { ShellBackend } from "./shell_backend.ts";
import { ensureSpaceIndex } from "../common/space_index.ts";
// // Important: load this before the actual plugs
// import {
// createSandbox as noSandboxFactory,
// runWithSystemLock,
// } from "../plugos/sandboxes/no_sandbox.ts";
// // Load list of builtin plugs
// import { plug as plugIndex } from "../dist_plug_bundle/_plug/index.plug.js";
// import { plug as plugFederation } from "../dist_plug_bundle/_plug/federation.plug.js";
// import { plug as plugQuery } from "../dist_plug_bundle/_plug/query.plug.js";
// import { plug as plugSearch } from "../dist_plug_bundle/_plug/search.plug.js";
// import { plug as plugTasks } from "../dist_plug_bundle/_plug/tasks.plug.js";
// import { plug as plugTemplate } from "../dist_plug_bundle/_plug/template.plug.js";
const fileListInterval = 30 * 1000; // 30s
const plugNameExtractRegex = /([^/]+)\.plug\.js$/;
@ -138,28 +152,29 @@ export class ServerSystem {
);
this.listInterval = setInterval(() => {
// runWithSystemLock(this.system, async () => {
// await space.updatePageList();
// });
space.updatePageList().catch(console.error);
}, fileListInterval);
eventHook.addLocalListener("file:changed", (path, localChange) => {
(async () => {
if (!localChange && path.endsWith(".md")) {
const pageName = path.slice(0, -3);
const data = await this.spacePrimitives.readFile(path);
console.log("Outside page change: reindexing", pageName);
// Change made outside of editor, trigger reindex
await eventHook.dispatchEvent("page:index_text", {
name: pageName,
text: new TextDecoder().decode(data.data),
});
}
eventHook.addLocalListener("file:changed", async (path, localChange) => {
if (!localChange && path.endsWith(".md")) {
const pageName = path.slice(0, -3);
const data = await this.spacePrimitives.readFile(path);
console.log("Outside page change: reindexing", pageName);
// Change made outside of editor, trigger reindex
await eventHook.dispatchEvent("page:index_text", {
name: pageName,
text: new TextDecoder().decode(data.data),
});
}
if (path.startsWith("_plug/") && path.endsWith(".plug.js")) {
console.log("Plug updated, reloading:", path);
this.system.unload(path);
await this.loadPlugFromSpace(path);
}
})().catch(console.error);
if (path.startsWith("_plug/") && path.endsWith(".plug.js")) {
console.log("Plug updated, reloading:", path);
this.system.unload(path);
await this.loadPlugFromSpace(path);
}
});
// Ensure a valid index
@ -168,10 +183,19 @@ export class ServerSystem {
await indexPromise;
}
// await runWithSystemLock(this.system, async () => {
await eventHook.dispatchEvent("system:ready");
// });
}
async loadPlugs() {
// await this.system.load("index", noSandboxFactory(plugIndex));
// await this.system.load("federation", noSandboxFactory(plugFederation));
// await this.system.load("query", noSandboxFactory(plugQuery));
// await this.system.load("search", noSandboxFactory(plugSearch));
// await this.system.load("tasks", noSandboxFactory(plugTasks));
// await this.system.load("template", noSandboxFactory(plugTemplate));
for (const { name } of await this.spacePrimitives.fetchFileList()) {
if (plugNameExtractRegex.test(name)) {
await this.loadPlugFromSpace(name);
@ -183,11 +207,12 @@ export class ServerSystem {
const { meta, data } = await this.spacePrimitives.readFile(path);
const plugName = path.match(plugNameExtractRegex)![1];
return this.system.load(
// Base64 encoding this to support `deno compile` mode
new URL(base64EncodedDataUrl("application/javascript", data)),
plugName,
createSandbox(
// Base64 encoding this to support `deno compile` mode
new URL(base64EncodedDataUrl("application/javascript", data)),
),
meta.lastModified,
createSandbox,
);
}

1
server/syscalls/shell.ts
View File

@ -1,4 +1,3 @@
import { shell } from "$sb/syscalls.ts";
import { SysCallMapping } from "../../plugos/system.ts";
import { ShellResponse } from "../../server/rpc.ts";
import { ShellBackend } from "../shell_backend.ts";

10
server/syscalls/space.ts
View File

@ -10,10 +10,7 @@ export function spaceSyscalls(space: Space): SysCallMapping {
"space.listPages": (): Promise<PageMeta[]> => {
return space.fetchPageList();
},
"space.readPage": async (
_ctx,
name: string,
): Promise<string> => {
"space.readPage": async (_ctx, name: string): Promise<string> => {
return (await space.readPage(name)).text;
},
"space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
@ -35,10 +32,7 @@ export function spaceSyscalls(space: Space): SysCallMapping {
"space.listAttachments": async (): Promise<AttachmentMeta[]> => {
return await space.fetchAttachmentList();
},
"space.readAttachment": async (
_ctx,
name: string,
): Promise<Uint8Array> => {
"space.readAttachment": async (_ctx, name: string): Promise<Uint8Array> => {
return (await space.readAttachment(name)).data;
},
"space.getAttachmentMeta": async (

13
web/client.ts
View File

@ -46,11 +46,12 @@ import { DataStoreSpacePrimitives } from "../common/spaces/datastore_space_primi
import {
EncryptedSpacePrimitives,
} from "../common/spaces/encrypted_space_primitives.ts";
import { LimitedMap } from "../common/limited_map.ts";
import {
ensureSpaceIndex,
markFullSpaceIndexComplete,
} from "../common/space_index.ts";
import { LimitedMap } from "$sb/lib/limited_map.ts";
const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
const autoSaveInterval = 1000;
@ -135,7 +136,7 @@ export class Client {
`${this.dbPrefix}_state`,
);
await stateKvPrimitives.init();
this.stateDataStore = new DataStore(stateKvPrimitives, true);
this.stateDataStore = new DataStore(stateKvPrimitives);
// Setup message queue
this.mq = new DataStoreMQ(this.stateDataStore);
@ -316,9 +317,13 @@ export class Client {
// We're going to look up the anchor through a API invocation
const matchingAnchor = await this.system.system.localSyscall(
"index",
"system.invokeFunction",
["getObjectByRef", pageName, "anchor", `${pageName}$${pos}`],
[
"index.getObjectByRef",
pageName,
"anchor",
`${pageName}$${pos}`,
],
);
if (!matchingAnchor) {

13
web/client_system.ts
View File

@ -133,10 +133,9 @@ export class ClientSystem {
console.log("Plug updated, reloading", plugName, "from", path);
this.system.unload(path);
const plug = await this.system.load(
new URL(`/${path}`, location.href),
plugName,
createSandbox(new URL(`/${path}`, location.href)),
newHash,
createSandbox,
);
if ((plug.manifest! as Manifest).syntax) {
// If there are syntax extensions, rebuild the markdown parser immediately
@ -201,10 +200,9 @@ export class ClientSystem {
try {
const plugName = plugNameExtractRegex.exec(plugMeta.name)![1];
await this.system.load(
new URL(plugMeta.name, location.origin),
plugName,
createSandbox(new URL(plugMeta.name, location.origin)),
plugMeta.lastModified,
createSandbox,
);
} catch (e: any) {
console.error(
@ -228,14 +226,13 @@ export class ClientSystem {
}
localSyscall(name: string, args: any[]) {
return this.system.localSyscall("editor", name, args);
return this.system.localSyscall(name, args);
}
queryObjects<T>(tag: string, query: Query): Promise<T[]> {
return this.system.localSyscall(
"index",
return this.localSyscall(
"system.invokeFunction",
["queryObjects", tag, query],
["index.queryObjects", tag, query],
);
}
}

2
web/cm_plugins/markdown_widget.ts
View File

@ -132,7 +132,7 @@ export class MarkdownWidget extends WidgetType {
buttons.filter((button) => !button.widgetTarget).map((button, idx) =>
`<button data-button="${idx}" title="${button.description}">${button.svg}</button> `
).join("")
}</div>${html}`;
}</div><div class="content">${html}</div>`;
}
private attachListeners(div: HTMLElement, buttons?: CodeWidgetButton[]) {

72
web/components/basic_modals.tsx
View File

@ -19,43 +19,41 @@ export function Prompt({
}) {
const [text, setText] = useState(defaultValue || "");
const returnEl = (
<div className="sb-modal-wrapper">
<div className="sb-modal-box">
<div className="sb-prompt">
<label>{message}</label>
<MiniEditor
text={defaultValue || ""}
vimMode={vimMode}
vimStartInInsertMode={true}
focus={true}
darkMode={darkMode}
completer={completer}
onEnter={(text) => {
callback(text);
return true;
}}
onEscape={() => {
callback();
}}
onChange={(text) => {
setText(text);
}}
/>
<button
onClick={() => {
callback(text);
}}
>
Ok
</button>
<button
onClick={() => {
callback();
}}
>
Cancel
</button>
</div>
<div className="sb-modal-box">
<div className="sb-prompt">
<label>{message}</label>
<MiniEditor
text={defaultValue || ""}
vimMode={vimMode}
vimStartInInsertMode={true}
focus={true}
darkMode={darkMode}
completer={completer}
onEnter={(text) => {
callback(text);
return true;
}}
onEscape={() => {
callback();
}}
onChange={(text) => {
setText(text);
}}
/>
<button
onClick={() => {
callback(text);
}}
>
Ok
</button>
<button
onClick={() => {
callback();
}}
>
Cancel
</button>
</div>
</div>
);

206
web/components/filter.tsx
View File

@ -95,115 +95,115 @@ export function FilterList({
}, []);
const returnEl = (
<div className="sb-modal-wrapper">
<div className="sb-modal-box">
<div
className="sb-header"
onClick={(e) => {
// Allow tapping/clicking the header without closing it
e.stopPropagation();
<div className="sb-modal-box">
<div
className="sb-header"
onClick={(e) => {
// Allow tapping/clicking the header without closing it
e.stopPropagation();
}}
>
<label>{label}</label>
<MiniEditor
text={text}
vimMode={vimMode}
vimStartInInsertMode={true}
focus={true}
darkMode={darkMode}
completer={completer}
placeholderText={placeholder}
onEnter={(_newText, shiftDown) => {
onSelect(
shiftDown ? { name: text } : matchingOptions[selectedOption],
);
return true;
}}
>
<label>{label}</label>
<MiniEditor
text={text}
vimMode={vimMode}
vimStartInInsertMode={true}
focus={true}
darkMode={darkMode}
completer={completer}
placeholderText={placeholder}
onEnter={(_newText, shiftDown) => {
onSelect(
shiftDown ? { name: text } : matchingOptions[selectedOption],
);
return true;
}}
onEscape={() => {
onSelect(undefined);
}}
onChange={(text) => {
setText(text);
}}
onKeyUp={(view, e) => {
// This event is triggered after the key has been processed by CM already
if (onKeyPress) {
onKeyPress(e.key, view.state.sliceDoc());
}
return false;
}}
onKeyDown={(view, e) => {
switch (e.key) {
case "ArrowUp":
setSelectionOption(Math.max(0, selectedOption - 1));
onEscape={() => {
onSelect(undefined);
}}
onChange={(text) => {
setText(text);
}}
onKeyUp={(view, e) => {
// This event is triggered after the key has been processed by CM already
if (onKeyPress) {
onKeyPress(e.key, view.state.sliceDoc());
}
return false;
}}
onKeyDown={(view, e) => {
switch (e.key) {
case "ArrowUp":
setSelectionOption(Math.max(0, selectedOption - 1));
return true;
case "ArrowDown":
setSelectionOption(
Math.min(matchingOptions.length - 1, selectedOption + 1),
);
return true;
case "PageUp":
setSelectionOption(Math.max(0, selectedOption - 5));
return true;
case "PageDown":
setSelectionOption(Math.max(0, selectedOption + 5));
return true;
case "Home":
setSelectionOption(0);
return true;
case "End":
setSelectionOption(matchingOptions.length - 1);
return true;
case " ": {
const text = view.state.sliceDoc();
if (completePrefix && text === "") {
setText(completePrefix);
// updateFilter(completePrefix);
return true;
case "ArrowDown":
setSelectionOption(
Math.min(matchingOptions.length - 1, selectedOption + 1),
);
return true;
case "PageUp":
setSelectionOption(Math.max(0, selectedOption - 5));
return true;
case "PageDown":
setSelectionOption(Math.max(0, selectedOption + 5));
return true;
case "Home":
setSelectionOption(0);
return true;
case "End":
setSelectionOption(matchingOptions.length - 1);
return true;
case " ": {
const text = view.state.sliceDoc();
if (completePrefix && text === "") {
setText(completePrefix);
// updateFilter(completePrefix);
return true;
}
break;
}
break;
}
return false;
}}
/>
</div>
<div
className="sb-help-text"
dangerouslySetInnerHTML={{ __html: helpText }}
>
</div>
<div className="sb-result-list">
{matchingOptions && matchingOptions.length > 0
? matchingOptions.map((option, idx) => (
<div
key={"" + idx}
ref={selectedOption === idx ? selectedElementRef : undefined}
className={selectedOption === idx
? "sb-selected-option"
: "sb-option"}
onMouseOver={(e) => {
}
return false;
}}
/>
</div>
<div
className="sb-help-text"
dangerouslySetInnerHTML={{ __html: helpText }}
>
</div>
<div className="sb-result-list">
{matchingOptions && matchingOptions.length > 0
? matchingOptions.map((option, idx) => (
<div
key={"" + idx}
ref={selectedOption === idx ? selectedElementRef : undefined}
className={selectedOption === idx
? "sb-selected-option"
: "sb-option"}
onMouseMove={(e) => {
if (selectedOption !== idx) {
setSelectionOption(idx);
}}
onClick={(e) => {
e.stopPropagation();
onSelect(option);
}}
>
{Icon && (
<span className="sb-icon">
<Icon width={16} height={16} />
</span>
)}
<span className="sb-name">
{option.name}
}
}}
onClick={(e) => {
e.stopPropagation();
onSelect(option);
}}
>
{Icon && (
<span className="sb-icon">
<Icon width={16} height={16} />
</span>
{option.hint && <span className="sb-hint">{option.hint}</span>}
<div className="sb-description">{option.description}</div>
</div>
))
: null}
</div>
)}
<span className="sb-name">
{option.name}
</span>
{option.hint && <span className="sb-hint">{option.hint}</span>}
<div className="sb-description">{option.description}</div>
</div>
))
: null}
</div>
</div>
);

7
web/components/fuse_search.ts
View File

@ -28,17 +28,18 @@ export const fuzzySearchAndSort = (
weight: 0.3,
}, {
name: "baseName",
weight: 0.7,
weight: 1,
}, {
name: "displayName",
weight: 0.3,
weight: 0.7,
}, {
name: "aliases",
weight: 0.7,
weight: 0.5,
}],
includeScore: true,
shouldSort: true,
isCaseSensitive: false,
ignoreLocation: true,
threshold: 0.6,
sortFn: (a, b): number => {
if (a.score === b.score) {

2
web/components/top_bar.tsx
View File

@ -142,8 +142,8 @@ export function TopBar({
<button
onClick={(e) => {
e.preventDefault();
actionButton.callback();
e.stopPropagation();
actionButton.callback();
}}
title={actionButton.description}
className={actionButton.class}

3
web/space.ts
View File

@ -4,9 +4,6 @@ import { safeRun } from "../common/util.ts";
import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts";
import { EventHook } from "../plugos/hooks/event.ts";
import { throttle } from "$sb/lib/async.ts";
import { DataStore } from "../plugos/lib/datastore.ts";
import { LimitedMap } from "../common/limited_map.ts";
const pageWatchInterval = 5000;

9
web/styles/editor.scss
View File

@ -428,12 +428,12 @@
margin-top: 10px;
}
.sb-markdown-top-widget:has(*) {
.sb-markdown-top-widget:has(*) .content {
max-height: 500px;
}
@media screen and (max-height: 1000px) {
.sb-markdown-top-widget:has(*) {
.sb-markdown-top-widget:has(*) .content {
max-height: 300px;
}
}
@ -441,13 +441,16 @@
.sb-markdown-widget,
.sb-markdown-top-widget:has(*),
.sb-markdown-bottom-widget:has(*) {
overflow-y: auto;
border: 1px solid var(--editor-widget-background-color);
border-radius: 5px;
white-space: normal;
position: relative;
min-height: 48px;
.content {
overflow-y: auto;
}
ul,
ol {
margin-top: 0;

26
web/styles/modals.scss
View File

@ -1,20 +1,16 @@
.sb-modal-wrapper {
position: absolute;
margin: auto;
max-width: 500px;
height: 600px;
left: 0;
right: 0;
top: 0;
bottom: 0;
max-height: 290px;
z-index: 100;
}
.sb-modal-box {
position: absolute;
// At the toppest of the toppest
z-index: 1000;
top: 60px;
left: 50%;
transform: translateX(-50%);
width: 700px;
max-width: 90%;
border-radius: 8px;
overflow: hidden;
margin: 10px;
.cm-content {
padding: 0;
@ -53,7 +49,7 @@
}
.sb-result-list {
max-height: 216px;
max-height: 250px;
overflow-y: scroll;
.sb-icon {

12
web/syscalls/clientStore.ts
View File

@ -7,14 +7,14 @@ export function clientStoreSyscalls(
prefix: KvKey = ["client"],
): SysCallMapping {
return {
"clientStore.get": (ctx, key: string): Promise<any> => {
return ds.get([...prefix, ctx.plug!.name!, key]);
"clientStore.get": (_ctx, key: string): Promise<any> => {
return ds.get([...prefix, key]);
},
"clientStore.set": (ctx, key: string, val: any): Promise<void> => {
return ds.set([...prefix, ctx.plug!.name!, key], val);
"clientStore.set": (_ctx, key: string, val: any): Promise<void> => {
return ds.set([...prefix, key], val);
},
"clientStore.delete": (ctx, key: string): Promise<void> => {
return ds.delete([...prefix, ctx.plug!.name!, key]);
"clientStore.delete": (_ctx, key: string): Promise<void> => {
return ds.delete([...prefix, key]);
},
};
}

37
web/syscalls/datastore.proxy.ts
View File

@ -1,48 +1,17 @@
import { KvQuery } from "$sb/types.ts";
import { LimitedMap } from "../../common/limited_map.ts";
import { LimitedMap } from "../../plug-api/lib/limited_map.ts";
import type { SysCallMapping } from "../../plugos/system.ts";
import type { Client } from "../client.ts";
import { proxySyscall, proxySyscalls } from "./util.ts";
export function dataStoreProxySyscalls(client: Client): SysCallMapping {
const syscalls = proxySyscalls(client, [
return proxySyscalls(client, [
"datastore.delete",
"datastore.set",
"datastore.batchSet",
"datastore.batchDelete",
"datastore.batchGet",
"datastore.query",
"datastore.get",
]);
// Add a cache for datastore.query
const queryCache = new LimitedMap<any>(5);
syscalls["datastore.query"] = async (ctx, query: KvQuery) => {
let cacheKey: string | undefined;
const cacheSecs = query.cacheSecs;
// Should we do caching?
if (cacheSecs) {
// Remove the cacheSecs from the query
query = { ...query, cacheSecs: undefined };
cacheKey = JSON.stringify(query);
const cachedResult = queryCache.get(cacheKey);
if (cachedResult) {
// Let's use the cached result
return cachedResult;
}
}
const result = await proxySyscall(
ctx,
client.httpSpacePrimitives,
"datastore.query",
[
query,
],
);
if (cacheKey) {
// Store in the cache
queryCache.set(cacheKey, result, cacheSecs! * 1000);
}
return result;
};
return syscalls;
}

5
web/syscalls/editor.ts
View File

@ -220,10 +220,7 @@ export function editorSyscalls(editor: Client): SysCallMapping {
): Promise<string | undefined> => {
return editor.prompt(message, defaultValue);
},
"editor.confirm": (
_ctx,
message: string,
): Promise<boolean> => {
"editor.confirm": (_ctx, message: string): Promise<boolean> => {
return editor.confirm(message);
},
"editor.getUiOption": (_ctx, key: string): any => {

10
web/syscalls/space.ts
View File

@ -7,10 +7,7 @@ export function spaceSyscalls(editor: Client): SysCallMapping {
"space.listPages": (): Promise<PageMeta[]> => {
return editor.space.fetchPageList();
},
"space.readPage": async (
_ctx,
name: string,
): Promise<string> => {
"space.readPage": async (_ctx, name: string): Promise<string> => {
return (await editor.space.readPage(name)).text;
},
"space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
@ -39,10 +36,7 @@ export function spaceSyscalls(editor: Client): SysCallMapping {
"space.listAttachments": async (): Promise<AttachmentMeta[]> => {
return await editor.space.fetchAttachmentList();
},
"space.readAttachment": async (
_ctx,
name: string,
): Promise<Uint8Array> => {
"space.readAttachment": async (_ctx, name: string): Promise<Uint8Array> => {
return (await editor.space.readAttachment(name)).data;
},
"space.getAttachmentMeta": async (

31
web/syscalls/system.ts
View File

@ -1,4 +1,3 @@
import type { Plug } from "../../plugos/plug.ts";
import { SysCallMapping, System } from "../../plugos/system.ts";
import type { Client } from "../client.ts";
import { CommandDef } from "../hooks/command.ts";
@ -11,30 +10,20 @@ export function systemSyscalls(
const api: SysCallMapping = {
"system.invokeFunction": (
ctx,
name: string,
fullName: string, // plug.function
...args: any[]
) => {
if (name === "server" || name === "client") {
// Backwards compatibility mode (previously there was an 'env' argument)
name = args[0];
args = args.slice(1);
const [plugName, functionName] = fullName.split(".");
if (!plugName || !functionName) {
throw Error(`Invalid function name ${fullName}`);
}
let plug: Plug<any> | undefined = ctx.plug;
const fullName = name;
// console.log("Invoking function", fullName, "on plug", plug);
if (name.includes(".")) {
// plug name in the name
const [plugName, functionName] = name.split(".");
plug = system.loadedPlugs.get(plugName);
if (!plug) {
throw Error(`Plug ${plugName} not found`);
}
name = functionName;
const plug = system.loadedPlugs.get(plugName);
if (!plug) {
throw Error(`Plug ${plugName} not found`);
}
const functionDef = plug?.manifest!.functions[name];
const functionDef = plug.manifest!.functions[functionName];
if (!functionDef) {
throw Error(`Function ${name} not found`);
throw Error(`Function ${functionName} not found`);
}
if (
client && functionDef.env && system.env &&
@ -48,7 +37,7 @@ export function systemSyscalls(
[fullName, ...args],
);
}
return plug.invoke(name, args);
return plug.invoke(functionName, args);
},
"system.invokeCommand": (_ctx, name: string, args?: string[]) => {
if (!client) {

5
web/syscalls/util.ts
View File

@ -18,8 +18,11 @@ export async function proxySyscall(
name: string,
args: any[],
): Promise<any> {
if (!ctx.plug) {
throw new Error(`Cannot proxy ${name} syscall without plug context`);
}
const resp = await httpSpacePrimitives.authenticatedFetch(
`${httpSpacePrimitives.url}/.rpc/${ctx.plug.name}/${name}`,
`${httpSpacePrimitives.url}/.rpc/${ctx.plug}/${name}`,
{
method: "POST",
body: JSON.stringify(args),