neil/silverbullet
1
0
Fork 0
Code Activity

Enabled back-end running of functions, moved indexing to server.

Browse Source
This commit is contained in:
Zef Hemel
2022-03-18 14:59:04 +01:00
parent 97b9d3a3e0
commit 8bff6d98e1
27 changed files with 538 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/package.json
+3
View File
@@ -24,9 +24,12 @@
"socket.io": "^4.4.1",
"socket.io-client": "^4.4.1",
"typescript": "^4.6.2",
"vm2": "^3.9.9",
"yargs": "^17.3.1"
},
"devDependencies": {
"@parcel/optimizer-data-url": "2.3.2",
"@parcel/transformer-inline-string": "2.3.2",
"@types/cors": "^2.8.12",
"@types/express": "^4.17.13",
"jest": "^27.5.1",
server/src/api.test.ts
+1 -1
View File
@@ -3,7 +3,7 @@ import { test, expect, beforeAll, afterAll, describe } from "@jest/globals";
import { createServer } from "http";
import { io as Client } from "socket.io-client";
import { Server } from "socket.io";
import { SocketServer } from "./api";
import { SocketServer } from "./api_server";
import * as path from "path";
import * as fs from "fs";
server/src/api.ts → server/src/api_server.ts
+32 -13
View File
@@ -3,6 +3,11 @@ import { Page } from "./types";
import * as path from "path";
import { IndexApi } from "./index_api";
import { PageApi } from "./page_api";
import { System } from "../../plugbox/src/runtime";
import { createSandbox } from "../../plugbox/src/node_sandbox";
import { NuggetHook } from "../../webapp/src/types";
import corePlug from "../../webapp/src/generated/core.plug.json";
import pageIndexSyscalls from "./syscalls/page_index";
export class ClientConnection {
openPages = new Set<string>();
@@ -15,27 +20,42 @@ export interface ApiProvider {
}
export class SocketServer {
rootPath: string;
openPages = new Map<string, Page>();
connectedSockets = new Set<Socket>();
serverSocket: Server;
private openPages = new Map<string, Page>();
private connectedSockets = new Set<Socket>();
private apis = new Map<string, ApiProvider>();
readonly rootPath: string;
private serverSocket: Server;
system: System<NuggetHook>;
constructor(rootPath: string, serverSocket: Server) {
this.rootPath = path.resolve(rootPath);
this.serverSocket = serverSocket;
this.system = new System<NuggetHook>();
}
async registerApi(name: string, apiProvider: ApiProvider) {
await apiProvider.init();
this.apis.set(name, apiProvider);
}
constructor(rootPath: string, serverSocket: Server) {
this.rootPath = path.resolve(rootPath);
this.serverSocket = serverSocket;
}
public async init() {
await this.registerApi("index", new IndexApi(this.rootPath));
const indexApi = new IndexApi(this.rootPath);
await this.registerApi("index", indexApi);
this.system.registerSyscalls(pageIndexSyscalls(indexApi.db));
await this.registerApi(
"page",
new PageApi(this.rootPath, this.connectedSockets)
new PageApi(
this.rootPath,
this.connectedSockets,
this.openPages,
this.system
)
);
let plug = await this.system.load(
"core",
corePlug,
createSandbox(this.system)
);
this.serverSocket.on("connection", (socket) => {
@@ -51,9 +71,8 @@ export class SocketServer {
});
socket.on("closePage", (pageName: string) => {
console.log("Closing page", pageName);
clientConn.openPages.delete(pageName);
disconnectPageSocket(pageName);
clientConn.openPages.delete(pageName);
});
const onCall = (
server/src/index_api.ts
+12 -45
View File
@@ -1,6 +1,7 @@
import { ApiProvider, ClientConnection } from "./api";
import { ApiProvider, ClientConnection } from "./api_server";
import knex, { Knex } from "knex";
import path from "path";
import pageIndexSyscalls from "./syscalls/page_index";
type IndexItem = {
page: string;
@@ -10,6 +11,7 @@ type IndexItem = {
export class IndexApi implements ApiProvider {
db: Knex;
constructor(rootPath: string) {
this.db = knex({
client: "better-sqlite3",
@@ -33,12 +35,13 @@ export class IndexApi implements ApiProvider {
}
api() {
const syscalls = pageIndexSyscalls(this.db);
return {
clearPageIndexForPage: async (
clientConn: ClientConnection,
page: string
) => {
await this.db<IndexItem>("page_index").where({ page }).del();
return syscalls["indexer.clearPageIndexForPage"](page);
},
set: async (
clientConn: ClientConnection,
@@ -46,77 +49,41 @@ export class IndexApi implements ApiProvider {
key: string,
value: any
) => {
let changed = await this.db<IndexItem>("page_index")
.where({ page, key })
.update("value", JSON.stringify(value));
if (changed === 0) {
await this.db<IndexItem>("page_index").insert({
page,
key,
value: JSON.stringify(value),
});
}
return syscalls["indexer.set"](page, key, value);
},
get: async (clientConn: ClientConnection, page: string, key: string) => {
let result = await this.db<IndexItem>("page_index")
.where({ page, key })
.select("value");
if (result.length) {
return JSON.parse(result[0].value);
} else {
return null;
}
return syscalls["indexer.get"](page, key);
},
delete: async (
clientConn: ClientConnection,
page: string,
key: string
) => {
await this.db<IndexItem>("page_index").where({ page, key }).del();
return syscalls["indexer.delete"](page, key);
},
scanPrefixForPage: async (
clientConn: ClientConnection,
page: string,
prefix: string
) => {
return (
await this.db<IndexItem>("page_index")
.where({ page })
.andWhereLike("key", `${prefix}%`)
.select("page", "key", "value")
).map(({ page, key, value }) => ({
page,
key,
value: JSON.parse(value),
}));
return syscalls["indexer.scanPrefixForPage"](page, prefix);
},
scanPrefixGlobal: async (
clientConn: ClientConnection,
prefix: string
) => {
return (
await this.db<IndexItem>("page_index")
.andWhereLike("key", `${prefix}%`)
.select("page", "key", "value")
).map(({ page, key, value }) => ({
page,
key,
value: JSON.parse(value),
}));
return syscalls["indexer.scanPrefixGlobal"](prefix);
},
deletePrefixForPage: async (
clientConn: ClientConnection,
page: string,
prefix: string
) => {
return this.db<IndexItem>("page_index")
.where({ page })
.andWhereLike("key", `${prefix}%`)
.del();
return syscalls["indexer.deletePrefixForPage"](page, prefix);
},
clearPageIndex: async (clientConn: ClientConnection) => {
return this.db<IndexItem>("page_index").del();
return syscalls["indexer.clearPageIndex"]();
},
};
}
server/src/page_api.ts
+25 -3
View File
@@ -1,7 +1,7 @@
import { ClientPageState, Page, PageMeta } from "./types";
import { ChangeSet } from "@codemirror/state";
import { Update } from "@codemirror/collab";
import { ApiProvider, ClientConnection } from "./api";
import { ApiProvider, ClientConnection } from "./api_server";
import { Socket } from "socket.io";
import { DiskStorage } from "./disk_storage";
import { safeRun } from "./util";
@@ -9,17 +9,27 @@ import fs from "fs";
import path from "path";
import { stat } from "fs/promises";
import { Cursor, cursorEffect } from "../../webapp/src/cursorEffect";
import { System } from "../../plugbox/src/runtime";
import { NuggetHook } from "../../webapp/src/types";
export class PageApi implements ApiProvider {
openPages = new Map<string, Page>();
openPages: Map<string, Page>;
pageStore: DiskStorage;
rootPath: string;
connectedSockets: Set<Socket>;
private system: System<NuggetHook>;
constructor(rootPath: string, connectedSockets: Set<Socket>) {
constructor(
rootPath: string,
connectedSockets: Set<Socket>,
openPages: Map<string, Page>,
system: System<NuggetHook>
) {
this.pageStore = new DiskStorage(rootPath);
this.rootPath = rootPath;
this.openPages = openPages;
this.connectedSockets = connectedSockets;
this.system = system;
}
async init(): Promise<void> {
@@ -45,6 +55,7 @@ export class PageApi implements ApiProvider {
}
disconnectClient(client: ClientPageState, page: Page) {
console.log("Disconnecting client");
page.clientStates.delete(client);
if (page.clientStates.size === 0) {
console.log("No more clients for", page.name, "flushing");
@@ -178,6 +189,12 @@ export class PageApi implements ApiProvider {
textChanged = true;
}
}
console.log(
"New version",
page.version,
"Updates buffered:",
page.updates.length
);
if (textChanged) {
if (page.saveTimer) {
@@ -185,6 +202,11 @@ export class PageApi implements ApiProvider {
}
page.saveTimer = setTimeout(() => {
console.log("This is the time to index a page");
this.system.dispatchEvent("page:index", {
name: pageName,
text: page!.text.sliceString(0),
});
this.flushPageToDisk(pageName, page!);
}, 1000);
}
server/src/server.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import express from "express";
import { readFile } from "fs/promises";
import http from "http";
import { Server } from "socket.io";
import { SocketServer } from "./api";
import { SocketServer } from "./api_server";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
server/src/syscalls/page_index.ts
+82
View File
@@ -0,0 +1,82 @@
import { Knex } from "knex";
type IndexItem = {
page: string;
key: string;
value: any;
};
export type KV = {
key: string;
value: any;
};
export default function (db: Knex) {
const setter = async (page: string, key: string, value: any) => {
let changed = await db<IndexItem>("page_index")
.where({ page, key })
.update("value", JSON.stringify(value));
if (changed === 0) {
await db<IndexItem>("page_index").insert({
page,
key,
value: JSON.stringify(value),
});
}
};
return {
"indexer.clearPageIndexForPage": async (page: string) => {
await db<IndexItem>("page_index").where({ page }).del();
},
"indexer.set": setter,
"indexer.batchSet": async (page: string, kvs: KV[]) => {
for (let { key, value } of kvs) {
await setter(page, key, value);
}
},
"indexer.get": async (page: string, key: string) => {
let result = await db<IndexItem>("page_index")
.where({ page, key })
.select("value");
if (result.length) {
return JSON.parse(result[0].value);
} else {
return null;
}
},
"indexer.delete": async (page: string, key: string) => {
await db<IndexItem>("page_index").where({ page, key }).del();
},
"indexer.scanPrefixForPage": async (page: string, prefix: string) => {
return (
await db<IndexItem>("page_index")
.where({ page })
.andWhereLike("key", `${prefix}%`)
.select("page", "key", "value")
).map(({ page, key, value }) => ({
page,
key,
value: JSON.parse(value),
}));
},
"indexer.scanPrefixGlobal": async (prefix: string) => {
return (
await db<IndexItem>("page_index")
.andWhereLike("key", `${prefix}%`)
.select("page", "key", "value")
).map(({ page, key, value }) => ({
page,
key,
value: JSON.parse(value),
}));
},
"indexer.deletePrefixForPage": async (page: string, prefix: string) => {
return db<IndexItem>("page_index")
.where({ page })
.andWhereLike("key", `${prefix}%`)
.del();
},
"indexer.clearPageIndex": async () => {
return db<IndexItem>("page_index").del();
},
};
}
server/yarn.lock
+41 -1
View File
@@ -706,6 +706,16 @@
cssnano "^5.0.15"
postcss "^8.4.5"
"@parcel/optimizer-data-url@2.3.2":
version "2.3.2"
resolved "https://registry.yarnpkg.com/@parcel/optimizer-data-url/-/optimizer-data-url-2.3.2.tgz#22c2951eb6bda7d7b589c28283d99f9d21dae568"
integrity sha512-q3Y1J3acGPf8yyAhEG+59qey7liB04T/U4i7nmvggDdDVG9S8aYgIeAjsPUKi/9fBoHLn4l8K/sJq3M7FFdcnw==
dependencies:
"@parcel/plugin" "2.3.2"
"@parcel/utils" "2.3.2"
isbinaryfile "^4.0.2"
mime "^2.4.4"
"@parcel/optimizer-htmlnano@2.3.2":
version "2.3.2"
resolved "https://registry.yarnpkg.com/@parcel/optimizer-htmlnano/-/optimizer-htmlnano-2.3.2.tgz#4086736866621182f5dd1a8abe78e9f5764e1a28"
@@ -940,6 +950,13 @@
"@parcel/workers" "2.3.2"
nullthrows "^1.1.1"
"@parcel/transformer-inline-string@2.3.2":
version "2.3.2"
resolved "https://registry.yarnpkg.com/@parcel/transformer-inline-string/-/transformer-inline-string-2.3.2.tgz#bec5d376d00b5c41abf11c8cf8e3d917036c0646"
integrity sha512-nitgU+YHnJpJjdUEyRqXD3DjIAstcdHDwwKgloTfpt7EDe2VspVuWhA054kH75Kn/Tvn4s0G4VGCTpDw5KxzSw==
dependencies:
"@parcel/plugin" "2.3.2"
"@parcel/transformer-js@2.3.2":
version "2.3.2"
resolved "https://registry.yarnpkg.com/@parcel/transformer-js/-/transformer-js-2.3.2.tgz#24bcb488d5f82678343a5630fe4bbe822789ac33"
@@ -1324,12 +1341,17 @@ acorn-walk@^7.1.1:
resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc"
integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==
acorn-walk@^8.2.0:
version "8.2.0"
resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1"
integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==
acorn@^7.1.1:
version "7.4.1"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==
acorn@^8.2.4, acorn@^8.5.0:
acorn@^8.2.4, acorn@^8.5.0, acorn@^8.7.0:
version "8.7.0"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.0.tgz#90951fde0f8f09df93549481e5fc141445b791cf"
integrity sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ==
@@ -2958,6 +2980,11 @@ isarray@~1.0.0:
resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
isbinaryfile@^4.0.2:
version "4.0.8"
resolved "https://registry.yarnpkg.com/isbinaryfile/-/isbinaryfile-4.0.8.tgz#5d34b94865bd4946633ecc78a026fc76c5b11fcf"
integrity sha512-53h6XFniq77YdW+spoRrebh0mnmTxRPTlcuIArO57lmMdq4uBKFKaeTjnb92oYWrSn/LVL+LT+Hap2tFQj8V+w==
isexe@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
@@ -3675,6 +3702,11 @@ mime@1.6.0:
resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
mime@^2.4.4:
version "2.6.0"
resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367"
integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==
mimic-fn@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
@@ -5163,6 +5195,14 @@ vary@^1, vary@~1.1.2:
resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"
integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=
vm2@^3.9.9:
version "3.9.9"
resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.9.tgz#c0507bc5fbb99388fad837d228badaaeb499ddc5"
integrity sha512-xwTm7NLh/uOjARRBs8/95H0e8fT3Ukw5D/JJWhxMbhKzNh1Nu981jQKvkep9iKYNxzlVrdzD0mlBGkDKZWprlw==
dependencies:
acorn "^8.7.0"
acorn-walk "^8.2.0"
w3c-hr-time@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd"