diff --git a/package-lock.json b/package-lock.json index e2bb9ca..ecea16f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2627,6 +2627,7 @@ "node_modules/@vscode/sqlite3": { "version": "5.0.8", "dev": true, + "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { "node-addon-api": "^4.2.0" @@ -9771,7 +9772,7 @@ "@codemirror/stream-parser": "^0.19.9", "@jest/globals": "^27.5.1", "@lezer/markdown": "^0.15.0", - "@silverbulletmd/web": "^0.0.1", + "@silverbulletmd/web": "^0.0.2", "better-sqlite3": "^7.5.0", "body-parser": "^1.19.2", "buffer": "^6.0.3", @@ -9815,7 +9816,7 @@ }, "packages/web": { "name": "@silverbulletmd/web", - "version": "0.0.1", + "version": "0.0.2", "license": "MIT", "dependencies": { "@codemirror/autocomplete": "^0.19.1", @@ -11351,7 +11352,7 @@ "@parcel/transformer-sass": "2.3.2", "@parcel/transformer-webmanifest": "2.3.2", "@parcel/validator-typescript": "2.3.2", - "@silverbulletmd/web": "^0.0.1", + "@silverbulletmd/web": "^0.0.2", "@types/cors": "^2.8.12", "@types/events": "^3.0.0", "@types/express": "^4.17.13", diff --git a/package.json b/package.json index 721c353..171525c 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "@lezer/common": "https://github.com/zefhemel/common.git#046c880d1fcab713cadad327a5b7d8bb5de6522c" }, "scripts": { - "watch": "rm -rf .parcel-cache && parcel watch packages/{web,server,plugos} desktop", + "watch": "rm -rf .parcel-cache && parcel watch --no-hmr packages/{web,server,plugos} desktop", "clean": "rm -rf .parcel-cache packages/*/dist", "build": "parcel build packages/{web,server,plugos}", "plugs": "cd packages/plugs && npm run watch", diff --git a/packages/common/spaces/http_space_primitives.ts b/packages/common/spaces/http_space_primitives.ts index 15a6e74..9bcb75d 100644 --- a/packages/common/spaces/http_space_primitives.ts +++ b/packages/common/spaces/http_space_primitives.ts @@ -119,7 +119,11 @@ export class HttpSpacePrimitives implements SpacePrimitives { if (req.headers.get("Content-length") === "0") { return; } - return await req.json(); + if (req.headers.get("Content-type") === "application/json") { + return await req.json(); + } else { + return await req.text(); + } } async getPageMeta(name: string): Promise { diff --git a/packages/plugos/bin/plugos-bundle.ts b/packages/plugos/bin/plugos-bundle.ts index e66982c..9a80b04 100755 --- a/packages/plugos/bin/plugos-bundle.ts +++ b/packages/plugos/bin/plugos-bundle.ts @@ -1,6 +1,5 @@ #!/usr/bin/env node -import esbuild from "esbuild"; import { readFile, unlink, watch, writeFile } from "fs/promises"; import path from "path"; @@ -9,53 +8,7 @@ import { hideBin } from "yargs/helpers"; import { Manifest } from "../types"; import YAML from "yaml"; import { mkdirSync } from "fs"; - -async function compile( - filePath: string, - functionName: string, - debug: boolean, - excludeModules: string[], - meta = false -) { - let outFile = "_out.tmp"; - let inFile = filePath; - - if (functionName) { - // Generate a new file importing just this one function and exporting it - inFile = "_in.js"; - await writeFile( - inFile, - `import {${functionName}} from "./${filePath}";export default ${functionName};` - ); - } - - // TODO: Figure out how to make source maps work correctly with eval() code - let result = await esbuild.build({ - entryPoints: [inFile], - bundle: true, - format: "iife", - globalName: "mod", - platform: "browser", - sourcemap: false, //sourceMap ? "inline" : false, - minify: !debug, - outfile: outFile, - metafile: true, - external: excludeModules, - }); - - if (meta) { - let text = await esbuild.analyzeMetafile(result.metafile); - console.log("Bundle info for", functionName, text); - } - - let jsCode = (await readFile(outFile)).toString(); - await unlink(outFile); - if (inFile !== filePath) { - await unlink(inFile); - } - return `(() => { ${jsCode} - return mod;})()`; -} +import { compile } from "../compile"; async function bundle( manifestPath: string, @@ -155,8 +108,8 @@ async function run() { })) { if ( filename.endsWith(".plug.yaml") || - filename.endsWith(".ts") || - (filename.endsWith(".js") && !filename.endsWith("_in.js")) + filename.endsWith(".js") || + (filename.endsWith(".ts") && !filename.endsWith("_in.ts")) ) { console.log("Change detected", eventType, filename); await buildAll(); diff --git a/packages/plugos/compile.ts b/packages/plugos/compile.ts new file mode 100644 index 0000000..dbc2ad4 --- /dev/null +++ b/packages/plugos/compile.ts @@ -0,0 +1,51 @@ +import esbuild from "esbuild"; +import { readFile, unlink, writeFile } from "fs/promises"; +import path from "path"; + +export async function compile( + filePath: string, + functionName: string = "", + debug: boolean = false, + excludeModules: string[] = [], + meta = false +): Promise { + let outFile = path.join(path.dirname(filePath), "_out.tmp"); + let inFile = filePath; + + if (functionName) { + // Generate a new file importing just this one function and exporting it + inFile = "_in.ts"; + await writeFile( + inFile, + `import {${functionName}} from "./${filePath}";export default ${functionName};` + ); + } + + // TODO: Figure out how to make source maps work correctly with eval() code + let result = await esbuild.build({ + entryPoints: [path.basename(inFile)], + bundle: true, + format: "iife", + globalName: "mod", + platform: "browser", + sourcemap: false, //sourceMap ? "inline" : false, + minify: !debug, + outfile: outFile, + metafile: true, + external: excludeModules, + absWorkingDir: path.resolve(path.dirname(inFile)), + }); + + if (meta) { + let text = await esbuild.analyzeMetafile(result.metafile); + console.log("Bundle info for", functionName, text); + } + + let jsCode = (await readFile(outFile)).toString(); + await unlink(outFile); + if (inFile !== filePath) { + await unlink(inFile); + } + return `(() => { ${jsCode} + return mod;})()`; +} diff --git a/packages/plugos/syscalls/esbuild.ts b/packages/plugos/syscalls/esbuild.ts new file mode 100644 index 0000000..786b966 --- /dev/null +++ b/packages/plugos/syscalls/esbuild.ts @@ -0,0 +1,44 @@ +import { compile } from "../compile"; +import { SysCallMapping } from "../system"; +import { tmpdir } from "os"; +import { mkdir, rm, symlink, writeFile } from "fs/promises"; +import { nodeModulesDir } from "../environments/node_sandbox"; + +const exposedModules = [ + "@silverbulletmd/plugos-silverbullet-syscall", + "@plugos/plugos-syscall", + "yaml", +]; + +export function esbuildSyscalls(): SysCallMapping { + return { + "esbuild.compile": async ( + ctx, + filename: string, + code: string + ): Promise => { + let tmpDir = `${tmpdir()}/plugos-${Math.random()}`; + await mkdir(tmpDir, { recursive: true }); + + const srcNodeModules = `${nodeModulesDir}/node_modules`; + const targetNodeModules = `${tmpDir}/node_modules`; + + await mkdir(`${targetNodeModules}/@silverbulletmd`, { recursive: true }); + await mkdir(`${targetNodeModules}/@plugos`, { recursive: true }); + for (const exposedModule of exposedModules) { + await symlink( + `${srcNodeModules}/${exposedModule}`, + `${targetNodeModules}/${exposedModule}`, + "dir" + ); + } + + await writeFile(`${tmpDir}/${filename}`, code); + console.log("Dir", tmpDir); + let jsCode = await compile(`${tmpDir}/${filename}`, "", false, ["yaml"]); + // console.log("JS code", jsCode); + await rm(tmpDir, { recursive: true }); + return jsCode; + }, + }; +} diff --git a/packages/plugos/yarn-error.log b/packages/plugos/yarn-error.log new file mode 100644 index 0000000..dec934c --- /dev/null +++ b/packages/plugos/yarn-error.log @@ -0,0 +1,113 @@ +Arguments: + /Users/zef/.nvm/versions/node/v16.13.2/bin/node /Users/zef/.nvm/versions/node/v16.13.2/bin/yarn add --json @parcel/transformer-babel -D -W + +PATH: + /Users/zef/git/silverbullet/packages/plugos/node_modules/.bin:/Users/zef/git/silverbullet/packages/node_modules/.bin:/Users/zef/git/silverbullet/node_modules/.bin:/Users/zef/git/node_modules/.bin:/Users/zef/node_modules/.bin:/Users/node_modules/.bin:/node_modules/.bin:/Users/zef/.nvm/versions/node/v16.13.2/lib/node_modules/npm/node_modules/@npmcli/run-script/lib/node-gyp-bin:/Users/zef/.nvm/versions/node/v16.13.2/bin:/Library/Frameworks/Python.framework/Versions/2.7/bin:/Users/zef/.local/share/solana/install/active_release/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin:/Users/zef/.nvm/versions/node/v16.13.2/bin:/Library/Frameworks/Python.framework/Versions/2.7/bin:/Users/zef/.local/share/solana/install/active_release/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/Users/zef/.cargo/bin + +Yarn version: + 1.22.18 + +Node version: + 16.13.2 + +Platform: + darwin arm64 + +Trace: + Error: ENOENT: no such file or directory, scandir '/Users/zef/git/silverbullet/node_modules/@plugos-silverbulet-syscall' + +npm manifest: + { + "name": "@plugos/plugos", + "author": { + "name": "Zef Hemel", + "email": "zef@zef.me" + }, + "version": "0.0.1", + "license": "MIT", + "bin": { + "plugos-bundle": "./dist/plugos/plugos-bundle.js", + "plugos-server": "./dist/plugos/plugos-server.js" + }, + "scripts": { + "watch": "rm -rf .parcel-cache && parcel watch", + "build": "parcel build", + "clean": "rm -rf dist", + "test": "jest dist/test" + }, + "targets": { + "plugos": { + "source": [ + "bin/plugos-bundle.ts", + "bin/plugos-server.ts" + ], + "outputFormat": "commonjs", + "isLibrary": true, + "context": "node" + }, + "test": { + "source": [ + "runtime.test.ts", + "hooks/endpoint.test.ts", + "syscalls/store.knex_node.test.ts", + "syscalls/store.dexie_browser.test.ts" + ], + "outputFormat": "commonjs", + "isLibrary": true, + "context": "node" + } + }, + "dependencies": { + "@jest/globals": "^27.5.1", + "@types/cors": "^2.8.12", + "@types/express": "^4.17.13", + "@types/jsonwebtoken": "^8.5.8", + "better-sqlite3": "^7.5.0", + "body-parser": "^1.19.2", + "cors": "^2.8.5", + "dexie": "^3.2.1", + "esbuild": "^0.14.27", + "express": "^4.17.3", + "fake-indexeddb": "^3.1.7", + "jest": "^27.5.1", + "jsonwebtoken": "^8.5.1", + "knex": "^1.0.4", + "node-cron": "^3.0.0", + "node-fetch": "2", + "node-watch": "^0.7.3", + "supertest": "^6.2.2", + "vm2": "^3.9.9", + "ws": "^8.5.0", + "yaml": "^1.10.2", + "yargs": "^17.3.1" + }, + "devDependencies": { + "@lezer/lr": "^0.15.0", + "@parcel/optimizer-data-url": "2.3.2", + "@parcel/packager-raw-url": "2.3.2", + "@parcel/service-worker": "2.3.2", + "@parcel/transformer-inline-string": "2.3.2", + "@parcel/transformer-sass": "2.3.2", + "@parcel/transformer-webmanifest": "2.3.2", + "@parcel/validator-typescript": "2.3.2", + "@types/events": "^3.0.0", + "@types/jest": "^27.4.1", + "@types/node": "^17.0.21", + "@types/node-cron": "^3.0.1", + "@types/node-fetch": "^2.6.1", + "@types/supertest": "^2.0.11", + "@types/yaml": "^1.9.7", + "@vscode/sqlite3": "^5.0.7", + "assert": "^2.0.0", + "events": "^3.3.0", + "parcel": "2.3.2", + "prettier": "^2.5.1", + "typescript": "^4.6.2" + } + } + +yarn manifest: + No manifest + +Lockfile: + No lockfile diff --git a/packages/plugs/draft/draft.plug.yaml b/packages/plugs/draft/draft.plug.yaml deleted file mode 100644 index 6935098..0000000 --- a/packages/plugs/draft/draft.plug.yaml +++ /dev/null @@ -1,5 +0,0 @@ -functions: - toggleDraftMode: - path: "./draft.ts:toggleMode" - command: - name: "Draft: Toggle Mode" diff --git a/packages/plugs/draft/draft.ts b/packages/plugs/draft/draft.ts deleted file mode 100644 index 44ef6ae..0000000 --- a/packages/plugs/draft/draft.ts +++ /dev/null @@ -1,7 +0,0 @@ -import * as clientStore from "@silverbulletmd/plugos-silverbullet-syscall/clientStore"; - -export async function toggleMode() { - let currentValue = !!(await clientStore.get("enableDraftMode")); - console.log("New draft mode", !currentValue); - await clientStore.set("enableDraftMode", !currentValue); -} diff --git a/packages/plugs/plugger/plugger.plug.yaml b/packages/plugs/plugger/plugger.plug.yaml new file mode 100644 index 0000000..6f4202c --- /dev/null +++ b/packages/plugs/plugger/plugger.plug.yaml @@ -0,0 +1,8 @@ +functions: + compile: + path: "./plugger.ts:compileCommand" + command: + name: "Plugger: Compile" + compileJS: + path: "./plugger.ts:compileJS" + env: server \ No newline at end of file diff --git a/packages/plugs/plugger/plugger.ts b/packages/plugs/plugger/plugger.ts new file mode 100644 index 0000000..ea24ac9 --- /dev/null +++ b/packages/plugs/plugger/plugger.ts @@ -0,0 +1,85 @@ +import type { Manifest } from "@silverbulletmd/common/manifest"; +import { + addParentPointers, + collectNodesOfType, + findNodeOfType, +} from "@silverbulletmd/common/tree"; +import { + getCurrentPage, + getText, +} from "@silverbulletmd/plugos-silverbullet-syscall/editor"; +import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown"; +import { writePage } from "@silverbulletmd/plugos-silverbullet-syscall/space"; +import { invokeFunction } from "@silverbulletmd/plugos-silverbullet-syscall/system"; +import YAML from "yaml"; +import { extractMeta } from "../query/data"; + +export async function compileCommand() { + let text = await getText(); + let tree = await parseMarkdown(text); + addParentPointers(tree); + let allHeaders = collectNodesOfType(tree, "ATXHeading2"); + let manifest: Manifest = { + functions: {}, + }; + for (let t of allHeaders) { + let parent = t.parent!; + let headerIdx = parent.children!.indexOf(t); + let headerTitle = t.children![1].text!.trim(); + if (!headerTitle.startsWith("function ")) { + continue; + } + let functionName = headerTitle + .substring("function ".length) + .replace(/[^\w]/g, "_"); + let meta: any; + let code: string | undefined; + let language = "js"; + for (let i = headerIdx + 1; i < parent.children!.length; i++) { + let child = parent.children![i]; + if (child.type === "FencedCode") { + let codeInfo = findNodeOfType(child, "CodeInfo")!.children![0].text!; + let codeText = findNodeOfType(child, "CodeText")!.children![0].text!; + if (codeInfo === "yaml") { + meta = YAML.parse(codeText); + continue; + } + if (codeInfo === "typescript" || codeInfo === "ts") { + language = "ts"; + } + code = codeText; + } + + if (child.type?.startsWith("ATXHeading")) { + break; + } + } + if (code) { + let compiled = await invokeFunction( + "server", + "compileJS", + `file.${language}`, + code + ); + manifest.functions[functionName] = meta; + manifest.functions[functionName].code = compiled; + } + } + + let pageMeta = extractMeta(tree); + + if (pageMeta.name) { + await writePage( + `_plug/${pageMeta.name}`, + JSON.stringify(manifest, null, 2) + ); + console.log("Wrote this plug", manifest); + } +} + +export async function compileJS( + filename: string, + code: string +): Promise { + return self.syscall("esbuild.compile", filename, code); +} diff --git a/packages/server/api_server.ts b/packages/server/api_server.ts index 63c7a3c..f3e3b3a 100644 --- a/packages/server/api_server.ts +++ b/packages/server/api_server.ts @@ -23,6 +23,7 @@ import { jwtSyscalls } from "@plugos/plugos/syscalls/jwt"; import buildMarkdown from "@silverbulletmd/web/parser"; import { loadMarkdownExtensions } from "@silverbulletmd/web/markdown_ext"; import http, { Server } from "http"; +import { esbuildSyscalls } from "@plugos/plugos/syscalls/esbuild"; export class ExpressServer { app: Express; @@ -72,6 +73,7 @@ export class ExpressServer { this.system.registerSyscalls([], spaceSyscalls(this.space)); this.system.registerSyscalls([], eventSyscalls(this.eventHook)); this.system.registerSyscalls([], markdownSyscalls(buildMarkdown([]))); + this.system.registerSyscalls([], esbuildSyscalls()); this.system.registerSyscalls([], jwtSyscalls()); this.system.addHook(new EndpointHook(this.app, "/_/")); @@ -268,12 +270,13 @@ export class ExpressServer { ); // Fallback, serve index.html - let cachedIndex: string | undefined = undefined; + // let cachedIndex: string | undefined = undefined; this.app.get("/*", async (req, res) => { - if (!cachedIndex) { - cachedIndex = await readFile(`${this.distDir}/index.html`, "utf8"); - } - res.status(200).header("Content-Type", "text/html").send(cachedIndex); + // if (!cachedIndex) { + // let cachedIndex = await readFile(`${this.distDir}/index.html`, "utf8"); + // } + res.sendFile(`${this.distDir}/index.html`, {}); + // res.status(200).header("Content-Type", "text/html").send(cachedIndex); }); this.server = http.createServer(this.app); diff --git a/packages/server/server.ts b/packages/server/server.ts index 86ff99d..f05a7be 100755 --- a/packages/server/server.ts +++ b/packages/server/server.ts @@ -5,6 +5,9 @@ import { hideBin } from "yargs/helpers"; import { ExpressServer } from "./api_server"; import { nodeModulesDir } from "@plugos/plugos/environments/node_sandbox"; import { preloadModules } from "@silverbulletmd/common/preload_modules"; +import path from "path"; +import { realpath } from "fs/promises"; +import { realpathSync } from "fs"; let args = yargs(hideBin(process.argv)) .option("port", { @@ -20,7 +23,11 @@ if (!args._.length) { const pagesPath = args._[0] as string; const port = args.port; -const webappDistDir = `${nodeModulesDir}/node_modules/@silverbulletmd/web/dist`; + +const webappDistDir = realpathSync( + `${nodeModulesDir}/node_modules/@silverbulletmd/web/dist` +); +console.log("Webapp dist dir", webappDistDir); const expressServer = new ExpressServer( port, diff --git a/packages/web/boot.ts b/packages/web/boot.ts index 23848ac..5e46068 100644 --- a/packages/web/boot.ts +++ b/packages/web/boot.ts @@ -1,17 +1,16 @@ import { Editor } from "./editor"; -import { safeRun } from "../common/util"; +import { safeRun } from "@silverbulletmd/common/util"; import { Space } from "@silverbulletmd/common/spaces/space"; import { HttpSpacePrimitives } from "@silverbulletmd/common/spaces/http_space_primitives"; // let localSpace = new Space(new IndexedDBSpacePrimitives("pages"), true); // localSpace.watch(); -// @ts-ignore -let isDesktop = typeof window.desktop !== "undefined"; - let serverSpace = new Space(new HttpSpacePrimitives(""), true); serverSpace.watch(); +console.log("Booting..."); + // // @ts-ignore // window.syncer = async () => { // let lastLocalSync = +(localStorage.getItem("lastLocalSync") || "0"), @@ -39,10 +38,13 @@ safeRun(async () => { // @ts-ignore window.editor = editor; -if (!isDesktop) { - navigator.serviceWorker - .register(new URL("service_worker.ts", import.meta.url), { type: "module" }) - .then((r) => { - console.log("Service worker registered", r); - }); -} +// if (!isDesktop) { +// if (localStorage.getItem("disable_sw") !== "true") { +navigator.serviceWorker + .register(new URL("service_worker.ts", import.meta.url), { type: "module" }) + .then((r) => { + console.log("Service worker registered..."); + }); +// } + +// } diff --git a/packages/web/editor.tsx b/packages/web/editor.tsx index 87d1053..151d11d 100644 --- a/packages/web/editor.tsx +++ b/packages/web/editor.tsx @@ -158,12 +158,6 @@ export class Editor { }, 100); this.space.on({ - pageCreated: (meta) => { - console.log("Page created", meta); - }, - pageDeleted: (meta) => { - console.log("Page delete", meta); - }, pageChanged: (meta) => { if (this.currentPage === meta.name) { console.log("Page changed on disk, reloading"); diff --git a/packages/web/service_worker.ts b/packages/web/service_worker.ts index b5387da..7a144f3 100644 --- a/packages/web/service_worker.ts +++ b/packages/web/service_worker.ts @@ -2,19 +2,23 @@ import { manifest, version } from "@parcel/service-worker"; async function install() { const cache = await caches.open(version); - // console.log("Installing", manifest); + console.log("Installing", manifest, "version", version); await cache.addAll(manifest); - // console.log("DOne"); + // @ts-ignore + self.skipWaiting(); // This automatically enables the service worker, preventing from caching stuff forever if there's a page open + console.log("Installed"); } + //@ts-ignore self.addEventListener("install", (e) => e.waitUntil(install())); async function activate() { const keys = await caches.keys(); - // console.log("Activating"); + // console.log("Activating for ", keys, "new version", version); await Promise.all(keys.map((key) => key !== version && caches.delete(key))); // console.log("DOne activating"); } + //@ts-ignore self.addEventListener("activate", (e) => e.waitUntil(activate()));