neil/silverbullet
1
0
Fork 0
Code Activity

SilverBullet pivot to become an offline-first PWA (#403)

Browse Source
This commit is contained in:
Zef Hemel
2023-05-23 20:53:53 +02:00
committed by GitHub
parent b256269897
commit 5f484bed57
389 changed files with 4484 additions and 291129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugos/asset_bundle/builder.ts
+17 -4
View File
@@ -1,4 +1,4 @@
import { globToRegExp, path, walk } from "../deps.ts";
import { globToRegExp, mime, path, walk } from "../deps.ts";
import { AssetBundle } from "./bundle.ts";
export async function bundleAssets(
@@ -23,21 +23,34 @@ export async function bundleAssets(
}
}
if (match) {
bundle.writeFileSync(cleanPath, await Deno.readFile(file.path));
bundle.writeFileSync(
cleanPath,
mime.getType(cleanPath) || "application/octet-stream",
await Deno.readFile(file.path),
);
}
}
return bundle;
}
export async function bundleFolder(rootPath: string, bundlePath: string) {
export async function bundleFolder(
rootPath: string,
bundlePath: string,
) {
const bundle = new AssetBundle();
await Deno.mkdir(path.dirname(bundlePath), { recursive: true });
for await (
const { path: filePath } of walk(rootPath, { includeDirs: false })
) {
console.log("Bundling", filePath);
const stat = await Deno.stat(filePath);
const cleanPath = filePath.substring(`${rootPath}/`.length);
bundle.writeFileSync(cleanPath, await Deno.readFile(filePath));
bundle.writeFileSync(
cleanPath,
mime.getType(filePath) || "application/octet-stream",
await Deno.readFile(filePath),
stat.mtime?.getTime(),
);
}
await Deno.writeTextFile(
bundlePath,
plugos/asset_bundle/bundle.test.ts
+2 -2
View File
@@ -3,14 +3,14 @@ import { assertEquals } from "../../test_deps.ts";
Deno.test("Asset bundle", () => {
const assetBundle = new AssetBundle();
assetBundle.writeTextFileSync("test.txt", "Sup yo");
assetBundle.writeTextFileSync("test.txt", "text/plain", "Sup yo");
assertEquals("text/plain", assetBundle.getMimeType("test.txt"));
assertEquals("Sup yo", assetBundle.readTextFileSync("test.txt"));
const buf = new Uint8Array(3);
buf[0] = 1;
buf[1] = 2;
buf[2] = 3;
assetBundle.writeFileSync("test.bin", buf);
assetBundle.writeFileSync("test.bin", "application/octet-stream", buf);
assertEquals("application/octet-stream", assetBundle.getMimeType("test.bin"));
assertEquals(buf, assetBundle.readFileSync("test.bin"));
});
plugos/asset_bundle/bundle.ts
+31 -12
View File
@@ -1,10 +1,9 @@
import { base64Decode, base64EncodedDataUrl } from "./base64.ts";
import { mime } from "../deps.ts";
type DataUrl = string;
// Mapping from path -> `data:mimetype;base64,base64-encoded-data` strings
export type AssetJson = Record<string, DataUrl>;
export type AssetJson = Record<string, { data: DataUrl; mtime: number }>;
export class AssetBundle {
readonly bundle: AssetJson;
@@ -28,7 +27,7 @@ export class AssetBundle {
if (!content) {
throw new Error(`No such file ${path}`);
}
const data = content.split(",", 2)[1];
const data = content.data.split(",", 2)[1];
return base64Decode(data);
}
@@ -37,7 +36,7 @@ export class AssetBundle {
if (!content) {
throw new Error(`No such file ${path}`);
}
return content;
return content.data;
}
readTextFileSync(
@@ -49,22 +48,42 @@ export class AssetBundle {
getMimeType(
path: string,
): string {
const content = this.bundle[path];
if (!content) {
const entry = this.bundle[path];
if (!entry) {
throw new Error(`No such file ${path}`);
}
return content.split(";")[0].split(":")[1];
return entry.data.split(";")[0].split(":")[1];
}
writeFileSync(path: string, data: Uint8Array) {
getMtime(path: string): number {
const entry = this.bundle[path];
if (!entry) {
throw new Error(`No such file ${path}`);
}
return entry.mtime;
}
writeFileSync(
path: string,
mimeType: string,
data: Uint8Array,
mtime: number = Date.now(),
) {
// Replace \ with / for windows
path = path.replaceAll("\\", "/");
const mimeType = mime.getType(path) || "application/octet-stream";
this.bundle[path] = base64EncodedDataUrl(mimeType, data);
this.bundle[path] = {
data: base64EncodedDataUrl(mimeType, data),
mtime,
};
}
writeTextFileSync(path: string, s: string) {
this.writeFileSync(path, new TextEncoder().encode(s));
writeTextFileSync(
path: string,
mimeType: string,
s: string,
mtime: number = Date.now(),
) {
this.writeFileSync(path, mimeType, new TextEncoder().encode(s), mtime);
}
toJSON(): AssetJson {
plugos/bin/plugos-bundle.ts
-202
View File
@@ -1,202 +0,0 @@
// The recommended way to use this for now is through `silverbullet bundle:build` until
// we fork out PlugOS as a whole
import { Manifest } from "../types.ts";
import { YAML } from "../../common/deps.ts";
import {
compile,
CompileOptions,
esbuild,
sandboxCompileModule,
} from "../compile.ts";
import { cacheDir, flags, path } from "../deps.ts";
import { bundleAssets } from "../asset_bundle/builder.ts";
export async function bundle(
manifestPath: string,
options: CompileOptions = {},
): Promise<Manifest<any>> {
const rootPath = path.dirname(manifestPath);
const manifest = YAML.parse(
await Deno.readTextFile(manifestPath),
) as Manifest<any>;
if (!manifest.name) {
throw new Error(`Missing 'name' in ${manifestPath}`);
}
// Dependencies
for (
const [name, moduleSpec] of Object.entries(manifest.dependencies || {})
) {
manifest.dependencies![name] = await sandboxCompileModule(moduleSpec);
}
// Assets
const assetsBundle = await bundleAssets(
path.resolve(rootPath),
manifest.assets as string[] || [],
);
manifest.assets = assetsBundle.toJSON();
// Imports
// Imports currently only "import" dependencies at this point, importing means: assume they're preloaded so we don't need to bundle them
const plugCache = path.join(cacheDir()!, "plugos-imports");
await Deno.mkdir(plugCache, { recursive: true });
// console.log("Cache dir", plugCache);
const imports: Manifest<any>[] = [];
for (const manifestUrl of manifest.imports || []) {
// Safe file name
const cachedManifestPath = manifestUrl.replaceAll(/[^a-zA-Z0-9]/g, "_");
try {
if (options.reload) {
throw new Error("Forced reload");
}
// Try to just load from the cache
const cachedManifest = JSON.parse(
await Deno.readTextFile(path.join(plugCache, cachedManifestPath)),
) as Manifest<any>;
imports.push(cachedManifest);
} catch {
// Otherwise, download and cache
console.log("Caching plug", manifestUrl, "to", plugCache);
const cachedManifest = await (await fetch(manifestUrl))
.json() as Manifest<any>;
await Deno.writeTextFile(
path.join(plugCache, cachedManifestPath),
JSON.stringify(cachedManifest),
);
imports.push(cachedManifest);
}
}
// Functions
for (const def of Object.values(manifest.functions || {})) {
if (!def.path) {
continue;
}
let jsFunctionName = "default",
filePath: string = def.path;
if (filePath.indexOf(":") !== -1) {
[filePath, jsFunctionName] = filePath.split(":");
}
// Resolve path
filePath = path.join(rootPath, filePath);
def.code = await compile(
filePath,
jsFunctionName,
{
...options,
imports: [
manifest,
...imports,
// This is mostly for testing
...options.imports || [],
],
},
);
delete def.path;
}
return manifest;
}
async function buildManifest(
manifestPath: string,
distPath: string,
options: CompileOptions = {},
) {
const generatedManifest = await bundle(manifestPath, options);
const outFile = manifestPath.substring(
0,
manifestPath.length - path.extname(manifestPath).length,
) + ".json";
const outPath = path.join(distPath, path.basename(outFile));
console.log("Emitting bundle to", outPath);
await Deno.writeTextFile(outPath, JSON.stringify(generatedManifest, null, 2));
return { generatedManifest, outPath };
}
export async function bundleRun(
manifestFiles: string[],
dist: string,
watch: boolean,
options: CompileOptions = {},
) {
let building = false;
async function buildAll() {
if (building) {
return;
}
console.log("Building", manifestFiles);
building = true;
Deno.mkdirSync(dist, { recursive: true });
const startTime = Date.now();
// Build all plugs in parallel
await Promise.all(manifestFiles.map(async (plugManifestPath) => {
const manifestPath = plugManifestPath as string;
try {
await buildManifest(
manifestPath,
dist,
options,
);
} catch (e) {
console.error(`Error building ${manifestPath}:`, e);
}
}));
console.log(`Done building plugs in ${Date.now() - startTime}ms`);
building = false;
}
await buildAll();
if (watch) {
console.log("Watching for changes...");
const watcher = Deno.watchFs(manifestFiles.map((p) => path.dirname(p)));
for await (const event of watcher) {
if (event.paths.length > 0) {
if (event.paths[0].endsWith(".json")) {
continue;
}
}
console.log("Change detected, rebuilding...");
buildAll();
}
}
}
if (import.meta.main) {
const args = flags.parse(Deno.args, {
boolean: ["debug", "watch", "reload", "info"],
string: ["dist", "importmap"],
alias: { w: "watch" },
});
if (args._.length === 0) {
console.log(
"Usage: plugos-bundle [--debug] [--reload] [--dist <path>] [--info] [--importmap import_map.json] [--exclude=package1,package2] <manifest.plug.yaml> <manifest2.plug.yaml> ...",
);
Deno.exit(1);
}
if (!args.dist) {
args.dist = path.resolve(".");
}
await bundleRun(
args._ as string[],
args.dist,
args.watch,
{
debug: args.debug,
reload: args.reload,
info: args.info,
importMap: args.importmap
? new URL(args.importmap, `file://${Deno.cwd()}/`)
: undefined,
},
);
esbuild.stop();
}
plugos/compile.ts
+152 -116
View File
@@ -1,145 +1,181 @@
// import { esbuild } from "../../mod.ts";
import * as esbuildWasm from "https://deno.land/x/esbuild@v0.14.54/wasm.js";
import * as esbuildNative from "https://deno.land/x/esbuild@v0.14.54/mod.js";
import { denoPlugins, esbuild, path, YAML } from "./deps.ts";
export const esbuild: typeof esbuildWasm = Deno.run === undefined
? esbuildWasm
: esbuildNative;
import { path } from "./deps.ts";
import { denoPlugin } from "./forked/esbuild_deno_loader/mod.ts";
import { patchDenoLibJS } from "./hack.ts";
import { bundleAssets } from "./asset_bundle/builder.ts";
import { Manifest } from "./types.ts";
import { version } from "../version.ts";
// const workerRuntimeUrl = new URL("./worker_runtime.ts", import.meta.url);
const workerRuntimeUrl =
`https://deno.land/x/silverbullet@${version}/plugos/worker_runtime.ts`;
export type CompileOptions = {
debug?: boolean;
imports?: Manifest<any>[];
importMap?: URL;
runtimeUrl?: string;
importMap?: string;
// Reload plug import cache
reload?: boolean;
// Print info on bundle size
info?: boolean;
};
function esBuildExternals(imports?: Manifest<any>[]) {
if (!imports) {
return [];
}
const externals: string[] = [];
for (const manifest of imports) {
for (const dep of Object.keys(manifest.dependencies || {})) {
if (!externals.includes(dep)) {
externals.push(dep);
}
}
}
return externals;
}
export async function compile(
filePath: string,
functionName: string | undefined = undefined,
export async function compileManifest(
manifestPath: string,
destPath: string,
options: CompileOptions = {},
): Promise<string> {
const outFile = await Deno.makeTempFile({ suffix: ".js" });
let inFile = filePath;
const rootPath = path.dirname(manifestPath);
const manifest = YAML.parse(
await Deno.readTextFile(manifestPath),
) as Manifest<any>;
if (functionName) {
// Generate a new file importing just this one function and exporting it
inFile = await Deno.makeTempFile({ suffix: ".ts" });
await Deno.writeTextFile(
inFile,
`import {${functionName}} from "file://${
if (!manifest.name) {
throw new Error(`Missing 'name' in ${manifestPath}`);
}
// Assets
const assetsBundle = await bundleAssets(
path.resolve(rootPath),
manifest.assets as string[] || [],
);
manifest.assets = assetsBundle.toJSON();
const jsFile = `
import { setupMessageListener } from "${
options.runtimeUrl || workerRuntimeUrl
}";
// Imports
${
Object.entries(manifest.functions).map(([funcName, def]) => {
if (!def.path) {
return "";
}
let [filePath, jsFunctionName] = def.path.split(":");
// Resolve path
filePath = path.join(rootPath, filePath);
return `import {${jsFunctionName} as ${funcName}} from "file://${
// Replacaing \ with / for Windows
path.resolve(filePath).replaceAll(
"\\",
"\\\\",
)}";export default ${functionName};`,
);
)}";\n`;
}).join("")
}
// console.log("External modules", excludeModules);
// Function mapping
export const functionMapping = {
${
Object.entries(manifest.functions).map(([funcName, def]) => {
if (!def.path) {
return "";
}
return ` ${funcName}: ${funcName},\n`;
}).join("")
}
};
try {
// TODO: Figure out how to make source maps work correctly with eval() code
const result = await esbuild.build({
entryPoints: [path.basename(inFile)],
bundle: true,
format: "iife",
globalName: "mod",
platform: "browser",
sourcemap: false, //debug ? "inline" : false,
minify: !options.debug,
outfile: outFile,
metafile: options.info,
external: esBuildExternals(options.imports),
treeShaking: true,
plugins: [
denoPlugin({
// TODO do this differently
importMapURL: options.importMap ||
new URL("./../import_map.json", import.meta.url),
loader: "native",
}),
],
absWorkingDir: path.resolve(path.dirname(inFile)),
});
const manifest = ${JSON.stringify(manifest, null, 2)};
if (options.info) {
const text = await esbuild.analyzeMetafile(result.metafile!);
console.log("Bundle info for", functionName, text);
setupMessageListener(functionMapping, manifest);
`;
// console.log("Code:", jsFile);
const inFile = await Deno.makeTempFile({ suffix: ".js" });
const outFile = `${destPath}/${manifest.name}.plug.js`;
await Deno.writeTextFile(inFile, jsFile);
const result = await esbuild.build({
entryPoints: [path.basename(inFile)],
bundle: true,
format: "iife",
globalName: "mod",
platform: "browser",
sourcemap: options.debug ? "linked" : false,
minify: !options.debug,
outfile: outFile,
metafile: options.info,
treeShaking: true,
plugins: [
{
name: "json",
setup: (build) =>
build.onLoad({ filter: /\.json$/ }, () => ({ loader: "json" })),
},
...denoPlugins({
// TODO do this differently
importMapURL: options.importMap ||
new URL("../import_map.json", import.meta.url).toString(),
loader: "native",
}),
],
absWorkingDir: path.resolve(path.dirname(inFile)),
});
if (options.info) {
const text = await esbuild.analyzeMetafile(result.metafile!);
console.log("Bundle info for", manifestPath, text);
}
let jsCode = await Deno.readTextFile(outFile);
jsCode = patchDenoLibJS(jsCode);
await Deno.writeTextFile(outFile, jsCode);
console.log(`Plug ${manifest.name} written to ${outFile}.`);
return outFile;
}
export async function compileManifests(
manifestFiles: string[],
dist: string,
watch: boolean,
options: CompileOptions = {},
) {
let building = false;
dist = path.resolve(dist);
async function buildAll() {
if (building) {
return;
}
console.log("Building", manifestFiles);
building = true;
Deno.mkdirSync(dist, { recursive: true });
const startTime = Date.now();
// Build all plugs in parallel
await Promise.all(manifestFiles.map(async (plugManifestPath) => {
const manifestPath = plugManifestPath as string;
try {
await compileManifest(
manifestPath,
dist,
options,
);
} catch (e) {
console.error(`Error building ${manifestPath}:`, e);
}
}));
console.log(`Done building plugs in ${Date.now() - startTime}ms`);
building = false;
}
let jsCode = await Deno.readTextFile(outFile);
jsCode = patchDenoLibJS(jsCode);
await Deno.remove(outFile);
return `(() => { ${jsCode} return mod;})()`;
} finally {
if (inFile !== filePath) {
await Deno.remove(inFile);
await buildAll();
if (watch) {
console.log("Watching for changes...");
const watcher = Deno.watchFs(manifestFiles.map((p) => path.dirname(p)));
for await (const event of watcher) {
if (event.paths.length > 0) {
if (event.paths[0].endsWith(".json")) {
continue;
}
}
console.log("Change detected, rebuilding...");
buildAll();
}
}
}
export async function compileModule(
cwd: string,
moduleName: string,
options: CompileOptions = {},
): Promise<string> {
const inFile = path.resolve(cwd, "_in.ts");
await Deno.writeTextFile(inFile, `export * from "${moduleName}";`);
const code = await compile(inFile, undefined, options);
await Deno.remove(inFile);
return code;
}
export async function sandboxCompile(
filename: string,
code: string,
functionName?: string,
options: CompileOptions = {},
): Promise<string> {
const tmpDir = await Deno.makeTempDir();
await Deno.writeTextFile(`${tmpDir}/${filename}`, code);
const jsCode = await compile(
`${tmpDir}/${filename}`,
functionName,
options,
);
await Deno.remove(tmpDir, { recursive: true });
return jsCode;
}
export async function sandboxCompileModule(
moduleUrl: string,
options: CompileOptions = {},
): Promise<string> {
await Deno.writeTextFile(
"_mod.ts",
`module.exports = require("${moduleUrl}");`,
);
const code = await compile("_mod.ts", undefined, options);
await Deno.remove("_mod.ts");
return code;
export function patchDenoLibJS(code: string): string {
// The Deno std lib has one occurence of a regex that Webkit JS doesn't (yet parse), we'll strip it because it's likely never invoked anyway, YOLO
return code.replaceAll("/(?<=\\n)/", "/()/");
}
plugos/deps.ts
+3 -2
View File
@@ -5,5 +5,6 @@ export { expandGlobSync } from "https://deno.land/std@0.165.0/fs/mod.ts";
export { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
export { default as cacheDir } from "https://deno.land/x/cache_dir@0.2.0/mod.ts";
export * as flags from "https://deno.land/std@0.165.0/flags/mod.ts";
export { CapacitorSQLite } from "https://esm.sh/@capacitor-community/sqlite@4.6.0?external=@capacitor/core";
export * as esbuild from "https://deno.land/x/esbuild@v0.17.18/mod.js";
export { denoPlugins } from "https://deno.land/x/esbuild_deno_loader@0.7.0/mod.ts";
export * as YAML from "https://deno.land/std@0.184.0/yaml/mod.ts";
plugos/environments/custom_logger.ts
-64
View File
@@ -1,64 +0,0 @@
export type LogLevel = "info" | "warn" | "error" | "log";
export class ConsoleLogger {
print: boolean;
callback: (level: LogLevel, entry: string) => void;
constructor(
callback: (level: LogLevel, entry: string) => void,
print: boolean = true,
) {
this.print = print;
this.callback = callback;
}
log(...args: any[]): void {
this.push("log", args);
}
warn(...args: any[]): void {
this.push("warn", args);
}
error(...args: any[]): void {
this.push("error", args);
}
info(...args: any[]): void {
this.push("info", args);
}
push(level: LogLevel, args: any[]) {
this.callback(level, this.logMessage(args));
if (this.print) {
console[level](...args);
}
}
logMessage(values: any[]): string {
const pieces: string[] = [];
for (const val of values) {
switch (typeof val) {
case "string":
case "number":
pieces.push("" + val);
break;
case "undefined":
pieces.push("undefined");
break;
default:
try {
let s = JSON.stringify(val, null, 2);
if (s.length > 500) {
s = s.substring(0, 500) + "...";
}
pieces.push(s);
} catch {
// May be cyclical reference
pieces.push("[circular object]");
}
}
}
return pieces.join(" ");
}
}
plugos/environments/deno_sandbox.ts
+20 -64
View File
@@ -1,69 +1,25 @@
import { safeRun } from "../util.ts";
import { Sandbox } from "../sandbox.ts";
import { WorkerLike } from "./worker.ts";
import { Plug } from "../plug.ts";
import { AssetBundle } from "../asset_bundle/bundle.ts";
class DenoWorkerWrapper implements WorkerLike {
private worker: Worker;
onMessage?: (message: any) => Promise<void>;
ready: Promise<void>;
constructor(worker: Worker) {
this.worker = worker;
this.worker.addEventListener("message", (evt: any) => {
const data = evt.data;
if (!data) return;
safeRun(async () => {
await this.onMessage!(data);
});
});
this.ready = Promise.resolve();
}
postMessage(message: any): void {
this.worker.postMessage(message);
}
terminate() {
return this.worker.terminate();
}
}
import workerBundleJson from "./worker_bundle.json" assert { type: "json" };
const workerBundle = new AssetBundle(workerBundleJson);
export function createSandbox(plug: Plug<any>) {
const workerHref = URL.createObjectURL(
new Blob([
workerBundle.readFileSync("worker.js"),
], {
type: "application/javascript",
}),
);
const worker = new Worker(
workerHref,
{
type: "module",
deno: {
permissions: {
// Disallow network access
net: false,
// This is required for console logging to work, apparently?
env: true,
// No talking to native code
ffi: false,
// No invocation of shell commands
run: false,
// No read access to the file system
read: false,
// No write access to the file system
write: false,
},
// Uses Deno's permissions to lock the worker down significantly
export function createSandbox<HookT>(plug: Plug<HookT>): Sandbox<HookT> {
return new Sandbox(plug, {
deno: {
permissions: {
// Disallow network access
net: false,
// This is required for console logging to work, apparently?
env: true,
// No talking to native code
ffi: false,
// No invocation of shell commands
run: false,
// No read access to the file system
read: false,
// No write access to the file system
write: false,
},
// Have to do this because the "deno" option is not standard and doesn't typecheck yet
} as any,
);
return new Sandbox(plug, new DenoWorkerWrapper(worker));
},
// Have to do this because the "deno" option is not standard and doesn't typecheck yet
} as any);
}
plugos/environments/sandbox_worker.ts
-165
View File
@@ -1,165 +0,0 @@
// IMPORTANT: After modifiying this file, run `deno task generate` in the SB root to regenerate the asset bundle (`worker_bundle.json`), which will be imported for the runtime.
import { safeRun } from "../util.ts";
import { ConsoleLogger } from "./custom_logger.ts";
import type { ControllerMessage, WorkerMessage } from "./worker.ts";
if (typeof Deno === "undefined") {
// @ts-ignore: Deno hack
self.Deno = {
args: [],
// @ts-ignore: Deno hack
build: {
arch: "x86_64",
},
env: {
// @ts-ignore: Deno hack
get() {
},
},
};
}
// deno-lint-ignore ban-types
const loadedFunctions = new Map<string, Function>();
const pendingRequests = new Map<
number,
{
resolve: (result: unknown) => void;
reject: (e: any) => void;
}
>();
function workerPostMessage(msg: ControllerMessage) {
if (typeof window !== "undefined" && window.parent !== window) {
window.parent.postMessage(msg, "*");
} else {
self.postMessage(msg);
}
}
declare global {
function syscall(name: string, ...args: any[]): Promise<any>;
// function require(moduleName: string): any;
}
let syscallReqId = 0;
self.syscall = async (name: string, ...args: any[]) => {
return await new Promise((resolve, reject) => {
syscallReqId++;
pendingRequests.set(syscallReqId, { resolve, reject });
workerPostMessage({
type: "syscall",
id: syscallReqId,
name,
args,
});
});
};
const loadedModules = new Map<string, any>();
// @ts-ignore: global to load dynamic imports
self.require = (moduleName: string): any => {
// console.log("Requiring", moduleName, loadedModules.get(moduleName));
const mod = loadedModules.get(moduleName);
if (!mod) {
throw new Error(
`Dynamically importing non-preloaded library ${moduleName}`,
);
}
return mod;
};
// @ts-ignore: global overwrite on purpose
self.console = new ConsoleLogger((level, message) => {
workerPostMessage({ type: "log", level, message });
}, false);
function wrapScript(code: string) {
return `return (${code})["default"]`;
}
self.addEventListener("message", (event: { data: WorkerMessage }) => {
safeRun(async () => {
const data = event.data;
switch (data.type) {
case "load":
{
const fn2 = new Function(wrapScript(data.code!));
loadedFunctions.set(data.name!, fn2());
workerPostMessage({
type: "inited",
name: data.name,
});
}
break;
case "load-dependency":
{
// console.log("Received dep", data.name);
const fn3 = new Function(`return ${data.code!}`);
const v = fn3();
loadedModules.set(data.name!, v);
// console.log("Dep val", v);
workerPostMessage({
type: "dependency-inited",
name: data.name,
});
}
break;
case "invoke":
{
const fn = loadedFunctions.get(data.name!);
if (!fn) {
throw new Error(`Function not loaded: ${data.name}`);
}
try {
const result = await Promise.resolve(fn(...(data.args || [])));
workerPostMessage({
type: "result",
id: data.id,
result: result,
} as ControllerMessage);
} catch (e: any) {
workerPostMessage({
type: "result",
id: data.id,
error: e.message,
stack: e.stack,
});
// console.error("Error invoking function", data.name, e.message);
// throw e;
}
}
break;
case "syscall-response":
{
const syscallId = data.id!;
const lookup = pendingRequests.get(syscallId);
if (!lookup) {
console.log(
"Current outstanding requests",
pendingRequests,
"looking up",
syscallId,
);
throw Error("Invalid request id");
}
pendingRequests.delete(syscallId);
if (data.error) {
lookup.reject(new Error(data.error));
} else {
lookup.resolve(data.result);
}
}
break;
}
});
});
import { monkeyPatchFetch } from "../../plug-api/plugos-syscall/fetch.ts";
monkeyPatchFetch();
plugos/environments/webworker_sandbox.ts
+2 -37
View File
@@ -1,41 +1,6 @@
import { safeRun } from "../util.ts";
import { Sandbox } from "../sandbox.ts";
import { WorkerLike } from "./worker.ts";
import type { Plug } from "../plug.ts";
class WebWorkerWrapper implements WorkerLike {
private worker: Worker;
onMessage?: (message: any) => Promise<void>;
ready: Promise<void>;
constructor(worker: Worker) {
this.worker = worker;
this.worker.addEventListener("message", (evt: any) => {
let data = evt.data;
if (!data) return;
safeRun(async () => {
await this.onMessage!(data);
});
});
this.ready = Promise.resolve();
}
postMessage(message: any): void {
this.worker.postMessage(message);
}
terminate() {
return this.worker.terminate();
}
}
export function createSandbox(plug: Plug<any>) {
const worker = new Worker(
import.meta.url
? new URL("sandbox_worker.ts", import.meta.url)
: new URL("worker.js", location.origin),
{
type: "module",
},
);
return new Sandbox(plug, new WebWorkerWrapper(worker));
export function createSandbox<HookT>(plug: Plug<HookT>): Sandbox<HookT> {
return new Sandbox(plug);
}
plugos/environments/worker.ts
-45
View File
@@ -1,45 +0,0 @@
import type { LogLevel } from "./custom_logger.ts";
export type ControllerMessageType =
| "inited"
| "dependency-inited"
| "result"
| "syscall"
| "log";
export type ControllerMessage = {
type: ControllerMessageType;
id?: number;
name?: string;
args?: any[];
error?: string;
stack?: string;
level?: LogLevel;
message?: string;
result?: any;
};
export interface WorkerLike {
ready: Promise<void>;
onMessage?: (message: any) => Promise<void>;
postMessage(message: any): void;
terminate(): void;
}
export type WorkerMessageType =
| "load"
| "load-dependency"
| "invoke"
| "syscall-response";
export type WorkerMessage = {
type: WorkerMessageType;
id?: number;
name?: string;
code?: string;
args?: any[];
result?: any;
error?: any;
};
plugos/environments/worker_bundle.json
-3
View File
@@ -1,3 +0,0 @@
{
"worker.js": "data:application/javascript;base64,KCgpID0+IHsgdmFyIG1vZD0oKCk9PntmdW5jdGlvbiBsKHQpe3QoKS5jYXRjaChlPT57Y29uc29sZS5lcnJvcigiQ2F1Z2h0IGVycm9yIixlLm1lc3NhZ2UpfSl9dmFyIGE9Y2xhc3N7Y29uc3RydWN0b3IoZSxuPSEwKXt0aGlzLnByaW50PW4sdGhpcy5jYWxsYmFjaz1lfWxvZyguLi5lKXt0aGlzLnB1c2goImxvZyIsZSl9d2FybiguLi5lKXt0aGlzLnB1c2goIndhcm4iLGUpfWVycm9yKC4uLmUpe3RoaXMucHVzaCgiZXJyb3IiLGUpfWluZm8oLi4uZSl7dGhpcy5wdXNoKCJpbmZvIixlKX1wdXNoKGUsbil7dGhpcy5jYWxsYmFjayhlLHRoaXMubG9nTWVzc2FnZShuKSksdGhpcy5wcmludCYmY29uc29sZVtlXSguLi5uKX1sb2dNZXNzYWdlKGUpe2xldCBuPVtdO2ZvcihsZXQgciBvZiBlKXN3aXRjaCh0eXBlb2Ygcil7Y2FzZSJzdHJpbmciOmNhc2UibnVtYmVyIjpuLnB1c2goIiIrcik7YnJlYWs7Y2FzZSJ1bmRlZmluZWQiOm4ucHVzaCgidW5kZWZpbmVkIik7YnJlYWs7ZGVmYXVsdDp0cnl7bGV0IHM9SlNPTi5zdHJpbmdpZnkocixudWxsLDIpO3MubGVuZ3RoPjUwMCYmKHM9cy5zdWJzdHJpbmcoMCw1MDApKyIuLi4iKSxuLnB1c2gocyl9Y2F0Y2h7bi5wdXNoKCJbY2lyY3VsYXIgb2JqZWN0XSIpfX1yZXR1cm4gbi5qb2luKCIgIil9fTtmdW5jdGlvbiBkKHQpe2xldCBlPWF0b2IodCksbj1lLmxlbmd0aCxyPW5ldyBVaW50OEFycmF5KG4pO2ZvcihsZXQgcz0wO3M8bjtzKyspcltzXT1lLmNoYXJDb2RlQXQocyk7cmV0dXJuIHJ9ZnVuY3Rpb24geSh0LGUpe3JldHVybiBzeXNjYWxsKCJzYW5kYm94RmV0Y2guZmV0Y2giLHQsZSl9ZnVuY3Rpb24gdSgpe2dsb2JhbFRoaXMuZmV0Y2g9YXN5bmMgZnVuY3Rpb24odCxlKXtsZXQgbj1hd2FpdCB5KHQsZSYme21ldGhvZDplLm1ldGhvZCxoZWFkZXJzOmUuaGVhZGVycyxib2R5OmUuYm9keX0pO3JldHVybiBuZXcgUmVzcG9uc2Uobi5iYXNlNjRCb2R5P2Qobi5iYXNlNjRCb2R5KTpudWxsLHtzdGF0dXM6bi5zdGF0dXMsaGVhZGVyczpuLmhlYWRlcnN9KX19dHlwZW9mIERlbm8+InUiJiYoc2VsZi5EZW5vPXthcmdzOltdLGJ1aWxkOnthcmNoOiJ4ODZfNjQifSxlbnY6e2dldCgpe319fSk7dmFyIGc9bmV3IE1hcCxpPW5ldyBNYXA7ZnVuY3Rpb24gbyh0KXt0eXBlb2Ygd2luZG93PCJ1IiYmd2luZG93LnBhcmVudCE9PXdpbmRvdz93aW5kb3cucGFyZW50LnBvc3RNZXNzYWdlKHQsIioiKTpzZWxmLnBvc3RNZXNzYWdlKHQpfXZhciBjPTA7c2VsZi5zeXNjYWxsPWFzeW5jKHQsLi4uZSk9PmF3YWl0IG5ldyBQcm9taXNlKChuLHIpPT57YysrLGkuc2V0KGMse3Jlc29sdmU6bixyZWplY3Q6cn0pLG8oe3R5cGU6InN5c2NhbGwiLGlkOmMsbmFtZTp0LGFyZ3M6ZX0pfSk7dmFyIHA9bmV3IE1hcDtzZWxmLnJlcXVpcmU9dD0+e2xldCBlPXAuZ2V0KHQpO2lmKCFlKXRocm93IG5ldyBFcnJvcihgRHluYW1pY2FsbHkgaW1wb3J0aW5nIG5vbi1wcmVsb2FkZWQgbGlicmFyeSAke3R9YCk7cmV0dXJuIGV9O3NlbGYuY29uc29sZT1uZXcgYSgodCxlKT0+e28oe3R5cGU6ImxvZyIsbGV2ZWw6dCxtZXNzYWdlOmV9KX0sITEpO2Z1bmN0aW9uIGgodCl7cmV0dXJuYHJldHVybiAoJHt0fSlbImRlZmF1bHQiXWB9c2VsZi5hZGRFdmVudExpc3RlbmVyKCJtZXNzYWdlIix0PT57bChhc3luYygpPT57bGV0IGU9dC5kYXRhO3N3aXRjaChlLnR5cGUpe2Nhc2UibG9hZCI6e2xldCBuPW5ldyBGdW5jdGlvbihoKGUuY29kZSkpO2cuc2V0KGUubmFtZSxuKCkpLG8oe3R5cGU6ImluaXRlZCIsbmFtZTplLm5hbWV9KX1icmVhaztjYXNlImxvYWQtZGVwZW5kZW5jeSI6e2xldCByPW5ldyBGdW5jdGlvbihgcmV0dXJuICR7ZS5jb2RlfWApKCk7cC5zZXQoZS5uYW1lLHIpLG8oe3R5cGU6ImRlcGVuZGVuY3ktaW5pdGVkIixuYW1lOmUubmFtZX0pfWJyZWFrO2Nhc2UiaW52b2tlIjp7bGV0IG49Zy5nZXQoZS5uYW1lKTtpZighbil0aHJvdyBuZXcgRXJyb3IoYEZ1bmN0aW9uIG5vdCBsb2FkZWQ6ICR7ZS5uYW1lfWApO3RyeXtsZXQgcj1hd2FpdCBQcm9taXNlLnJlc29sdmUobiguLi5lLmFyZ3N8fFtdKSk7byh7dHlwZToicmVzdWx0IixpZDplLmlkLHJlc3VsdDpyfSl9Y2F0Y2gocil7byh7dHlwZToicmVzdWx0IixpZDplLmlkLGVycm9yOnIubWVzc2FnZSxzdGFjazpyLnN0YWNrfSl9fWJyZWFrO2Nhc2Uic3lzY2FsbC1yZXNwb25zZSI6e2xldCBuPWUuaWQscj1pLmdldChuKTtpZighcil0aHJvdyBjb25zb2xlLmxvZygiQ3VycmVudCBvdXRzdGFuZGluZyByZXF1ZXN0cyIsaSwibG9va2luZyB1cCIsbiksRXJyb3IoIkludmFsaWQgcmVxdWVzdCBpZCIpO2kuZGVsZXRlKG4pLGUuZXJyb3I/ci5yZWplY3QobmV3IEVycm9yKGUuZXJyb3IpKTpyLnJlc29sdmUoZS5yZXN1bHQpfWJyZWFrfX0pfSk7dSgpO30pKCk7CiByZXR1cm4gbW9kO30pKCk="
}
plugos/forked/esbuild_deno_loader/LICENSE
-21
View File
@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2021 Luca Casonato
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
plugos/forked/esbuild_deno_loader/README.md
-21
View File
@@ -1,21 +0,0 @@
# esbuild_deno_loader
Deno module resolution for `esbuild`.
## Example
This example bundles an entrypoint into a single ESM output.
```js
import * as esbuild from "https://deno.land/x/esbuild@v0.14.51/mod.js";
import { denoPlugin } from "https://deno.land/x/esbuild_deno_loader@0.5.2/mod.ts";
await esbuild.build({
plugins: [denoPlugin()],
entryPoints: ["https://deno.land/std@0.150.0/hash/sha1.ts"],
outfile: "./dist/sha1.esm.js",
bundle: true,
format: "esm",
});
esbuild.stop();
```
plugos/forked/esbuild_deno_loader/deps.ts
-13
View File
@@ -1,13 +0,0 @@
import type * as esbuild from "https://deno.land/x/esbuild@v0.14.54/mod.d.ts";
export type { esbuild };
export {
fromFileUrl,
resolve,
toFileUrl,
} from "https://deno.land/std@0.150.0/path/mod.ts";
export { basename, extname } from "https://deno.land/std@0.165.0/path/mod.ts";
export {
resolveImportMap,
resolveModuleSpecifier,
} from "https://deno.land/x/importmap@0.2.1/mod.ts";
export type { ImportMap } from "https://deno.land/x/importmap@0.2.1/mod.ts";
plugos/forked/esbuild_deno_loader/examples/bundle.ts
-11
View File
@@ -1,11 +0,0 @@
import * as esbuild from "https://deno.land/x/esbuild@v0.14.51/mod.js";
import { denoPlugin } from "https://deno.land/x/esbuild_deno_loader@0.5.2/mod.ts";
await esbuild.build({
plugins: [denoPlugin()],
entryPoints: ["https://deno.land/std@0.150.0/hash/sha1.ts"],
outfile: "./dist/sha1.esm.js",
bundle: true,
format: "esm",
});
esbuild.stop();
plugos/forked/esbuild_deno_loader/makefile
-11
View File
@@ -1,11 +0,0 @@
test:
deno test -A
lint:
deno lint
fmt:
deno fmt
fmt/check:
deno fmt --check
plugos/forked/esbuild_deno_loader/mod.ts
-116
View File
@@ -1,116 +0,0 @@
import {
esbuild,
ImportMap,
resolveImportMap,
resolveModuleSpecifier,
toFileUrl,
} from "./deps.ts";
import { load as nativeLoad } from "./src/native_loader.ts";
import { load as portableLoad } from "./src/portable_loader.ts";
import { ModuleEntry } from "./src/deno.ts";
export interface DenoPluginOptions {
/**
* Specify the URL to an import map to use when resolving import specifiers.
* The URL must be fetchable with `fetch`.
*/
importMapURL?: URL;
/**
* Specify which loader to use. By default this will use the `native` loader,
* unless `Deno.run` is not available.
*
* - `native`: Shells out to the Deno execuatble under the hood to load
* files. Requires --allow-read and --allow-run.
* - `portable`: Do module downloading and caching with only Web APIs.
* Requires --allow-net.
*/
loader?: "native" | "portable";
}
/** The default loader to use. */
export const DEFAULT_LOADER: "native" | "portable" =
typeof Deno.run === "function" ? "native" : "portable";
export function denoPlugin(options: DenoPluginOptions = {}): esbuild.Plugin {
const loader = options.loader ?? DEFAULT_LOADER;
return {
name: "deno",
setup(build) {
const infoCache = new Map<string, ModuleEntry>();
let importMap: ImportMap | null = null;
build.onStart(async function onStart() {
if (options.importMapURL !== undefined) {
const resp = await fetch(options.importMapURL.href);
const txt = await resp.text();
importMap = resolveImportMap(JSON.parse(txt), options.importMapURL);
} else {
importMap = null;
}
});
build.onResolve(
{ filter: /.*/ },
function onResolve(
args: esbuild.OnResolveArgs,
): esbuild.OnResolveResult | null | undefined {
// console.log("To resolve", args.path);
const resolveDir = args.resolveDir
? `${toFileUrl(args.resolveDir).href}/`
: "";
const referrer = args.importer || resolveDir;
let resolved: URL;
if (importMap !== null) {
const res = resolveModuleSpecifier(
args.path,
importMap,
new URL(referrer) || undefined,
);
resolved = new URL(res);
} else {
resolved = new URL(args.path, referrer);
}
// console.log("Resolved", resolved.href);
if (build.initialOptions.external) {
for (const external of build.initialOptions.external) {
if (resolved.href.startsWith(external)) {
// console.log("Got external", args.path, resolved.href);
return { path: resolved.href, external: true };
}
}
}
const href = resolved.href;
// Don't use the deno loader for any of the specific loader file extensions
const loaderExts = Object.keys(build.initialOptions.loader || {});
for (const ext of loaderExts) {
if (href.endsWith(ext)) {
console.log("Skipping", href);
return {
path: resolved.href.substring("file://".length),
};
}
}
return { path: resolved.href, namespace: "deno" };
},
);
build.onLoad(
{ filter: /.*/ },
function onLoad(
args: esbuild.OnLoadArgs,
): Promise<esbuild.OnLoadResult | null> {
if (args.path.endsWith(".css")) {
return Promise.resolve(null);
}
const url = new URL(args.path);
switch (loader) {
case "native":
return nativeLoad(infoCache, url, options);
case "portable":
return portableLoad(url, options);
}
},
);
},
};
}
plugos/forked/esbuild_deno_loader/src/deno.ts
-89
View File
@@ -1,89 +0,0 @@
// Lifted from https://raw.githubusercontent.com/denoland/deno_graph/89affe43c9d3d5c9165c8089687c107d53ed8fe1/lib/media_type.ts
export type MediaType =
| "JavaScript"
| "Mjs"
| "Cjs"
| "JSX"
| "TypeScript"
| "Mts"
| "Cts"
| "Dts"
| "Dmts"
| "Dcts"
| "TSX"
| "Json"
| "Wasm"
| "TsBuildInfo"
| "SourceMap"
| "Unknown";
export interface InfoOutput {
roots: string[];
modules: ModuleEntry[];
redirects: Record<string, string>;
}
export interface ModuleEntry {
specifier: string;
size: number;
mediaType?: MediaType;
local?: string;
checksum?: string;
emit?: string;
map?: string;
error?: string;
}
interface DenoInfoOptions {
importMap?: string;
}
let tempDir: null | string;
export async function info(
specifier: URL,
options: DenoInfoOptions,
): Promise<InfoOutput> {
const cmd = [
Deno.execPath(),
"info",
"--json",
];
if (options.importMap !== undefined) {
cmd.push("--import-map", options.importMap);
}
cmd.push(specifier.href);
if (!tempDir) {
tempDir = Deno.makeTempDirSync();
}
let proc;
try {
proc = Deno.run({
cmd,
stdout: "piped",
cwd: tempDir,
});
const raw = await proc.output();
const status = await proc.status();
if (!status.success) {
throw new Error(`Failed to call 'deno info' on '${specifier.href}'`);
}
const txt = new TextDecoder().decode(raw);
return JSON.parse(txt);
} finally {
try {
proc?.stdout.close();
} catch (err) {
if (err instanceof Deno.errors.BadResource) {
// ignore the error
} else {
// deno-lint-ignore no-unsafe-finally
throw err;
}
}
proc?.close();
}
}
plugos/forked/esbuild_deno_loader/src/native_loader.ts
-65
View File
@@ -1,65 +0,0 @@
import { esbuild, fromFileUrl } from "../deps.ts";
import * as deno from "./deno.ts";
import { mediaTypeToLoader, transformRawIntoContent } from "./shared.ts";
export interface LoadOptions {
importMapURL?: URL;
}
export async function load(
infoCache: Map<string, deno.ModuleEntry>,
url: URL,
options: LoadOptions,
): Promise<esbuild.OnLoadResult | null> {
switch (url.protocol) {
case "http:":
case "https:":
case "data:":
return await loadFromCLI(infoCache, url, options);
case "file:": {
const res = await loadFromCLI(infoCache, url, options);
res.watchFiles = [fromFileUrl(url.href)];
return res;
}
}
return null;
}
async function loadFromCLI(
infoCache: Map<string, deno.ModuleEntry>,
specifier: URL,
options: LoadOptions,
): Promise<esbuild.OnLoadResult> {
const specifierRaw = specifier.href;
if (!infoCache.has(specifierRaw)) {
const { modules, redirects } = await deno.info(specifier, {
importMap: options.importMapURL?.href,
});
for (const module of modules) {
infoCache.set(module.specifier, module);
}
for (const [specifier, redirect] of Object.entries(redirects)) {
const redirected = infoCache.get(redirect);
if (!redirected) {
throw new TypeError("Unreachable.");
}
infoCache.set(specifier, redirected);
}
}
const module = infoCache.get(specifierRaw);
if (!module) {
throw new TypeError("Unreachable.");
}
if (module.error) throw new Error(module.error);
if (!module.local) throw new Error("Module not downloaded yet.");
const mediaType = module.mediaType ?? "Unknown";
const loader = mediaTypeToLoader(mediaType);
const raw = await Deno.readFile(module.local);
const contents = transformRawIntoContent(raw, mediaType);
return { contents, loader };
}
plugos/forked/esbuild_deno_loader/src/portable_loader.ts
-194
View File
@@ -1,194 +0,0 @@
import { esbuild, extname, fromFileUrl } from "../deps.ts";
import * as deno from "./deno.ts";
import { mediaTypeToLoader, transformRawIntoContent } from "./shared.ts";
export interface LoadOptions {
importMapURL?: URL;
}
export async function load(
url: URL,
_options: LoadOptions,
): Promise<esbuild.OnLoadResult | null> {
switch (url.protocol) {
case "http:":
case "https:":
case "data:":
return await loadWithFetch(url);
case "file:": {
const res = await loadWithReadFile(url);
res.watchFiles = [fromFileUrl(url.href)];
return res;
}
}
return null;
}
async function loadWithFetch(
specifier: URL,
): Promise<esbuild.OnLoadResult> {
const specifierRaw = specifier.href;
// TODO(lucacasonato): redirects!
const resp = await fetch(specifierRaw);
if (!resp.ok) {
throw new Error(
`Encountered status code ${resp.status} while fetching ${specifierRaw}.`,
);
}
const contentType = resp.headers.get("content-type");
const mediaType = mapContentType(
new URL(resp.url || specifierRaw),
contentType,
);
const loader = mediaTypeToLoader(mediaType);
const raw = new Uint8Array(await resp.arrayBuffer());
const contents = transformRawIntoContent(raw, mediaType);
return { contents, loader };
}
async function loadWithReadFile(specifier: URL): Promise<esbuild.OnLoadResult> {
const path = fromFileUrl(specifier);
const mediaType = mapContentType(specifier, null);
const loader = mediaTypeToLoader(mediaType);
const raw = await Deno.readFile(path);
const contents = transformRawIntoContent(raw, mediaType);
return { contents, loader };
}
function mapContentType(
specifier: URL,
contentType: string | null,
): deno.MediaType {
if (contentType !== null) {
const contentTypes = contentType.split(";");
const mediaType = contentTypes[0].toLowerCase();
switch (mediaType) {
case "application/typescript":
case "text/typescript":
case "video/vnd.dlna.mpeg-tts":
case "video/mp2t":
case "application/x-typescript":
return mapJsLikeExtension(specifier, "TypeScript");
case "application/javascript":
case "text/javascript":
case "application/ecmascript":
case "text/ecmascript":
case "application/x-javascript":
case "application/node":
return mapJsLikeExtension(specifier, "JavaScript");
case "text/jsx":
return "JSX";
case "text/tsx":
return "TSX";
case "application/json":
case "text/json":
return "Json";
case "application/wasm":
return "Wasm";
case "text/plain":
case "application/octet-stream":
return mediaTypeFromSpecifier(specifier);
default:
return "Unknown";
}
} else {
return mediaTypeFromSpecifier(specifier);
}
}
function mapJsLikeExtension(
specifier: URL,
defaultType: deno.MediaType,
): deno.MediaType {
const path = specifier.pathname;
switch (extname(path)) {
case ".jsx":
return "JSX";
case ".mjs":
return "Mjs";
case ".cjs":
return "Cjs";
case ".tsx":
return "TSX";
case ".ts":
if (path.endsWith(".d.ts")) {
return "Dts";
} else {
return defaultType;
}
case ".mts": {
if (path.endsWith(".d.mts")) {
return "Dmts";
} else {
return defaultType == "JavaScript" ? "Mjs" : "Mts";
}
}
case ".cts": {
if (path.endsWith(".d.cts")) {
return "Dcts";
} else {
return defaultType == "JavaScript" ? "Cjs" : "Cts";
}
}
default:
return defaultType;
}
}
function mediaTypeFromSpecifier(specifier: URL): deno.MediaType {
const path = specifier.pathname;
switch (extname(path)) {
case "":
if (path.endsWith("/.tsbuildinfo")) {
return "TsBuildInfo";
} else {
return "Unknown";
}
case ".ts":
if (path.endsWith(".d.ts")) {
return "Dts";
} else {
return "TypeScript";
}
case ".mts":
if (path.endsWith(".d.mts")) {
return "Dmts";
} else {
return "Mts";
}
case ".cts":
if (path.endsWith(".d.cts")) {
return "Dcts";
} else {
return "Cts";
}
case ".tsx":
return "TSX";
case ".js":
return "JavaScript";
case ".jsx":
return "JSX";
case ".mjs":
return "Mjs";
case ".cjs":
return "Cjs";
case ".json":
return "Json";
case ".wasm":
return "Wasm";
case ".tsbuildinfo":
return "TsBuildInfo";
case ".map":
return "SourceMap";
default:
return "Unknown";
}
}
plugos/forked/esbuild_deno_loader/src/shared.ts
-40
View File
@@ -1,40 +0,0 @@
import { esbuild } from "../deps.ts";
import { MediaType } from "./deno.ts";
export function mediaTypeToLoader(mediaType: MediaType): esbuild.Loader {
switch (mediaType) {
case "JavaScript":
case "Mjs":
return "js";
case "JSX":
return "jsx";
case "TypeScript":
case "Mts":
return "ts";
case "TSX":
return "tsx";
case "Json":
return "js";
default:
throw new Error(`Unhandled media type ${mediaType}.`);
}
}
export function transformRawIntoContent(
raw: Uint8Array,
mediaType: MediaType,
): string | Uint8Array {
switch (mediaType) {
case "Json":
return jsonToESM(raw);
default:
return raw;
}
}
function jsonToESM(source: Uint8Array): string {
const sourceString = new TextDecoder().decode(source);
let json = JSON.stringify(JSON.parse(sourceString), null, 2);
json = json.replaceAll(`"__proto__":`, `["__proto__"]:`);
return `export default ${json};`;
}
plugos/forked/esbuild_deno_loader/test_deps.ts
-6
View File
@@ -1,6 +0,0 @@
import * as esbuild from "https://deno.land/x/esbuild@v0.14.51/mod.js";
export { esbuild };
export {
assert,
assertEquals,
} from "https://deno.land/std@0.150.0/testing/asserts.ts";
plugos/gen.ts
-37
View File
@@ -1,37 +0,0 @@
import { AssetBundle } from "./asset_bundle/bundle.ts";
import { compile } from "./compile.ts";
console.log("Generating sandbox worker...");
const bundlePath =
new URL("./environments/worker_bundle.json", import.meta.url).pathname;
const workerPath =
new URL("./environments/sandbox_worker.ts", import.meta.url).pathname;
const workerCode = await compile(workerPath);
const assetBundle = new AssetBundle();
assetBundle.writeTextFileSync("worker.js", workerCode);
Deno.writeTextFile(
bundlePath,
JSON.stringify(assetBundle.toJSON(), null, 2),
);
console.log(`Wrote updated bundle to ${bundlePath}`);
console.log("Now generating SQLite worker...");
const sqliteBundlePath =
new URL("./sqlite/worker_bundle.json", import.meta.url).pathname;
const sqliteWorkerPath =
new URL("./sqlite/worker.ts", import.meta.url).pathname;
const sqliteWorkerCode = await compile(sqliteWorkerPath);
const sqliteAssetBundle = new AssetBundle();
sqliteAssetBundle.writeTextFileSync("worker.js", sqliteWorkerCode);
Deno.writeTextFile(
sqliteBundlePath,
JSON.stringify(sqliteAssetBundle.toJSON(), null, 2),
);
console.log(`Wrote updated bundle to ${sqliteBundlePath}`);
Deno.exit(0);
plugos/hack.ts
-4
View File
@@ -1,4 +0,0 @@
export function patchDenoLibJS(code: string): string {
// The Deno std lib has one occurence of a regex that Webkit JS doesn't (yet parse), we'll strip it because it's likely never invoked anyway, YOLO
return code.replaceAll("/(?<=\\n)/", "/()/");
}
plugos/hooks/cron.ts
+2 -3
View File
@@ -1,6 +1,5 @@
import { Hook, Manifest } from "../types.ts";
import { Cron } from "https://cdn.jsdelivr.net/gh/hexagon/croner@4/src/croner.js";
import { safeRun } from "../util.ts";
import { System } from "../system.ts";
export type CronHookT = {
cron?: string | string[];
@@ -51,13 +50,13 @@ export class CronHook implements Hook<CronHookT> {
this.tasks.push(
new Cron(cronDef, () => {
// console.log("Now acting on cron", cronDef);
safeRun(async () => {
(async () => {
try {
await plug.invoke(name, [cronDef]);
} catch (e: any) {
console.error("Execution of cron function failed", e);
}
});
})().catch(console.error);
}),
);
}
plugos/hooks/endpoint.test.ts
+36 -36
View File
@@ -6,43 +6,43 @@ import { System } from "../system.ts";
import { Application } from "../../server/deps.ts";
import { assertEquals } from "../../test_deps.ts";
Deno.test("Run a plugos endpoint server", async () => {
const system = new System<EndpointHookT>("server");
await system.load(
{
name: "test",
functions: {
testhandler: {
http: {
path: "/",
},
code: `(() => {
return {
default: (req) => {
console.log("Req", req);
return {status: 200, body: [1, 2, 3], headers: {"Content-type": "application/json"}};
}
};
})()`,
},
},
} as Manifest<EndpointHookT>,
createSandbox,
);
// Deno.test("Run a plugos endpoint server", async () => {
// const system = new System<EndpointHookT>("server");
// await system.load(
// {
// name: "test",
// functions: {
// testhandler: {
// http: {
// path: "/",
// },
// code: `(() => {
// return {
// default: (req) => {
// console.log("Req", req);
// return {status: 200, body: [1, 2, 3], headers: {"Content-type": "application/json"}};
// }
// };
// })()`,
// },
// },
// } as Manifest<EndpointHookT>,
// createSandbox,
// );
const app = new Application();
const port = 3123;
// const app = new Application();
// const port = 3123;
system.addHook(new EndpointHook(app, "/_"));
// system.addHook(new EndpointHook(app, "/_"));
const controller = new AbortController();
app.listen({ port: port, signal: controller.signal });
// const controller = new AbortController();
// app.listen({ port: port, signal: controller.signal });
const res = await fetch(`http://localhost:${port}/_/test/?name=Pete`);
assertEquals(res.status, 200);
assertEquals(res.headers.get("Content-type"), "application/json");
assertEquals(await res.json(), [1, 2, 3]);
console.log("Aborting");
controller.abort();
await system.unloadAll();
});
// const res = await fetch(`http://localhost:${port}/_/test/?name=Pete`);
// assertEquals(res.status, 200);
// assertEquals(res.headers.get("Content-type"), "application/json");
// assertEquals(await res.json(), [1, 2, 3]);
// console.log("Aborting");
// controller.abort();
// await system.unloadAll();
// });
plugos/hooks/event.ts
+5 -7
View File
@@ -1,6 +1,5 @@
import type { Hook, Manifest } from "../types.ts";
import { System } from "../system.ts";
import { safeRun } from "../util.ts";
// System events:
// - plug:load (plugName: string)
@@ -48,14 +47,15 @@ export class EventHook implements Hook<EventHookT> {
}
const responses: any[] = [];
for (const plug of this.system.loadedPlugs.values()) {
const manifest = await plug.manifest;
for (
const [name, functionDef] of Object.entries(
plug.manifest!.functions,
manifest!.functions,
)
) {
if (functionDef.events && functionDef.events.includes(eventName)) {
// Only dispatch functions that can run in this environment
if (plug.canInvoke(name)) {
if (await plug.canInvoke(name)) {
const result = await plug.invoke(name, [data]);
if (result !== undefined) {
responses.push(result);
@@ -80,10 +80,8 @@ export class EventHook implements Hook<EventHookT> {
apply(system: System<EventHookT>): void {
this.system = system;
this.system.on({
plugLoaded: (plug) => {
safeRun(async () => {
await this.dispatchEvent("plug:load", plug.name);
});
plugLoaded: async (plug) => {
await this.dispatchEvent("plug:load", plug.name);
},
});
}
plugos/lib/kv_store.dexie.ts
+80
View File
@@ -0,0 +1,80 @@
import Dexie, { Table } from "dexie";
import type { KV, KVStore } from "./kv_store.ts";
export class DexieKVStore implements KVStore {
db: Dexie;
items: Table<KV, string>;
constructor(
private dbName: string,
private tableName: string,
private indexedDB?: any,
) {
this.db = new Dexie(dbName, {
indexedDB,
});
this.db.version(1).stores({
[tableName]: "key",
});
this.items = this.db.table<KV, string>(tableName);
}
async del(key: string) {
await this.items.delete(key);
}
async deletePrefix(prefix: string) {
await this.items.where("key").startsWith(prefix).delete();
}
async deleteAll() {
await this.items.clear();
}
async set(key: string, value: any) {
await this.items.put({
key,
value,
});
}
async batchSet(kvs: KV[]) {
await this.items.bulkPut(
kvs.map(({ key, value }) => ({
key,
value,
})),
);
}
async batchDelete(keys: string[]) {
await this.items.bulkDelete(keys);
}
async batchGet(
keys: string[],
): Promise<(any | undefined)[]> {
return (await this.items.bulkGet(keys)).map((result) => result?.value);
}
async get(key: string): Promise<any | null> {
const result = await this.items.get({ key });
return result ? result.value : null;
}
async has(key: string): Promise<boolean> {
return await this.items.get({
key,
}) !== undefined;
}
async queryPrefix(
keyPrefix: string,
): Promise<{ key: string; value: any }[]> {
const results = await this.items.where("key").startsWith(keyPrefix)
.toArray();
return results.map((result) => ({
key: result.key,
value: result.value,
}));
}
}
plugos/lib/kv_store.ts
+59
View File
@@ -0,0 +1,59 @@
export type KV = {
key: string;
value: any;
};
/**
* An interface to any simple key-value store.
*/
export interface KVStore {
/**
* Deletes the value associated with a given key.
*/
del(key: string): Promise<void>;
/**
* Deletes all keys that start with a specific prefix.
*/
deletePrefix(prefix: string): Promise<void>;
/**
* Deletes all keys in the store.
*/
deleteAll(): Promise<void>;
/**
* Sets the value for a given key.
*/
set(key: string, value: any): Promise<void>;
/**
* Sets the values for a list of key-value pairs.
*/
batchSet(kvs: KV[]): Promise<void>;
/**
* Deletes a list of keys.
*/
batchDelete(keys: string[]): Promise<void>;
/**
* Gets the values for a list of keys.
*/
batchGet(keys: string[]): Promise<(any | undefined)[]>;
/**
* Gets the value for a given key.
*/
get(key: string): Promise<any | null>;
/**
* Checks whether a given key exists in the store.
*/
has(key: string): Promise<boolean>;
/**
* Gets all key-value pairs where the key starts with a specific prefix.
*/
queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]>;
}
plugos/plug.ts
+37 -54
View File
@@ -4,80 +4,66 @@ import { System } from "./system.ts";
import { AssetBundle, AssetJson } from "./asset_bundle/bundle.ts";
export class Plug<HookT> {
system: System<HookT>;
sandbox?: Sandbox;
readonly runtimeEnv?: RuntimeEnvironment;
public grantedPermissions: string[] = [];
public sandbox: Sandbox<HookT>;
// Resolves once the worker has been loaded
ready: Promise<void>;
// Only available after ready resolves
public manifest?: Manifest<HookT>;
public assets?: AssetBundle;
private sandboxFactory: (plug: Plug<HookT>) => Sandbox;
readonly runtimeEnv?: RuntimeEnvironment;
grantedPermissions: string[] = [];
name: string;
version: number;
constructor(
system: System<HookT>,
name: string,
sandboxFactory: (plug: Plug<HookT>) => Sandbox,
private system: System<HookT>,
public workerUrl: URL,
private sandboxFactory: (plug: Plug<HookT>) => Sandbox<HookT>,
) {
this.system = system;
this.name = name;
this.sandboxFactory = sandboxFactory;
// this.sandbox = sandboxFactory(this);
this.runtimeEnv = system.env;
this.version = new Date().getTime();
}
private sandboxInitialized: Promise<void> | undefined = undefined;
// Lazy load sandbox, guarantees that the sandbox is loaded
lazyInitSandbox(): Promise<void> {
if (this.sandboxInitialized) {
return this.sandboxInitialized;
}
this.sandboxInitialized = Promise.resolve().then(async () => {
console.log("Now starting sandbox for", this.name);
// Kick off worker
this.sandbox = this.sandboxFactory(this);
// Push in any dependencies
for (
const [dep, code] of Object.entries(this.manifest!.dependencies || {})
) {
await this.sandbox.loadDependency(dep, code);
}
await this.system.emit("sandboxInitialized", this.sandbox, this);
// Kick off worker
this.sandbox = this.sandboxFactory(this);
this.ready = this.sandbox.ready.then(() => {
this.manifest = this.sandbox.manifest!;
this.assets = new AssetBundle(
this.manifest.assets ? this.manifest.assets as AssetJson : {},
);
// TODO: These need to be explicitly granted, not just taken
this.grantedPermissions = this.manifest.requiredPermissions || [];
});
return this.sandboxInitialized;
}
load(manifest: Manifest<HookT>) {
this.manifest = manifest;
this.assets = new AssetBundle(
manifest.assets ? manifest.assets as AssetJson : {},
);
// TODO: These need to be explicitly granted, not just taken
this.grantedPermissions = manifest.requiredPermissions || [];
get name(): string | undefined {
return this.manifest?.name;
}
// Invoke a syscall
syscall(name: string, args: any[]): Promise<any> {
return this.system.syscallWithContext({ plug: this }, name, args);
}
canInvoke(name: string) {
if (!this.manifest) {
return false;
}
const funDef = this.manifest.functions[name];
// Checks if a function can be invoked (it may be restricted on its execution environment)
async canInvoke(name: string) {
await this.ready;
const funDef = this.manifest!.functions[name];
if (!funDef) {
throw new Error(`Function ${name} not found in manifest`);
}
return !funDef.env || !this.runtimeEnv || funDef.env === this.runtimeEnv;
}
// Invoke a function
async invoke(name: string, args: any[]): Promise<any> {
// Ensure the worker is fully up and running
await this.ready;
// Before we access the manifest
const funDef = this.manifest!.functions[name];
if (!funDef) {
throw new Error(`Function ${name} not found in manifest`);
}
await this.lazyInitSandbox();
const sandbox = this.sandbox!;
if (funDef.redirect) {
// Function redirect, look up
@@ -95,13 +81,10 @@ export class Plug<HookT> {
}
return plug.invoke(name, args);
}
if (!sandbox.isLoaded(name)) {
if (!this.canInvoke(name)) {
throw new Error(
`Function ${name} is not available in ${this.runtimeEnv}`,
);
}
await sandbox.load(name, funDef.code!);
if (!await this.canInvoke(name)) {
throw new Error(
`Function ${name} is not available in ${this.runtimeEnv}`,
);
}
return await sandbox.invoke(name, args);
}
plugos/protocol.ts
+40
View File
@@ -0,0 +1,40 @@
import type { Manifest } from "../common/manifest.ts";
// Messages received from the worker
export type ControllerMessage =
| {
// Parsed manifest when worker is initialized
type: "manifest";
manifest: Manifest;
}
| {
// Function invocation result
type: "invr";
id: number;
error?: string;
result?: any;
}
| {
// Syscall
type: "sys";
id: number;
name: string;
args: any[];
};
// Messages received inside the worker
export type WorkerMessage =
| {
// Function invocation
type: "inv";
id: number;
name: string;
args: any[];
}
| {
// Syscall result
type: "sysr";
id: number;
result?: any;
error?: any;
};
plugos/runtime.test.ts
+17 -141
View File
@@ -1,10 +1,8 @@
import { createSandbox } from "./environments/deno_sandbox.ts";
import { System } from "./system.ts";
import {
assert,
assertEquals,
} from "https://deno.land/std@0.165.0/testing/asserts.ts";
import { assertEquals } from "../test_deps.ts";
import { compileManifest } from "./compile.ts";
import { esbuild } from "./deps.ts";
Deno.test("Run a deno sandbox", async () => {
const system = new System("server");
@@ -26,148 +24,26 @@ Deno.test("Run a deno sandbox", async () => {
return "yay";
},
});
const tempDir = await Deno.makeTempDir();
const workerPath = await compileManifest(
new URL("test.plug.yaml", import.meta.url).pathname,
tempDir,
);
const plug = await system.load(
{
name: "test",
requiredPermissions: ["dangerous"],
functions: {
addTen: {
code: `(() => {
return {
default: (n) => {
return n + 10;
}
};
})()`,
},
redirectTest: {
redirect: "addTen",
},
redirectTest2: {
redirect: "test.addTen",
},
addNumbersSyscall: {
code: `(() => {
return {
default: async (a, b) => {
return await self.syscall("addNumbers", a, b);
}
};
})()`,
},
errorOut: {
code: `(() => {
return {
default: () => {
throw Error("BOOM");
}
};
})()`,
},
errorOutSys: {
code: `(() => {
return {
default: async () => {
await self.syscall("failingSyscall");
}
};
})()`,
},
restrictedTest: {
code: `(() => {
return {
default: async () => {
await self.syscall("restrictedSyscall");
}
};
})()`,
},
dangerousTest: {
code: `(() => {
return {
default: async () => {
return await self.syscall("dangerousSyscall");
}
};
})()`,
},
},
},
new URL(`file://${workerPath}`),
createSandbox,
);
assertEquals(await plug.invoke("addTen", [10]), 20);
assertEquals(await plug.invoke("redirectTest", [10]), 20);
assertEquals(await plug.invoke("redirectTest2", [10]), 20);
for (let i = 0; i < 100; i++) {
assertEquals(await plug.invoke("addNumbersSyscall", [10, i]), 10 + i);
}
try {
await plug.invoke("errorOut", []);
assert(false);
} catch (e: any) {
assert(e.message.indexOf("BOOM") !== -1);
}
try {
await plug.invoke("errorOutSys", []);
assert(false);
} catch (e: any) {
assert(e.message.indexOf("#fail") !== -1);
}
try {
await plug.invoke("restrictedTest", []);
assert(false);
} catch (e: any) {
assert(
e.message.indexOf(
"Missing permission 'restricted' for syscall restrictedSyscall",
) !== -1,
);
}
assertEquals(await plug.invoke("dangerousTest", []), "yay");
console.log("Plug", plug.manifest);
assertEquals("hello", await plug.invoke("boot", []));
await system.unloadAll();
});
import { bundle as plugOsBundle } from "./bin/plugos-bundle.ts";
import { esbuild } from "./compile.ts";
import { urlToPathname } from "./util.ts";
await Deno.remove(tempDir, { recursive: true });
const __dirname = urlToPathname(new URL(".", import.meta.url));
Deno.test("Preload dependencies", async () => {
const globalModules = await plugOsBundle(
`${__dirname}../plugs/global.plug.yaml`,
);
const testPlugManifest = await plugOsBundle(
`${__dirname}test.plug.yaml`,
{
imports: [globalModules],
},
);
esbuild.stop();
const system = new System("server");
system.on({
sandboxInitialized: async (sandbox) => {
for (
const [modName, code] of Object.entries(globalModules.dependencies!)
) {
await sandbox.loadDependency(modName, code as string);
}
},
});
// Load test module
console.log("Loading test module");
const testPlug = await system.load(
testPlugManifest,
createSandbox,
);
console.log("Running");
const result = await testPlug.invoke("boot", []);
console.log("Result", result);
await system.unloadAll();
});
plugos/sandbox.ts
+36 -107
View File
@@ -1,160 +1,89 @@
import type { LogLevel } from "./environments/custom_logger.ts";
import {
ControllerMessage,
WorkerLike,
WorkerMessage,
} from "./environments/worker.ts";
import { Manifest } from "./types.ts";
import { ControllerMessage, WorkerMessage } from "./protocol.ts";
import { Plug } from "./plug.ts";
export type SandboxFactory<HookT> = (plug: Plug<HookT>) => Sandbox;
export type SandboxFactory<HookT> = (plug: Plug<HookT>) => Sandbox<HookT>;
export type LogEntry = {
level: LogLevel;
message: string;
date: number;
};
export class Sandbox {
protected worker: WorkerLike;
protected reqId = 0;
protected outstandingInits = new Map<string, () => void>();
protected outstandingDependencyInits = new Map<string, () => void>();
protected outstandingInvocations = new Map<
/**
* Represents a "safe" execution environment for plug code
* Effectively this wraps a web worker, the reason to have this split from Plugs is to allow plugs to manage multiple sandboxes, e.g. for performance in the future
*/
export class Sandbox<HookT> {
private worker: Worker;
private reqId = 0;
private outstandingInvocations = new Map<
number,
{ resolve: (result: any) => void; reject: (e: any) => void }
>();
protected loadedFunctions = new Set<string>();
protected plug: Plug<any>;
public logBuffer: LogEntry[] = [];
public maxLogBufferSize = 100;
constructor(plug: Plug<any>, worker: WorkerLike) {
worker.onMessage = this.onMessage.bind(this);
this.worker = worker;
this.plug = plug;
}
public ready: Promise<void>;
public manifest?: Manifest<HookT>;
isLoaded(name: string) {
return this.loadedFunctions.has(name);
}
async load(name: string, code: string): Promise<void> {
await this.worker.ready;
const outstandingInit = this.outstandingInits.get(name);
if (outstandingInit) {
// Load already in progress, let's wait for it...
return new Promise((resolve) => {
this.outstandingInits.set(name, () => {
outstandingInit!();
resolve();
});
});
}
this.worker.postMessage({
type: "load",
name: name,
code: code,
} as WorkerMessage);
return new Promise((resolve) => {
this.outstandingInits.set(name, () => {
this.loadedFunctions.add(name);
this.outstandingInits.delete(name);
resolve();
});
constructor(
readonly plug: Plug<HookT>,
workerOptions = {},
) {
this.worker = new Worker(plug.workerUrl, {
...workerOptions,
type: "module",
});
}
this.ready = new Promise((resolve) => {
this.worker.onmessage = (ev) => {
if (ev.data.type === "manifest") {
this.manifest = ev.data.manifest;
resolve();
return;
}
loadDependency(name: string, code: string): Promise<void> {
// console.log("Loading dependency", name);
this.worker.postMessage({
type: "load-dependency",
name: name,
code: code,
} as WorkerMessage);
return new Promise((resolve) => {
// console.log("Loaded dependency", name);
this.outstandingDependencyInits.set(name, () => {
this.outstandingDependencyInits.delete(name);
resolve();
});
this.onMessage(ev.data);
};
});
}
async onMessage(data: ControllerMessage) {
switch (data.type) {
case "inited": {
const initCb = this.outstandingInits.get(data.name!);
initCb && initCb();
this.outstandingInits.delete(data.name!);
break;
}
case "dependency-inited": {
const depInitCb = this.outstandingDependencyInits.get(data.name!);
depInitCb && depInitCb();
this.outstandingDependencyInits.delete(data.name!);
break;
}
case "syscall":
case "sys":
try {
const result = await this.plug.syscall(data.name!, data.args!);
this.worker.postMessage({
type: "syscall-response",
type: "sysr",
id: data.id,
result: result,
} as WorkerMessage);
} catch (e: any) {
// console.error("Syscall fail", e);
this.worker.postMessage({
type: "syscall-response",
type: "sysr",
id: data.id,
error: e.message,
} as WorkerMessage);
}
break;
case "result": {
case "invr": {
const resultCbs = this.outstandingInvocations.get(data.id!);
this.outstandingInvocations.delete(data.id!);
if (data.error) {
resultCbs &&
resultCbs.reject(
new Error(`${data.error}\nStack trace: ${data.stack}`),
);
resultCbs.reject(new Error(data.error));
} else {
resultCbs && resultCbs.resolve(data.result);
}
break;
}
case "log": {
this.log(data.level!, data.message!);
break;
}
default:
console.error("Unknown message type", data);
}
}
log(level: string, ...messageBits: any[]) {
const message = messageBits.map((a) => "" + a).join(" ");
this.logBuffer.push({
message,
level: level as LogLevel,
date: Date.now(),
});
if (this.logBuffer.length > this.maxLogBufferSize) {
this.logBuffer.shift();
}
console.log(`[Sandbox ${level}]`, message);
}
invoke(name: string, args: any[]): Promise<any> {
this.reqId++;
this.worker.postMessage({
type: "invoke",
type: "inv",
id: this.reqId,
name,
args,
});
} as WorkerMessage);
return new Promise((resolve, reject) => {
this.outstandingInvocations.set(this.reqId, { resolve, reject });
});
plugos/sqlite/async_sqlite.test.ts
-16
View File
@@ -1,16 +0,0 @@
import { AsyncSQLite } from "./async_sqlite.ts";
import { assertEquals } from "../../test_deps.ts";
Deno.test("Async SQLite test", async () => {
const db = new AsyncSQLite(":memory:");
await db.init();
await db.execute("CREATE TABLE test (name TEXT)");
await db.execute("INSERT INTO test (name) VALUES (?)", "test");
await db.execute("INSERT INTO test (name) VALUES (?)", "test 2");
assertEquals(await db.query("SELECT * FROM test ORDER BY name"), [{
name: "test",
}, {
name: "test 2",
}]);
db.stop();
});
plugos/sqlite/async_sqlite.ts
-71
View File
@@ -1,71 +0,0 @@
import { AssetBundle } from "../asset_bundle/bundle.ts";
import { ISQLite } from "./sqlite_interface.ts";
import workerBundleJson from "./worker_bundle.json" assert { type: "json" };
const workerBundle = new AssetBundle(workerBundleJson);
export class AsyncSQLite implements ISQLite {
worker: Worker;
requestId = 0;
outstandingRequests = new Map<
number,
{ resolve: (val: any) => void; reject: (error: Error) => void }
>();
constructor(readonly dbPath: string) {
const workerHref = URL.createObjectURL(
new Blob([
workerBundle.readFileSync("worker.js"),
], {
type: "application/javascript",
}),
);
this.worker = new Worker(
workerHref,
{
type: "module",
},
);
this.worker.addEventListener("message", (event: MessageEvent) => {
const { data } = event;
// console.log("Got data back", data);
const { id, result, error } = data;
const req = this.outstandingRequests.get(id);
if (!req) {
console.error("Invalid request id", id);
return;
}
if (result !== undefined) {
req.resolve(result);
} else if (error) {
req.reject(new Error(error));
}
this.outstandingRequests.delete(id);
});
}
private request(message: Record<string, any>): Promise<any> {
this.requestId++;
return new Promise((resolve, reject) => {
this.outstandingRequests.set(this.requestId, { resolve, reject });
// console.log("Sending request", message);
this.worker.postMessage({ ...message, id: this.requestId });
});
}
init(): Promise<void> {
return this.request({ type: "init", dbPath: this.dbPath });
}
execute(query: string, ...params: any[]): Promise<number> {
return this.request({ type: "execute", query, params });
}
query(query: string, ...params: any[]): Promise<any[]> {
return this.request({ type: "query", query, params });
}
stop() {
this.worker.terminate();
}
}
plugos/sqlite/capacitor_sqlite.ts
-36
View File
@@ -1,36 +0,0 @@
import { Capacitor } from "../../mobile/deps.ts";
import { CapacitorSQLite } from "../deps.ts";
import { ISQLite } from "./sqlite_interface.ts";
export class CapacitorDb implements ISQLite {
constructor(readonly name: string) {
}
async init() {
await CapacitorSQLite.createConnection({
database: this.name,
});
await CapacitorSQLite.open({
database: this.name,
});
}
async query(sql: string, ...args: any[]) {
const result = await CapacitorSQLite.query({
statement: sql,
database: this.name,
values: args,
});
if (Capacitor.getPlatform() === "ios") {
return result.values!.slice(1);
}
return result.values!;
}
async execute(sql: string, ...args: any[]): Promise<number> {
return (await CapacitorSQLite.run({
statement: sql,
database: this.name,
values: args,
})).changes!.changes!;
}
}
plugos/sqlite/deno-sqlite/.gitattributes
-1
View File
@@ -1 +0,0 @@
build/lib/** linguist-vendored
plugos/sqlite/deno-sqlite/.gitignore
-23
View File
@@ -1,23 +0,0 @@
# System files
.DS_Store
# Debug builds
debug.js
# Misc
hack.*
*.br
*.db
*.db-journal
*.db-shm
*.sqlite
*.sqlite-journal
*.sqlite-shm
*.gz
build/sqlite-src
build/wasi-sdk
deno
# NPM garbage
node_modules
package-lock.json
plugos/sqlite/deno-sqlite/CONTRIBUTING.md
-88
View File
@@ -1,88 +0,0 @@
# Contribute to SQLite for Deno
> Note: this is a draft
Thank you for considering to contribute to the SQLite for Deno module! Below are
a few guidelines on how to contribute.
## Prerequisites
To work on the JavaScript/ TypeScript wrapper module, all you need is a
[deno](https://deno.land) runtime.
To change the compiled SQLite WASM binary, you will require to download the
[WASI SDK][wasi-sdk]. This process should function fully automatically for most
users.
**To install build dependencies** go to the `build` folder (`cd build`), then
run `make setup`.
**To compile the binary** run `make release` (or `make debug` for a debug
build). If you changed any build flags of SQLite, also run `make amalgamation`,
before building.
If you are interested in more details regarding the compilation setup, also see
[this blog post][compile-wasm-blog].
## Code Style, Review, and Dependencies
This project uses the `deno fmt` code style.
This project uses no external dependencies (with the exception of a copy of the
SQLite C library).
For testing purposes, Deno standard library modules may be used.
## Documentation
Any user-facing interfaces should be documented. To document such interfaces,
include a **JSDoc comment**, which should be formatted as follows:
```javascript
/**
* A short but complete description, formatted
* as markdown.
*/
functionName(arg1, arg2) {
// ...
}
```
Comments with this format will be automatically parsed by `deno doc`.
These comments should not include examples unless they are essential to
illustrating an important point.
## Tests and Benchmarks
Any important functionality should be tested. Tests are in the `test.ts` file.
Changes will not be merged unless all tests pass.
Benchmarks are in the `bench.ts` file.
## Technical Direction
The goal of this module is to provide a **simple and predictable** interface to
SQLite. The interface should feel like a JavaScript library, but also
immediately make sense to someone who knows the SQLite C/C++ interface. Features
and interfaces should generally be orthogonal.
This is a low-level library, which provides access to running SQL queries and
retrieving the results of these queries. This library will only wrap SQLite C
API functions, but never try to provide a higher level interface to the database
than plain SQL. It is meant to serve as a building block for constructing higher
level interfaces, or for people who need an easy way to execute SQL queries on
their SQLite database.
The library should be easy to use and behave as any regular JavaScript library
would in Deno. This means, it should only need the required permissions (e.g. if
only in-memory databases are used, no permissions should be necessary. If a
database is opened in read-only mode, `--allow-read` should be sufficient).
## License
By making contributions, you agree that anything you submit will be distributed
under the projects license (see `LICENSE`).
[wasi-sdk]: https://github.com/CraneStation/wasi-sdk/releases
[compile-wasm-blog]: https://tilman.xyz/blog/2019/12/building-webassembly-for-deno/
plugos/sqlite/deno-sqlite/LICENSE
-19
View File
@@ -1,19 +0,0 @@
Copyright (c) 2019 - 2022 Tilman Roeder and contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
plugos/sqlite/deno-sqlite/README.md
-107
View File
@@ -1,107 +0,0 @@
# Deno SQLite Module
[![test status](https://github.com/dyedgreen/deno-sqlite/workflows/tests/badge.svg?branch=master)](https://github.com/dyedgreen/deno-sqlite/actions)
[![deno doc](https://doc.deno.land/badge.svg)](https://deno.land/x/sqlite/mod.ts)
This is an SQLite module for JavaScript and TypeScript. The wrapper is targeted
at [Deno](https://deno.land) and uses a version of SQLite3 compiled to
WebAssembly (WASM). This module focuses on correctness, ease of use and
performance.
This module guarantees API compatibility according to
[semantic versioning](https://semver.org). Please report any issues you
encounter. Note that the `master` branch might contain new or breaking features.
The versioning guarantee applies only to
[tagged releases](https://github.com/dyedgreen/deno-sqlite/releases).
## Documentation
Documentation is available [Deno Docs](https://deno.land/x/sqlite). There is
also a list of examples in the [`examples`](./examples) folder.
## Example
```javascript
import { DB } from "https://deno.land/x/sqlite/mod.ts";
// Open a database
const db = new DB("test.db");
db.execute(`
CREATE TABLE IF NOT EXISTS people (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT
)
`);
const names = ["Peter Parker", "Clark Kent", "Bruce Wayne"];
// Run a simple query
for (const name of names) {
db.query("INSERT INTO people (name) VALUES (?)", [name]);
}
// Print out data in table
for (const [name] of db.query("SELECT name FROM people")) {
console.log(name);
}
// Close connection
db.close();
```
## Comparison to Plugin based Modules
### TL;DR
If you want something that just works (and is fast), use this library.
Depending on your specific needs, there is also
[sqlite3](https://github.com/denodrivers/sqlite3), however using this module
requires the `--allow-ffi` and `--unstable` flags, which means the database
connection may bypass e.g. file access permissions.
### Advantages
- Security: benefit from Denos security settings, without the need to trust a
third party
- Portability: runs everywhere Deno runs and can even run in the browser
- Ease of Use: takes full advantage of Denos module cache and does not require
any network access after initial download
- Speed: thanks to WASM, the database performance is comparable to native
bindings in most situations and the API is carefully designed to provide
optimal performance
### Disadvantages
- Weaker Persistence Guarantees: due to limitations in Denos file system APIs,
SQLite can't acquire file locks or memory map files (e.g. this module does not
support WAL mode)
## Browser Version (Experimental)
There is **experimental** support for using `deno-sqlite` in the browser. You
can generate a browser compatible module by running:
```bash
deno bundle --import-map browser/import_map.json browser/mod.ts [output_bundle_path]
```
The modules documentation can be seen by running
```bash
deno doc browser/mod.ts
```
Databases created in the browser are persisted using
[indexedDB](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API).
## Users
- [cotton](https://github.com/rahmanfadhil/cotton)
- [deno-nessie](https://github.com/halvardssm/deno-nessie)
- [denodb](https://github.com/eveningkid/denodb)
- [denolib/typeorm](https://github.com/denolib/typeorm)
- [small-orm-sqlite](https://github.com/enimatek-nl/small-orm-sqlite)
_(listed in alphabetical order, please submit a PR if you are using this library
and are not included)_
plugos/sqlite/deno-sqlite/bench.ts
-88
View File
@@ -1,88 +0,0 @@
import {
bench,
runBenchmarks,
} from "https://deno.land/std@0.135.0/testing/bench.ts";
import { DB } from "./mod.ts";
if (Deno.args[0]) {
try {
await Deno.remove(Deno.args[0]);
} catch (_) {
// ignore
}
}
const dbFile = Deno.args[0] || ":memory:";
const db = new DB(dbFile);
db.query(
"CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, balance INTEGER)",
);
/** Performance of insert statements (1 insert). */
const names = "Deno Land Peter Parker Clark Kent Robert Parr".split(" ");
bench({
name: "insert 10 000 (named)",
runs: 100,
func: (b): void => {
b.start();
const query = db.prepareQuery(
"INSERT INTO users (name, balance) VALUES (:name, :balance)",
);
db.query("begin");
for (let i = 0; i < 10_000; i++) {
query.execute({ name: names[i % names.length], balance: i });
}
db.query("commit");
b.stop();
},
});
bench({
name: "insert 10 000 (positional)",
runs: 100,
func: (b): void => {
b.start();
const query = db.prepareQuery(
"INSERT INTO users (name, balance) VALUES (?, ?)",
);
db.query("begin");
for (let i = 0; i < 10_000; i++) {
query.execute([names[i % names.length], i]);
}
db.query("commit");
b.stop();
},
});
/** Performance of select statements (select all; 10_000). */
bench({
name: "select 10 000 (select all)",
runs: 100,
func: (b): void => {
b.start();
db.query(
"SELECT name, balance FROM users LIMIT 10000",
);
b.stop();
},
});
/** Performance of select statements (select individually; 10_000). */
bench({
name: "select 10 000 (select first)",
runs: 100,
func: (b): void => {
b.start();
const query = db.prepareQuery(
"SELECT name, balance FROM users WHERE id = ?",
);
for (let id = 1; id <= 10_000; id++) {
query.first([id]);
}
b.stop();
},
});
runBenchmarks();
plugos/sqlite/deno-sqlite/browser/import_map.json
-5
View File
@@ -1,5 +0,0 @@
{
"imports": {
"../build/vfs.js": "../browser/vfs.js"
}
}
plugos/sqlite/deno-sqlite/browser/mod.ts
-38
View File
@@ -1,38 +0,0 @@
import { DB } from "../src/db.ts";
import { loadFile, writeFile } from "./vfs.js";
import { compile, instantiateBrowser } from "../build/sqlite.js";
export { SqliteError } from "../src/error.ts";
export { Status } from "../src/constants.ts";
const hasCompiled = compile();
/**
* Opens a database with the given name. If `file` is
* not provided or `:memory:`, an in-memory database
* is returned which will not persist after the database
* is closed.
*/
export async function open(file?: string): Promise<DB> {
if (file != null && file !== ":memory:") await loadFile(file);
await hasCompiled;
await instantiateBrowser();
return new DB(file);
}
/**
* Overwrite a given file with arbitrary data. This can be used
* to import a database which can later be opened.
*/
export async function write(file: string, data: Uint8Array): Promise<void> {
await writeFile(file, data);
}
/**
* Read the data currently stored for a given file. This can be used
* to export a database which has been created or modified.
*/
export async function read(file: string): Promise<Uint8Array | null> {
const buffer = await loadFile(file);
return buffer?.toUint8Array()?.slice();
}
plugos/sqlite/deno-sqlite/browser/vfs.js
-208
View File
@@ -1,208 +0,0 @@
import { getStr } from "../src/wasm.ts";
const DB_NAME = "sqlitevfs";
const LOADED_FILES = new Map();
const OPEN_FILES = new Map();
function nextRid() {
const rid = (nextRid?.LAST_RID ?? 0) + 1;
nextRid.LAST_RID = rid;
return rid;
}
function getOpenFile(rid) {
if (!OPEN_FILES.has(rid)) {
throw new Error(`Resource ID ${rid} does not exist.`);
}
return OPEN_FILES.get(rid);
}
const MIN_GROW_BYTES = 2048;
const MAX_GROW_BYTES = 65536;
class Buffer {
constructor(data) {
this._data = data ?? new Uint8Array();
this._size = this._data.length;
}
get size() {
return this._size;
}
read(offset, buffer) {
if (offset >= this._size) return 0;
const toCopy = this._data.subarray(
offset,
Math.min(this._size, offset + buffer.length),
);
buffer.set(toCopy);
return toCopy.length;
}
reserve(capacity) {
if (this._data.length >= capacity) return;
const neededBytes = capacity - this._data.length;
const growBy = Math.min(
MAX_GROW_BYTES,
Math.max(MIN_GROW_BYTES, this._data.length),
);
const newArray = new Uint8Array(
this._data.length + Math.max(growBy, neededBytes),
);
newArray.set(this._data);
this._data = newArray;
}
write(offset, buffer) {
this.reserve(offset + buffer.length);
this._data.set(buffer, offset);
this._size = Math.max(this._size, offset + buffer.length);
return buffer.length;
}
truncate(size) {
this._size = size;
}
toUint8Array() {
return this._data.subarray(0, this._size);
}
}
const indexedDB = window.indexedDB || window.mozIndexedDB ||
window.webkitIndexedDB || window.msIndexedDB || window.shimIndexedDB;
// Web browser indexedDB database
const database = new Promise((resolve, reject) => {
const request = indexedDB.open(DB_NAME, 1);
request.onupgradeneeded = () =>
request.result.createObjectStore("files", { keyPath: "name" });
request.onsuccess = () => resolve(request.result);
request.onerror = () => reject(request.error);
});
export async function loadFile(fileName) {
const db = await database;
const file = await new Promise((resolve, reject) => {
const store = db.transaction("files", "readonly").objectStore("files");
const request = store.get(fileName);
request.onsuccess = () => resolve(request.result);
request.onerror = () => reject(request.error);
});
if (file != null && !LOADED_FILES.has(fileName)) {
const buffer = new Buffer(file.data);
LOADED_FILES.set(fileName, buffer);
return buffer;
} else if (LOADED_FILES.has(fileName)) {
return LOADED_FILES.get(fileName);
} else {
return null;
}
}
async function syncFile(fileName, data) {
const db = await database;
await new Promise((resolve, reject) => {
const store = db.transaction("files", "readwrite").objectStore("files");
const request = store.put({ name: fileName, data });
request.onsuccess = () => resolve();
request.onerror = () => reject(request.error);
});
}
async function deleteFile(fileName) {
const db = await database;
await new Promise((resolve, reject) => {
const store = db.transaction("files", "readwrite").objectStore("files");
const request = store.delete(fileName);
request.onsuccess = () => resolve();
request.onerror = () => reject(request.error);
});
}
export async function writeFile(fileName, data) {
await syncFile(fileName, data);
if (LOADED_FILES.has(fileName)) {
const buffer = LOADED_FILES.get(fileName);
buffer.truncate(0);
buffer.write(0, data);
}
}
// Closure to return an environment that links
// the current wasm context. This is a modified
// version, suitable for use within browsers.
export default function env(inst) {
const env = {
js_print: (str_ptr) => {
const text = getStr(inst.exports, str_ptr);
console.log(text[text.length - 1] === "\n" ? text.slice(0, -1) : text);
},
js_open: (path_ptr, mode, _flags) => {
if (mode === 1 /* temp file */) {
const rid = nextRid();
OPEN_FILES.set(rid, { path: null, buffer: new Buffer() });
return rid;
} else if (mode === 0 /* regular file */) {
const path = getStr(inst.exports, path_ptr);
const buffer = LOADED_FILES.get(path) ?? new Buffer();
if (!LOADED_FILES.has(path)) LOADED_FILES.set(path, buffer);
const rid = nextRid();
OPEN_FILES.set(rid, { path, buffer });
return rid;
}
},
js_close: (rid) => {
OPEN_FILES.delete(rid);
},
js_delete: (path_ptr) => {
const path = getStr(inst.exports, path_ptr);
LOADED_FILES.delete(path);
deleteFile(path);
},
js_read: (rid, buffer_ptr, offset, amount) => {
const buffer = new Uint8Array(
inst.exports.memory.buffer,
buffer_ptr,
amount,
);
const file = getOpenFile(rid);
return file.buffer.read(offset, buffer);
},
js_write: (rid, buffer_ptr, offset, amount) => {
const buffer = new Uint8Array(
inst.exports.memory.buffer,
buffer_ptr,
amount,
);
const file = getOpenFile(rid);
return file.buffer.write(offset, buffer);
},
js_truncate: (rid, size) => {
getOpenFile(rid).buffer.truncate(size);
},
js_sync: (rid) => {
const file = getOpenFile(rid);
if (file.path != null) syncFile(file.path, file.buffer.toUint8Array());
},
js_size: (rid) => {
return getOpenFile(rid).buffer.size;
},
js_lock: (_rid, _exclusive) => {},
js_unlock: (_rid) => {},
js_time: () => {
return Date.now();
},
js_timezone: () => {
return (new Date()).getTimezoneOffset();
},
js_exists: (path_ptr) => {
const path = getStr(inst.exports, path_ptr);
return LOADED_FILES.has(path) ? 1 : 0;
},
js_access: (_path_ptr) => 1,
};
return { env };
}
plugos/sqlite/deno-sqlite/build/.checksums
-3
View File
@@ -1,3 +0,0 @@
b9365f4aa1d3047a8d80d6bfe90e705c80e158c3 sqlite_dl.zip
022ae45d50b124b9df68be065d65b9a607923362 wasi_dl_linux.tar.gz
bc76d264214c21a603fc38adb405622a2fcda8b3 wasi_dl_darwin.tar.gz
plugos/sqlite/deno-sqlite/build/Makefile
-131
View File
@@ -1,131 +0,0 @@
DENO ?= deno
WASI ?= ./wasi-sdk
CC = $(WASI)/bin/clang
OUT_WA = "sqlite.wasm"
OUT_BN = "sqlite.js"
OUT_TY = "sqlite.d.ts"
SQLITE_DLD = "https://sqlite.org/2022/sqlite-src-3390200.zip"
SQLITE_DIR = "sqlite-src-3390200"
WASI_DLD = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-16/wasi-sdk-16.0-linux.tar.gz"
WASI_TAR = wasi_dl_linux.tar.gz
ifeq ($(shell uname), Darwin)
WASI_DLD = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-16/wasi-sdk-16.0-macos.tar.gz"
WASI_TAR = wasi_dl_darwin.tar.gz
endif
CSRC = ""
CSRC += $(shell find ./src -name "*.c")
CSRC += $(shell find ./lib -name "*.c")
CSRC += $(shell find ./hask -name "*.c")
FLGS = -Wall
RFLG = -Os
DFLG = -DDEBUG_BUILD
WAFLG = --target=wasm32-unknown-wasi -Wl,--no-entry -nostartfiles --sysroot $(WASI)/share/wasi-sysroot\
-DWASI_BUILD -Wl,--export,malloc -Wl,--export,free -Wl,--allow-undefined-file=vfs.syms
INCS = -Ilib
# Location of wrapper library which contains all c-land export
CWRP = "./src/wrapper.c"
# Configure sqlite for out use-case
SQLFLG = -DSQLITE_DQS=0 -DSQLITE_THREADSAFE=0 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS\
-DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_TEMP_STORE=2\
-DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_UTF16 -DSQLITE_OMIT_SHARED_CACHE\
-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_TRACE\
-DSQLITE_OS_OTHER=1 -DSQLITE_OMIT_COMPLETE -DSQLITE_OMIT_WAL\
-DNDEBUG=1 -DSQLITE_ENABLE_COLUMN_METADATA -DHAVE_LOCALTIME_R\
-DSQLITE_OMIT_DESERIALIZE -DSQLITE_ENABLE_FTS5
# Rational:
# SQLITE_DQS -> we do not need to have backwards comp
# SQLITE_THREADSAFE -> we run single-threaded
# SQLITE_LIKE_DOESNT_MATCH_BLOBS -> faster (is recommended if no backwards comp)
# SQLITE_DEFAULT_FOREIGN_KEYS -> this should be the default
# SQLITE_TEMP_STORE -> in memory is faster, so it's the better default
# SQLITE_OMIT_DEPRECATED -> we do not need to have backwards comp
# SQLITE_OMIT_UTF16 -> we only support utf-8 encoded strings
# SQLITE_OMIT_SHARED_CACHE -> we only ever open one connection
# SQLITE_OMIT_LOAD_EXTENSION -> we don't use it
# SQLITE_OMIT_PROGRESS_CALLBACK -> we don't use it
# SQLITE_OMIT_TRACE -> we make no use of these
# SQLITE_OS_OTHER -> we provide our own vfs
# SQLITE_OMIT_COMPLETE -> we don't need these
# SQLITE_OMIT_WAL -> this is doggy, as we can not memory map files
# DNDEBUG -> "use for maximum speed"
# SQLITE_ENABLE_COLUMN_METADATA -> we depend on column metadata interfaces (`sqlite3_column_table_name` and `sqlite3_column_origin_name`)
# SQLITE_OMIT_DESERIALIZE -> we don't use these interfaces
all: release
build:
$(DENO) run --allow-read --allow-write hack/gen_syms.js vfs.syms
$(CC) $(WAFLG) $(FLGS) $(INCS) $(CSRC) $(SQLFLG) -o $(OUT_WA)
bundle:
$(DENO) run --allow-read --allow-write hack/bundle.js $(OUT_WA) $(OUT_BN)
$(DENO) fmt $(OUT_BN)
types:
$(DENO) run --allow-read --allow-write hack/gen_types.ts $(CWRP) $(OUT_TY)
$(DENO) fmt $(OUT_TY)
size.txt: sqlite.js sqlite.wasm
rm -f size.txt *.gz *.br
gzip --best < sqlite.js > sqlite.js.gz
gzip --best < sqlite.wasm > sqlite.wasm.gz
brotli --best -o sqlite.js.br < sqlite.js && \
brotli --best -o sqlite.wasm.br < sqlite.wasm || \
echo "WARN: brotli size comparison unavailable"
ls -l sqlite.* | awk '{printf "%-15s➜%7s bytes\n",$$9,$$5}' | tee size.txt
debug: FLGS += $(DFLG)
debug: build
debug: bundle
debug: types
release: FLGS += $(RFLG)
release: build
release: bundle
release: types
release: size.txt
amalgamation:
make -C sqlite-src clean
make -C sqlite-src sqlite3.c SQLFLG="$(SQLFLG)"
mv sqlite-src/sqlite3.c lib/sqlite3.c
mv sqlite-src/sqlite3.h lib/sqlite3.h
dlsqlite:
curl "$(SQLITE_DLD)" -o "sqlite_dl.zip"
sed -n '/sqlite_dl.zip/p' ".checksums" | shasum -c -
rm -rf "sqlite-src"
unzip "sqlite_dl.zip"
mv $(SQLITE_DIR) "sqlite-src"
cp "Makefile.sqlite" "sqlite-src/Makefile"
rm "sqlite_dl.zip"
dlwasi:
curl -L "$(WASI_DLD)" -o "$(WASI_TAR)"
sed -n '/${WASI_TAR}/p' ".checksums" | shasum -c -
rm -rf "wasi-sdk"
tar -xzvf "$(WASI_TAR)"
mv "wasi-sdk-16.0" "wasi-sdk"
rm "${WASI_TAR}"
testdb:
gcc -Ilib lib/sqlite3.c hack/gen_test_db.c -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o gen_test_db
rm -f 2GB_test.db
./gen_test_db
rm gen_test_db
setup: dlsqlite
setup: dlwasi
clean:
rm -rf sqlite-src
rm -rf wasi-sdk
rm -f 2GB_test.db
.PHONY: build amalgamation dlsqlite dlwasi setup clean
plugos/sqlite/deno-sqlite/build/hack/bundle.js
-62
View File
@@ -1,62 +0,0 @@
const [src, dest] = Deno.args;
const wasm = await Deno.readFile(src);
function encode(bytes) {
let binary = "";
for (let i = 0; i < bytes.length; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary).replace(/\n/g, "");
}
await Deno.writeFile(
dest,
new TextEncoder().encode(
`/// <reference types="./sqlite.d.ts" />
/* This file is automatically generated. Do not edit directly. */
import env from "./vfs.js";
const wasm =
"${encode(wasm)}";
function decode(base64) {
const bytesStr = atob(base64);
const bytes = new Uint8Array(bytesStr.length);
for (let i = 0, c = bytesStr.length; i < c; i++) {
bytes[i] = bytesStr.charCodeAt(i);
}
return bytes;
}
const moduleOrInstance = {
module: null,
instances: [],
};
export async function compile() {
moduleOrInstance.module = await WebAssembly.compile(decode(wasm));
}
export async function instantiateBrowser() {
const placeholder = { exports: null };
const instance = await WebAssembly.instantiate(moduleOrInstance.module, env(placeholder));
placeholder.exports = instance.exports;
instance.exports.seed_rng(Date.now());
moduleOrInstance.instances.push(instance);
}
export function instantiate() {
if (moduleOrInstance.instances.length) {
return moduleOrInstance.instances.pop();
} else {
const placeholder = { exports: null };
const instance = new WebAssembly.Instance(moduleOrInstance.module, env(placeholder));
placeholder.exports = instance.exports;
instance.exports.seed_rng(Date.now());
return instance;
}
}`,
),
);
plugos/sqlite/deno-sqlite/build/hack/gen_syms.js
-10
View File
@@ -1,10 +0,0 @@
// Generate available import symbols
// from vfs.js file
import env from "../vfs.js";
let symbols = "";
for (const symbol of Object.keys(env().env)) {
symbols += `${symbol}\n`;
}
await Deno.writeFile(Deno.args[0], new TextEncoder().encode(symbols));
plugos/sqlite/deno-sqlite/build/hack/gen_test_db.c
-98
View File
@@ -1,98 +0,0 @@
#include <stdio.h>
#include <stdlib.h>
#include <sqlite3.h>
#define TRUE 1
#define FALSE 0
#define TEST_DB_FILE "2GB_test.db"
#define SQL_CREATE_TBL "CREATE TABLE test (id INTEGER PRIMARY KEY, value TEXT)"
#define SQL_INSERT_VAL "INSERT INTO test (value) VALUES (?)"
#define VAL_LEN 65536
#define VAL_NUM 45000
void rand_str(char *dest, size_t length) {
char charset[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
while (length --> 0) {
size_t index = (double) rand() / RAND_MAX * (sizeof charset - 1);
*dest++ = charset[index];
}
*dest = '\0';
}
int execute_query(sqlite3* db, char* query) {
sqlite3_stmt* stmt_create;
if (sqlite3_prepare_v2(db, query, -1, &stmt_create, NULL) != SQLITE_OK) {
printf("Failed to prepare query statement: %s\n", sqlite3_errmsg(db));
return FALSE;
}
if (sqlite3_step(stmt_create) != SQLITE_DONE) {
printf("Failed to run query statement: %s\n", sqlite3_errmsg(db));
return FALSE;
}
sqlite3_finalize(stmt_create);
return TRUE;
}
int main(int argc, char* argv[]) {
sqlite3* db;
if (sqlite3_open(TEST_DB_FILE, &db) != SQLITE_OK) {
printf("Failed to open database: %s\n", sqlite3_errmsg(db));
return 1;
}
// create database table
sqlite3_stmt* stmt_create;
if (sqlite3_prepare_v2(db, SQL_CREATE_TBL, -1, &stmt_create, NULL) != SQLITE_OK) {
printf("Failed to prepare create table statement: %s\n", sqlite3_errmsg(db));
return 1;
}
if (sqlite3_step(stmt_create) != SQLITE_DONE) {
printf("Failed to run create table statement: %s\n", sqlite3_errmsg(db));
return 1;
}
sqlite3_finalize(stmt_create);
// insert values to reach 2GB
sqlite3_stmt* stmt_insert;
if (sqlite3_prepare_v2(db, SQL_INSERT_VAL, -1, &stmt_insert, NULL) != SQLITE_OK) {
printf("Failed to prepare insert table statement: %s\n", sqlite3_errmsg(db));
return 1;
}
// begin transaction
if (!execute_query(db, "begin")) {
return 1;
}
char* buffer = malloc(VAL_LEN + 1);
for (int64_t i = 0; i < VAL_NUM; i++) {
rand_str(buffer, VAL_LEN);
if (sqlite3_bind_text(stmt_insert, 1, buffer, VAL_LEN, NULL) != SQLITE_OK) {
printf("Failed to bind value `%s`: %s\n", buffer, sqlite3_errmsg(db));
return 1;
}
if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
printf("Failed to run insert statement: %s\n", sqlite3_errmsg(db));
return 1;
}
if (sqlite3_reset(stmt_insert) != SQLITE_OK) {
printf("Failed to reset statement: %s\n", sqlite3_errmsg(db));
return 1;
}
}
// end transaction
if (!execute_query(db, "commit")) {
return 1;
}
sqlite3_finalize(stmt_insert);
sqlite3_close(db);
printf("Database created successfully.\n");
return 0;
}
plugos/sqlite/deno-sqlite/build/hack/gen_types.ts
-156
View File
@@ -1,156 +0,0 @@
interface Item {
name: string;
arguments: Argument[];
returnType: Type;
}
interface Argument {
name: string;
type: Type;
}
enum Type {
Void,
VoidPtr,
StringPtr,
StatementPtr,
Double,
Int,
}
const items = [
// exported manually in compiler invocation
{
name: "malloc",
arguments: [{ name: "size", type: Type.Int }],
returnType: Type.VoidPtr,
},
{
name: "free",
arguments: [{ name: "ptr", type: Type.VoidPtr }],
returnType: Type.Void,
},
];
const [src, dest] = Deno.args;
const wrapperSrc = await Deno.readTextFile(src);
// int EXPORT(bind_int) (sqlite3_stmt* stmt, int idx, double value)
const typeRegexp =
`(const +)?(sqlite3_stmt\\*|char\\*|void\\*|int|uint32_t|double|void)`;
const argRegexp = `${typeRegexp} +[a-z_]+`;
const exportSignature = new RegExp(
`${typeRegexp} +EXPORT\\([a-z_]+\\) +\\(((${argRegexp}( *, *${argRegexp})*)|)\\)`,
);
function nullThrows<T>(value: T | null | undefined): T {
if (value == null) {
throw new Error("Got a null value");
}
return value as T;
}
function typeFromCType(cType: string): Type {
cType = cType.replace("const", "").replace(/ /g, "");
switch (cType) {
case "void":
return Type.Void;
case "void*":
return Type.VoidPtr;
case "char*":
return Type.StringPtr;
case "sqlite3_stmt*":
return Type.StatementPtr;
case "double":
return Type.Double;
case "int":
case "uint32_t":
return Type.Int;
default:
throw new Error("Unknown type");
}
}
function getReturnType(line: string): Type {
const regexp = new RegExp(typeRegexp);
const [, _const, cType] = nullThrows(regexp.exec(line));
return typeFromCType(cType);
}
function getName(line: string): string {
const [, name] = nullThrows(/EXPORT\(([a-z_]+)\)/.exec(line));
return name;
}
function getArguments(line: string): Argument[] {
const [, argList] = nullThrows(/EXPORT\([a-z_]+\) *\(([^)]*)\)/.exec(line));
if (argList.length === 0) {
return [];
} else {
return argList.split(",").map((arg) => {
const regexp = new RegExp(`${typeRegexp} +([a-z_]+)`);
const [, _const, cType, name] = nullThrows(regexp.exec(arg));
return {
name,
type: typeFromCType(cType),
};
});
}
}
function generateType(tp: Type): string {
switch (tp) {
case Type.Void:
return "void";
case Type.VoidPtr:
return "VoidPtr";
case Type.StringPtr:
return "StringPtr";
case Type.StatementPtr:
return "StatementPtr";
case Type.Int:
case Type.Double:
return "number";
default:
throw new Error("Unknown type");
}
}
function generateDecl(item: Item): string {
const args = item.arguments.map((arg) =>
`${arg.name}: ${generateType(arg.type)}`
).join(", ");
return `${item.name}: (${args}) => ${generateType(item.returnType)}`;
}
const exportLines = wrapperSrc.split("\n").filter((line) =>
exportSignature.test(line)
);
for (const line of exportLines) {
const name = getName(line);
const returnType = getReturnType(line);
const args = getArguments(line);
items.push({ name, returnType, arguments: args });
}
const typeDeclaration =
`/* This file is automatically generated. Do not edit directly. */
export type VoidPtr = number;
export type StringPtr = number;
export type StatementPtr = number;
export interface Wasm {
memory: WebAssembly.Memory;
${items.map(generateDecl).join(";\n ")};
}
export function compile(): Promise<void>;
export function instantiateBrowser(): Promise<void>;
export function instantiate(): { exports: Wasm };
`;
await Deno.writeTextFile(dest, typeDeclaration);
plugos/sqlite/deno-sqlite/build/lib/pcg.c
-34
View File
@@ -1,34 +0,0 @@
#include "pcg.h"
// Random number generator.
// Based on:
// *Really* minimal PCG32 code / (c) 2014 M.E. O'Neill / pcg-random.org
// Licensed under Apache License 2.0 (NO WARRANTY, etc. see website)
uint64_t state = 0x853c49e6748fea9bULL;
uint64_t inc = 0xda3e39cb94b95bdbULL;
// Update seed of generator.
void pcg_seed(uint64_t seed) {
state = seed;
}
// Generate random integer.
uint32_t pcg_rand() {
uint64_t oldstate = state;
// Advance internal state
state = oldstate * 6364136223846793005ULL + (inc|1);
// Calculate output function (XSH RR), uses old state for max ILP
uint32_t xorshifted = ((oldstate >> 18u) ^ oldstate) >> 27u;
uint32_t rot = oldstate >> 59u;
return (xorshifted >> rot) | (xorshifted << ((-rot) & 31));
}
// Fill out buffer with size random bytes.
void pcg_bytes(char* out, int size) {
// TODO: We can be more efficient by using all 4
// pieces of the random number returned.
for (int i = 0; i < size; i ++) {
out[i] = (char)pcg_rand();
}
}
plugos/sqlite/deno-sqlite/build/lib/pcg.h
-13
View File
@@ -1,13 +0,0 @@
#ifndef PCG_H
#define PCG_H
#include <stdint.h>
// Seed the random number generator
void pcg_seed(uint64_t seed);
// Get random numbers and random bits
uint32_t pcg_rand();
void pcg_bytes(char* out, int size);
#endif // PCG_H
plugos/sqlite/deno-sqlite/build/lib/printf.c
-914
View File
@@ -1,914 +0,0 @@
///////////////////////////////////////////////////////////////////////////////
// \author (c) Marco Paland (info@paland.com)
// 2014-2019, PALANDesign Hannover, Germany
//
// \license The MIT License (MIT)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//
// \brief Tiny printf, sprintf and (v)snprintf implementation, optimized for speed on
// embedded systems with a very limited resources. These routines are thread
// safe and reentrant!
// Use this instead of the bloated standard/newlib printf cause these use
// malloc for printf (and may not be thread safe).
//
///////////////////////////////////////////////////////////////////////////////
#include <stdbool.h>
#include <stdint.h>
#include "printf.h"
// define this globally (e.g. gcc -DPRINTF_INCLUDE_CONFIG_H ...) to include the
// printf_config.h header file
// default: undefined
#ifdef PRINTF_INCLUDE_CONFIG_H
#include "printf_config.h"
#endif
// 'ntoa' conversion buffer size, this must be big enough to hold one converted
// numeric number including padded zeros (dynamically created on stack)
// default: 32 byte
#ifndef PRINTF_NTOA_BUFFER_SIZE
#define PRINTF_NTOA_BUFFER_SIZE 32U
#endif
// 'ftoa' conversion buffer size, this must be big enough to hold one converted
// float number including padded zeros (dynamically created on stack)
// default: 32 byte
#ifndef PRINTF_FTOA_BUFFER_SIZE
#define PRINTF_FTOA_BUFFER_SIZE 32U
#endif
// support for the floating point type (%f)
// default: activated
#ifndef PRINTF_DISABLE_SUPPORT_FLOAT
#define PRINTF_SUPPORT_FLOAT
#endif
// support for exponential floating point notation (%e/%g)
// default: activated
#ifndef PRINTF_DISABLE_SUPPORT_EXPONENTIAL
#define PRINTF_SUPPORT_EXPONENTIAL
#endif
// define the default floating point precision
// default: 6 digits
#ifndef PRINTF_DEFAULT_FLOAT_PRECISION
#define PRINTF_DEFAULT_FLOAT_PRECISION 6U
#endif
// define the largest float suitable to print with %f
// default: 1e9
#ifndef PRINTF_MAX_FLOAT
#define PRINTF_MAX_FLOAT 1e9
#endif
// support for the long long types (%llu or %p)
// default: activated
#ifndef PRINTF_DISABLE_SUPPORT_LONG_LONG
#define PRINTF_SUPPORT_LONG_LONG
#endif
// support for the ptrdiff_t type (%t)
// ptrdiff_t is normally defined in <stddef.h> as long or long long type
// default: activated
#ifndef PRINTF_DISABLE_SUPPORT_PTRDIFF_T
#define PRINTF_SUPPORT_PTRDIFF_T
#endif
///////////////////////////////////////////////////////////////////////////////
// internal flag definitions
#define FLAGS_ZEROPAD (1U << 0U)
#define FLAGS_LEFT (1U << 1U)
#define FLAGS_PLUS (1U << 2U)
#define FLAGS_SPACE (1U << 3U)
#define FLAGS_HASH (1U << 4U)
#define FLAGS_UPPERCASE (1U << 5U)
#define FLAGS_CHAR (1U << 6U)
#define FLAGS_SHORT (1U << 7U)
#define FLAGS_LONG (1U << 8U)
#define FLAGS_LONG_LONG (1U << 9U)
#define FLAGS_PRECISION (1U << 10U)
#define FLAGS_ADAPT_EXP (1U << 11U)
// import float.h for DBL_MAX
#if defined(PRINTF_SUPPORT_FLOAT)
#include <float.h>
#endif
// output function type
typedef void (*out_fct_type)(char character, void* buffer, size_t idx, size_t maxlen);
// wrapper (used as buffer) for output function type
typedef struct {
void (*fct)(char character, void* arg);
void* arg;
} out_fct_wrap_type;
// internal buffer output
static inline void _out_buffer(char character, void* buffer, size_t idx, size_t maxlen)
{
if (idx < maxlen) {
((char*)buffer)[idx] = character;
}
}
// internal null output
static inline void _out_null(char character, void* buffer, size_t idx, size_t maxlen)
{
(void)character; (void)buffer; (void)idx; (void)maxlen;
}
// internal _putchar wrapper
static inline void _out_char(char character, void* buffer, size_t idx, size_t maxlen)
{
(void)buffer; (void)idx; (void)maxlen;
if (character) {
_putchar(character);
}
}
// internal output function wrapper
static inline void _out_fct(char character, void* buffer, size_t idx, size_t maxlen)
{
(void)idx; (void)maxlen;
if (character) {
// buffer is the output fct pointer
((out_fct_wrap_type*)buffer)->fct(character, ((out_fct_wrap_type*)buffer)->arg);
}
}
// internal secure strlen
// \return The length of the string (excluding the terminating 0) limited by 'maxsize'
static inline unsigned int _strnlen_s(const char* str, size_t maxsize)
{
const char* s;
for (s = str; *s && maxsize--; ++s);
return (unsigned int)(s - str);
}
// internal test if char is a digit (0-9)
// \return true if char is a digit
static inline bool _is_digit(char ch)
{
return (ch >= '0') && (ch <= '9');
}
// internal ASCII string to unsigned int conversion
static unsigned int _atoi(const char** str)
{
unsigned int i = 0U;
while (_is_digit(**str)) {
i = i * 10U + (unsigned int)(*((*str)++) - '0');
}
return i;
}
// output the specified string in reverse, taking care of any zero-padding
static size_t _out_rev(out_fct_type out, char* buffer, size_t idx, size_t maxlen, const char* buf, size_t len, unsigned int width, unsigned int flags)
{
const size_t start_idx = idx;
// pad spaces up to given width
if (!(flags & FLAGS_LEFT) && !(flags & FLAGS_ZEROPAD)) {
for (size_t i = len; i < width; i++) {
out(' ', buffer, idx++, maxlen);
}
}
// reverse string
while (len) {
out(buf[--len], buffer, idx++, maxlen);
}
// append pad spaces up to given width
if (flags & FLAGS_LEFT) {
while (idx - start_idx < width) {
out(' ', buffer, idx++, maxlen);
}
}
return idx;
}
// internal itoa format
static size_t _ntoa_format(out_fct_type out, char* buffer, size_t idx, size_t maxlen, char* buf, size_t len, bool negative, unsigned int base, unsigned int prec, unsigned int width, unsigned int flags)
{
// pad leading zeros
if (!(flags & FLAGS_LEFT)) {
if (width && (flags & FLAGS_ZEROPAD) && (negative || (flags & (FLAGS_PLUS | FLAGS_SPACE)))) {
width--;
}
while ((len < prec) && (len < PRINTF_NTOA_BUFFER_SIZE)) {
buf[len++] = '0';
}
while ((flags & FLAGS_ZEROPAD) && (len < width) && (len < PRINTF_NTOA_BUFFER_SIZE)) {
buf[len++] = '0';
}
}
// handle hash
if (flags & FLAGS_HASH) {
if (!(flags & FLAGS_PRECISION) && len && ((len == prec) || (len == width))) {
len--;
if (len && (base == 16U)) {
len--;
}
}
if ((base == 16U) && !(flags & FLAGS_UPPERCASE) && (len < PRINTF_NTOA_BUFFER_SIZE)) {
buf[len++] = 'x';
}
else if ((base == 16U) && (flags & FLAGS_UPPERCASE) && (len < PRINTF_NTOA_BUFFER_SIZE)) {
buf[len++] = 'X';
}
else if ((base == 2U) && (len < PRINTF_NTOA_BUFFER_SIZE)) {
buf[len++] = 'b';
}
if (len < PRINTF_NTOA_BUFFER_SIZE) {
buf[len++] = '0';
}
}
if (len < PRINTF_NTOA_BUFFER_SIZE) {
if (negative) {
buf[len++] = '-';
}
else if (flags & FLAGS_PLUS) {
buf[len++] = '+'; // ignore the space if the '+' exists
}
else if (flags & FLAGS_SPACE) {
buf[len++] = ' ';
}
}
return _out_rev(out, buffer, idx, maxlen, buf, len, width, flags);
}
// internal itoa for 'long' type
static size_t _ntoa_long(out_fct_type out, char* buffer, size_t idx, size_t maxlen, unsigned long value, bool negative, unsigned long base, unsigned int prec, unsigned int width, unsigned int flags)
{
char buf[PRINTF_NTOA_BUFFER_SIZE];
size_t len = 0U;
// no hash for 0 values
if (!value) {
flags &= ~FLAGS_HASH;
}
// write if precision != 0 and value is != 0
if (!(flags & FLAGS_PRECISION) || value) {
do {
const char digit = (char)(value % base);
buf[len++] = digit < 10 ? '0' + digit : (flags & FLAGS_UPPERCASE ? 'A' : 'a') + digit - 10;
value /= base;
} while (value && (len < PRINTF_NTOA_BUFFER_SIZE));
}
return _ntoa_format(out, buffer, idx, maxlen, buf, len, negative, (unsigned int)base, prec, width, flags);
}
// internal itoa for 'long long' type
#if defined(PRINTF_SUPPORT_LONG_LONG)
static size_t _ntoa_long_long(out_fct_type out, char* buffer, size_t idx, size_t maxlen, unsigned long long value, bool negative, unsigned long long base, unsigned int prec, unsigned int width, unsigned int flags)
{
char buf[PRINTF_NTOA_BUFFER_SIZE];
size_t len = 0U;
// no hash for 0 values
if (!value) {
flags &= ~FLAGS_HASH;
}
// write if precision != 0 and value is != 0
if (!(flags & FLAGS_PRECISION) || value) {
do {
const char digit = (char)(value % base);
buf[len++] = digit < 10 ? '0' + digit : (flags & FLAGS_UPPERCASE ? 'A' : 'a') + digit - 10;
value /= base;
} while (value && (len < PRINTF_NTOA_BUFFER_SIZE));
}
return _ntoa_format(out, buffer, idx, maxlen, buf, len, negative, (unsigned int)base, prec, width, flags);
}
#endif // PRINTF_SUPPORT_LONG_LONG
#if defined(PRINTF_SUPPORT_FLOAT)
#if defined(PRINTF_SUPPORT_EXPONENTIAL)
// forward declaration so that _ftoa can switch to exp notation for values > PRINTF_MAX_FLOAT
static size_t _etoa(out_fct_type out, char* buffer, size_t idx, size_t maxlen, double value, unsigned int prec, unsigned int width, unsigned int flags);
#endif
// internal ftoa for fixed decimal floating point
static size_t _ftoa(out_fct_type out, char* buffer, size_t idx, size_t maxlen, double value, unsigned int prec, unsigned int width, unsigned int flags)
{
char buf[PRINTF_FTOA_BUFFER_SIZE];
size_t len = 0U;
double diff = 0.0;
// powers of 10
static const double pow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 };
// test for special values
if (value != value)
return _out_rev(out, buffer, idx, maxlen, "nan", 3, width, flags);
if (value < -DBL_MAX)
return _out_rev(out, buffer, idx, maxlen, "fni-", 4, width, flags);
if (value > DBL_MAX)
return _out_rev(out, buffer, idx, maxlen, (flags & FLAGS_PLUS) ? "fni+" : "fni", (flags & FLAGS_PLUS) ? 4U : 3U, width, flags);
// test for very large values
// standard printf behavior is to print EVERY whole number digit -- which could be 100s of characters overflowing your buffers == bad
if ((value > PRINTF_MAX_FLOAT) || (value < -PRINTF_MAX_FLOAT)) {
#if defined(PRINTF_SUPPORT_EXPONENTIAL)
return _etoa(out, buffer, idx, maxlen, value, prec, width, flags);
#else
return 0U;
#endif
}
// test for negative
bool negative = false;
if (value < 0) {
negative = true;
value = 0 - value;
}
// set default precision, if not set explicitly
if (!(flags & FLAGS_PRECISION)) {
prec = PRINTF_DEFAULT_FLOAT_PRECISION;
}
// limit precision to 9, cause a prec >= 10 can lead to overflow errors
while ((len < PRINTF_FTOA_BUFFER_SIZE) && (prec > 9U)) {
buf[len++] = '0';
prec--;
}
int whole = (int)value;
double tmp = (value - whole) * pow10[prec];
unsigned long frac = (unsigned long)tmp;
diff = tmp - frac;
if (diff > 0.5) {
++frac;
// handle rollover, e.g. case 0.99 with prec 1 is 1.0
if (frac >= pow10[prec]) {
frac = 0;
++whole;
}
}
else if (diff < 0.5) {
}
else if ((frac == 0U) || (frac & 1U)) {
// if halfway, round up if odd OR if last digit is 0
++frac;
}
if (prec == 0U) {
diff = value - (double)whole;
if ((!(diff < 0.5) || (diff > 0.5)) && (whole & 1)) {
// exactly 0.5 and ODD, then round up
// 1.5 -> 2, but 2.5 -> 2
++whole;
}
}
else {
unsigned int count = prec;
// now do fractional part, as an unsigned number
while (len < PRINTF_FTOA_BUFFER_SIZE) {
--count;
buf[len++] = (char)(48U + (frac % 10U));
if (!(frac /= 10U)) {
break;
}
}
// add extra 0s
while ((len < PRINTF_FTOA_BUFFER_SIZE) && (count-- > 0U)) {
buf[len++] = '0';
}
if (len < PRINTF_FTOA_BUFFER_SIZE) {
// add decimal
buf[len++] = '.';
}
}
// do whole part, number is reversed
while (len < PRINTF_FTOA_BUFFER_SIZE) {
buf[len++] = (char)(48 + (whole % 10));
if (!(whole /= 10)) {
break;
}
}
// pad leading zeros
if (!(flags & FLAGS_LEFT) && (flags & FLAGS_ZEROPAD)) {
if (width && (negative || (flags & (FLAGS_PLUS | FLAGS_SPACE)))) {
width--;
}
while ((len < width) && (len < PRINTF_FTOA_BUFFER_SIZE)) {
buf[len++] = '0';
}
}
if (len < PRINTF_FTOA_BUFFER_SIZE) {
if (negative) {
buf[len++] = '-';
}
else if (flags & FLAGS_PLUS) {
buf[len++] = '+'; // ignore the space if the '+' exists
}
else if (flags & FLAGS_SPACE) {
buf[len++] = ' ';
}
}
return _out_rev(out, buffer, idx, maxlen, buf, len, width, flags);
}
#if defined(PRINTF_SUPPORT_EXPONENTIAL)
// internal ftoa variant for exponential floating-point type, contributed by Martijn Jasperse <m.jasperse@gmail.com>
static size_t _etoa(out_fct_type out, char* buffer, size_t idx, size_t maxlen, double value, unsigned int prec, unsigned int width, unsigned int flags)
{
// check for NaN and special values
if ((value != value) || (value > DBL_MAX) || (value < -DBL_MAX)) {
return _ftoa(out, buffer, idx, maxlen, value, prec, width, flags);
}
// determine the sign
const bool negative = value < 0;
if (negative) {
value = -value;
}
// default precision
if (!(flags & FLAGS_PRECISION)) {
prec = PRINTF_DEFAULT_FLOAT_PRECISION;
}
// determine the decimal exponent
// based on the algorithm by David Gay (https://www.ampl.com/netlib/fp/dtoa.c)
union {
uint64_t U;
double F;
} conv;
conv.F = value;
int exp2 = (int)((conv.U >> 52U) & 0x07FFU) - 1023; // effectively log2
conv.U = (conv.U & ((1ULL << 52U) - 1U)) | (1023ULL << 52U); // drop the exponent so conv.F is now in [1,2)
// now approximate log10 from the log2 integer part and an expansion of ln around 1.5
int expval = (int)(0.1760912590558 + exp2 * 0.301029995663981 + (conv.F - 1.5) * 0.289529654602168);
// now we want to compute 10^expval but we want to be sure it won't overflow
exp2 = (int)(expval * 3.321928094887362 + 0.5);
const double z = expval * 2.302585092994046 - exp2 * 0.6931471805599453;
const double z2 = z * z;
conv.U = (uint64_t)(exp2 + 1023) << 52U;
// compute exp(z) using continued fractions, see https://en.wikipedia.org/wiki/Exponential_function#Continued_fractions_for_ex
conv.F *= 1 + 2 * z / (2 - z + (z2 / (6 + (z2 / (10 + z2 / 14)))));
// correct for rounding errors
if (value < conv.F) {
expval--;
conv.F /= 10;
}
// the exponent format is "%+03d" and largest value is "307", so set aside 4-5 characters
unsigned int minwidth = ((expval < 100) && (expval > -100)) ? 4U : 5U;
// in "%g" mode, "prec" is the number of *significant figures* not decimals
if (flags & FLAGS_ADAPT_EXP) {
// do we want to fall-back to "%f" mode?
if ((value >= 1e-4) && (value < 1e6)) {
if ((int)prec > expval) {
prec = (unsigned)((int)prec - expval - 1);
}
else {
prec = 0;
}
flags |= FLAGS_PRECISION; // make sure _ftoa respects precision
// no characters in exponent
minwidth = 0U;
expval = 0;
}
else {
// we use one sigfig for the whole part
if ((prec > 0) && (flags & FLAGS_PRECISION)) {
--prec;
}
}
}
// will everything fit?
unsigned int fwidth = width;
if (width > minwidth) {
// we didn't fall-back so subtract the characters required for the exponent
fwidth -= minwidth;
} else {
// not enough characters, so go back to default sizing
fwidth = 0U;
}
if ((flags & FLAGS_LEFT) && minwidth) {
// if we're padding on the right, DON'T pad the floating part
fwidth = 0U;
}
// rescale the float value
if (expval) {
value /= conv.F;
}
// output the floating part
const size_t start_idx = idx;
idx = _ftoa(out, buffer, idx, maxlen, negative ? -value : value, prec, fwidth, flags & ~FLAGS_ADAPT_EXP);
// output the exponent part
if (minwidth) {
// output the exponential symbol
out((flags & FLAGS_UPPERCASE) ? 'E' : 'e', buffer, idx++, maxlen);
// output the exponent value
idx = _ntoa_long(out, buffer, idx, maxlen, (expval < 0) ? -expval : expval, expval < 0, 10, 0, minwidth-1, FLAGS_ZEROPAD | FLAGS_PLUS);
// might need to right-pad spaces
if (flags & FLAGS_LEFT) {
while (idx - start_idx < width) out(' ', buffer, idx++, maxlen);
}
}
return idx;
}
#endif // PRINTF_SUPPORT_EXPONENTIAL
#endif // PRINTF_SUPPORT_FLOAT
// internal vsnprintf
static int _vsnprintf(out_fct_type out, char* buffer, const size_t maxlen, const char* format, va_list va)
{
unsigned int flags, width, precision, n;
size_t idx = 0U;
if (!buffer) {
// use null output function
out = _out_null;
}
while (*format)
{
// format specifier? %[flags][width][.precision][length]
if (*format != '%') {
// no
out(*format, buffer, idx++, maxlen);
format++;
continue;
}
else {
// yes, evaluate it
format++;
}
// evaluate flags
flags = 0U;
do {
switch (*format) {
case '0': flags |= FLAGS_ZEROPAD; format++; n = 1U; break;
case '-': flags |= FLAGS_LEFT; format++; n = 1U; break;
case '+': flags |= FLAGS_PLUS; format++; n = 1U; break;
case ' ': flags |= FLAGS_SPACE; format++; n = 1U; break;
case '#': flags |= FLAGS_HASH; format++; n = 1U; break;
default : n = 0U; break;
}
} while (n);
// evaluate width field
width = 0U;
if (_is_digit(*format)) {
width = _atoi(&format);
}
else if (*format == '*') {
const int w = va_arg(va, int);
if (w < 0) {
flags |= FLAGS_LEFT; // reverse padding
width = (unsigned int)-w;
}
else {
width = (unsigned int)w;
}
format++;
}
// evaluate precision field
precision = 0U;
if (*format == '.') {
flags |= FLAGS_PRECISION;
format++;
if (_is_digit(*format)) {
precision = _atoi(&format);
}
else if (*format == '*') {
const int prec = (int)va_arg(va, int);
precision = prec > 0 ? (unsigned int)prec : 0U;
format++;
}
}
// evaluate length field
switch (*format) {
case 'l' :
flags |= FLAGS_LONG;
format++;
if (*format == 'l') {
flags |= FLAGS_LONG_LONG;
format++;
}
break;
case 'h' :
flags |= FLAGS_SHORT;
format++;
if (*format == 'h') {
flags |= FLAGS_CHAR;
format++;
}
break;
#if defined(PRINTF_SUPPORT_PTRDIFF_T)
case 't' :
flags |= (sizeof(ptrdiff_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG);
format++;
break;
#endif
case 'j' :
flags |= (sizeof(intmax_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG);
format++;
break;
case 'z' :
flags |= (sizeof(size_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG);
format++;
break;
default :
break;
}
// evaluate specifier
switch (*format) {
case 'd' :
case 'i' :
case 'u' :
case 'x' :
case 'X' :
case 'o' :
case 'b' : {
// set the base
unsigned int base;
if (*format == 'x' || *format == 'X') {
base = 16U;
}
else if (*format == 'o') {
base = 8U;
}
else if (*format == 'b') {
base = 2U;
}
else {
base = 10U;
flags &= ~FLAGS_HASH; // no hash for dec format
}
// uppercase
if (*format == 'X') {
flags |= FLAGS_UPPERCASE;
}
// no plus or space flag for u, x, X, o, b
if ((*format != 'i') && (*format != 'd')) {
flags &= ~(FLAGS_PLUS | FLAGS_SPACE);
}
// ignore '0' flag when precision is given
if (flags & FLAGS_PRECISION) {
flags &= ~FLAGS_ZEROPAD;
}
// convert the integer
if ((*format == 'i') || (*format == 'd')) {
// signed
if (flags & FLAGS_LONG_LONG) {
#if defined(PRINTF_SUPPORT_LONG_LONG)
const long long value = va_arg(va, long long);
idx = _ntoa_long_long(out, buffer, idx, maxlen, (unsigned long long)(value > 0 ? value : 0 - value), value < 0, base, precision, width, flags);
#endif
}
else if (flags & FLAGS_LONG) {
const long value = va_arg(va, long);
idx = _ntoa_long(out, buffer, idx, maxlen, (unsigned long)(value > 0 ? value : 0 - value), value < 0, base, precision, width, flags);
}
else {
const int value = (flags & FLAGS_CHAR) ? (char)va_arg(va, int) : (flags & FLAGS_SHORT) ? (short int)va_arg(va, int) : va_arg(va, int);
idx = _ntoa_long(out, buffer, idx, maxlen, (unsigned int)(value > 0 ? value : 0 - value), value < 0, base, precision, width, flags);
}
}
else {
// unsigned
if (flags & FLAGS_LONG_LONG) {
#if defined(PRINTF_SUPPORT_LONG_LONG)
idx = _ntoa_long_long(out, buffer, idx, maxlen, va_arg(va, unsigned long long), false, base, precision, width, flags);
#endif
}
else if (flags & FLAGS_LONG) {
idx = _ntoa_long(out, buffer, idx, maxlen, va_arg(va, unsigned long), false, base, precision, width, flags);
}
else {
const unsigned int value = (flags & FLAGS_CHAR) ? (unsigned char)va_arg(va, unsigned int) : (flags & FLAGS_SHORT) ? (unsigned short int)va_arg(va, unsigned int) : va_arg(va, unsigned int);
idx = _ntoa_long(out, buffer, idx, maxlen, value, false, base, precision, width, flags);
}
}
format++;
break;
}
#if defined(PRINTF_SUPPORT_FLOAT)
case 'f' :
case 'F' :
if (*format == 'F') flags |= FLAGS_UPPERCASE;
idx = _ftoa(out, buffer, idx, maxlen, va_arg(va, double), precision, width, flags);
format++;
break;
#if defined(PRINTF_SUPPORT_EXPONENTIAL)
case 'e':
case 'E':
case 'g':
case 'G':
if ((*format == 'g')||(*format == 'G')) flags |= FLAGS_ADAPT_EXP;
if ((*format == 'E')||(*format == 'G')) flags |= FLAGS_UPPERCASE;
idx = _etoa(out, buffer, idx, maxlen, va_arg(va, double), precision, width, flags);
format++;
break;
#endif // PRINTF_SUPPORT_EXPONENTIAL
#endif // PRINTF_SUPPORT_FLOAT
case 'c' : {
unsigned int l = 1U;
// pre padding
if (!(flags & FLAGS_LEFT)) {
while (l++ < width) {
out(' ', buffer, idx++, maxlen);
}
}
// char output
out((char)va_arg(va, int), buffer, idx++, maxlen);
// post padding
if (flags & FLAGS_LEFT) {
while (l++ < width) {
out(' ', buffer, idx++, maxlen);
}
}
format++;
break;
}
case 's' : {
const char* p = va_arg(va, char*);
unsigned int l = _strnlen_s(p, precision ? precision : (size_t)-1);
// pre padding
if (flags & FLAGS_PRECISION) {
l = (l < precision ? l : precision);
}
if (!(flags & FLAGS_LEFT)) {
while (l++ < width) {
out(' ', buffer, idx++, maxlen);
}
}
// string output
while ((*p != 0) && (!(flags & FLAGS_PRECISION) || precision--)) {
out(*(p++), buffer, idx++, maxlen);
}
// post padding
if (flags & FLAGS_LEFT) {
while (l++ < width) {
out(' ', buffer, idx++, maxlen);
}
}
format++;
break;
}
case 'p' : {
width = sizeof(void*) * 2U;
flags |= FLAGS_ZEROPAD | FLAGS_UPPERCASE;
#if defined(PRINTF_SUPPORT_LONG_LONG)
const bool is_ll = sizeof(uintptr_t) == sizeof(long long);
if (is_ll) {
idx = _ntoa_long_long(out, buffer, idx, maxlen, (uintptr_t)va_arg(va, void*), false, 16U, precision, width, flags);
}
else {
#endif
idx = _ntoa_long(out, buffer, idx, maxlen, (unsigned long)((uintptr_t)va_arg(va, void*)), false, 16U, precision, width, flags);
#if defined(PRINTF_SUPPORT_LONG_LONG)
}
#endif
format++;
break;
}
case '%' :
out('%', buffer, idx++, maxlen);
format++;
break;
default :
out(*format, buffer, idx++, maxlen);
format++;
break;
}
}
// termination
out((char)0, buffer, idx < maxlen ? idx : maxlen - 1U, maxlen);
// return written chars without terminating \0
return (int)idx;
}
///////////////////////////////////////////////////////////////////////////////
int printf_(const char* format, ...)
{
va_list va;
va_start(va, format);
char buffer[1];
const int ret = _vsnprintf(_out_char, buffer, (size_t)-1, format, va);
va_end(va);
return ret;
}
int sprintf_(char* buffer, const char* format, ...)
{
va_list va;
va_start(va, format);
const int ret = _vsnprintf(_out_buffer, buffer, (size_t)-1, format, va);
va_end(va);
return ret;
}
int snprintf_(char* buffer, size_t count, const char* format, ...)
{
va_list va;
va_start(va, format);
const int ret = _vsnprintf(_out_buffer, buffer, count, format, va);
va_end(va);
return ret;
}
int vprintf_(const char* format, va_list va)
{
char buffer[1];
return _vsnprintf(_out_char, buffer, (size_t)-1, format, va);
}
int vsnprintf_(char* buffer, size_t count, const char* format, va_list va)
{
return _vsnprintf(_out_buffer, buffer, count, format, va);
}
int fctprintf(void (*out)(char character, void* arg), void* arg, const char* format, ...)
{
va_list va;
va_start(va, format);
const out_fct_wrap_type out_fct_wrap = { out, arg };
const int ret = _vsnprintf(_out_fct, (char*)(uintptr_t)&out_fct_wrap, (size_t)-1, format, va);
va_end(va);
return ret;
}
plugos/sqlite/deno-sqlite/build/lib/printf.h
-117
View File
@@ -1,117 +0,0 @@
///////////////////////////////////////////////////////////////////////////////
// \author (c) Marco Paland (info@paland.com)
// 2014-2019, PALANDesign Hannover, Germany
//
// \license The MIT License (MIT)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//
// \brief Tiny printf, sprintf and snprintf implementation, optimized for speed on
// embedded systems with a very limited resources.
// Use this instead of bloated standard/newlib printf.
// These routines are thread safe and reentrant.
//
///////////////////////////////////////////////////////////////////////////////
#ifndef _PRINTF_H_
#define _PRINTF_H_
#include <stdarg.h>
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* Output a character to a custom device like UART, used by the printf() function
* This function is declared here only. You have to write your custom implementation somewhere
* \param character Character to output
*/
void _putchar(char character);
/**
* Tiny printf implementation
* You have to implement _putchar if you use printf()
* To avoid conflicts with the regular printf() API it is overridden by macro defines
* and internal underscore-appended functions like printf_() are used
* \param format A string that specifies the format of the output
* \return The number of characters that are written into the array, not counting the terminating null character
*/
#define printf printf_
int printf_(const char* format, ...);
/**
* Tiny sprintf implementation
* Due to security reasons (buffer overflow) YOU SHOULD CONSIDER USING (V)SNPRINTF INSTEAD!
* \param buffer A pointer to the buffer where to store the formatted string. MUST be big enough to store the output!
* \param format A string that specifies the format of the output
* \return The number of characters that are WRITTEN into the buffer, not counting the terminating null character
*/
#define sprintf sprintf_
int sprintf_(char* buffer, const char* format, ...);
/**
* Tiny snprintf/vsnprintf implementation
* \param buffer A pointer to the buffer where to store the formatted string
* \param count The maximum number of characters to store in the buffer, including a terminating null character
* \param format A string that specifies the format of the output
* \param va A value identifying a variable arguments list
* \return The number of characters that COULD have been written into the buffer, not counting the terminating
* null character. A value equal or larger than count indicates truncation. Only when the returned value
* is non-negative and less than count, the string has been completely written.
*/
#define snprintf snprintf_
#define vsnprintf vsnprintf_
int snprintf_(char* buffer, size_t count, const char* format, ...);
int vsnprintf_(char* buffer, size_t count, const char* format, va_list va);
/**
* Tiny vprintf implementation
* \param format A string that specifies the format of the output
* \param va A value identifying a variable arguments list
* \return The number of characters that are WRITTEN into the buffer, not counting the terminating null character
*/
#define vprintf vprintf_
int vprintf_(const char* format, va_list va);
/**
* printf with output function
* You may use this as dynamic alternative to printf() with its fixed _putchar() output
* \param out An output function which takes one character and an argument pointer
* \param arg An argument pointer for user data passed to output function
* \param format A string that specifies the format of the output
* \return The number of characters that are sent to the output function, not counting the terminating null character
*/
int fctprintf(void (*out)(char character, void* arg), void* arg, const char* format, ...);
#ifdef __cplusplus
}
#endif
#endif // _PRINTF_H_
plugos/sqlite/deno-sqlite/build/lib/sqlite3.c
-241619
View File
File diff suppressed because it is too large Load Diff
plugos/sqlite/deno-sqlite/build/lib/sqlite3.h
-12836
View File
File diff suppressed because it is too large Load Diff
plugos/sqlite/deno-sqlite/build/size.txt
-7
View File
@@ -1,7 +0,0 @@
sqlite.d.ts ➜ 2213 bytes
sqlite.js ➜1116045 bytes
sqlite.js.br ➜ 374501 bytes
sqlite.js.gz ➜ 457972 bytes
sqlite.wasm ➜ 836080 bytes
sqlite.wasm.br ➜ 299791 bytes
sqlite.wasm.gz ➜ 352828 bytes
plugos/sqlite/deno-sqlite/build/sqlite.d.ts
Vendored
-59
View File
@@ -1,59 +0,0 @@
/* This file is automatically generated. Do not edit directly. */
export type VoidPtr = number;
export type StringPtr = number;
export type StatementPtr = number;
export interface Wasm {
memory: WebAssembly.Memory;
malloc: (size: number) => VoidPtr;
free: (ptr: VoidPtr) => void;
str_len: (str: StringPtr) => number;
seed_rng: (seed: number) => void;
get_status: () => number;
open: (filename: StringPtr, flags: number) => number;
close: () => number;
get_sqlite_error_str: () => StringPtr;
prepare: (sql: StringPtr) => StatementPtr;
finalize: (stmt: StatementPtr) => number;
reset: (stmt: StatementPtr) => number;
clear_bindings: (stmt: StatementPtr) => number;
exec: (sql: StringPtr) => number;
bind_int: (stmt: StatementPtr, idx: number, value: number) => number;
bind_double: (stmt: StatementPtr, idx: number, value: number) => number;
bind_text: (stmt: StatementPtr, idx: number, value: StringPtr) => number;
bind_blob: (
stmt: StatementPtr,
idx: number,
value: VoidPtr,
size: number,
) => number;
bind_big_int: (
stmt: StatementPtr,
idx: number,
sign: number,
high: number,
low: number,
) => number;
bind_null: (stmt: StatementPtr, idx: number) => number;
bind_parameter_index: (stmt: StatementPtr, name: StringPtr) => number;
step: (stmt: StatementPtr) => number;
column_count: (stmt: StatementPtr) => number;
column_type: (stmt: StatementPtr, col: number) => number;
column_int: (stmt: StatementPtr, col: number) => number;
column_double: (stmt: StatementPtr, col: number) => number;
column_text: (stmt: StatementPtr, col: number) => StringPtr;
column_blob: (stmt: StatementPtr, col: number) => VoidPtr;
column_bytes: (stmt: StatementPtr, col: number) => number;
column_name: (stmt: StatementPtr, col: number) => StringPtr;
column_origin_name: (stmt: StatementPtr, col: number) => StringPtr;
column_table_name: (stmt: StatementPtr, col: number) => StringPtr;
last_insert_rowid: () => number;
changes: () => number;
total_changes: () => number;
}
export function compile(): Promise<void>;
export function instantiateBrowser(): Promise<void>;
export function instantiate(): { exports: Wasm };
plugos/sqlite/deno-sqlite/build/sqlite.js
-51
View File
File diff suppressed because one or more lines are too long
plugos/sqlite/deno-sqlite/build/sqlite.wasm
BIN
View File
Binary file not shown.
plugos/sqlite/deno-sqlite/build/src/debug.h
-27
View File
@@ -1,27 +0,0 @@
#ifndef DEBUG_H
#define DEBUG_H
#ifdef DEBUG_BUILD
#include <stdlib.h>
#include <printf.h>
#include "imports.h"
// Print debug messages
#define debug_printf(...) { \
char* __debug_msg = malloc(2048); \
if (!__debug_msg) { \
js_print("ERROR: No memory for debug message.\n"); \
} else { \
size_t __used = snprintf(__debug_msg, 2048, "DEBUG: %s:%d:%s(): ", __FILE__, __LINE__, __func__); \
snprintf(&__debug_msg[__used], 2048 - __used, __VA_ARGS__); \
js_print(__debug_msg); \
free(__debug_msg); \
} \
}
#else // DEBUG_BUILD
#define debug_printf(...)
#endif // DEBUG_BUILD
#endif // DEBUG_H
plugos/sqlite/deno-sqlite/build/src/imports.h
-22
View File
@@ -1,22 +0,0 @@
#ifndef IMPORTS_H
#define IMPORTS_H
// WASM imports specified in vfs.syms
extern void js_print(const char*);
extern int js_open(const char*, int, int);
extern void js_close(int);
extern void js_delete(const char*);
extern int js_read(int, const char*, double, int);
extern int js_write(int, const char*, double, int);
extern void js_truncate(int, double);
extern void js_sync(int);
extern double js_size(int);
extern void js_lock(int, int);
extern void js_unlock(int);
extern double js_time();
extern int js_timezone();
extern int js_exists(const char*);
extern int js_access(const char*);
#endif // DEBUG_H
plugos/sqlite/deno-sqlite/build/src/vfs.c
-302
View File
@@ -1,302 +0,0 @@
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <time.h>
#include <sqlite3.h>
#include <pcg.h>
#include "debug.h"
#include "imports.h"
// SQLite VFS component.
// Based on demoVFS from SQLlite.
// https://www.sqlite.org/src/doc/trunk/src/test_demovfs.c
#define MAXPATHNAME 1024
#define JS_MAX_SAFE_INTEGER 9007199254740991
// When using this VFS, the sqlite3_file* handles that SQLite uses are
// actually pointers to instances of type DenoFile.
typedef struct DenoFile DenoFile;
struct DenoFile {
sqlite3_file base;
// Deno file resource id
int rid;
};
static int denoClose(sqlite3_file *pFile) {
DenoFile* p = (DenoFile*)pFile;
js_close(p->rid);
debug_printf("closing file (rid %i)\n", p->rid);
return SQLITE_OK;
}
// Read data from a file.
static int denoRead(sqlite3_file *pFile, void *zBuf, int iAmt, sqlite_int64 iOfst) {
DenoFile *p = (DenoFile*)pFile;
int read_bytes = 0;
if (iOfst <= JS_MAX_SAFE_INTEGER) {
// Read bytes from buffer
read_bytes = js_read(p->rid, (char*)zBuf, (double)iOfst, iAmt);
debug_printf("attempt to read from file (rid %i, amount %i, offset %lli, read %i)\n",
p->rid, iAmt, iOfst, read_bytes);
} else {
debug_printf("read offset %lli overflows JS_MAX_SAFE_INTEGER\n", iOfst);
}
// Zero memory if read was short
if (read_bytes < iAmt)
memset(&((char*)zBuf)[read_bytes], 0, iAmt-read_bytes);
return read_bytes < iAmt ? SQLITE_IOERR_SHORT_READ : SQLITE_OK;
}
// Write data to a file.
static int denoWrite(sqlite3_file *pFile, const void *zBuf, int iAmt, sqlite_int64 iOfst) {
DenoFile *p = (DenoFile*)pFile;
int write_bytes = 0;
if (iOfst <= JS_MAX_SAFE_INTEGER) {
// Write bytes to buffer
write_bytes = js_write(p->rid, (char*)zBuf, (double)iOfst, iAmt);
debug_printf("attempt to write to file (rid %i, amount %i, offset %lli, written %i)\n",
p->rid, iAmt, iOfst, write_bytes);
} else {
debug_printf("write offset %lli overflows JS_MAX_SAFE_INTEGER\n", iOfst);
}
return write_bytes == iAmt ? SQLITE_OK : SQLITE_IOERR_WRITE;
}
// Truncate file.
static int denoTruncate(sqlite3_file *pFile, sqlite_int64 size) {
DenoFile *p = (DenoFile*)pFile;
if (size <= JS_MAX_SAFE_INTEGER) {
js_truncate(p->rid, (double)size);
debug_printf("truncating file (rid %i, size: %lli)\n", p->rid, size);
return SQLITE_OK;
} else {
debug_printf("truncate length %lli overflows JS_MAX_SAFE_INTEGER\n", size);
return SQLITE_IOERR;
}
}
// Deno provides no explicit sync for us, so we
// just have a no-op here.
// TODO(dyedgreen): Investigate if there is a better way
static int denoSync(sqlite3_file *pFile, int flags) {
DenoFile *p = (DenoFile*)pFile;
js_sync(p->rid);
debug_printf("syncing file (rid %i)\n", p->rid);
return SQLITE_OK;
}
// Write the size of the file in bytes to *pSize.
static int denoFileSize(sqlite3_file *pFile, sqlite_int64 *pSize) {
DenoFile *p = (DenoFile*)pFile;
*pSize = (sqlite_int64)js_size(p->rid);
debug_printf("read file size: %lli (rid %i)\n", *pSize, p->rid);
return SQLITE_OK;
}
// File locking
static int denoLock(sqlite3_file *pFile, int eLock) {
DenoFile *p = (DenoFile*)pFile;
switch (eLock) {
case SQLITE_LOCK_NONE:
// no op
break;
case SQLITE_LOCK_SHARED:
case SQLITE_LOCK_RESERVED: // one WASM process <-> one open database
js_lock(p->rid, 0);
break;
case SQLITE_LOCK_PENDING:
case SQLITE_LOCK_EXCLUSIVE:
js_lock(p->rid, 1);
break;
}
return SQLITE_OK;
}
static int denoUnlock(sqlite3_file *pFile, int eLock) {
DenoFile *p = (DenoFile*)pFile;
switch (eLock) {
case SQLITE_LOCK_NONE:
// no op
break;
case SQLITE_LOCK_SHARED:
case SQLITE_LOCK_RESERVED:
case SQLITE_LOCK_PENDING:
case SQLITE_LOCK_EXCLUSIVE:
js_unlock(p->rid);
break;
}
return SQLITE_OK;
}
static int denoCheckReservedLock(sqlite3_file *pFile, int *pResOut) {
*pResOut = 0;
return SQLITE_OK;
}
// No xFileControl() verbs are implemented by this VFS.
static int denoFileControl(sqlite3_file *pFile, int op, void *pArg) {
return SQLITE_NOTFOUND;
}
// TODO(dyedgreen): Should we try to get these?
static int denoSectorSize(sqlite3_file *pFile) {
return 0;
}
static int denoDeviceCharacteristics(sqlite3_file *pFile) {
return 0;
}
// Open a file handle.
static int denoOpen(
sqlite3_vfs *pVfs, /* VFS */
const char *zName, /* File to open, or 0 for a temp file */
sqlite3_file *pFile, /* Pointer to DenoFile struct to populate */
int flags, /* Input SQLITE_OPEN_XXX flags */
int *pOutFlags /* Output SQLITE_OPEN_XXX flags (or NULL) */
) {
static const sqlite3_io_methods denoio = {
1, /* iVersion */
denoClose, /* xClose */
denoRead, /* xRead */
denoWrite, /* xWrite */
denoTruncate, /* xTruncate */
denoSync, /* xSync */
denoFileSize, /* xFileSize */
denoLock, /* xLock */
denoUnlock, /* xUnlock */
denoCheckReservedLock, /* xCheckReservedLock */
denoFileControl, /* xFileControl */
denoSectorSize, /* xSectorSize */
denoDeviceCharacteristics /* xDeviceCharacteristics */
};
DenoFile *p = (DenoFile*)pFile;
p->base.pMethods = &denoio;
// TODO(dyedgreen): The current approach is to raise
// the permission error on the vfs.js side of things,
// should the error be propagates through the wrapper
// and be raised on the wrapper side of things?
p->rid = js_open(zName, zName ? 0 : 1, flags);
if (pOutFlags) {
*pOutFlags = flags;
}
debug_printf("opened file (rid %i)\n", p->rid);
debug_printf("file path name: '%s'\n", zName);
return SQLITE_OK;
}
// Delete the file at the path.
static int denoDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync) {
js_delete(zPath);
return SQLITE_OK;
}
// All valid id files are accessible.
static int denoAccess(sqlite3_vfs *pVfs, const char *zPath, int flags, int *pResOut) {
switch (flags) {
case SQLITE_ACCESS_EXISTS:
*pResOut = js_exists(zPath);
break;
default:
*pResOut = js_access(zPath);
break;
}
debug_printf("determining file access (path %s, access %i)\n", zPath, *pResOut);
return SQLITE_OK;
}
// TODO(dyedgreen): Actually resolve the full path name
static int denoFullPathname(sqlite3_vfs *pVfs, const char *zPath, int nPathOut, char *zPathOut) {
sqlite3_snprintf(nPathOut, zPathOut, "%s", zPath);
debug_printf("requesting full path name for path: %s\n", zPath);
return SQLITE_OK;
}
// We don't support shared objects
static void *denoDlOpen(sqlite3_vfs *pVfs, const char *zPath) {
return 0;
}
static void denoDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg) {
sqlite3_snprintf(nByte, zErrMsg, "Loadable extensions are not supported");
zErrMsg[nByte-1] = '\0';
}
static void (*denoDlSym(sqlite3_vfs *pVfs, void *pH, const char *z))(void) {
return 0;
}
static void denoDlClose(sqlite3_vfs *pVfs, void *pHandle) {
return;
}
// Generate pseudo-random data
static int denoRandomness(sqlite3_vfs *pVfs, int nByte, char *zByte) {
pcg_bytes(zByte, nByte);
return SQLITE_OK;
}
// TODO(dyedgreen): Can anything be done here?
static int denoSleep(sqlite3_vfs *pVfs, int nMicro) {
return 0;
}
// Retrieve the current time
static int denoCurrentTime(sqlite3_vfs *pVfs, double *pTime) {
*pTime = js_time() / 1000 / 86400.0 + 2440587.5;
return SQLITE_OK;
}
// Implement localtime_r
struct tm* localtime_r(const time_t *time, struct tm *result) {
debug_printf("running localtime_r");
time_t shifted = *time - 60 * js_timezone();
return gmtime_r(&shifted, result);
}
// This function returns a pointer to the VFS implemented in this file.
sqlite3_vfs *sqlite3_denovfs(void) {
static sqlite3_vfs denovfs = {
3, /* iVersion */
sizeof(DenoFile), /* szOsFile */
MAXPATHNAME, /* mxPathname */
0, /* pNext */
"deno", /* zName */
0, /* pAppData */
denoOpen, /* xOpen */
denoDelete, /* xDelete */
denoAccess, /* xAccess */
denoFullPathname, /* xFullPathname */
denoDlOpen, /* xDlOpen */
denoDlError, /* xDlError */
denoDlSym, /* xDlSym */
denoDlClose, /* xDlClose */
denoRandomness, /* xRandomness */
denoSleep, /* xSleep */
denoCurrentTime, /* xCurrentTime */
0, /* xGetLastError */
0, /* xCurrentTimeInt64 */
0, /* xSetSystemCall */
0, /* xGetSystemCall */
0, /* xNextSystemCall */
};
return &denovfs;
}
int sqlite3_os_init(void) {
debug_printf("running sqlite3_os_init\n");
// Register VFS
return sqlite3_vfs_register(sqlite3_denovfs(), 1);
}
int sqlite3_os_end(void) {
return SQLITE_OK;
}
plugos/sqlite/deno-sqlite/build/src/wrapper.c
-247
View File
@@ -1,247 +0,0 @@
#include <stdlib.h>
#include <sqlite3.h>
#include <pcg.h>
#include "debug.h"
#define EXPORT(name) __attribute__((used)) __attribute__((export_name (#name))) name
#define ERROR_VAL -1
#define BIG_INT_TYPE 6
#define JS_MAX_SAFE_INTEGER 9007199254740991
#define JS_MIN_SAFE_INTEGER (-JS_MAX_SAFE_INTEGER)
// Status returned by last instruction
int last_status = SQLITE_OK;
// Database handle for this instance
sqlite3* database = NULL;
// Return length of string pointed to by str.
int EXPORT(str_len) (const char* str) {
int len;
for (len = 0; str[len] != '\0'; len ++);
return len;
}
// Seed the random number generator. We pass a double, to
// get as many bytes from the JS number as possible.
void EXPORT(seed_rng) (double seed) {
pcg_seed((uint64_t)seed);
}
// Return last status encountered.
int EXPORT(get_status) () {
return last_status;
}
// Initialize the database and return the status.
int EXPORT(open) (const char* filename, int flags) {
// Return error is database is already open
if (database) {
last_status = SQLITE_MISUSE;
return last_status;
}
// Open SQLite db connection
last_status = sqlite3_open_v2(filename, &database, flags, NULL);
if (last_status != SQLITE_OK) {
debug_printf("failed to open database with status %i\n", last_status);
return last_status;
}
debug_printf("opened database at path '%s'\n", filename);
return last_status;
}
// Attempt to close the database connection.
int EXPORT(close) () {
last_status = sqlite3_close(database);
if (last_status == SQLITE_OK) {
database = NULL;
debug_printf("closed database");
} else {
debug_printf("failed to close database with status %i\n", last_status);
}
return last_status;
}
// Return most recent SQLite error as a string
const char* EXPORT(get_sqlite_error_str) () {
if (!database)
return "No open database.";
return sqlite3_errmsg(database);
}
// Wraps sqlite3_prepare. Returns statement id.
sqlite3_stmt* EXPORT(prepare) (const char* sql) {
// Prepare sqlite statement
sqlite3_stmt* stmt;
last_status = sqlite3_prepare_v2(database, sql, -1, &stmt, NULL);
debug_printf("prepared sql statement (status %i)\n", last_status);
if (last_status != SQLITE_OK)
return NULL;
return stmt;
}
// Destruct the given statement/ transaction. This will destruct the SQLite
// statement and free up it's transaction slot. Regardless of returned
// status, the statement id will be freed up.
int EXPORT(finalize) (sqlite3_stmt* stmt) {
last_status = sqlite3_finalize(stmt);
debug_printf("finalized statement (status %i)\n", last_status);
return last_status;
}
// Reset a given statement so it can be re-used.
int EXPORT(reset) (sqlite3_stmt* stmt) {
last_status = sqlite3_reset(stmt);
debug_printf("reset statement (status %i)\n", last_status);
return last_status;
}
// Resets all bound parameter values for this statement.
int EXPORT(clear_bindings) (sqlite3_stmt* stmt) {
last_status = sqlite3_clear_bindings(stmt);
debug_printf("clear bindings (status %i)\n", last_status);
return last_status;
}
// Execute multiple statements from a single string. This ignores any result
// rows.
int EXPORT(exec) (const char* sql) {
last_status = sqlite3_exec(database, sql, NULL, NULL, NULL);
debug_printf("ran exec (status %i)\n", last_status);
return last_status;
}
// Wrappers for bind statements, these return the status directly
int EXPORT(bind_int) (sqlite3_stmt* stmt, int idx, double value) {
// we use double to pass in the value, as JS does not support 64 bit integers,
// but handles floats and we can contain a 32 bit in in a 64 bit float, so there
// should be no loss.
last_status = sqlite3_bind_int64(stmt, idx, (sqlite3_int64)value);
debug_printf("binding int %lli (status %i)\n", (sqlite3_int64)value, last_status);
return last_status;
}
int EXPORT(bind_double) (sqlite3_stmt* stmt, int idx, double value) {
last_status = sqlite3_bind_double(stmt, idx, value);
debug_printf("binding double %f (status %i)\n", value, last_status);
return last_status;
}
int EXPORT(bind_text) (sqlite3_stmt* stmt, int idx, const char* value) {
// SQLite retrains the string until we execute the statement, but any strings
// passed in from JS are freed when the function returns. Thus we need to mark
// is as transient.
last_status = sqlite3_bind_text(stmt, idx, value, -1, SQLITE_TRANSIENT);
debug_printf("binding text '%s' (status %i)\n", value, last_status);
return last_status;
}
int EXPORT(bind_blob) (sqlite3_stmt* stmt, int idx, void* value, int size) {
// SQLite retrains the pointer until we execute the statement, but any pointers
// passed in from JS are freed when the function returns. Thus we need to mark
// is as transient.
last_status = sqlite3_bind_blob(stmt, idx, value, size, SQLITE_TRANSIENT);
debug_printf("binding blob '%s' (status %i)\n", value, last_status);
return last_status;
}
int EXPORT(bind_big_int) (sqlite3_stmt* stmt, int idx, int sign, uint32_t high, uint32_t low) {
// Bind a big integer within the 64 bit integer range by passing it as two 32
// bit integers. The integers are assumed to be positive, and a sign is passed
// separately.
sqlite3_int64 int_val = ((sqlite3_int64)low + ((sqlite3_int64)high << 32)) * (sqlite3_int64)sign;
debug_printf("binding big_int %lld", int_val);
last_status = sqlite3_bind_int64(stmt, idx, int_val);
return last_status;
}
int EXPORT(bind_null) (sqlite3_stmt* stmt, int idx) {
last_status = sqlite3_bind_null(stmt, idx);
debug_printf("binding null (status %i)\n", last_status);
return last_status;
}
// Determine parameter index for named parameters
int EXPORT(bind_parameter_index) (sqlite3_stmt* stmt, const char* name) {
int index = sqlite3_bind_parameter_index(stmt, name);
if (index == 0) {
debug_printf("parameter '%s' does not exist", name);
// Normalize SQLite returning 0 for not found to ERROR_VAL
return ERROR_VAL;
}
debug_printf("obtained parameter index (param '%s', index %i)\n", name, index);
return index;
}
// Wraps running statements, this returns the status directly
int EXPORT(step) (sqlite3_stmt* stmt) {
last_status = sqlite3_step(stmt);
debug_printf("stepping statement (status %i)\n", last_status);
return last_status;
}
// Count columns returned by statement.
int EXPORT(column_count) (sqlite3_stmt* stmt) {
return sqlite3_column_count(stmt);
}
// Determine type of column. Returns SQLITE column types.
int EXPORT(column_type) (sqlite3_stmt* stmt, int col) {
int type = sqlite3_column_type(stmt, col);
if (type == SQLITE_INTEGER) {
// handle integers that exceed JS_MAX_SAFE_INTEGER
sqlite3_int64 col_val = sqlite3_column_int64(stmt, col);
if (col_val > JS_MAX_SAFE_INTEGER || col_val < JS_MIN_SAFE_INTEGER) {
debug_printf("detected big integer: %lld\n", col_val);
return BIG_INT_TYPE;
}
}
return type;
}
// Wrap result returning functions.
double EXPORT(column_int) (sqlite3_stmt* stmt, int col) {
return (double)sqlite3_column_int64(stmt, col);
}
double EXPORT(column_double) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_double(stmt, col);
}
const char* EXPORT(column_text) (sqlite3_stmt* stmt, int col) {
return (const char*)sqlite3_column_text(stmt, col);
}
const void* EXPORT(column_blob) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_blob(stmt, col);
}
int EXPORT(column_bytes) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_bytes(stmt, col);
}
const char* EXPORT(column_name) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_name(stmt, col);
}
const char* EXPORT(column_origin_name) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_origin_name(stmt, col);
}
const char* EXPORT(column_table_name) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_table_name(stmt, col);
}
double EXPORT(last_insert_rowid) () {
return (double)sqlite3_last_insert_rowid(database);
}
double EXPORT(changes) () {
return (double)sqlite3_changes(database);
}
double EXPORT(total_changes) () {
return (double)sqlite3_total_changes(database);
}
plugos/sqlite/deno-sqlite/build/vfs.js
-120
View File
@@ -1,120 +0,0 @@
import { getStr } from "../src/wasm.ts";
const isWindows = Deno.build.os === "windows";
// Closure to return an environment that links
// the current wasm context
export default function env(inst) {
// Exported environment
const env = {
// Print a string pointer to console
js_print: (str_ptr) => {
const text = getStr(inst.exports, str_ptr);
console.log(text[text.length - 1] === "\n" ? text.slice(0, -1) : text);
},
// Open the file at path, mode = 0 is open RW, mode = 1 is open TEMP
js_open: (path_ptr, mode, flags) => {
let path;
switch (mode) {
case 0:
path = getStr(inst.exports, path_ptr);
break;
case 1:
path = Deno.makeTempFileSync({ prefix: "deno_sqlite" });
break;
}
const write = !!(flags & 0x00000002);
const create = !!(flags & 0x00000004);
const rid = Deno.openSync(path, { read: true, write, create }).rid;
return rid;
},
// Close a file
js_close: (rid) => {
Deno.close(rid);
},
// Delete file at path
js_delete: (path_ptr) => {
const path = getStr(inst.exports, path_ptr);
Deno.removeSync(path);
},
// Read from a file to a buffer in the module
js_read: (rid, buffer_ptr, offset, amount) => {
const buffer = new Uint8Array(
inst.exports.memory.buffer,
buffer_ptr,
amount,
);
Deno.seekSync(rid, offset, Deno.SeekMode.Start);
return Deno.readSync(rid, buffer);
},
// Write to a file from a buffer in the module
js_write: (rid, buffer_ptr, offset, amount) => {
const buffer = new Uint8Array(
inst.exports.memory.buffer,
buffer_ptr,
amount,
);
Deno.seekSync(rid, offset, Deno.SeekMode.Start);
return Deno.writeSync(rid, buffer);
},
// Truncate the given file
js_truncate: (rid, size) => {
Deno.ftruncateSync(rid, size);
},
// Sync file data to disk
js_sync: (rid) => {
Deno.fdatasyncSync(rid);
},
// Retrieve the size of the given file
js_size: (rid) => {
return Deno.fstatSync(rid).size;
},
// Acquire a SHARED or EXCLUSIVE file lock
js_lock: (rid, exclusive) => {
// this is unstable and has issues on Windows ...
if (Deno.flockSync && !isWindows) Deno.flockSync(rid, exclusive !== 0);
},
// Release a file lock
js_unlock: (rid) => {
// this is unstable and has issues on Windows ...
if (Deno.funlockSync && !isWindows) Deno.funlockSync(rid);
},
// Return current time in ms since UNIX epoch
js_time: () => {
return Date.now();
},
// Return the timezone offset in minutes for
// the current locale.
js_timezone: () => {
return (new Date()).getTimezoneOffset();
},
// Determine if a path exists
js_exists: (path_ptr) => {
const path = getStr(inst.exports, path_ptr);
try {
Deno.statSync(path);
} catch (e) {
if (e instanceof Deno.errors.NotFound) {
return 0;
}
}
return 1;
},
// Determine if a path is accessible i.e. if it has read/write permissions
// TODO(dyedgreen): Properly determine if there are read permissions
js_access: (path_ptr) => {
const path = getStr(inst.exports, path_ptr);
try {
Deno.statSync(path);
} catch (e) {
if (e instanceof Deno.errors.PermissionDenied) {
return 0;
}
}
return 1;
},
};
return { env };
}
plugos/sqlite/deno-sqlite/build/vfs.syms
-15
View File
@@ -1,15 +0,0 @@
js_print
js_open
js_close
js_delete
js_read
js_write
js_truncate
js_sync
js_size
js_lock
js_unlock
js_time
js_timezone
js_exists
js_access
plugos/sqlite/deno-sqlite/examples/cli.ts
-79
View File
@@ -1,79 +0,0 @@
/**
* cli.ts
*
* A simple clone of the sqlite3 command line
* interface, build using deno-sqlite.
*
* This is an example, meant to illustrate using
* the API provided by deno-sqlite.
*/
import { readLines, writeAll } from "https://deno.land/std@0.134.0/io/mod.ts";
import AsciiTable from "https://deno.land/x/ascii_table@v0.1.0/mod.ts";
import { DB } from "../mod.ts";
const db = new DB(Deno.args[0] ?? undefined);
async function print(str: string) {
const enc = new TextEncoder();
await writeAll(Deno.stdout, enc.encode(str));
}
async function prompt() {
await print("sqlite> ");
}
const tablesQuery = db.prepareQuery<[string]>(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
);
const commands: Record<string, () => Promise<void>> = {
"tables": async () => {
for (const [name] of tablesQuery.iter()) {
await print(`${name}\n`);
}
},
"quit": async () => {
await print("\n");
Deno.exit(0);
},
"help": async () => {
await print(
"Type an SQL query or run a command.\nThe following commands are available:\n",
);
for (const key in commands) {
await print(`.${key}\n`);
}
},
};
await prompt();
for await (const cmd of readLines(Deno.stdin)) {
if (cmd[0] === ".") {
const action = commands[cmd.slice(1)] ??
(() => print("Unrecognized command, try .help\n"));
await action();
} else {
try {
const query = db.prepareQuery(cmd);
const rows = query.all();
const cols = query.columns();
query.finalize();
if (cols.length) {
const table = new AsciiTable();
table.setHeading("#", ...cols.map(({ name }) => name));
for (const [idx, row] of rows.entries()) {
table.addRow(idx + 1, ...row);
}
print(table.toString());
print("\n");
} else {
print(`Executed query: ${db.changes} changes\n`);
}
} catch (err) {
console.error(err);
}
}
await prompt();
}
plugos/sqlite/deno-sqlite/examples/notes.ts
-57
View File
@@ -1,57 +0,0 @@
/**
* notes.ts
*
* A command line tool to manage a set
* of simple notes.
*
* This is an example, meant to illustrate using
* the API provided by deno-sqlite.
*/
import { DB } from "../mod.ts";
const commands: Record<string, (...args: string[]) => Promise<void> | void> = {
"create": (file: string) => {
const db = new DB(file, { mode: "create" });
db.query(`
CREATE TABLE notes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
note TEXT NOT NULL,
created_at TEXT NOT NULL
)
`);
db.close();
console.log("Database created!");
},
"record": (file: string, note: string) => {
const db = new DB(file, { mode: "write" });
db.query("INSERT INTO notes (note, created_at) VALUES (?, ?)", [
note,
new Date(),
]);
db.close();
console.log("Note recorded!");
},
"delete": (file: string, noteId: string) => {
const db = new DB(file, { mode: "write" });
db.query("DELETE FROM notes WHERE id = ?", [noteId]);
db.close();
console.log("Note deleted!");
},
"list": (file: string) => {
const db = new DB(file, { mode: "read" });
const query = db.prepareQuery<[number, string, string]>(
"SELECT id, note, created_at FROM notes ORDER BY created_at DESC",
);
for (const [id, note, createdAt] of query.iter()) {
const date = new Date(createdAt);
console.log(`Note #${id} (recorded ${date.toLocaleString()})\n${note}\n`);
}
query.finalize();
db.close();
},
};
const command = commands[Deno.args[0]] ??
(() => console.error(`Unknown command '${Deno.args[0]}'.`));
await command(...Deno.args.slice(1));
plugos/sqlite/deno-sqlite/examples/server.ts
-42
View File
@@ -1,42 +0,0 @@
/**
* server.ts
*
* A server which returns the number
* of hits to any given path since
* the server started running.
*
* This is an example, meant to illustrate using
* the API provided by deno-sqlite.
*/
import { serve } from "https://deno.land/std@0.134.0/http/mod.ts";
import { DB } from "../mod.ts";
const db = new DB();
db.query(`
CREATE TABLE visits (
id INTEGER PRIMARY KEY AUTOINCREMENT,
url TEXT NOT NULL,
visited_at TEXT NOT NULL
)
`);
const addVisitQuery = db.prepareQuery(
"INSERT INTO visits (url, visited_at) VALUES (:url, :time)",
);
const countVisitsQuery = db.prepareQuery<[number]>(
"SELECT COUNT(*) FROM visits WHERE url = :url",
);
console.log("Running server on localhost:8080");
await serve((req) => {
addVisitQuery.execute({
url: req.url,
time: new Date(),
});
const [count] = countVisitsQuery.one({ url: req.url });
return new Response(`This page was visited ${count} times!`);
}, { port: 8080 });
plugos/sqlite/deno-sqlite/mod.ts
-13
View File
@@ -1,13 +0,0 @@
export { DB } from "./src/db.ts";
export { SqliteError } from "./src/error.ts";
export { Status } from "./src/constants.ts";
export type { SqliteOptions } from "./src/db.ts";
export type {
ColumnName,
PreparedQuery,
QueryParameter,
QueryParameterSet,
Row,
RowObject,
} from "./src/query.ts";
plugos/sqlite/deno-sqlite/src/constants.ts
-63
View File
@@ -1,63 +0,0 @@
/**
* Status codes which can be returned
* by SQLite.
*
* Also see https://www.sqlite.org/rescode.html.
*/
export enum Status {
Unknown = -1, // Unknown status
SqliteOk = 0, // Successful result
SqliteError = 1, // Generic error
SqliteInternal = 2, // Internal logic error in SQLite
SqlitePerm = 3, // Access permission denied
SqliteAbort = 4, // Callback routine requested an abort
SqliteBusy = 5, // The database file is locked
SqliteLocked = 6, // A table in the database is locked
SqliteNoMem = 7, // A malloc() failed
SqliteReadOnly = 8, // Attempt to write a readonly database
SqliteInterrupt = 9, // Operation terminated by sqlite3_interrupt()
SqliteIOErr = 10, // Some kind of disk I/O error occurred
SqliteCorrupt = 11, // The database disk image is malformed
SqliteNotFound = 12, // Unknown opcode in sqlite3_file_control()
SqliteFull = 13, // Insertion failed because database is full
SqliteCantOpen = 14, // Unable to open the database file
SqliteProtocol = 15, // Database lock protocol error
SqliteEmpty = 16, // Internal use only
SqliteSchema = 17, // The database schema changed
SqliteTooBig = 18, // String or BLOB exceeds size limit
SqliteConstraint = 19, // Abort due to constraint violation
SqliteMismatch = 20, // Data type mismatch
SqliteMisuse = 21, // Library used incorrectly
SqliteNoLFS = 22, // Uses OS features not supported on host
SqliteAuth = 23, // Authorization denied
SqliteFormat = 24, // Not used
SqliteRange = 25, // 2nd parameter to sqlite3_bind out of range
SqliteNotADB = 26, // File opened that is not a database file
SqliteNotice = 27, // Notifications from sqlite3_log()
SqliteWarning = 28, // Warnings from sqlite3_log()
SqliteRow = 100, // sqlite3_step() has another row ready
SqliteDone = 101, // sqlite3_step() has finished executing
}
export enum OpenFlags {
ReadOnly = 0x00000001,
ReadWrite = 0x00000002,
Create = 0x00000004,
Uri = 0x00000040,
Memory = 0x00000080,
}
export enum Types {
Integer = 1,
Float = 2,
Text = 3,
Blob = 4,
Null = 5,
BigInteger = 6,
}
export enum Values {
Error = -1,
Null = 0,
}
plugos/sqlite/deno-sqlite/src/db.ts
-393
View File
@@ -1,393 +0,0 @@
import { instantiate, StatementPtr, Wasm } from "../build/sqlite.js";
import { setStr } from "./wasm.ts";
import { OpenFlags, Status, Values } from "./constants.ts";
import { SqliteError } from "./error.ts";
import { PreparedQuery, QueryParameterSet, Row, RowObject } from "./query.ts";
/**
* Options for opening a database.
*/
export interface SqliteOptions {
/**
* Mode in which to open the database.
*
* - `read`: read-only, throws an error if
* the database file does not exists
* - `write`: read-write, throws an error
* if the database file does not exists
* - `create`: read-write, create the database
* if the file does not exist
*
* `create` is the default if no mode is
* specified.
*/
mode?: "read" | "write" | "create";
/**
* Force the database to be in-memory. When
* this option is set, the database is opened
* in memory, regardless of the specified
* filename.
*/
memory?: boolean;
/**
* Interpret the file name as a URI.
* See https://sqlite.org/uri.html
* for more information.
*/
uri?: boolean;
}
/**
* A database handle that can be used to run
* queries.
*/
export class DB {
private _wasm: Wasm;
private _open: boolean;
private _statements: Set<StatementPtr>;
private _transactionDepth: number;
/**
* Create a new database. The file at the
* given path will be opened with the
* mode specified in options. The default
* mode is `create`.
*
* If no path is given, or if the `memory`
* option is set, the database is opened in
* memory.
*
* # Examples
*
* Create an in-memory database.
* ```typescript
* const db = new DB();
* ```
*
* Open a database backed by a file on disk.
* ```typescript
* const db = new DB("path/to/database.sqlite");
* ```
*
* Pass options to open a read-only database.
* ```typescript
* const db = new DB("path/to/database.sqlite", { mode: "read" });
* ```
*/
constructor(path: string = ":memory:", options: SqliteOptions = {}) {
this._wasm = instantiate().exports;
this._open = false;
this._statements = new Set();
this._transactionDepth = 0;
// Configure flags
let flags = 0;
switch (options.mode) {
case "read":
flags = OpenFlags.ReadOnly;
break;
case "write":
flags = OpenFlags.ReadWrite;
break;
case "create": // fall through
default:
flags = OpenFlags.ReadWrite | OpenFlags.Create;
break;
}
if (options.memory === true) {
flags |= OpenFlags.Memory;
}
if (options.uri === true) {
flags |= OpenFlags.Uri;
}
// Try to open the database
const status = setStr(
this._wasm,
path,
(ptr) => this._wasm.open(ptr, flags),
);
if (status !== Status.SqliteOk) {
throw new SqliteError(this._wasm, status);
}
this._open = true;
}
/**
* Query the database and return all matching
* rows.
*
* This is equivalent to calling `all` on
* a prepared query which is then immediately
* finalized.
*
* The type parameter `R` may be supplied by
* the user to indicated the type for the rows returned
* by the query. Notice that the user is responsible
* for ensuring the correctness of the supplied type.
*
* To avoid SQL injection, user-provided values
* should always be passed to the database through
* a query parameter.
*
* See `QueryParameterSet` for documentation on
* how values can be bound to SQL statements.
*
* See `QueryParameter` for documentation on how
* values are returned from the database.
*
* # Examples
*
* ```typescript
* const rows = db.query<[string, number]>("SELECT name, age FROM people WHERE city = ?", [city]);
* // rows = [["Peter Parker", 21], ...]
* ```
*
* ```typescript
* const rows = db.query<[string, number]>(
* "SELECT name, age FROM people WHERE city = :city",
* { city },
* );
* // rows = [["Peter Parker", 21], ...]
* ```
*/
query<R extends Row = Row>(
sql: string,
params?: QueryParameterSet,
): Array<R> {
const query = this.prepareQuery<R>(sql);
try {
const rows = query.all(params);
query.finalize();
return rows;
} catch (err) {
query.finalize();
throw err;
}
}
/**
* Like `query` except each row is returned
* as an object containing key-value pairs.
*
* # Examples
*
* ```typescript
* const rows = db.queryEntries<{ name: string, age: number }>("SELECT name, age FROM people");
* // rows = [{ name: "Peter Parker", age: 21 }, ...]
* ```
*
* ```typescript
* const rows = db.queryEntries<{ name: string, age: number }>(
* "SELECT name, age FROM people WHERE age >= :minAge",
* { minAge },
* );
* // rows = [{ name: "Peter Parker", age: 21 }, ...]
* ```
*/
queryEntries<O extends RowObject = RowObject>(
sql: string,
params?: QueryParameterSet,
): Array<O> {
const query = this.prepareQuery<Row, O>(sql);
try {
const rows = query.allEntries(params);
query.finalize();
return rows;
} catch (err) {
query.finalize();
throw err;
}
}
/**
* Prepares the given SQL query, so that it
* can be run multiple times and potentially
* with different parameters.
*
* If a query will be issued a lot, this is more
* efficient than using `query`. A prepared
* query also provides more control over how
* the query is run, as well as access to meta-data
* about the issued query.
*
* The returned `PreparedQuery` object must be
* finalized by calling its `finalize` method
* once it is no longer needed.
*
* # Typing Queries
*
* Prepared query objects accept three type parameters
* to specify precise types for returned data and
* query parameters.
*
* + The first type parameter `R` indicates the tuple type
* for rows returned by the query.
*
* + The second type parameter `O` indicates the record type
* for rows returned as entries (mappings from column names
* to values).
*
* + The third type parameter `P` indicates the type this query
* accepts as parameters.
*
* Note, that the correctness of those types must
* be guaranteed by the caller of this function.
*
* # Examples
*
* ```typescript
* const query = db.prepareQuery<
* [string, number],
* { name: string, age: number },
* { city: string },
* >("SELECT name, age FROM people WHERE city = :city");
*
* // use query ...
*
* query.finalize();
* ```
*/
prepareQuery<
R extends Row = Row,
O extends RowObject = RowObject,
P extends QueryParameterSet = QueryParameterSet,
>(
sql: string,
): PreparedQuery<R, O, P> {
if (!this._open) {
throw new SqliteError("Database was closed.");
}
const stmt = setStr(
this._wasm,
sql,
(ptr) => this._wasm.prepare(ptr),
);
if (stmt === Values.Null) {
throw new SqliteError(this._wasm);
}
this._statements.add(stmt);
return new PreparedQuery<R, O, P>(this._wasm, stmt, this._statements);
}
/**
* Run multiple semicolon-separated statements from a single
* string.
*
* This method cannot bind any query parameters, and any
* result rows are discarded. It is only for running a chunk
* of raw SQL; for example, to initialize a database.
*
* # Examples
*
* ```typescript
* db.execute(`
* CREATE TABLE people (
* id INTEGER PRIMARY KEY AUTOINCREMENT,
* name TEXT,
* age REAL,
* city TEXT
* );
* INSERT INTO people (name, age, city) VALUES ("Peter Parker", 21, "nyc");
* `);
* ```
*/
execute(sql: string) {
const status = setStr(
this._wasm,
sql,
(ptr) => this._wasm.exec(ptr),
);
if (status !== Status.SqliteOk) {
throw new SqliteError(this._wasm, status);
}
}
/**
* Run a function within the context of a database
* transaction. If the function throws an error,
* the transaction is rolled back. Otherwise, the
* transaction is committed when the function returns.
*
* Calls to `transaction` may be nested. Nested transactions
* behave like SQLite save points.
*/
transaction<V>(closure: () => V): V {
this._transactionDepth += 1;
this.query(`SAVEPOINT _deno_sqlite_sp_${this._transactionDepth}`);
let value;
try {
value = closure();
} catch (err) {
this.query(`ROLLBACK TO _deno_sqlite_sp_${this._transactionDepth}`);
this._transactionDepth -= 1;
throw err;
}
this.query(`RELEASE _deno_sqlite_sp_${this._transactionDepth}`);
this._transactionDepth -= 1;
return value;
}
/**
* Close the database. This must be called if
* the database is no longer used to avoid leaking
* open file descriptors.
*
* If `force = true` is passed, any non-finalized
* `PreparedQuery` objects will be finalized. Otherwise,
* this throws if there are active queries.
*
* `close` may safely be called multiple
* times.
*/
close(force = false) {
if (!this._open) {
return;
}
if (force) {
for (const stmt of this._statements) {
if (this._wasm.finalize(stmt) !== Status.SqliteOk) {
throw new SqliteError(this._wasm);
}
}
}
if (this._wasm.close() !== Status.SqliteOk) {
throw new SqliteError(this._wasm);
}
this._open = false;
}
/**
* Get last inserted row id. This corresponds to
* the SQLite function `sqlite3_last_insert_rowid`.
*
* Before a row is inserted for the first time (since
* the database was opened), this returns `0`.
*/
get lastInsertRowId(): number {
return this._wasm.last_insert_rowid();
}
/**
* Return the number of rows modified, inserted or
* deleted by the most recently completed query.
* This corresponds to the SQLite function
* `sqlite3_changes`.
*/
get changes(): number {
return this._wasm.changes();
}
/**
* Return the number of rows modified, inserted or
* deleted since the database was opened.
* This corresponds to the SQLite function
* `sqlite3_total_changes`.
*/
get totalChanges(): number {
return this._wasm.total_changes();
}
}
plugos/sqlite/deno-sqlite/src/error.ts
-57
View File
@@ -1,57 +0,0 @@
import { Wasm } from "../build/sqlite.js";
import { getStr } from "./wasm.ts";
import { Status } from "./constants.ts";
/**
* Errors which can occur while interacting with
* a database.
*/
export class SqliteError extends Error {
/**
* Extension over the standard JS Error object
* to also contain class members for error code
* and error code name.
*
* Instances of this class should not be constructed
* directly and should only be obtained
* from exceptions raised in this module.
*/
constructor(context: Wasm | string, code?: Status) {
let message;
let status;
if (typeof context === "string") {
message = context;
status = Status.Unknown;
} else {
message = getStr(context, context.get_sqlite_error_str());
status = context.get_status();
}
super(message);
this.code = code ?? status;
this.name = "SqliteError";
}
/**
* The SQLite status code which caused this error.
*
* Errors that originate in the JavaScript part of
* the library will not have an associated status
* code. For these errors, the code will be
* `Status.Unknown`.
*
* These codes are accessible via
* the exported `Status` object.
*/
code: Status;
/**
* Key of code in exported `status`
* object.
*
* E.g. if `code` is `19`,
* `codeName` would be `SqliteConstraint`.
*/
get codeName(): keyof typeof Status {
return Status[this.code] as keyof typeof Status;
}
}
plugos/sqlite/deno-sqlite/src/query.ts
-692
View File
@@ -1,692 +0,0 @@
import { StatementPtr, Wasm } from "../build/sqlite.js";
import { getStr, setArr, setStr } from "./wasm.ts";
import { Status, Types, Values } from "./constants.ts";
import { SqliteError } from "./error.ts";
/**
* The default type for returned rows.
*/
export type Row = Array<unknown>;
/**
* The default type for row returned
* as objects.
*/
export type RowObject = Record<string, unknown>;
/**
* Possible parameter values to be bound to a query.
*
* When values are bound to a query, they are
* converted between JavaScript and SQLite types
* in the following way:
*
* | JS type in | SQL type | JS type out |
* |------------|-----------------|------------------|
* | number | INTEGER or REAL | number or bigint |
* | bigint | INTEGER | number or bigint |
* | boolean | INTEGER | number |
* | string | TEXT | string |
* | Date | TEXT | string |
* | Uint8Array | BLOB | Uint8Array |
* | null | NULL | null |
* | undefined | NULL | null |
*
* If no value is provided for a given parameter,
* SQLite will default to NULL.
*
* If a `bigint` is bound, it is converted to a
* signed 64 bit integer, which may overflow.
*
* If an integer value is read from the database, which
* is too big to safely be contained in a `number`, it
* is automatically returned as a `bigint`.
*
* If a `Date` is bound, it will be converted to
* an ISO 8601 string: `YYYY-MM-DDTHH:MM:SS.SSSZ`.
* This format is understood by built-in SQLite
* date-time functions. Also see https://sqlite.org/lang_datefunc.html.
*/
export type QueryParameter =
| boolean
| number
| bigint
| string
| null
| undefined
| Date
| Uint8Array;
/**
* A set of query parameters.
*
* When a query is constructed, it can contain
* either positional or named parameters. For
* more information see https://www.sqlite.org/lang_expr.html#parameters.
*
* A set of parameters can be passed to
* a query method either as an array of
* parameters (in positional order), or
* as an object which maps parameter names
* to their values:
*
* | SQL Parameter | QueryParameterSet |
* |---------------|-------------------------|
* | `?NNN` or `?` | NNN-th value in array |
* | `:AAAA` | value `AAAA` or `:AAAA` |
* | `@AAAA` | value `@AAAA` |
* | `$AAAA` | value `$AAAA` |
*
* See `QueryParameter` for documentation on
* how values are converted between SQL
* and JavaScript types.
*/
export type QueryParameterSet =
| Record<string, QueryParameter>
| Array<QueryParameter>;
/**
* Name of a column in a database query.
*/
export interface ColumnName {
/**
* Name of the returned column.
*/
name: string;
/**
* Name of the database column that stores
* the data returned from this query.
*
* This might be different from `name` if a
* columns was renamed using e.g. as in
* `SELECT foo AS bar FROM table`.
*/
originName: string;
/**
* Name of the table that stores the data
* returned from this query.
*/
tableName: string;
}
interface RowsIterator<R> {
next: () => IteratorResult<R>;
[Symbol.iterator]: () => RowsIterator<R>;
}
/**
* A prepared query which can be executed many
* times.
*/
export class PreparedQuery<
R extends Row = Row,
O extends RowObject = RowObject,
P extends QueryParameterSet = QueryParameterSet,
> {
private _wasm: Wasm;
private _stmt: StatementPtr;
private _openStatements: Set<StatementPtr>;
private _status: number;
private _iterKv: boolean;
private _rowKeys?: Array<string>;
private _finalized: boolean;
/**
* This constructor should never be used directly.
* Instead a prepared query can be obtained by
* calling `DB.prepareQuery`.
*/
constructor(
wasm: Wasm,
stmt: StatementPtr,
openStatements: Set<StatementPtr>,
) {
this._wasm = wasm;
this._stmt = stmt;
this._openStatements = openStatements;
this._status = Status.Unknown;
this._iterKv = false;
this._finalized = false;
}
private startQuery(params?: P) {
if (this._finalized) {
throw new SqliteError("Query is finalized.");
}
// Reset query
this._wasm.reset(this._stmt);
this._wasm.clear_bindings(this._stmt);
// Prepare parameter array
let parameters = [];
if (Array.isArray(params)) {
parameters = params;
} else if (typeof params === "object") {
// Resolve parameter index for named parameter
for (const key of Object.keys(params)) {
let name = key;
// blank names default to ':'
if (name[0] !== ":" && name[0] !== "@" && name[0] !== "$") {
name = `:${name}`;
}
const idx = setStr(
this._wasm,
name,
(ptr) => this._wasm.bind_parameter_index(this._stmt, ptr),
);
if (idx === Values.Error) {
throw new SqliteError(`No parameter named '${name}'.`);
}
parameters[idx - 1] = params[key];
}
}
// Bind parameters
for (let i = 0; i < parameters.length; i++) {
let value = parameters[i];
let status;
switch (typeof value) {
case "boolean":
value = value ? 1 : 0;
// fall through
case "number":
if (Number.isSafeInteger(value)) {
status = this._wasm.bind_int(this._stmt, i + 1, value);
} else {
status = this._wasm.bind_double(this._stmt, i + 1, value);
}
break;
case "bigint":
// bigint is bound as two 32bit integers and reassembled on the C side
if (value > 9223372036854775807n || value < -9223372036854775808n) {
throw new SqliteError(
`BigInt value ${value} overflows 64 bit integer.`,
);
} else {
const posVal = value >= 0n ? value : -value;
const sign = value >= 0n ? 1 : -1;
const upper = Number(BigInt.asUintN(32, posVal >> 32n));
const lower = Number(BigInt.asUintN(32, posVal));
status = this._wasm.bind_big_int(
this._stmt,
i + 1,
sign,
upper,
lower,
);
}
break;
case "string":
status = setStr(
this._wasm,
value,
(ptr) => this._wasm.bind_text(this._stmt, i + 1, ptr),
);
break;
default:
if (value instanceof Date) {
// Dates are allowed and bound to TEXT, formatted `YYYY-MM-DDTHH:MM:SS.SSSZ`
status = setStr(
this._wasm,
value.toISOString(),
(ptr) => this._wasm.bind_text(this._stmt, i + 1, ptr),
);
} else if (value instanceof Uint8Array) {
// Uint8Arrays are allowed and bound to BLOB
const size = value.length;
status = setArr(
this._wasm,
value,
(ptr) => this._wasm.bind_blob(this._stmt, i + 1, ptr, size),
);
} else if (value === null || value === undefined) {
// Both null and undefined result in a NULL entry
status = this._wasm.bind_null(this._stmt, i + 1);
} else {
throw new SqliteError(`Can not bind ${typeof value}.`);
}
break;
}
if (status !== Status.SqliteOk) {
throw new SqliteError(this._wasm, status);
}
}
}
private getQueryRow(): R {
if (this._finalized) {
throw new SqliteError("Query is finalized.");
}
const columnCount = this._wasm.column_count(this._stmt);
const row: Row = [];
for (let i = 0; i < columnCount; i++) {
switch (this._wasm.column_type(this._stmt, i)) {
case Types.Integer:
row.push(this._wasm.column_int(this._stmt, i));
break;
case Types.Float:
row.push(this._wasm.column_double(this._stmt, i));
break;
case Types.Text:
row.push(
getStr(
this._wasm,
this._wasm.column_text(this._stmt, i),
),
);
break;
case Types.Blob: {
const ptr = this._wasm.column_blob(this._stmt, i);
if (ptr === 0) {
// Zero pointer results in null
row.push(null);
} else {
const length = this._wasm.column_bytes(this._stmt, i);
// Slice should copy the bytes, as it makes a shallow copy
row.push(
new Uint8Array(this._wasm.memory.buffer, ptr, length).slice(),
);
}
break;
}
case Types.BigInteger: {
const ptr = this._wasm.column_text(this._stmt, i);
row.push(BigInt(getStr(this._wasm, ptr)));
break;
}
default:
// TODO(dyedgreen): Differentiate between NULL and not-recognized?
row.push(null);
break;
}
}
return row as R;
}
private makeRowObject(row: Row): O {
if (this._rowKeys == null) {
const rowCount = this._wasm.column_count(this._stmt);
this._rowKeys = [];
for (let i = 0; i < rowCount; i++) {
this._rowKeys.push(
getStr(this._wasm, this._wasm.column_name(this._stmt, i)),
);
}
}
const obj = row.reduce<RowObject>((obj, val, idx) => {
obj[this._rowKeys![idx]] = val;
return obj;
}, {});
return obj as O;
}
/**
* Binds the given parameters to the query
* and returns an iterator over rows.
*
* Using an iterator avoids loading all returned
* rows into memory and hence allows to process a large
* number of rows.
*
* Calling `iter`, `all`, or `first` invalidates any iterators
* previously returned from this prepared query.
*
* # Examples
*
* ```typescript
* const query = db.prepareQuery<[number, string]>("SELECT id, name FROM people");
* for (const [id, name] of query.iter()) {
* // ...
* }
* ```
*
* To avoid SQL injection, user-provided values
* should always be passed to the database through
* a query parameter.
*
* ```typescript
* const query = db.prepareQuery("SELECT id FROM people WHERE name = ?");
* preparedQuery.iter([name]);
* ```
*
* ```typescript
* const query = db.prepareQuery("SELECT id FROM people WHERE name = :name");
* preparedQuery.iter({ name });
* ```
*
* See `QueryParameterSet` for documentation on
* how values can be bound to SQL statements.
*
* See `QueryParameter` for documentation on how
* values are returned from the database.
*/
iter(params?: P): RowsIterator<R> {
this.startQuery(params);
this._status = this._wasm.step(this._stmt);
if (
this._status !== Status.SqliteRow && this._status !== Status.SqliteDone
) {
throw new SqliteError(this._wasm, this._status);
}
this._iterKv = false;
return this as RowsIterator<R>;
}
/**
* Like `iter` except each row is returned
* as an object containing key-value pairs.
*
* # Examples
*
* ```typescript
* const query = db.prepareQuery<_, { id: number, name: string }>("SELECT id, name FROM people");
* for (const { id, name } of query.iter()) {
* // ...
* }
* ```
*/
iterEntries(params?: P): RowsIterator<O> {
this.iter(params);
this._iterKv = true;
return this as RowsIterator<O>;
}
/**
* @ignore
*
* Implements the iterable protocol. It is
* a bug to call this method directly.
*/
[Symbol.iterator](): RowsIterator<R | O> {
return this;
}
/**
* @ignore
*
* Implements the iterator protocol. It is
* a bug to call this method directly.
*/
next(): IteratorResult<R | O> {
if (this._status === Status.SqliteRow) {
const value = this.getQueryRow();
this._status = this._wasm.step(this._stmt);
if (this._iterKv) {
return { value: this.makeRowObject(value), done: false };
} else {
return { value, done: false };
}
} else if (this._status === Status.SqliteDone) {
return { value: null, done: true };
} else {
throw new SqliteError(this._wasm, this._status);
}
}
/**
* Binds the given parameters to the query
* and returns an array containing all resulting
* rows.
*
* # Examples
*
* ```typescript
* const query = db.prepareQuery<[number, string]>("SELECT id, name FROM people");
* const rows = query.all();
* // [[1, "Peter"], ...]
* ```
*
* To avoid SQL injection, user-provided values
* should always be passed to the database through
* a query parameter.
*
* ```typescript
* const query = db.prepareQuery("SELECT id FROM people WHERE name = ?");
* preparedQuery.all([name]);
* ```
*
* ```typescript
* const query = db.prepareQuery("SELECT id FROM people WHERE name = :name");
* preparedQuery.all({ name });
* ```
*
* See `QueryParameterSet` for documentation on
* how values can be bound to SQL statements.
*
* See `QueryParameter` for documentation on how
* values are returned from the database.
*/
all(params?: P): Array<R> {
this.startQuery(params);
const rows: Array<R> = [];
this._status = this._wasm.step(this._stmt);
while (this._status === Status.SqliteRow) {
rows.push(this.getQueryRow());
this._status = this._wasm.step(this._stmt);
}
if (this._status !== Status.SqliteDone) {
throw new SqliteError(this._wasm, this._status);
}
return rows;
}
/**
* Like `all` except each row is returned
* as an object containing key-value pairs.
*
* # Examples
*
* ```typescript
* const query = db.prepareQuery<_, { id: number, name: string }>("SELECT id, name FROM people");
* const rows = query.all();
* // [{ id: 1, name: "Peter" }, ...]
* ```
*/
allEntries(params?: P): Array<O> {
return this.all(params).map((row) => this.makeRowObject(row));
}
/**
* Binds the given parameters to the query
* and returns the first resulting row or
* `undefined` when there are no rows returned
* by the query.
*
* # Examples
*
* ```typescript
* const query = db.prepareQuery<[number, string]>("SELECT id, name FROM people");
* const person = query.first();
* // [1, "Peter"]
* ```
*
* ```typescript
* const query = db.prepareQuery("SELECT id, name FROM people WHERE name = ?");
* const person = query.first(["not a name"]);
* // undefined
* ```
*
* To avoid SQL injection, user-provided values
* should always be passed to the database through
* a query parameter.
*
* ```typescript
* const query = db.prepareQuery("SELECT id FROM people WHERE name = ?");
* preparedQuery.first([name]);
* ```
*
* ```typescript
* const query = db.prepareQuery("SELECT id FROM people WHERE name = :name");
* preparedQuery.first({ name });
* ```
*
* See `QueryParameterSet` for documentation on
* how values can be bound to SQL statements.
*
* See `QueryParameter` for documentation on how
* values are returned from the database.
*/
first(params?: P): R | undefined {
this.startQuery(params);
this._status = this._wasm.step(this._stmt);
let row = undefined;
if (this._status === Status.SqliteRow) {
row = this.getQueryRow();
}
while (this._status === Status.SqliteRow) {
this._status = this._wasm.step(this._stmt);
}
if (this._status !== Status.SqliteDone) {
throw new SqliteError(this._wasm, this._status);
}
return row;
}
/**
* Like `first` except the row is returned
* as an object containing key-value pairs.
*
* # Examples
*
* ```typescript
* const query = db.prepareQuery<_, { id: number, name: string }>("SELECT id, name FROM people");
* const person = query.first();
* // { id: 1, name: "Peter" }
* ```
*/
firstEntry(params?: P): O | undefined {
const row = this.first(params);
return row === undefined ? undefined : this.makeRowObject(row);
}
/**
* **Deprecated:** prefer `first`.
*/
one(params?: P): R {
const rows = this.all(params);
if (rows.length === 0) {
throw new SqliteError("The query did not return any rows.");
} else if (rows.length > 1) {
throw new SqliteError("The query returned more than one row.");
} else {
return rows[0];
}
}
/**
* **Deprecated:** prefer `firstEntry`.
*/
oneEntry(params?: P): O {
return this.makeRowObject(this.one(params));
}
/**
* Binds the given parameters to the query and
* executes the query, ignoring any rows which
* might be returned.
*
* Using this method is more efficient when the
* rows returned by a query are not needed or
* the query does not return any rows.
*
* # Examples
*
* ```typescript
* const query = db.prepareQuery<_, _, [string]>("INSERT INTO people (name) VALUES (?)");
* query.execute(["Peter"]);
* ```
*
* ```typescript
* const query = db.prepareQuery<_, _, { name: string }>("INSERT INTO people (name) VALUES (:name)");
* query.execute({ name: "Peter" });
* ```
*
* See `QueryParameterSet` for documentation on
* how values can be bound to SQL statements.
*
* See `QueryParameter` for documentation on how
* values are returned from the database.
*/
execute(params?: P) {
this.startQuery(params);
this._status = this._wasm.step(this._stmt);
while (this._status === Status.SqliteRow) {
this._status = this._wasm.step(this._stmt);
}
if (this._status !== Status.SqliteDone) {
throw new SqliteError(this._wasm, this._status);
}
}
/**
* Closes the prepared query. This must be
* called once the query is no longer needed
* to avoid leaking resources.
*
* After a prepared query has been finalized,
* calls to `iter`, `all`, `first`, `execute`,
* or `columns` will fail.
*
* Using iterators which were previously returned
* from the finalized query will fail.
*
* `finalize` may safely be called multiple
* times.
*/
finalize() {
if (!this._finalized) {
this._wasm.finalize(this._stmt);
this._openStatements.delete(this._stmt);
this._finalized = true;
}
}
/**
* Returns the column names for the query
* results.
*
* This method returns an array of objects,
* where each object has the following properties:
*
* | Property | Value |
* |--------------|--------------------------------------------|
* | `name` | the result of `sqlite3_column_name` |
* | `originName` | the result of `sqlite3_column_origin_name` |
* | `tableName` | the result of `sqlite3_column_table_name` |
*/
columns(): Array<ColumnName> {
if (this._finalized) {
throw new SqliteError(
"Unable to retrieve column names from finalized transaction.",
);
}
const columnCount = this._wasm.column_count(this._stmt);
const columns: Array<ColumnName> = [];
for (let i = 0; i < columnCount; i++) {
const name = getStr(
this._wasm,
this._wasm.column_name(this._stmt, i),
);
const originName = getStr(
this._wasm,
this._wasm.column_origin_name(this._stmt, i),
);
const tableName = getStr(
this._wasm,
this._wasm.column_table_name(this._stmt, i),
);
columns.push({ name, originName, tableName });
}
return columns;
}
}
plugos/sqlite/deno-sqlite/src/wasm.ts
-87
View File
@@ -1,87 +0,0 @@
import { Wasm } from "../build/sqlite.js";
import { SqliteError } from "./error.ts";
// Move string to C
export function setStr<T>(
wasm: Wasm,
str: string,
closure: (ptr: number) => T,
): T {
const bytes = new TextEncoder().encode(str);
const ptr = wasm.malloc(bytes.length + 1);
if (ptr === 0) {
throw new SqliteError("Out of memory.");
}
const mem = new Uint8Array(wasm.memory.buffer, ptr, bytes.length + 1);
mem.set(bytes);
mem[bytes.length] = 0; // \0 terminator
try {
const result = closure(ptr);
wasm.free(ptr);
return result;
} catch (error) {
wasm.free(ptr);
throw error;
}
}
// Move Uint8Array to C
export function setArr<T>(
wasm: Wasm,
arr: Uint8Array,
closure: (ptr: number) => T,
): T {
const ptr = wasm.malloc(arr.length);
if (ptr === 0) {
throw new SqliteError("Out of memory.");
}
const mem = new Uint8Array(wasm.memory.buffer, ptr, arr.length);
mem.set(arr);
try {
const result = closure(ptr);
wasm.free(ptr);
return result;
} catch (error) {
wasm.free(ptr);
throw error;
}
}
// Read string from C
export function getStr(wasm: Wasm, ptr: number): string {
const len = wasm.str_len(ptr);
const bytes = new Uint8Array(wasm.memory.buffer, ptr, len);
if (len > 16) {
return new TextDecoder().decode(bytes);
} else {
// This optimization is lifted from EMSCRIPTEN's glue code
let str = "";
let idx = 0;
while (idx < len) {
let u0 = bytes[idx++];
if (!(u0 & 0x80)) {
str += String.fromCharCode(u0);
continue;
}
const u1 = bytes[idx++] & 63;
if ((u0 & 0xE0) == 0xC0) {
str += String.fromCharCode(((u0 & 31) << 6) | u1);
continue;
}
const u2 = bytes[idx++] & 63;
if ((u0 & 0xF0) == 0xE0) {
u0 = ((u0 & 15) << 12) | (u1 << 6) | u2;
} else {
// cut warning
u0 = ((u0 & 7) << 18) | (u1 << 12) | (u2 << 6) | (bytes[idx++] & 63);
}
if (u0 < 0x10000) {
str += String.fromCharCode(u0);
} else {
const ch = u0 - 0x10000;
str += String.fromCharCode(0xD800 | (ch >> 10), 0xDC00 | (ch & 0x3FF));
}
}
return str;
}
}
plugos/sqlite/sqlite_interface.ts
-4
View File
@@ -1,4 +0,0 @@
export interface ISQLite {
execute(query: string, ...params: any[]): Promise<number>;
query(query: string, ...params: any[]): Promise<any[]>;
}
plugos/sqlite/worker.ts
-61
View File
@@ -1,61 +0,0 @@
// This file is never loaded directly, it's loaded via a bundle. Run `deno task generate` to update.
import { DB } from "./deno-sqlite/mod.ts";
let db: DB | undefined;
import { compile } from "./deno-sqlite/build/sqlite.js";
const ready = compile();
globalThis.addEventListener("message", (event: MessageEvent) => {
const { data } = event;
// console.log("Got message", data);
ready.then(() => {
switch (data.type) {
case "init": {
try {
db = new DB(data.dbPath);
} catch (e: any) {
// console.error("Error!!!", e, data);
respondError(data.id, e);
break;
}
respond(data.id, true);
break;
}
case "execute": {
if (!db) {
respondError(data.id, new Error("Not initialized"));
break;
}
try {
db.query(data.query, data.params);
respond(data.id, db.changes);
} catch (e: any) {
respondError(data.id, e);
}
break;
}
case "query": {
if (!db) {
respondError(data.id, new Error("Not initialized"));
break;
}
try {
const result = db.queryEntries(data.query, data.params);
respond(data.id, result);
} catch (e: any) {
respondError(data.id, e);
}
break;
}
}
}).catch(console.error);
});
function respond(id: number, result: any) {
globalThis.postMessage({ id, result });
}
function respondError(id: number, error: Error) {
globalThis.postMessage({ id, error: error.message });
}
plugos/sqlite/worker_bundle.json
-3
View File
File diff suppressed because one or more lines are too long
plugos/syscalls/asset.ts
+1 -1
View File
@@ -6,7 +6,7 @@ export default function assetSyscalls(system: System<any>): SysCallMapping {
ctx,
name: string,
): string => {
return system.loadedPlugs.get(ctx.plug.name)!.assets!.readFileAsDataUrl(
return system.loadedPlugs.get(ctx.plug.name!)!.assets!.readFileAsDataUrl(
name,
);
},
plugos/syscalls/esbuild.ts
-46
View File
@@ -1,46 +0,0 @@
import { sandboxCompile, sandboxCompileModule } from "../compile.ts";
import { SysCallMapping } from "../system.ts";
import { Manifest } from "../types.ts";
import importMap from "../../import_map.json" assert { type: "json" };
import { base64EncodedDataUrl } from "../asset_bundle/base64.ts";
export function esbuildSyscalls(
imports: Manifest<any>[],
): SysCallMapping {
return {
"esbuild.compile": async (
_ctx,
filename: string,
code: string,
functionName?: string,
): Promise<string> => {
// Override this to point to a URL
importMap.imports["$sb/"] = "https://deno.land/x/silverbullet/plug-api/";
const importUrl = new URL(
base64EncodedDataUrl(
"application/json",
new TextEncoder().encode(JSON.stringify(importMap)),
),
);
return await sandboxCompile(
filename,
code,
functionName,
{
debug: true,
imports,
importMap: importUrl,
},
);
},
"esbuild.compileModule": async (
_ctx,
moduleName: string,
): Promise<string> => {
return await sandboxCompileModule(moduleName, {
imports,
});
},
};
}
plugos/syscalls/fetch.ts
-39
View File
@@ -1,39 +0,0 @@
import type {
SandboxFetchRequest,
SandboxFetchResponse,
} from "../../plug-api/plugos-syscall/fetch.ts";
import { base64Encode } from "../asset_bundle/base64.ts";
import { SysCallMapping } from "../system.ts";
export async function sandboxFetch(
url: string,
req?: SandboxFetchRequest,
): Promise<SandboxFetchResponse> {
const result = await fetch(
url,
req && {
method: req.method,
headers: req.headers,
body: req.body,
},
);
const body = await (await result.blob()).arrayBuffer();
return {
ok: result.ok,
status: result.status,
headers: Object.fromEntries(result.headers.entries()),
base64Body: base64Encode(new Uint8Array(body)),
};
}
export function sandboxFetchSyscalls(): SysCallMapping {
return {
"sandboxFetch.fetch": (
_ctx,
url: string,
options?: SandboxFetchRequest,
): Promise<SandboxFetchResponse> => {
return sandboxFetch(url, options);
},
};
}
plugos/syscalls/fs.deno.test.ts
+1 -2
View File
@@ -2,13 +2,12 @@ import { assert } from "../../test_deps.ts";
import { FileMeta } from "../../common/types.ts";
import { path } from "../deps.ts";
import fileSystemSyscalls from "./fs.deno.ts";
import { urlToPathname } from "../util.ts";
const fakeCtx = {} as any;
Deno.test("Test FS operations", async () => {
const thisFolder = path.resolve(
path.dirname(urlToPathname(new URL(import.meta.url))),
path.dirname(new URL(import.meta.url).pathname),
);
const syscalls = fileSystemSyscalls(thisFolder);
const allFiles: FileMeta[] = await syscalls["fs.listFiles"](
plugos/syscalls/fulltext.sqlite.ts
-64
View File
@@ -1,64 +0,0 @@
import { FullTextSearchOptions } from "../../plug-api/plugos-syscall/fulltext.ts";
import { ISQLite } from "../sqlite/sqlite_interface.ts";
import { SysCallMapping } from "../system.ts";
export async function ensureFTSTable(
db: ISQLite,
tableName: string,
) {
const result = await db.query(
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
tableName,
);
if (result.length === 0) {
await db.execute(
`CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`,
);
// console.log(`Created fts5 table ${tableName}`);
}
}
export function fullTextSearchSyscalls(
db: ISQLite,
tableName: string,
): SysCallMapping {
return {
"fulltext.index": async (_ctx, key: string, value: string) => {
await db.execute(`DELETE FROM ${tableName} WHERE key = ?`, key);
await db.execute(
`INSERT INTO ${tableName} (key, value) VALUES (?, ?)`,
key,
value,
);
},
"fulltext.delete": async (_ctx, key: string) => {
await db.execute(`DELETE FROM ${tableName} WHERE key = ?`, key);
},
"fulltext.search": async (
_ctx,
phrase: string,
options: FullTextSearchOptions,
) => {
return (
await db.query(
`SELECT key, bm25(fts) AS score, snippet(fts, 1, ?, ?, ?, ?) as snippet
FROM ${tableName}
WHERE value
MATCH ?
ORDER BY score LIMIT ?`,
options.highlightPrefix || "",
options.highlightPostfix || "",
options.highlightEllipsis || "...",
options.summaryMaxLength || 50,
phrase,
options.limit || 20,
)
).map((item) => ({
name: item.key,
score: item.score,
snippet: item.snippet,
}));
},
};
}
plugos/syscalls/sandbox.ts
-17
View File
@@ -1,17 +0,0 @@
import type { LogEntry } from "../sandbox.ts";
import type { SysCallMapping, System } from "../system.ts";
export default function sandboxSyscalls(system: System<any>): SysCallMapping {
return {
"sandbox.getLogs": (): LogEntry[] => {
let allLogs: LogEntry[] = [];
for (const plug of system.loadedPlugs.values()) {
if (plug.sandbox) {
allLogs = allLogs.concat(plug.sandbox.logBuffer);
}
}
allLogs = allLogs.sort((a, b) => a.date - b.date);
return allLogs;
},
};
}
plugos/syscalls/store.deno.test.ts
-108
View File
@@ -1,108 +0,0 @@
import { assertEquals } from "../../test_deps.ts";
import { createSandbox } from "../environments/deno_sandbox.ts";
import { System } from "../system.ts";
import { ensureTable, storeSyscalls } from "./store.sqlite.ts";
import { AsyncSQLite } from "../sqlite/async_sqlite.ts";
Deno.test("Test store", async () => {
const db = new AsyncSQLite(":memory:");
await db.init();
await ensureTable(db, "test_table");
const system = new System("server");
const syscalls = storeSyscalls(db, "test_table");
system.registerSyscalls([], syscalls);
const plug = await system.load(
{
name: "test",
functions: {
test1: {
code: `(() => {
return {
default: async () => {
await self.syscall("store.set", "name", "Pete");
return await self.syscall("store.get", "name");
}
};
})()`,
},
},
},
createSandbox,
);
assertEquals(await plug.invoke("test1", []), "Pete");
await system.unloadAll();
const dummyCtx: any = {};
await syscalls["store.deleteAll"](dummyCtx);
await syscalls["store.batchSet"](dummyCtx, [
{
key: "pete",
value: {
age: 20,
firstName: "Pete",
lastName: "Roberts",
},
},
{
key: "petejr",
value: {
age: 8,
firstName: "Pete Jr",
lastName: "Roberts",
},
},
{
key: "petesr",
value: {
age: 78,
firstName: "Pete Sr",
lastName: "Roberts",
},
},
]);
let allRoberts = await syscalls["store.query"](dummyCtx, {
filter: [{ op: "=", prop: "lastName", value: "Roberts" }],
orderBy: "age",
orderDesc: true,
});
assertEquals(allRoberts.length, 3);
assertEquals(allRoberts[0].key, "petesr");
allRoberts = await syscalls["store.query"](dummyCtx, {
filter: [{ op: "=", prop: "lastName", value: "Roberts" }],
orderBy: "age",
limit: 1,
});
assertEquals(allRoberts.length, 1);
assertEquals(allRoberts[0].key, "petejr");
allRoberts = await syscalls["store.query"](dummyCtx, {
filter: [
{ op: ">", prop: "age", value: 10 },
{ op: "<", prop: "age", value: 30 },
],
orderBy: "age",
});
assertEquals(allRoberts.length, 1);
assertEquals(allRoberts[0].key, "pete");
// Delete the middle one
await syscalls["store.deleteQuery"](dummyCtx, {
filter: [
{ op: ">", prop: "age", value: 10 },
{ op: "<", prop: "age", value: 30 },
],
});
allRoberts = await syscalls["store.query"](dummyCtx, {});
// console.log("All Roberts", allRoberts);
assertEquals(allRoberts.length, 2);
db.stop();
});
plugos/syscalls/store.dexie_browser.ts
+32 -43
View File
@@ -1,66 +1,55 @@
import Dexie from "https://esm.sh/dexie@3.2.2";
import { SysCallMapping } from "../system.ts";
export type KV = {
key: string;
value: any;
};
import { DexieKVStore } from "../lib/kv_store.dexie.ts";
import { KV } from "../lib/kv_store.ts";
export function storeSyscalls(
dbName: string,
tableName: string,
db: DexieKVStore,
): SysCallMapping {
const db = new Dexie(dbName);
db.version(1).stores({
[tableName]: "key",
});
const items = db.table(tableName);
return {
"store.delete": async (_ctx, key: string) => {
await items.delete(key);
"store.delete": (_ctx, key: string) => {
return db.del(key);
},
"store.deletePrefix": async (_ctx, prefix: string) => {
await items.where("key").startsWith(prefix).delete();
"store.deletePrefix": (_ctx, prefix: string) => {
return db.deletePrefix(prefix);
},
"store.deleteAll": async () => {
await items.clear();
"store.deleteAll": () => {
return db.deleteAll();
},
"store.set": async (_ctx, key: string, value: any) => {
await items.put({
key,
value,
});
"store.set": (_ctx, key: string, value: any) => {
return db.set(key, value);
},
"store.batchSet": async (_ctx, kvs: KV[]) => {
await items.bulkPut(
kvs.map(({ key, value }) => ({
key,
value,
})),
);
"store.batchSet": (_ctx, kvs: KV[]) => {
return db.batchSet(kvs);
},
"store.get": async (_ctx, key: string): Promise<any | null> => {
const result = await items.get({
key,
});
return result ? result.value : null;
"store.batchDelete": (_ctx, keys: string[]) => {
return db.batchDelete(keys);
},
"store.queryPrefix": async (
"store.batchGet": (
_ctx,
keys: string[],
): Promise<(any | undefined)[]> => {
return db.batchGet(keys);
},
"store.get": (_ctx, key: string): Promise<any | null> => {
return db.get(key);
},
"store.has": (_ctx, key: string): Promise<boolean> => {
return db.has(key);
},
"store.queryPrefix": (
_ctx,
keyPrefix: string,
): Promise<{ key: string; value: any }[]> => {
const results = await items.where("key").startsWith(keyPrefix).toArray();
return results.map((result) => ({
key: result.key,
value: result.value,
}));
return db.queryPrefix(keyPrefix);
},
};
}
plugos/syscalls/store.sqlite.ts
-170
View File
@@ -1,170 +0,0 @@
import { ISQLite } from "../sqlite/sqlite_interface.ts";
import { SysCallMapping } from "../system.ts";
export type Item = {
page: string;
key: string;
value: any;
};
export type KV = {
key: string;
value: any;
};
export async function ensureTable(db: ISQLite, tableName: string) {
const result = await db.query(
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
tableName,
);
if (result.length === 0) {
await db.execute(
`CREATE TABLE ${tableName} (key STRING PRIMARY KEY, value TEXT);`,
);
// console.log(`Created table ${tableName}`);
}
}
export type Query = {
filter?: Filter[];
orderBy?: string;
orderDesc?: boolean;
limit?: number;
select?: string[];
};
export type Filter = {
op: string;
prop: string;
value: any;
};
export function queryToSql(
query: Query,
): { sql: string; params: any[] } {
const whereClauses: string[] = [];
const clauses: string[] = [];
const params: any[] = [];
if (query.filter) {
for (const filter of query.filter) {
whereClauses.push(
`json_extract(value, '$.${filter.prop}') ${filter.op} ?`,
);
params.push(filter.value);
}
}
if (query.orderBy) {
clauses.push(
`ORDER BY json_extract(value, '$.${query.orderBy}') ${
query.orderDesc ? "desc" : "asc"
}`,
);
}
if (query.limit) {
clauses.push(`LIMIT ${query.limit}`);
}
return {
sql: whereClauses.length > 0
? `WHERE ${whereClauses.join(" AND ")} ${clauses.join(" ")}`
: clauses.join(" "),
params,
};
}
export function storeSyscalls(
db: ISQLite,
tableName: string,
): SysCallMapping {
const apiObj: SysCallMapping = {
"store.delete": async (_ctx, key: string) => {
await db.execute(`DELETE FROM ${tableName} WHERE key = ?`, key);
},
"store.deletePrefix": async (_ctx, prefix: string) => {
await db.execute(
`DELETE FROM ${tableName} WHERE key LIKE ?`,
`${prefix}%`,
);
},
"store.deleteQuery": async (_ctx, query: Query) => {
const { sql, params } = queryToSql(query);
await db.execute(`DELETE FROM ${tableName} ${sql}`, ...params);
},
"store.deleteAll": async () => {
await db.execute(`DELETE FROM ${tableName}`);
},
"store.set": async (_ctx, key: string, value: any) => {
await db.execute(
`INSERT INTO ${tableName}
(key, value)
VALUES (?, ?)
ON CONFLICT(key)
DO UPDATE SET value=excluded.value`,
key,
JSON.stringify(value),
);
},
"store.batchSet": async (_ctx, kvs: KV[]) => {
if (kvs.length === 0) {
return;
}
const values = kvs.flatMap((
kv,
) => [kv.key, JSON.stringify(kv.value)]);
await db.execute(
`INSERT INTO ${tableName}
(key, value)
VALUES ${kvs.map((_) => "(?, ?)").join(",")}
ON CONFLICT(key)
DO UPDATE SET value=excluded.value`,
...values,
);
},
"store.batchDelete": async (ctx, keys: string[]) => {
for (const key of keys) {
await apiObj["store.delete"](ctx, key);
}
},
"store.get": async (_ctx, key: string): Promise<any | null> => {
const result = await db.query(
`SELECT value FROM ${tableName} WHERE key = ?`,
key,
);
if (result.length) {
return JSON.parse(result[0].value);
} else {
return null;
}
},
"store.has": async (_ctx, key: string): Promise<boolean> => {
const result = await db.query(
`SELECT count(value) as cnt FROM ${tableName} WHERE key = ?`,
key,
);
return result[0].cnt === 1;
},
"store.queryPrefix": async (_ctx, prefix: string) => {
return (
await db.query(
`SELECT key, value FROM ${tableName} WHERE key LIKE ?`,
`${prefix}%`,
)
).map(({ key, value }) => ({
key,
value: JSON.parse(value),
}));
},
"store.query": async (_ctx, query: Query) => {
const { sql, params } = queryToSql(query);
return (
await db.query(
`SELECT key, value FROM ${tableName} ${sql}`,
...params,
)
).map(({ key, value }: { key: string; value: string }) => ({
key,
value: JSON.parse(value),
}));
},
};
return apiObj;
}
plugos/system.ts
+22 -42
View File
@@ -1,20 +1,18 @@
import { Hook, Manifest, RuntimeEnvironment } from "./types.ts";
import { Hook, RuntimeEnvironment } from "./types.ts";
import { EventEmitter } from "./event.ts";
import { Sandbox, SandboxFactory } from "./sandbox.ts";
import type { SandboxFactory } from "./sandbox.ts";
import { Plug } from "./plug.ts";
export interface SysCallMapping {
[key: string]: (ctx: SyscallContext, ...args: any) => Promise<any> | any;
}
export type SystemJSON<HookT> = Manifest<HookT>[];
export type SystemEvents<HookT> = {
plugLoaded: (plug: Plug<HookT>) => void | Promise<void>;
sandboxInitialized(sandbox: Sandbox, plug: Plug<HookT>): void | Promise<void>;
plugUnloaded: (name: string) => void | Promise<void>;
};
// Passed to every syscall, allows to pass in additional context that the syscall may use
export type SyscallContext = {
plug: Plug<any>;
};
@@ -95,63 +93,45 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
}
async load(
manifest: Manifest<HookT>,
workerUrl: URL,
sandboxFactory: SandboxFactory<HookT>,
): Promise<Plug<HookT>> {
const name = manifest.name;
if (this.plugs.has(name)) {
await this.unload(name);
}
// Validate
const plug = new Plug(this, workerUrl, sandboxFactory);
// Wait for worker to boot, and pass back its manifest
await plug.ready;
// and there it is!
const manifest = plug.manifest!;
// Validate the manifest
let errors: string[] = [];
for (const feature of this.enabledHooks) {
errors = [...errors, ...feature.validateManifest(manifest)];
errors = [...errors, ...feature.validateManifest(plug.manifest!)];
}
if (errors.length > 0) {
throw new Error(`Invalid manifest: ${errors.join(", ")}`);
}
// Ok, let's load this thing!
const plug = new Plug(this, name, sandboxFactory);
console.log("Loading", name);
plug.load(manifest);
this.plugs.set(name, plug);
if (this.plugs.has(manifest.name)) {
this.unload(manifest.name);
}
console.log("Loaded plug", manifest.name);
this.plugs.set(manifest.name, plug);
await this.emit("plugLoaded", plug);
return plug;
}
async unload(name: string) {
unload(name: string) {
// console.log("Unloading", name);
const plug = this.plugs.get(name);
if (!plug) {
throw Error(`Plug ${name} not found`);
return;
}
await plug.stop();
plug.stop();
this.emit("plugUnloaded", name);
this.plugs.delete(name);
}
toJSON(): SystemJSON<HookT> {
const plugJSON: Manifest<HookT>[] = [];
for (const [_, plug] of this.plugs) {
if (!plug.manifest) {
continue;
}
plugJSON.push(plug.manifest);
}
return plugJSON;
}
async replaceAllFromJSON(
json: SystemJSON<HookT>,
sandboxFactory: SandboxFactory<HookT>,
) {
await this.unloadAll();
for (const manifest of json) {
// console.log("Loading plug", manifest.name);
await this.load(manifest, sandboxFactory);
}
}
unloadAll(): Promise<void[]> {
return Promise.all(
Array.from(this.plugs.keys()).map(this.unload.bind(this)),
plugos/types.ts
-7
View File
@@ -4,12 +4,7 @@ import { AssetJson } from "./asset_bundle/bundle.ts";
export interface Manifest<HookT> {
name: string;
requiredPermissions?: string[];
// URLs to plugs whose dependencies are presumed to already be loaded (main use case: global.plug.json)
imports?: string[];
assets?: string[] | AssetJson;
dependencies?: {
[key: string]: string;
};
functions: {
[key: string]: FunctionDef<HookT>;
};
@@ -22,7 +17,6 @@ export type FunctionDef<HookT> = {
// Reuse an
// Format: plugName.functionName
redirect?: string;
code?: string;
env?: RuntimeEnvironment;
} & HookT;
@@ -30,6 +24,5 @@ export type RuntimeEnvironment = "client" | "server";
export interface Hook<HookT> {
validateManifest(manifest: Manifest<HookT>): string[];
apply(system: System<HookT>): void;
}
plugos/util.ts
-12
View File
@@ -1,12 +0,0 @@
export function safeRun(fn: () => Promise<void>) {
fn().catch((e: any) => {
console.error("Caught error", e.message);
// throw e;
});
}
export function urlToPathname(url: URL) {
// For Windows, remove prefix /
return url.pathname.replace(/^\/(\w:)/, "$1");
}
plugos/worker_runtime.ts
+114
View File
@@ -0,0 +1,114 @@
// This is the runtime imported from the compiled plug worker code
import type { ControllerMessage, WorkerMessage } from "./protocol.ts";
import type { Manifest } from "../common/manifest.ts";
declare global {
function syscall(name: string, ...args: any[]): Promise<any>;
}
if (typeof Deno === "undefined") {
// @ts-ignore: Deno hack
self.Deno = {
args: [],
// @ts-ignore: Deno hack
build: {
arch: "x86_64",
},
env: {
// @ts-ignore: Deno hack
get() {
},
},
};
}
const pendingRequests = new Map<
number,
{
resolve: (result: unknown) => void;
reject: (e: any) => void;
}
>();
let syscallReqId = 0;
function workerPostMessage(msg: ControllerMessage) {
self.postMessage(msg);
}
self.syscall = async (name: string, ...args: any[]) => {
return await new Promise((resolve, reject) => {
syscallReqId++;
pendingRequests.set(syscallReqId, { resolve, reject });
workerPostMessage({
type: "sys",
id: syscallReqId,
name,
args,
});
});
};
export function setupMessageListener(
// deno-lint-ignore ban-types
functionMapping: Record<string, Function>,
manifest: Manifest,
) {
self.addEventListener("message", (event: { data: WorkerMessage }) => {
(async () => {
const data = event.data;
switch (data.type) {
case "inv":
{
const fn = functionMapping[data.name!];
if (!fn) {
throw new Error(`Function not loaded: ${data.name}`);
}
try {
const result = await Promise.resolve(fn(...(data.args || [])));
workerPostMessage({
type: "invr",
id: data.id,
result: result,
} as ControllerMessage);
} catch (e: any) {
console.error(e);
workerPostMessage({
type: "invr",
id: data.id!,
error: e.message,
});
}
}
break;
case "sysr":
{
const syscallId = data.id;
const lookup = pendingRequests.get(syscallId);
if (!lookup) {
throw Error("Invalid request id");
}
pendingRequests.delete(syscallId);
if (data.error) {
lookup.reject(new Error(data.error));
} else {
lookup.resolve(data.result);
}
}
break;
}
})().catch(console.error);
});
// Signal initialization with manifest
workerPostMessage({
type: "manifest",
manifest,
});
}
// Monkey patch fetch()
import { monkeyPatchFetch } from "../plug-api/plugos-syscall/fetch.ts";
monkeyPatchFetch();