Migrate to Deno (#86)

Big bang migration to Deno 🤯
This commit is contained in:
Zef Hemel
2022-10-10 14:50:21 +02:00
committed by GitHub
parent 78f83c70d8
commit 561aa6891f
287 changed files with 4577 additions and 25087 deletions
@@ -0,0 +1,86 @@
import { Plug } from "../../plugos/plug.ts";
import {
AssetBundle,
assetReadFileSync,
} from "../../plugos/asset_bundle_reader.ts";
import { FileMeta } from "../types.ts";
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
constructor(
private wrapped: SpacePrimitives,
private assetBundle: AssetBundle,
) {
}
async fetchFileList(): Promise<FileMeta[]> {
const l = await this.wrapped.fetchFileList();
return Object.entries(this.assetBundle).filter(([k]) =>
k.startsWith("_plug/")
).map(([_, v]) => v.meta).concat(l);
}
readFile(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
if (this.assetBundle[name]) {
const data = assetReadFileSync(this.assetBundle, name);
// console.log("Requested encoding", encoding);
return Promise.resolve({
data: encoding === "string" ? new TextDecoder().decode(data) : data,
meta: {
lastModified: 0,
size: data.byteLength,
perm: "ro",
contentType: "application/json",
} as FileMeta,
});
}
return this.wrapped.readFile(name, encoding);
}
getFileMeta(name: string): Promise<FileMeta> {
if (this.assetBundle[name]) {
const data = assetReadFileSync(this.assetBundle, name);
return Promise.resolve({
lastModified: 0,
size: data.byteLength,
perm: "ro",
contentType: "application/json",
} as FileMeta);
}
return this.wrapped.getFileMeta(name);
}
writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate?: boolean | undefined,
): Promise<FileMeta> {
return this.wrapped.writeFile(name, encoding, data, selfUpdate);
}
deleteFile(name: string): Promise<void> {
if (this.assetBundle[name]) {
// Quietly ignore
return Promise.resolve();
}
return this.wrapped.deleteFile(name);
}
// deno-lint-ignore no-explicit-any
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return this.wrapped.proxySyscall(plug, name, args);
}
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[],
): Promise<any> {
return this.wrapped.invokeFunction(plug, env, name, args);
}
}
+1
View File
@@ -0,0 +1 @@
export const plugPrefix = "_plug/";
+191
View File
@@ -0,0 +1,191 @@
// import { mkdir, readdir, readFile, stat, unlink, writeFile } from "fs/promises";
import { path } from "../deps.ts";
import { readAll } from "../deps.ts";
import { FileMeta } from "../types.ts";
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
import { Plug } from "../../plugos/plug.ts";
import { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
import { base64Decode, base64Encode } from "../../plugos/base64.ts";
function lookupContentType(path: string): string {
return mime.getType(path) || "application/octet-stream";
}
export class DiskSpacePrimitives implements SpacePrimitives {
rootPath: string;
constructor(rootPath: string) {
this.rootPath = Deno.realPathSync(rootPath);
}
safePath(p: string): string {
const realPath = path.resolve(p);
if (!realPath.startsWith(this.rootPath)) {
throw Error(`Path ${p} is not in the space`);
}
return realPath;
}
filenameToPath(pageName: string) {
return this.safePath(path.join(this.rootPath, pageName));
}
pathToFilename(fullPath: string): string {
return fullPath.substring(this.rootPath.length + 1);
}
async readFile(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
const localPath = this.filenameToPath(name);
try {
const s = await Deno.stat(localPath);
let data: FileData | null = null;
const contentType = lookupContentType(name);
switch (encoding) {
case "string":
data = await Deno.readTextFile(localPath);
break;
case "dataurl":
{
const f = await Deno.open(localPath, { read: true });
const buf = base64Encode(await readAll(f));
Deno.close(f.rid);
data = `data:${contentType};base64,${buf}`;
}
break;
case "arraybuffer":
{
const f = await Deno.open(localPath, { read: true });
const buf = await readAll(f);
Deno.close(f.rid);
data = buf.buffer;
}
break;
}
return {
data,
meta: {
name: name,
lastModified: s.mtime!.getTime(),
perm: "rw",
size: s.size,
contentType: contentType,
},
};
} catch (e) {
// console.error("Error while reading file", name, e);
throw Error(`Could not read file ${name}`);
}
}
async writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate?: boolean,
): Promise<FileMeta> {
let localPath = this.filenameToPath(name);
try {
// Ensure parent folder exists
await Deno.mkdir(path.dirname(localPath), { recursive: true });
// Actually write the file
switch (encoding) {
case "string":
await Deno.writeTextFile(localPath, data as string);
break;
case "dataurl":
await Deno.writeFile(
localPath,
base64Decode((data as string).split(",")[1]),
);
break;
case "arraybuffer":
await Deno.writeFile(localPath, new Uint8Array(data as ArrayBuffer));
break;
}
// Fetch new metadata
const s = await Deno.stat(localPath);
return {
name: name,
size: s.size,
contentType: lookupContentType(name),
lastModified: s.mtime!.getTime(),
perm: "rw",
};
} catch (e) {
console.error("Error while writing file", name, e);
throw Error(`Could not write ${name}`);
}
}
async getFileMeta(name: string): Promise<FileMeta> {
const localPath = this.filenameToPath(name);
try {
const s = await Deno.stat(localPath);
return {
name: name,
size: s.size,
contentType: lookupContentType(name),
lastModified: s.mtime!.getTime(),
perm: "rw",
};
} catch (e) {
// console.error("Error while getting page meta", pageName, e);
throw Error(`Could not get meta for ${name}`);
}
}
async deleteFile(name: string): Promise<void> {
const localPath = this.filenameToPath(name);
await Deno.remove(localPath);
}
async fetchFileList(): Promise<FileMeta[]> {
const fileList: FileMeta[] = [];
const walkPath = async (dir: string) => {
for await (const file of Deno.readDir(dir)) {
if (file.name.startsWith(".")) {
continue;
}
const fullPath = path.join(dir, file.name);
let s = await Deno.stat(fullPath);
if (file.isDirectory) {
await walkPath(fullPath);
} else {
if (!file.name.startsWith(".")) {
fileList.push({
name: this.pathToFilename(fullPath),
size: s.size,
contentType: lookupContentType(fullPath),
lastModified: s.mtime!.getTime(),
perm: "rw",
});
}
}
}
};
await walkPath(this.rootPath);
return fileList;
}
// Plugs
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[],
): Promise<any> {
return plug.invoke(name, args);
}
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return plug.syscall(name, args);
}
}
+91
View File
@@ -0,0 +1,91 @@
import { EventHook } from "../../plugos/hooks/event.ts";
import { Plug } from "../../plugos/plug.ts";
import { FileMeta } from "../types.ts";
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
export class EventedSpacePrimitives implements SpacePrimitives {
constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {}
fetchFileList(): Promise<FileMeta[]> {
return this.wrapped.fetchFileList();
}
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return this.wrapped.proxySyscall(plug, name, args);
}
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[]
): Promise<any> {
return this.wrapped.invokeFunction(plug, env, name, args);
}
readFile(
name: string,
encoding: FileEncoding
): Promise<{ data: FileData; meta: FileMeta }> {
return this.wrapped.readFile(name, encoding);
}
async writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate: boolean
): Promise<FileMeta> {
const newMeta = await this.wrapped.writeFile(
name,
encoding,
data,
selfUpdate
);
// This can happen async
if (name.endsWith(".md")) {
const pageName = name.substring(0, name.length - 3);
let text = "";
switch (encoding) {
case "string":
text = data as string;
break;
case "arraybuffer":
{
const decoder = new TextDecoder("utf-8");
text = decoder.decode(data as ArrayBuffer);
}
break;
case "dataurl":
throw Error("Data urls not supported in this context");
}
this.eventHook
.dispatchEvent("page:saved", pageName)
.then(() => {
return this.eventHook.dispatchEvent("page:index_text", {
name: pageName,
text,
});
})
.catch((e) => {
console.error("Error dispatching page:saved event", e);
});
}
return newMeta;
}
getFileMeta(name: string): Promise<FileMeta> {
return this.wrapped.getFileMeta(name);
}
async deleteFile(name: string): Promise<void> {
if (name.endsWith(".md")) {
const pageName = name.substring(0, name.length - 3);
await this.eventHook.dispatchEvent("page:deleted", pageName);
}
return this.wrapped.deleteFile(name);
}
}
+209
View File
@@ -0,0 +1,209 @@
import { AttachmentMeta, FileMeta, PageMeta } from "../types.ts";
import { Plug } from "../../plugos/plug.ts";
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
export class HttpSpacePrimitives implements SpacePrimitives {
fsUrl: string;
private plugUrl: string;
token?: string;
constructor(url: string, token?: string) {
this.fsUrl = url + "/fs";
this.plugUrl = url + "/plug";
this.token = token;
}
private async authenticatedFetch(
url: string,
options: any
): Promise<Response> {
if (this.token) {
options.headers = options.headers || {};
options.headers["Authorization"] = `Bearer ${this.token}`;
}
let result = await fetch(url, options);
if (result.status === 401) {
throw Error("Unauthorized");
}
return result;
}
public async fetchFileList(): Promise<FileMeta[]> {
let req = await this.authenticatedFetch(this.fsUrl, {
method: "GET",
});
let result: FileMeta[] = await req.json();
return result;
}
async readFile(
name: string,
encoding: FileEncoding
): Promise<{ data: FileData; meta: FileMeta }> {
let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
method: "GET",
});
if (res.status === 404) {
throw new Error(`Page not found`);
}
let data: FileData | null = null;
switch (encoding) {
case "arraybuffer":
{
let abBlob = await res.blob();
data = await abBlob.arrayBuffer();
}
break;
case "dataurl":
{
let dUBlob = await res.blob();
data = arrayBufferToDataUrl(await dUBlob.arrayBuffer());
}
break;
case "string":
data = await res.text();
break;
}
return {
data: data,
meta: this.responseToMeta(name, res),
};
}
async writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate?: boolean
): Promise<FileMeta> {
let body: any = null;
switch (encoding) {
case "arraybuffer":
case "string":
body = data;
break;
case "dataurl":
data = dataUrlToArrayBuffer(data as string);
break;
}
let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
method: "PUT",
headers: {
"Content-type": "application/octet-stream",
},
body,
});
const newMeta = this.responseToMeta(name, res);
return newMeta;
}
async deleteFile(name: string): Promise<void> {
let req = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
method: "DELETE",
});
if (req.status !== 200) {
throw Error(`Failed to delete file: ${req.statusText}`);
}
}
async getFileMeta(name: string): Promise<FileMeta> {
let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
method: "OPTIONS",
});
if (res.status === 404) {
throw new Error(`File not found`);
}
return this.responseToMeta(name, res);
}
private responseToMeta(name: string, res: Response): FileMeta {
return {
name,
size: +res.headers.get("Content-length")!,
contentType: res.headers.get("Content-type")!,
lastModified: +(res.headers.get("Last-Modified") || "0"),
perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
};
}
// Plugs
async proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
let req = await this.authenticatedFetch(
`${this.plugUrl}/${plug.name}/syscall/${name}`,
{
method: "POST",
headers: {
"Content-type": "application/json",
},
body: JSON.stringify(args),
}
);
if (req.status !== 200) {
let error = await req.text();
throw Error(error);
}
if (req.headers.get("Content-length") === "0") {
return;
}
return await req.json();
}
async invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[]
): Promise<any> {
// Invoke locally
if (!env || env === "client") {
return plug.invoke(name, args);
}
// Or dispatch to server
let req = await this.authenticatedFetch(
`${this.plugUrl}/${plug.name}/function/${name}`,
{
method: "POST",
headers: {
"Content-type": "application/json",
},
body: JSON.stringify(args),
}
);
if (req.status !== 200) {
let error = await req.text();
throw Error(error);
}
if (req.headers.get("Content-length") === "0") {
return;
}
if (req.headers.get("Content-type")?.includes("application/json")) {
return await req.json();
} else {
return await req.text();
}
}
}
function dataUrlToArrayBuffer(dataUrl: string): ArrayBuffer {
var binary_string = window.atob(dataUrl.split(",")[1]);
var len = binary_string.length;
var bytes = new Uint8Array(len);
for (var i = 0; i < len; i++) {
bytes[i] = binary_string.charCodeAt(i);
}
return bytes.buffer;
}
function arrayBufferToDataUrl(buffer: ArrayBuffer): string {
var binary = "";
var bytes = new Uint8Array(buffer);
var len = bytes.byteLength;
for (var i = 0; i < len; i++) {
binary += String.fromCharCode(bytes[i]);
}
return `data:application/octet-stream,${window.btoa(binary)}`;
}
+226
View File
@@ -0,0 +1,226 @@
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
import { AttachmentMeta, FileMeta, PageMeta } from "../types.ts";
import { EventEmitter } from "../../plugos/event.ts";
import { Plug } from "../../plugos/plug.ts";
import { plugPrefix } from "./constants.ts";
import { safeRun } from "../util.ts";
const pageWatchInterval = 2000;
export type SpaceEvents = {
pageCreated: (meta: PageMeta) => void;
pageChanged: (meta: PageMeta) => void;
pageDeleted: (name: string) => void;
pageListUpdated: (pages: Set<PageMeta>) => void;
};
export class Space extends EventEmitter<SpaceEvents> {
pageMetaCache = new Map<string, PageMeta>();
watchedPages = new Set<string>();
private initialPageListLoad = true;
private saving = false;
constructor(private space: SpacePrimitives) {
super();
}
public async updatePageList() {
let newPageList = await this.fetchPageList();
// console.log("Updating page list", newPageList);
let deletedPages = new Set<string>(this.pageMetaCache.keys());
newPageList.forEach((meta) => {
const pageName = meta.name;
const oldPageMeta = this.pageMetaCache.get(pageName);
const newPageMeta: PageMeta = {
name: pageName,
lastModified: meta.lastModified,
perm: meta.perm,
};
if (
!oldPageMeta &&
(pageName.startsWith(plugPrefix) || !this.initialPageListLoad)
) {
this.emit("pageCreated", newPageMeta);
} else if (
oldPageMeta &&
oldPageMeta.lastModified !== newPageMeta.lastModified
) {
this.emit("pageChanged", newPageMeta);
}
// Page found, not deleted
deletedPages.delete(pageName);
// Update in cache
this.pageMetaCache.set(pageName, newPageMeta);
});
for (const deletedPage of deletedPages) {
this.pageMetaCache.delete(deletedPage);
this.emit("pageDeleted", deletedPage);
}
this.emit("pageListUpdated", this.listPages());
this.initialPageListLoad = false;
}
watch() {
setInterval(() => {
safeRun(async () => {
if (this.saving) {
return;
}
for (const pageName of this.watchedPages) {
const oldMeta = this.pageMetaCache.get(pageName);
if (!oldMeta) {
// No longer in cache, meaning probably deleted let's unwatch
this.watchedPages.delete(pageName);
continue;
}
// This seems weird, but simply fetching it will compare to local cache and trigger an event if necessary
await this.getPageMeta(pageName);
}
});
}, pageWatchInterval);
this.updatePageList().catch(console.error);
}
async deletePage(name: string, deleteDate?: number): Promise<void> {
await this.getPageMeta(name); // Check if page exists, if not throws Error
await this.space.deleteFile(`${name}.md`);
this.pageMetaCache.delete(name);
this.emit("pageDeleted", name);
this.emit("pageListUpdated", new Set([...this.pageMetaCache.values()]));
}
async getPageMeta(name: string): Promise<PageMeta> {
let oldMeta = this.pageMetaCache.get(name);
let newMeta = fileMetaToPageMeta(
await this.space.getFileMeta(`${name}.md`)
);
if (oldMeta) {
if (oldMeta.lastModified !== newMeta.lastModified) {
// Changed on disk, trigger event
this.emit("pageChanged", newMeta);
}
}
return this.metaCacher(name, newMeta);
}
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[]
): Promise<any> {
return this.space.invokeFunction(plug, env, name, args);
}
listPages(): Set<PageMeta> {
return new Set(this.pageMetaCache.values());
}
async listPlugs(): Promise<string[]> {
let allFiles = await this.space.fetchFileList();
return allFiles
.filter((fileMeta) => fileMeta.name.endsWith(".plug.json"))
.map((fileMeta) => fileMeta.name);
}
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return this.space.proxySyscall(plug, name, args);
}
async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
let pageData = await this.space.readFile(`${name}.md`, "string");
let previousMeta = this.pageMetaCache.get(name);
let newMeta = fileMetaToPageMeta(pageData.meta);
if (previousMeta) {
if (previousMeta.lastModified !== newMeta.lastModified) {
// Page changed since last cached metadata, trigger event
this.emit("pageChanged", newMeta);
}
}
let meta = this.metaCacher(name, newMeta);
return {
text: pageData.data as string,
meta: meta,
};
}
watchPage(pageName: string) {
this.watchedPages.add(pageName);
}
unwatchPage(pageName: string) {
this.watchedPages.delete(pageName);
}
async writePage(
name: string,
text: string,
selfUpdate?: boolean
): Promise<PageMeta> {
try {
this.saving = true;
let pageMeta = fileMetaToPageMeta(
await this.space.writeFile(`${name}.md`, "string", text, selfUpdate)
);
if (!selfUpdate) {
this.emit("pageChanged", pageMeta);
}
return this.metaCacher(name, pageMeta);
} finally {
this.saving = false;
}
}
async fetchPageList(): Promise<PageMeta[]> {
return (await this.space.fetchFileList())
.filter((fileMeta) => fileMeta.name.endsWith(".md"))
.map(fileMetaToPageMeta);
}
async fetchAttachmentList(): Promise<AttachmentMeta[]> {
return (await this.space.fetchFileList()).filter(
(fileMeta) =>
!fileMeta.name.endsWith(".md") && !fileMeta.name.endsWith(".plug.json")
);
}
readAttachment(
name: string,
encoding: FileEncoding
): Promise<{ data: FileData; meta: AttachmentMeta }> {
return this.space.readFile(name, encoding);
}
getAttachmentMeta(name: string): Promise<AttachmentMeta> {
return this.space.getFileMeta(name);
}
writeAttachment(
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate?: boolean | undefined
): Promise<AttachmentMeta> {
return this.space.writeFile(name, encoding, data, selfUpdate);
}
deleteAttachment(name: string): Promise<void> {
return this.space.deleteFile(name);
}
private metaCacher(name: string, meta: PageMeta): PageMeta {
this.pageMetaCache.set(name, meta);
return meta;
}
}
function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta {
return {
...fileMeta,
name: fileMeta.name.substring(0, fileMeta.name.length - 3),
} as PageMeta;
}
+30
View File
@@ -0,0 +1,30 @@
import { Plug } from "../../plugos/plug.ts";
import { FileMeta } from "../types.ts";
export type FileEncoding = "string" | "arraybuffer" | "dataurl";
export type FileData = ArrayBuffer | string;
export interface SpacePrimitives {
// Pages
fetchFileList(): Promise<FileMeta[]>;
readFile(
name: string,
encoding: FileEncoding
): Promise<{ data: FileData; meta: FileMeta }>;
getFileMeta(name: string): Promise<FileMeta>;
writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate?: boolean
): Promise<FileMeta>;
deleteFile(name: string): Promise<void>;
// Plugs
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any>;
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[]
): Promise<any>;
}