Fixes #97: SQLite is now async, optimized, tests

2022-10-21 10:00:43 +02:00
parent b9da9b7965
commit c1a78e0105
65 changed files with 329 additions and 142 deletions
@@ -0,0 +1,62 @@
+const [src, dest] = Deno.args;
+
+const wasm = await Deno.readFile(src);
+
+function encode(bytes) {
+  let binary = "";
+  for (let i = 0; i < bytes.length; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return btoa(binary).replace(/\n/g, "");
+}
+
+await Deno.writeFile(
+  dest,
+  new TextEncoder().encode(
+    `/// <reference types="./sqlite.d.ts" />
+
+/* This file is automatically generated. Do not edit directly. */
+import env from "./vfs.js";
+
+const wasm =
+  "${encode(wasm)}";
+
+function decode(base64) {
+  const bytesStr = atob(base64);
+  const bytes = new Uint8Array(bytesStr.length);
+  for (let i = 0, c = bytesStr.length; i < c; i++) {
+    bytes[i] = bytesStr.charCodeAt(i);
+  }
+  return bytes;
+}
+
+const moduleOrInstance = {
+  module: null,
+  instances: [],
+};
+
+export async function compile() {
+  moduleOrInstance.module = await WebAssembly.compile(decode(wasm));
+}
+
+export async function instantiateBrowser() {
+  const placeholder = { exports: null };
+  const instance = await WebAssembly.instantiate(moduleOrInstance.module, env(placeholder));
+  placeholder.exports = instance.exports;
+  instance.exports.seed_rng(Date.now());
+  moduleOrInstance.instances.push(instance);
+}
+
+export function instantiate() {
+  if (moduleOrInstance.instances.length) {
+    return moduleOrInstance.instances.pop();
+  } else {
+    const placeholder = { exports: null };
+    const instance = new WebAssembly.Instance(moduleOrInstance.module, env(placeholder));
+    placeholder.exports = instance.exports;
+    instance.exports.seed_rng(Date.now());
+    return instance;
+  }
+}`,
+  ),
+);
@@ -0,0 +1,10 @@
+// Generate available import symbols
+// from vfs.js file
+
+import env from "../vfs.js";
+
+let symbols = "";
+for (const symbol of Object.keys(env().env)) {
+  symbols += `${symbol}\n`;
+}
+await Deno.writeFile(Deno.args[0], new TextEncoder().encode(symbols));
@@ -0,0 +1,98 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sqlite3.h>
+
+#define TRUE 1
+#define FALSE 0
+
+#define TEST_DB_FILE "2GB_test.db"
+
+#define SQL_CREATE_TBL "CREATE TABLE test (id INTEGER PRIMARY KEY, value TEXT)"
+#define SQL_INSERT_VAL "INSERT INTO test (value) VALUES (?)"
+
+#define VAL_LEN 65536
+#define VAL_NUM 45000
+
+void rand_str(char *dest, size_t length) {
+  char charset[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+  while (length --> 0) {
+    size_t index = (double) rand() / RAND_MAX * (sizeof charset - 1);
+    *dest++ = charset[index];
+  }
+  *dest = '\0';
+}
+
+int execute_query(sqlite3* db, char* query) {
+  sqlite3_stmt* stmt_create;
+  if (sqlite3_prepare_v2(db, query, -1, &stmt_create, NULL) != SQLITE_OK) {
+    printf("Failed to prepare query statement: %s\n", sqlite3_errmsg(db));
+    return FALSE;
+  }
+  if (sqlite3_step(stmt_create) != SQLITE_DONE) {
+    printf("Failed to run query statement: %s\n", sqlite3_errmsg(db));
+    return FALSE;
+  }
+  sqlite3_finalize(stmt_create);
+  return TRUE;
+}
+
+int main(int argc, char* argv[]) {
+  sqlite3* db;
+  if (sqlite3_open(TEST_DB_FILE, &db) != SQLITE_OK) {
+    printf("Failed to open database: %s\n", sqlite3_errmsg(db));
+    return 1;
+  }
+
+  // create database table
+  sqlite3_stmt* stmt_create;
+  if (sqlite3_prepare_v2(db, SQL_CREATE_TBL, -1, &stmt_create, NULL) != SQLITE_OK) {
+    printf("Failed to prepare create table statement: %s\n", sqlite3_errmsg(db));
+    return 1;
+  }
+  if (sqlite3_step(stmt_create) != SQLITE_DONE) {
+    printf("Failed to run create table statement: %s\n", sqlite3_errmsg(db));
+    return 1;
+  }
+  sqlite3_finalize(stmt_create);
+
+  // insert values to reach 2GB
+  sqlite3_stmt* stmt_insert;
+  if (sqlite3_prepare_v2(db, SQL_INSERT_VAL, -1, &stmt_insert, NULL) != SQLITE_OK) {
+    printf("Failed to prepare insert table statement: %s\n", sqlite3_errmsg(db));
+    return 1;
+  }
+
+  // begin transaction
+  if (!execute_query(db, "begin")) {
+    return 1;
+  }
+
+  char* buffer = malloc(VAL_LEN + 1);
+  for (int64_t i = 0; i < VAL_NUM; i++) {
+    rand_str(buffer, VAL_LEN);
+    if (sqlite3_bind_text(stmt_insert, 1, buffer, VAL_LEN, NULL) != SQLITE_OK) {
+      printf("Failed to bind value `%s`: %s\n", buffer, sqlite3_errmsg(db));
+      return 1;
+    }
+    if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
+      printf("Failed to run insert statement: %s\n", sqlite3_errmsg(db));
+      return 1;
+    }
+    if (sqlite3_reset(stmt_insert) != SQLITE_OK) {
+      printf("Failed to reset statement: %s\n", sqlite3_errmsg(db));
+      return 1;
+    }
+  }
+
+  // end transaction
+  if (!execute_query(db, "commit")) {
+    return 1;
+  }
+
+  sqlite3_finalize(stmt_insert);
+  sqlite3_close(db);
+
+  printf("Database created successfully.\n");
+
+  return 0;
+}
@@ -0,0 +1,156 @@
+interface Item {
+  name: string;
+  arguments: Argument[];
+  returnType: Type;
+}
+
+interface Argument {
+  name: string;
+  type: Type;
+}
+
+enum Type {
+  Void,
+  VoidPtr,
+  StringPtr,
+  StatementPtr,
+  Double,
+  Int,
+}
+
+const items = [
+  // exported manually in compiler invocation
+  {
+    name: "malloc",
+    arguments: [{ name: "size", type: Type.Int }],
+    returnType: Type.VoidPtr,
+  },
+  {
+    name: "free",
+    arguments: [{ name: "ptr", type: Type.VoidPtr }],
+    returnType: Type.Void,
+  },
+];
+
+const [src, dest] = Deno.args;
+const wrapperSrc = await Deno.readTextFile(src);
+
+// int EXPORT(bind_int) (sqlite3_stmt* stmt, int idx, double value)
+const typeRegexp =
+  `(const +)?(sqlite3_stmt\\*|char\\*|void\\*|int|uint32_t|double|void)`;
+const argRegexp = `${typeRegexp} +[a-z_]+`;
+const exportSignature = new RegExp(
+  `${typeRegexp} +EXPORT\\([a-z_]+\\) +\\(((${argRegexp}( *, *${argRegexp})*)|)\\)`,
+);
+
+function nullThrows<T>(value: T | null | undefined): T {
+  if (value == null) {
+    throw new Error("Got a null value");
+  }
+  return value as T;
+}
+
+function typeFromCType(cType: string): Type {
+  cType = cType.replace("const", "").replace(/ /g, "");
+  switch (cType) {
+    case "void":
+      return Type.Void;
+    case "void*":
+      return Type.VoidPtr;
+    case "char*":
+      return Type.StringPtr;
+    case "sqlite3_stmt*":
+      return Type.StatementPtr;
+    case "double":
+      return Type.Double;
+    case "int":
+    case "uint32_t":
+      return Type.Int;
+    default:
+      throw new Error("Unknown type");
+  }
+}
+
+function getReturnType(line: string): Type {
+  const regexp = new RegExp(typeRegexp);
+  const [, _const, cType] = nullThrows(regexp.exec(line));
+  return typeFromCType(cType);
+}
+
+function getName(line: string): string {
+  const [, name] = nullThrows(/EXPORT\(([a-z_]+)\)/.exec(line));
+  return name;
+}
+
+function getArguments(line: string): Argument[] {
+  const [, argList] = nullThrows(/EXPORT\([a-z_]+\) *\(([^)]*)\)/.exec(line));
+  if (argList.length === 0) {
+    return [];
+  } else {
+    return argList.split(",").map((arg) => {
+      const regexp = new RegExp(`${typeRegexp} +([a-z_]+)`);
+      const [, _const, cType, name] = nullThrows(regexp.exec(arg));
+      return {
+        name,
+        type: typeFromCType(cType),
+      };
+    });
+  }
+}
+
+function generateType(tp: Type): string {
+  switch (tp) {
+    case Type.Void:
+      return "void";
+    case Type.VoidPtr:
+      return "VoidPtr";
+    case Type.StringPtr:
+      return "StringPtr";
+    case Type.StatementPtr:
+      return "StatementPtr";
+    case Type.Int:
+    case Type.Double:
+      return "number";
+    default:
+      throw new Error("Unknown type");
+  }
+}
+
+function generateDecl(item: Item): string {
+  const args = item.arguments.map((arg) =>
+    `${arg.name}: ${generateType(arg.type)}`
+  ).join(", ");
+  return `${item.name}: (${args}) => ${generateType(item.returnType)}`;
+}
+
+const exportLines = wrapperSrc.split("\n").filter((line) =>
+  exportSignature.test(line)
+);
+
+for (const line of exportLines) {
+  const name = getName(line);
+  const returnType = getReturnType(line);
+  const args = getArguments(line);
+
+  items.push({ name, returnType, arguments: args });
+}
+
+const typeDeclaration =
+  `/* This file is automatically generated. Do not edit directly. */
+
+export type VoidPtr = number;
+export type StringPtr = number;
+export type StatementPtr = number;
+
+export interface Wasm {
+  memory: WebAssembly.Memory;
+
+  ${items.map(generateDecl).join(";\n  ")};
+}
+
+export function compile(): Promise<void>;
+export function instantiateBrowser(): Promise<void>;
+export function instantiate(): { exports: Wasm };
+`;
+
+await Deno.writeTextFile(dest, typeDeclaration);