Fixes #97: SQLite is now async, optimized, tests

This commit is contained in:
Zef Hemel
2022-10-21 10:00:43 +02:00
parent b9da9b7965
commit c1a78e0105
65 changed files with 329 additions and 142 deletions
@@ -0,0 +1,27 @@
#ifndef DEBUG_H
#define DEBUG_H
#ifdef DEBUG_BUILD
#include <stdlib.h>
#include <printf.h>
#include "imports.h"
// Print debug messages
#define debug_printf(...) { \
char* __debug_msg = malloc(2048); \
if (!__debug_msg) { \
js_print("ERROR: No memory for debug message.\n"); \
} else { \
size_t __used = snprintf(__debug_msg, 2048, "DEBUG: %s:%d:%s(): ", __FILE__, __LINE__, __func__); \
snprintf(&__debug_msg[__used], 2048 - __used, __VA_ARGS__); \
js_print(__debug_msg); \
free(__debug_msg); \
} \
}
#else // DEBUG_BUILD
#define debug_printf(...)
#endif // DEBUG_BUILD
#endif // DEBUG_H
@@ -0,0 +1,22 @@
#ifndef IMPORTS_H
#define IMPORTS_H
// WASM imports specified in vfs.syms
extern void js_print(const char*);
extern int js_open(const char*, int, int);
extern void js_close(int);
extern void js_delete(const char*);
extern int js_read(int, const char*, double, int);
extern int js_write(int, const char*, double, int);
extern void js_truncate(int, double);
extern void js_sync(int);
extern double js_size(int);
extern void js_lock(int, int);
extern void js_unlock(int);
extern double js_time();
extern int js_timezone();
extern int js_exists(const char*);
extern int js_access(const char*);
#endif // DEBUG_H
+302
View File
@@ -0,0 +1,302 @@
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <time.h>
#include <sqlite3.h>
#include <pcg.h>
#include "debug.h"
#include "imports.h"
// SQLite VFS component.
// Based on demoVFS from SQLlite.
// https://www.sqlite.org/src/doc/trunk/src/test_demovfs.c
#define MAXPATHNAME 1024
#define JS_MAX_SAFE_INTEGER 9007199254740991
// When using this VFS, the sqlite3_file* handles that SQLite uses are
// actually pointers to instances of type DenoFile.
typedef struct DenoFile DenoFile;
struct DenoFile {
sqlite3_file base;
// Deno file resource id
int rid;
};
static int denoClose(sqlite3_file *pFile) {
DenoFile* p = (DenoFile*)pFile;
js_close(p->rid);
debug_printf("closing file (rid %i)\n", p->rid);
return SQLITE_OK;
}
// Read data from a file.
static int denoRead(sqlite3_file *pFile, void *zBuf, int iAmt, sqlite_int64 iOfst) {
DenoFile *p = (DenoFile*)pFile;
int read_bytes = 0;
if (iOfst <= JS_MAX_SAFE_INTEGER) {
// Read bytes from buffer
read_bytes = js_read(p->rid, (char*)zBuf, (double)iOfst, iAmt);
debug_printf("attempt to read from file (rid %i, amount %i, offset %lli, read %i)\n",
p->rid, iAmt, iOfst, read_bytes);
} else {
debug_printf("read offset %lli overflows JS_MAX_SAFE_INTEGER\n", iOfst);
}
// Zero memory if read was short
if (read_bytes < iAmt)
memset(&((char*)zBuf)[read_bytes], 0, iAmt-read_bytes);
return read_bytes < iAmt ? SQLITE_IOERR_SHORT_READ : SQLITE_OK;
}
// Write data to a file.
static int denoWrite(sqlite3_file *pFile, const void *zBuf, int iAmt, sqlite_int64 iOfst) {
DenoFile *p = (DenoFile*)pFile;
int write_bytes = 0;
if (iOfst <= JS_MAX_SAFE_INTEGER) {
// Write bytes to buffer
write_bytes = js_write(p->rid, (char*)zBuf, (double)iOfst, iAmt);
debug_printf("attempt to write to file (rid %i, amount %i, offset %lli, written %i)\n",
p->rid, iAmt, iOfst, write_bytes);
} else {
debug_printf("write offset %lli overflows JS_MAX_SAFE_INTEGER\n", iOfst);
}
return write_bytes == iAmt ? SQLITE_OK : SQLITE_IOERR_WRITE;
}
// Truncate file.
static int denoTruncate(sqlite3_file *pFile, sqlite_int64 size) {
DenoFile *p = (DenoFile*)pFile;
if (size <= JS_MAX_SAFE_INTEGER) {
js_truncate(p->rid, (double)size);
debug_printf("truncating file (rid %i, size: %lli)\n", p->rid, size);
return SQLITE_OK;
} else {
debug_printf("truncate length %lli overflows JS_MAX_SAFE_INTEGER\n", size);
return SQLITE_IOERR;
}
}
// Deno provides no explicit sync for us, so we
// just have a no-op here.
// TODO(dyedgreen): Investigate if there is a better way
static int denoSync(sqlite3_file *pFile, int flags) {
DenoFile *p = (DenoFile*)pFile;
js_sync(p->rid);
debug_printf("syncing file (rid %i)\n", p->rid);
return SQLITE_OK;
}
// Write the size of the file in bytes to *pSize.
static int denoFileSize(sqlite3_file *pFile, sqlite_int64 *pSize) {
DenoFile *p = (DenoFile*)pFile;
*pSize = (sqlite_int64)js_size(p->rid);
debug_printf("read file size: %lli (rid %i)\n", *pSize, p->rid);
return SQLITE_OK;
}
// File locking
static int denoLock(sqlite3_file *pFile, int eLock) {
DenoFile *p = (DenoFile*)pFile;
switch (eLock) {
case SQLITE_LOCK_NONE:
// no op
break;
case SQLITE_LOCK_SHARED:
case SQLITE_LOCK_RESERVED: // one WASM process <-> one open database
js_lock(p->rid, 0);
break;
case SQLITE_LOCK_PENDING:
case SQLITE_LOCK_EXCLUSIVE:
js_lock(p->rid, 1);
break;
}
return SQLITE_OK;
}
static int denoUnlock(sqlite3_file *pFile, int eLock) {
DenoFile *p = (DenoFile*)pFile;
switch (eLock) {
case SQLITE_LOCK_NONE:
// no op
break;
case SQLITE_LOCK_SHARED:
case SQLITE_LOCK_RESERVED:
case SQLITE_LOCK_PENDING:
case SQLITE_LOCK_EXCLUSIVE:
js_unlock(p->rid);
break;
}
return SQLITE_OK;
}
static int denoCheckReservedLock(sqlite3_file *pFile, int *pResOut) {
*pResOut = 0;
return SQLITE_OK;
}
// No xFileControl() verbs are implemented by this VFS.
static int denoFileControl(sqlite3_file *pFile, int op, void *pArg) {
return SQLITE_NOTFOUND;
}
// TODO(dyedgreen): Should we try to get these?
static int denoSectorSize(sqlite3_file *pFile) {
return 0;
}
static int denoDeviceCharacteristics(sqlite3_file *pFile) {
return 0;
}
// Open a file handle.
static int denoOpen(
sqlite3_vfs *pVfs, /* VFS */
const char *zName, /* File to open, or 0 for a temp file */
sqlite3_file *pFile, /* Pointer to DenoFile struct to populate */
int flags, /* Input SQLITE_OPEN_XXX flags */
int *pOutFlags /* Output SQLITE_OPEN_XXX flags (or NULL) */
) {
static const sqlite3_io_methods denoio = {
1, /* iVersion */
denoClose, /* xClose */
denoRead, /* xRead */
denoWrite, /* xWrite */
denoTruncate, /* xTruncate */
denoSync, /* xSync */
denoFileSize, /* xFileSize */
denoLock, /* xLock */
denoUnlock, /* xUnlock */
denoCheckReservedLock, /* xCheckReservedLock */
denoFileControl, /* xFileControl */
denoSectorSize, /* xSectorSize */
denoDeviceCharacteristics /* xDeviceCharacteristics */
};
DenoFile *p = (DenoFile*)pFile;
p->base.pMethods = &denoio;
// TODO(dyedgreen): The current approach is to raise
// the permission error on the vfs.js side of things,
// should the error be propagates through the wrapper
// and be raised on the wrapper side of things?
p->rid = js_open(zName, zName ? 0 : 1, flags);
if (pOutFlags) {
*pOutFlags = flags;
}
debug_printf("opened file (rid %i)\n", p->rid);
debug_printf("file path name: '%s'\n", zName);
return SQLITE_OK;
}
// Delete the file at the path.
static int denoDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync) {
js_delete(zPath);
return SQLITE_OK;
}
// All valid id files are accessible.
static int denoAccess(sqlite3_vfs *pVfs, const char *zPath, int flags, int *pResOut) {
switch (flags) {
case SQLITE_ACCESS_EXISTS:
*pResOut = js_exists(zPath);
break;
default:
*pResOut = js_access(zPath);
break;
}
debug_printf("determining file access (path %s, access %i)\n", zPath, *pResOut);
return SQLITE_OK;
}
// TODO(dyedgreen): Actually resolve the full path name
static int denoFullPathname(sqlite3_vfs *pVfs, const char *zPath, int nPathOut, char *zPathOut) {
sqlite3_snprintf(nPathOut, zPathOut, "%s", zPath);
debug_printf("requesting full path name for path: %s\n", zPath);
return SQLITE_OK;
}
// We don't support shared objects
static void *denoDlOpen(sqlite3_vfs *pVfs, const char *zPath) {
return 0;
}
static void denoDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg) {
sqlite3_snprintf(nByte, zErrMsg, "Loadable extensions are not supported");
zErrMsg[nByte-1] = '\0';
}
static void (*denoDlSym(sqlite3_vfs *pVfs, void *pH, const char *z))(void) {
return 0;
}
static void denoDlClose(sqlite3_vfs *pVfs, void *pHandle) {
return;
}
// Generate pseudo-random data
static int denoRandomness(sqlite3_vfs *pVfs, int nByte, char *zByte) {
pcg_bytes(zByte, nByte);
return SQLITE_OK;
}
// TODO(dyedgreen): Can anything be done here?
static int denoSleep(sqlite3_vfs *pVfs, int nMicro) {
return 0;
}
// Retrieve the current time
static int denoCurrentTime(sqlite3_vfs *pVfs, double *pTime) {
*pTime = js_time() / 1000 / 86400.0 + 2440587.5;
return SQLITE_OK;
}
// Implement localtime_r
struct tm* localtime_r(const time_t *time, struct tm *result) {
debug_printf("running localtime_r");
time_t shifted = *time - 60 * js_timezone();
return gmtime_r(&shifted, result);
}
// This function returns a pointer to the VFS implemented in this file.
sqlite3_vfs *sqlite3_denovfs(void) {
static sqlite3_vfs denovfs = {
3, /* iVersion */
sizeof(DenoFile), /* szOsFile */
MAXPATHNAME, /* mxPathname */
0, /* pNext */
"deno", /* zName */
0, /* pAppData */
denoOpen, /* xOpen */
denoDelete, /* xDelete */
denoAccess, /* xAccess */
denoFullPathname, /* xFullPathname */
denoDlOpen, /* xDlOpen */
denoDlError, /* xDlError */
denoDlSym, /* xDlSym */
denoDlClose, /* xDlClose */
denoRandomness, /* xRandomness */
denoSleep, /* xSleep */
denoCurrentTime, /* xCurrentTime */
0, /* xGetLastError */
0, /* xCurrentTimeInt64 */
0, /* xSetSystemCall */
0, /* xGetSystemCall */
0, /* xNextSystemCall */
};
return &denovfs;
}
int sqlite3_os_init(void) {
debug_printf("running sqlite3_os_init\n");
// Register VFS
return sqlite3_vfs_register(sqlite3_denovfs(), 1);
}
int sqlite3_os_end(void) {
return SQLITE_OK;
}
@@ -0,0 +1,247 @@
#include <stdlib.h>
#include <sqlite3.h>
#include <pcg.h>
#include "debug.h"
#define EXPORT(name) __attribute__((used)) __attribute__((export_name (#name))) name
#define ERROR_VAL -1
#define BIG_INT_TYPE 6
#define JS_MAX_SAFE_INTEGER 9007199254740991
#define JS_MIN_SAFE_INTEGER (-JS_MAX_SAFE_INTEGER)
// Status returned by last instruction
int last_status = SQLITE_OK;
// Database handle for this instance
sqlite3* database = NULL;
// Return length of string pointed to by str.
int EXPORT(str_len) (const char* str) {
int len;
for (len = 0; str[len] != '\0'; len ++);
return len;
}
// Seed the random number generator. We pass a double, to
// get as many bytes from the JS number as possible.
void EXPORT(seed_rng) (double seed) {
pcg_seed((uint64_t)seed);
}
// Return last status encountered.
int EXPORT(get_status) () {
return last_status;
}
// Initialize the database and return the status.
int EXPORT(open) (const char* filename, int flags) {
// Return error is database is already open
if (database) {
last_status = SQLITE_MISUSE;
return last_status;
}
// Open SQLite db connection
last_status = sqlite3_open_v2(filename, &database, flags, NULL);
if (last_status != SQLITE_OK) {
debug_printf("failed to open database with status %i\n", last_status);
return last_status;
}
debug_printf("opened database at path '%s'\n", filename);
return last_status;
}
// Attempt to close the database connection.
int EXPORT(close) () {
last_status = sqlite3_close(database);
if (last_status == SQLITE_OK) {
database = NULL;
debug_printf("closed database");
} else {
debug_printf("failed to close database with status %i\n", last_status);
}
return last_status;
}
// Return most recent SQLite error as a string
const char* EXPORT(get_sqlite_error_str) () {
if (!database)
return "No open database.";
return sqlite3_errmsg(database);
}
// Wraps sqlite3_prepare. Returns statement id.
sqlite3_stmt* EXPORT(prepare) (const char* sql) {
// Prepare sqlite statement
sqlite3_stmt* stmt;
last_status = sqlite3_prepare_v2(database, sql, -1, &stmt, NULL);
debug_printf("prepared sql statement (status %i)\n", last_status);
if (last_status != SQLITE_OK)
return NULL;
return stmt;
}
// Destruct the given statement/ transaction. This will destruct the SQLite
// statement and free up it's transaction slot. Regardless of returned
// status, the statement id will be freed up.
int EXPORT(finalize) (sqlite3_stmt* stmt) {
last_status = sqlite3_finalize(stmt);
debug_printf("finalized statement (status %i)\n", last_status);
return last_status;
}
// Reset a given statement so it can be re-used.
int EXPORT(reset) (sqlite3_stmt* stmt) {
last_status = sqlite3_reset(stmt);
debug_printf("reset statement (status %i)\n", last_status);
return last_status;
}
// Resets all bound parameter values for this statement.
int EXPORT(clear_bindings) (sqlite3_stmt* stmt) {
last_status = sqlite3_clear_bindings(stmt);
debug_printf("clear bindings (status %i)\n", last_status);
return last_status;
}
// Execute multiple statements from a single string. This ignores any result
// rows.
int EXPORT(exec) (const char* sql) {
last_status = sqlite3_exec(database, sql, NULL, NULL, NULL);
debug_printf("ran exec (status %i)\n", last_status);
return last_status;
}
// Wrappers for bind statements, these return the status directly
int EXPORT(bind_int) (sqlite3_stmt* stmt, int idx, double value) {
// we use double to pass in the value, as JS does not support 64 bit integers,
// but handles floats and we can contain a 32 bit in in a 64 bit float, so there
// should be no loss.
last_status = sqlite3_bind_int64(stmt, idx, (sqlite3_int64)value);
debug_printf("binding int %lli (status %i)\n", (sqlite3_int64)value, last_status);
return last_status;
}
int EXPORT(bind_double) (sqlite3_stmt* stmt, int idx, double value) {
last_status = sqlite3_bind_double(stmt, idx, value);
debug_printf("binding double %f (status %i)\n", value, last_status);
return last_status;
}
int EXPORT(bind_text) (sqlite3_stmt* stmt, int idx, const char* value) {
// SQLite retrains the string until we execute the statement, but any strings
// passed in from JS are freed when the function returns. Thus we need to mark
// is as transient.
last_status = sqlite3_bind_text(stmt, idx, value, -1, SQLITE_TRANSIENT);
debug_printf("binding text '%s' (status %i)\n", value, last_status);
return last_status;
}
int EXPORT(bind_blob) (sqlite3_stmt* stmt, int idx, void* value, int size) {
// SQLite retrains the pointer until we execute the statement, but any pointers
// passed in from JS are freed when the function returns. Thus we need to mark
// is as transient.
last_status = sqlite3_bind_blob(stmt, idx, value, size, SQLITE_TRANSIENT);
debug_printf("binding blob '%s' (status %i)\n", value, last_status);
return last_status;
}
int EXPORT(bind_big_int) (sqlite3_stmt* stmt, int idx, int sign, uint32_t high, uint32_t low) {
// Bind a big integer within the 64 bit integer range by passing it as two 32
// bit integers. The integers are assumed to be positive, and a sign is passed
// separately.
sqlite3_int64 int_val = ((sqlite3_int64)low + ((sqlite3_int64)high << 32)) * (sqlite3_int64)sign;
debug_printf("binding big_int %lld", int_val);
last_status = sqlite3_bind_int64(stmt, idx, int_val);
return last_status;
}
int EXPORT(bind_null) (sqlite3_stmt* stmt, int idx) {
last_status = sqlite3_bind_null(stmt, idx);
debug_printf("binding null (status %i)\n", last_status);
return last_status;
}
// Determine parameter index for named parameters
int EXPORT(bind_parameter_index) (sqlite3_stmt* stmt, const char* name) {
int index = sqlite3_bind_parameter_index(stmt, name);
if (index == 0) {
debug_printf("parameter '%s' does not exist", name);
// Normalize SQLite returning 0 for not found to ERROR_VAL
return ERROR_VAL;
}
debug_printf("obtained parameter index (param '%s', index %i)\n", name, index);
return index;
}
// Wraps running statements, this returns the status directly
int EXPORT(step) (sqlite3_stmt* stmt) {
last_status = sqlite3_step(stmt);
debug_printf("stepping statement (status %i)\n", last_status);
return last_status;
}
// Count columns returned by statement.
int EXPORT(column_count) (sqlite3_stmt* stmt) {
return sqlite3_column_count(stmt);
}
// Determine type of column. Returns SQLITE column types.
int EXPORT(column_type) (sqlite3_stmt* stmt, int col) {
int type = sqlite3_column_type(stmt, col);
if (type == SQLITE_INTEGER) {
// handle integers that exceed JS_MAX_SAFE_INTEGER
sqlite3_int64 col_val = sqlite3_column_int64(stmt, col);
if (col_val > JS_MAX_SAFE_INTEGER || col_val < JS_MIN_SAFE_INTEGER) {
debug_printf("detected big integer: %lld\n", col_val);
return BIG_INT_TYPE;
}
}
return type;
}
// Wrap result returning functions.
double EXPORT(column_int) (sqlite3_stmt* stmt, int col) {
return (double)sqlite3_column_int64(stmt, col);
}
double EXPORT(column_double) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_double(stmt, col);
}
const char* EXPORT(column_text) (sqlite3_stmt* stmt, int col) {
return (const char*)sqlite3_column_text(stmt, col);
}
const void* EXPORT(column_blob) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_blob(stmt, col);
}
int EXPORT(column_bytes) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_bytes(stmt, col);
}
const char* EXPORT(column_name) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_name(stmt, col);
}
const char* EXPORT(column_origin_name) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_origin_name(stmt, col);
}
const char* EXPORT(column_table_name) (sqlite3_stmt* stmt, int col) {
return sqlite3_column_table_name(stmt, col);
}
double EXPORT(last_insert_rowid) () {
return (double)sqlite3_last_insert_rowid(database);
}
double EXPORT(changes) () {
return (double)sqlite3_changes(database);
}
double EXPORT(total_changes) () {
return (double)sqlite3_total_changes(database);
}