Backporting a bunch of optimizations from db-only branch

plugs/editor/complete.ts

@@ -18,7 +18,9 @@ export async function pageComplete(completeEvent: CompleteEvent) {
       completeEvent.linePrefix,
     );
   const tagToQuery = isInTemplateContext ? "template" : "page";
-  let allPages: PageMeta[] = await queryObjects<PageMeta>(tagToQuery, {});
+  let allPages: PageMeta[] = await queryObjects<PageMeta>(tagToQuery, {
+    cacheSecs: 5,
+  });
   const prefix = match[1];
   if (prefix.startsWith("!")) {
     // Federation prefix, let's first see if we're matching anything from federation that is locally synced

plugs/index/anchor.ts

@@ -43,7 +43,10 @@ export async function anchorComplete(completeEvent: CompleteEvent) {
     // "bare" anchor, match any page for completion purposes
     filter = undefined;
   }
-  const allAnchors = await queryObjects<AnchorObject>("anchor", { filter });
+  const allAnchors = await queryObjects<AnchorObject>("anchor", {
+    filter,
+    cacheSecs: 5,
+  });
   return {
     from: completeEvent.pos - match[1].length,
     options: allAnchors.map((a) => ({

plugs/index/api.ts

@@ -64,7 +64,7 @@ export async function clearIndex(): Promise<void> {
 /**
  * Indexes entities in the data store
  */
-export async function indexObjects<T>(
+export function indexObjects<T>(
   page: string,
   objects: ObjectValue<T>[],
 ): Promise<void> {
@@ -127,14 +127,12 @@ export async function indexObjects<T>(
     }
   }
   if (allAttributes.size > 0) {
-    await indexObjects<AttributeObject>(
-      page,
-      [...allAttributes].map(([key, value]) => {
-        const [tagName, name] = key.split(":");
-        const attributeType = value.startsWith("!")
-          ? value.substring(1)
-          : value;
-        return {
+    [...allAttributes].forEach(([key, value]) => {
+      const [tagName, name] = key.split(":");
+      const attributeType = value.startsWith("!") ? value.substring(1) : value;
+      kvs.push({
+        key: ["attribute", cleanKey(key, page)],
+        value: {
           ref: key,
           tag: "attribute",
           tagName,
@@ -142,11 +140,15 @@ export async function indexObjects<T>(
           attributeType,
           readOnly: value.startsWith("!"),
           page,
-        };
-      }),
-    );
+        } as T,
+      });
+    });
+  }
+  if (kvs.length > 0) {
+    return batchSet(page, kvs);
+  } else {
+    return Promise.resolve();
   }
-  return batchSet(page, kvs);
 }
 
 function cleanKey(ref: string, page: string) {

plugs/index/attributes.ts

@@ -42,23 +42,17 @@ export function determineType(v: any): string {
 export async function objectAttributeCompleter(
   attributeCompleteEvent: AttributeCompleteEvent,
 ): Promise<AttributeCompletion[]> {
-  const prefixFilter: QueryExpression = ["call", "startsWith", [[
-    "attr",
-    "name",
-  ], ["string", attributeCompleteEvent.prefix]]];
   const attributeFilter: QueryExpression | undefined =
     attributeCompleteEvent.source === ""
-      ? prefixFilter
-      : ["and", prefixFilter, ["=", ["attr", "tagName"], [
-        "string",
-        attributeCompleteEvent.source,
-      ]]];
+      ? undefined
+      : ["=", ["attr", "tagName"], ["string", attributeCompleteEvent.source]];
   const allAttributes = await queryObjects<AttributeObject>("attribute", {
     filter: attributeFilter,
     distinct: true,
     select: [{ name: "name" }, { name: "attributeType" }, { name: "tag" }, {
       name: "readOnly",
     }],
+    cacheSecs: 5,
   });
   return allAttributes.map((value) => {
     return {

plugs/index/builtins.ts

@@ -90,31 +90,28 @@ export const builtins: Record<string, Record<string, string>> = {
 
 export async function loadBuiltinsIntoIndex() {
   console.log("Loading builtins attributes into index");
-  const allTags: ObjectValue<TagObject>[] = [];
+  const allObjects: ObjectValue<any>[] = [];
   for (const [tagName, attributes] of Object.entries(builtins)) {
-    allTags.push({
+    allObjects.push({
       ref: tagName,
       tag: "tag",
       name: tagName,
       page: builtinPseudoPage,
       parent: "builtin",
     });
-    await indexObjects<AttributeObject>(
-      builtinPseudoPage,
-      Object.entries(attributes).map(([name, attributeType]) => {
-        return {
-          ref: `${tagName}:${name}`,
-          tag: "attribute",
-          tagName,
-          name,
-          attributeType: attributeType.startsWith("!")
-            ? attributeType.substring(1)
-            : attributeType,
-          readOnly: attributeType.startsWith("!"),
-          page: builtinPseudoPage,
-        };
-      }),
+    allObjects.push(
+      ...Object.entries(attributes).map(([name, attributeType]) => ({
+        ref: `${tagName}:${name}`,
+        tag: "attribute",
+        tagName,
+        name,
+        attributeType: attributeType.startsWith("!")
+          ? attributeType.substring(1)
+          : attributeType,
+        readOnly: attributeType.startsWith("!"),
+        page: builtinPseudoPage,
+      })),
     );
   }
-  await indexObjects(builtinPseudoPage, allTags);
+  await indexObjects(builtinPseudoPage, allObjects);
 }

plugs/index/index.plug.yaml

@@ -23,13 +23,12 @@ functions:
     env: server
   query:
     path: api.ts:query
-    env: server
   indexObjects:
     path: api.ts:indexObjects
     env: server
   queryObjects:
     path: api.ts:queryObjects
-    env: server
+    # Note: not setting env: server to allow for client-side datastore query caching
   getObjectByRef:
     path: api.ts:getObjectByRef
     env: server

plugs/index/lint.ts

@@ -14,21 +14,24 @@ export async function lintYAML({ tree }: LintEvent): Promise<LintDiagnostic[]> {
   const diagnostics: LintDiagnostic[] = [];
   const frontmatter = await extractFrontmatter(tree);
   const tags = ["page", ...frontmatter.tags || []];
-  // Query all readOnly attributes for pages with this tag set
-  const readOnlyAttributes = await queryObjects<AttributeObject>("attribute", {
-    filter: ["and", ["=", ["attr", "tagName"], [
-      "array",
-      tags.map((tag): QueryExpression => ["string", tag]),
-    ]], [
-      "=",
-      ["attr", "readOnly"],
-      ["boolean", true],
-    ]],
-    distinct: true,
-    select: [{ name: "name" }],
-  });
   await traverseTreeAsync(tree, async (node) => {
     if (node.type === "FrontMatterCode") {
+      // Query all readOnly attributes for pages with this tag set
+      const readOnlyAttributes = await queryObjects<AttributeObject>(
+        "attribute",
+        {
+          filter: ["and", ["=", ["attr", "tagName"], [
+            "array",
+            tags.map((tag): QueryExpression => ["string", tag]),
+          ]], [
+            "=",
+            ["attr", "readOnly"],
+            ["boolean", true],
+          ]],
+          distinct: true,
+          select: [{ name: "name" }],
+        },
+      );
       const lintResult = await lintYaml(
         renderToText(node),
         node.from!,

plugs/index/tags.ts

@@ -73,6 +73,7 @@ export async function tagComplete(completeEvent: CompleteEvent) {
     filter: ["=", ["attr", "parent"], ["string", parent]],
     select: [{ name: "name" }],
     distinct: true,
+    cacheSecs: 5,
   });
 
   if (parent === "page") {

plugs/query/query.ts

@@ -91,7 +91,6 @@ export async function performQuery(parsedQuery: Query, pageObject: PageMeta) {
 export async function lintQuery(
   { name, tree }: LintEvent,
 ): Promise<LintDiagnostic[]> {
-  const pageObject = await loadPageObject(name);
   const diagnostics: LintDiagnostic[] = [];
   await traverseTreeAsync(tree, async (node) => {
     if (node.type === "FencedCode") {
@@ -111,6 +110,7 @@ export async function lintQuery(
       }
       const bodyText = codeText.children![0].text!;
       try {
+        const pageObject = await loadPageObject(name);
         const parsedQuery = await parseQuery(
           await replaceTemplateVars(bodyText, pageObject),
         );

plugs/tasks/complete.ts

@@ -9,7 +9,9 @@ export async function completeTaskState(completeEvent: CompleteEvent) {
   if (!taskMatch) {
     return null;
   }
-  const allStates = await queryObjects<TaskStateObject>("taskstate", {});
+  const allStates = await queryObjects<TaskStateObject>("taskstate", {
+    cacheSecs: 5,
+  });
   const states = [...new Set(allStates.map((s) => s.state))];
 
   return {

plugs/template/complete.ts

@@ -56,6 +56,7 @@ export async function templateSlashComplete(
       "boolean",
       false,
     ]]],
+    cacheSecs: 5,
   });
   return allTemplates.map((template) => ({
     label: template.trigger!,