refactor drawing

implement shareable cache logic
improve caching
fix drawing bug

app/src-tauri/tauri.conf.json

@@ -7,7 +7,7 @@
     "withGlobalTauri": false
   },
   "package": {
-    "productName": "tempblade-creator",
+    "productName": "tempblade Creator",
     "version": "0.1.0"
   },
   "tauri": {

app/src/components/Canvas/index.tsx

@@ -1,4 +1,4 @@
-import { FC } from "react";
+import { FC, useMemo } from "react";
 import { useEffect, useRef, useState } from "react";
 import { invoke } from "@tauri-apps/api";
 import { useTimelineStore } from "stores/timeline.store";
@@ -16,6 +16,7 @@ import drawStaggeredText, {
   calculateLetters,
 } from "drawers/staggered-text";
 import useMap from "hooks/useMap";
+import { Drawer } from "drawers/draw";
 
 type CanvasProps = {};
 
@@ -28,13 +29,7 @@ function typedArrayToBuffer(array: Uint8Array): ArrayBuffer {
 
 const CanvasComponent: FC<CanvasProps> = () => {
   const canvas = useRef<HTMLCanvasElement>(null);
-
+  const [didInit, setDidInit] = useState(false);
-  const [loading, setLoading] = useState(true);
-  const [canvasKit, setCanvasKit] = useState<CanvasKit>();
-  const [fontData, setFontData] = useState<ArrayBuffer>();
-  const surface = useRef<Surface>();
-  const staggeredTextCache = useMap<string, StaggeredTextCache>();
-  const isLocked = useRef<boolean>(false);
   const renderState = useRenderStateStore((store) => store.renderState);
   const { fps, size, duration } = useTimelineStore((store) => ({
     fps: store.fps,
@@ -46,126 +41,24 @@ const CanvasComponent: FC<CanvasProps> = () => {
     updateEntityById: store.updateEntityById,
   }));
 
+  const drawer = useMemo(() => new Drawer(), []);
+
   useEffect(() => {
-    InitCanvasKit({
+    if (canvas.current && !didInit) {
-      locateFile: (file) =>
+      drawer
-        "https://unpkg.com/canvaskit-wasm@latest/bin/" + file,
+        .init(canvas.current)
-    }).then((CanvasKit) => {
+        .then(() => {
-      setCanvasKit(CanvasKit);
+          setDidInit(true);
-
+        })
-      /*   fetch("https://storage.googleapis.com/skia-cdn/misc/Roboto-Regular.ttf")
+        .catch((e) => console.error(e));
-        .then((response) => response.arrayBuffer())
-        .then((arrayBuffer) => {
-          setLoading(false);
-          setFontData(arrayBuffer);
-        }); */
-
-      invoke("get_system_font", { fontName: "Helvetica-Bold" }).then((data) => {
-        console.log(data);
-
-        if (Array.isArray(data)) {
-          const u8 = new Uint8Array(data as any);
-          const buffer = typedArrayToBuffer(u8);
-          setFontData(buffer);
-          setLoading(false);
-
-          if (canvas.current) {
-            const CSurface = CanvasKit.MakeWebGLCanvasSurface(canvas.current);
-            if (CSurface) {
-              surface.current = CSurface;
     }
-          }
-        }
-      });
-    });
   }, []);
 
   useEffect(() => {
-    //console.time("calculation");
+    if (didInit) {
-    const parsedEntities = AnimatedEntities.parse(entities);
+      drawer.update(entities);
-
-    if (!loading && !isLocked.current) {
-      isLocked.current = true;
-      invoke("calculate_timeline_entities_at_frame", {
-        timeline: {
-          entities: parsedEntities,
-          render_state: renderState,
-          fps,
-          size,
-          duration,
-        },
-      }).then((data) => {
-        const entitiesResult = Entities.safeParse(data);
-
-        if (canvasKit && canvas.current && surface.current && fontData) {
-          surface.current.flush();
-
-          surface.current.requestAnimationFrame((skCanvas) => {
-            skCanvas.clear(canvasKit.WHITE);
-            if (entitiesResult.success) {
-              const entities = entitiesResult.data;
-
-              entities.reverse().forEach((entity) => {
-                switch (entity.type) {
-                  case EntityType.Enum.Rect:
-                    drawRect(canvasKit, skCanvas, entity);
-                    break;
-                  case EntityType.Enum.Ellipse:
-                    drawEllipse(canvasKit, skCanvas, entity);
-                    break;
-                  case EntityType.Enum.Text:
-                    drawText(canvasKit, skCanvas, entity, fontData);
-                    break;
-                  case EntityType.Enum.StaggeredText:
-                    {
-                      let cache: StaggeredTextCache;
-                      if (!entity.cache.valid) {
-                        const _cache = staggeredTextCache[0].get(entity.id);
-
-                        if (_cache !== undefined) {
-                          canvasKit.Free(_cache.glyphs);
     }
-
+  }, [entities, renderState.curr_frame, didInit]);
-                        cache = calculateLetters(canvasKit, entity, fontData);
-
-                        staggeredTextCache[1].set(entity.id, cache);
-                        updateEntityById(entity.id, { cache: { valid: true } });
-                      } else {
-                        const _cache = staggeredTextCache[0].get(entity.id);
-                        if (_cache) {
-                          cache = _cache;
-                        } else {
-                          cache = calculateLetters(canvasKit, entity, fontData);
-                        }
-                      }
-
-                      drawStaggeredText(
-                        canvasKit,
-                        skCanvas,
-                        entity,
-                        cache.font,
-                        cache.letterMeasures,
-                        cache.metrics
-                      );
-                    }
-
-                    break;
-                  default:
-                    break;
-                }
-
-                isLocked.current = false;
-              });
-            } else {
-              isLocked.current = false;
-              console.log(entitiesResult.error);
-            }
-          });
-        }
-        //console.timeEnd("draw");
-      });
-    }
-  }, [entities, loading, renderState.curr_frame]);
 
   return (
     <div>

app/src/drawers/cache.ts

@@ -0,0 +1,35 @@
+import { BaseEntity } from "primitives/Entities";
+import { z } from "zod";
+
+export interface EntityCache<T> {
+  build: () => T;
+  get: () => T | undefined;
+  set: (id: string, cache: T) => void;
+  cleanup: (cache: T) => void;
+}
+
+export function handleEntityCache<
+  E extends z.output<typeof BaseEntity>,
+  C,
+  EC extends EntityCache<C>
+>(entity: E, cache: EC): C {
+  const cached = cache.get();
+
+  if (!entity.cache.valid) {
+    console.log("Invalid cache");
+    if (cached) {
+      cache.cleanup(cached);
+    }
+    return cache.build();
+  } else {
+    if (!cached) {
+      const nextCache = cache.build();
+
+      cache.set(entity.id, nextCache);
+
+      return nextCache;
+    } else {
+      return cached;
+    }
+  }
+}

app/src/drawers/draw.ts

@@ -1,49 +1,115 @@
 import { invoke } from "@tauri-apps/api";
-import InitCanvasKit, { CanvasKit } from "canvaskit-wasm";
+import InitCanvasKit, { Canvas, CanvasKit, Surface } from "canvaskit-wasm";
 import { AnimatedEntities } from "primitives/AnimatedEntities";
-import { Entities } from "primitives/Entities";
+import { Entities, EntityType, StaggeredText } from "primitives/Entities";
 import { useRenderStateStore } from "stores/render-state.store";
 import { useTimelineStore } from "stores/timeline.store";
 import { z } from "zod";
-import { StaggeredTextCache } from "./staggered-text";
+import drawStaggeredText, {
+  StaggeredTextCache,
+  StaggeredTextEntityCache,
+  calculateLetters,
+} from "./staggered-text";
+import drawText from "./text";
+import drawEllipse from "./ellipse";
+import drawRect from "./rect";
+import { useEntitiesStore } from "stores/entities.store";
+import { handleEntityCache } from "./cache";
+
+function typedArrayToBuffer(array: Uint8Array): ArrayBuffer {
+  return array.buffer.slice(
+    array.byteOffset,
+    array.byteLength + array.byteOffset
+  );
+}
 
 /**
  *
- * TODO Add dependency logic for e.g. dynamically loading fonts, images etc.
+ * TODO Add more sophisticated dependency logic for e.g. dynamically loading fonts, images etc.
  */
 
 export class Drawer {
-  private readonly didLoad: boolean;
+  private didLoad: boolean;
   private entities: z.output<typeof Entities> | undefined;
   private ckDidLoad: boolean;
   private dependenciesDidLoad: boolean;
+  drawCount: number;
   private CanvasKit: CanvasKit | undefined;
   cache: { staggeredText: Map<string, StaggeredTextCache> };
+  surface: Surface | undefined;
+  fontData: ArrayBuffer | undefined;
+  raf: number | undefined;
+  isLocked: boolean;
 
   constructor() {
     this.entities = undefined;
     this.CanvasKit = undefined;
     this.ckDidLoad = false;
     this.dependenciesDidLoad = false;
+    this.drawCount = 0;
+    this.surface = undefined;
+    this.fontData = undefined;
     this.cache = {
       staggeredText: new Map(),
     };
+    this.isLocked = false;
+    this.raf = undefined;
     this.didLoad = this.ckDidLoad && this.dependenciesDidLoad;
   }
 
-  init() {
+  async init(canvas: HTMLCanvasElement) {
-    InitCanvasKit({
+    await this.loadCanvasKit(canvas);
+    await this.loadDependencies(false);
+
+    this.didLoad = this.ckDidLoad && this.dependenciesDidLoad;
+  }
+
+  async loadCanvasKit(canvas: HTMLCanvasElement) {
+    await InitCanvasKit({
       locateFile: (file) =>
         "https://unpkg.com/canvaskit-wasm@latest/bin/" + file,
     }).then((CanvasKit) => {
+      if (canvas) {
+        const CSurface = CanvasKit.MakeWebGLCanvasSurface(canvas);
+        if (CSurface) {
           this.CanvasKit = CanvasKit;
+          this.surface = CSurface;
+          this.ckDidLoad = true;
+        }
+      }
     });
   }
 
+  async loadDependencies(remote: boolean) {
+    if (remote) {
+      await fetch(
+        "https://storage.googleapis.com/skia-cdn/misc/Roboto-Regular.ttf"
+      )
+        .then((response) => response.arrayBuffer())
+        .then((arrayBuffer) => {
+          this.fontData = arrayBuffer;
+          this.dependenciesDidLoad = true;
+        });
+    } else {
+      await invoke("get_system_font", { fontName: "Helvetica-Bold" }).then(
+        (data) => {
+          if (Array.isArray(data)) {
+            const u8 = new Uint8Array(data as any);
+            const buffer = typedArrayToBuffer(u8);
+            this.fontData = buffer;
+            this.dependenciesDidLoad = true;
+          }
+        }
+      );
+    }
+  }
+
   /**
    * Updates the entities based on the input
    */
   update(animatedEntities: z.input<typeof AnimatedEntities>) {
+    console.time("calculate");
+
     const parsedAnimatedEntities = AnimatedEntities.parse(animatedEntities);
 
     if (this.didLoad) {
@@ -59,9 +125,17 @@ export class Drawer {
           duration,
         },
       }).then((data) => {
+        console.timeEnd("calculate");
         const parsedEntities = Entities.safeParse(data);
         if (parsedEntities.success) {
           this.entities = parsedEntities.data;
+
+          const isCached = this.entities.reduce(
+            (prev, curr) => prev && curr.cache.valid,
+            true
+          );
+
+          this.requestRedraw(!isCached);
         } else {
           console.error(parsedEntities.error);
         }
@@ -69,8 +143,79 @@ export class Drawer {
     }
   }
 
-  draw() {
+  requestRedraw(rebuild: boolean) {
-    if (this.didLoad) {
+    if (this.didLoad && this.surface) {
+      if (rebuild && this.raf !== undefined) {
+        cancelAnimationFrame(this.raf);
+        this.surface.flush();
+        this.raf = this.surface.requestAnimationFrame((canvas) =>
+          this.draw(canvas)
+        );
+      } else {
+        this.surface.flush();
+        this.raf = this.surface.requestAnimationFrame((canvas) =>
+          this.draw(canvas)
+        );
+      }
+    }
+  }
+
+  draw(canvas: Canvas) {
+    if (this.CanvasKit && this.entities && this.fontData && !this.isLocked) {
+      this.isLocked = true;
+      console.time("draw");
+      const CanvasKit = this.CanvasKit;
+      const fontData = this.fontData;
+
+      canvas.clear(CanvasKit.WHITE);
+
+      this.drawCount++;
+
+      [...this.entities].reverse().forEach((entity) => {
+        switch (entity.type) {
+          case EntityType.Enum.Rect:
+            drawRect(CanvasKit, canvas, entity);
+            break;
+          case EntityType.Enum.Ellipse:
+            drawEllipse(CanvasKit, canvas, entity);
+            break;
+          case EntityType.Enum.Text:
+            drawText(CanvasKit, canvas, entity, fontData);
+            break;
+          case EntityType.Enum.StaggeredText:
+            {
+              const cache = handleEntityCache<
+                z.output<typeof StaggeredText>,
+                StaggeredTextCache,
+                StaggeredTextEntityCache
+              >(entity, {
+                build: () => {
+                  const cache = calculateLetters(CanvasKit, entity, fontData);
+                  useEntitiesStore
+                    .getState()
+                    .updateEntityById(entity.id, { cache: { valid: true } });
+
+                  return cache;
+                },
+                get: () => this.cache.staggeredText.get(entity.id),
+                set: (id, cache) => this.cache.staggeredText.set(id, cache),
+                cleanup: (cache) => {
+                  cache.font.delete();
+                  cache.typeface.delete();
+                  CanvasKit.Free(cache.glyphs);
+                },
+              });
+
+              drawStaggeredText(CanvasKit, canvas, entity, cache);
+            }
+
+            break;
+          default:
+            break;
+        }
+      });
+      this.isLocked = false;
+      console.timeEnd("draw");
     }
   }
 }

app/src/drawers/staggered-text.ts

@@ -10,6 +10,7 @@ import {
 import { StaggeredText } from "primitives/Entities";
 import { z } from "zod";
 import { buildPaintStyle } from "./paint";
+import { EntityCache } from "./cache";
 
 export type StaggeredTextCache = {
   letterMeasures: Array<LetterMeasures>;
@@ -19,6 +20,8 @@ export type StaggeredTextCache = {
   glyphs: MallocObj;
 };
 
+export type StaggeredTextEntityCache = EntityCache<StaggeredTextCache>;
+
 function getUniqueCharacters(str: string): string {
   const uniqueCharacters: string[] = [];
 
@@ -55,12 +58,9 @@ function measureLetters(
       currentWidth = 0;
     }
 
-    const glyph = glyphArr.subarray(i, i + 1) as unknown as MallocObj;
-
     measuredLetters.push({
       bounds: nextGlyph,
       line: currentLine,
-      glyph,
       offset: {
         x: currentWidth - nextGlyphWidth,
       },
@@ -89,7 +89,6 @@ type LetterMeasures = {
     x: number;
   };
   line: number;
-  glyph: MallocObj;
   bounds: LetterBounds;
 };
 
@@ -98,8 +97,6 @@ export function calculateLetters(
   entity: z.output<typeof StaggeredText>,
   fontData: ArrayBuffer
 ): StaggeredTextCache {
-  console.log("Called");
-
   const typeface = CanvasKit.Typeface.MakeFreeTypeFaceFromData(
     fontData
   ) as Typeface;
@@ -110,7 +107,7 @@ export function calculateLetters(
 
   const glyphIDs = font.getGlyphIDs(entity.text);
 
-  font.setLinearMetrics(true);
+  // font.setLinearMetrics(true);
   font.setSubpixel(true);
   font.setHinting(CanvasKit.FontHinting.None);
 
@@ -193,20 +190,22 @@ export default function drawStaggeredText(
   CanvasKit: CanvasKit,
   canvas: Canvas,
   entity: z.output<typeof StaggeredText>,
-  font: Font,
+  cache: StaggeredTextCache
-  measuredLetters: Array<LetterMeasures>,
-  metrics: FontMetrics
 ) {
   const paint = new CanvasKit.Paint();
 
+  const { letterMeasures: measuredLetters, font, glyphs, metrics } = cache;
+
   buildPaintStyle(CanvasKit, paint, entity.letter.paint);
 
   // Draw all those runs.
   for (let i = 0; i < measuredLetters.length; i++) {
     const measuredLetter = measuredLetters[i];
 
+    const glyph = glyphs.subarray(i, i + 1);
+
     const blob = CanvasKit.TextBlob.MakeFromGlyphs(
-      measuredLetters[i].glyph as unknown as Array<number>,
+      glyph as unknown as Array<number>,
       font
     );
     if (blob) {
@@ -273,6 +272,8 @@ export default function drawStaggeredText(
       canvas.drawTextBlob(blob, entityOrigin[0], entityOrigin[1], paint);
 
       canvas.restore();
+
+      blob.delete();
     }
   }
 }

app/src/drawers/text.ts

@@ -1,7 +1,14 @@
-import { Canvas, CanvasKit } from "canvaskit-wasm";
+import { Canvas, CanvasKit, Font } from "canvaskit-wasm";
 import { TextEntity } from "primitives/Entities";
 import { convertToFloat } from "@tempblade/common";
 import { z } from "zod";
+import { EntityCache } from "./cache";
+
+export type TextCache = {
+  font: Font;
+};
+
+export type TextEntityCache = EntityCache<TextCache>;
 
 export default function drawText(
   CanvasKit: CanvasKit,

app/src/example.ts

@@ -186,7 +186,7 @@ function buildStaggeredText(
   return {
     type: "StaggeredText",
     text,
-    cache: {},
+    cache: { valid: false },
     id: uuid(),
     origin: staticAnimatedVec2(1280 / 2, 720 / 2),
     transform: {
@@ -273,7 +273,7 @@ export const EXAMPLE_ANIMATED_ENTITIES: Array<z.input<typeof AnimatedEntity>> =
     buildStaggeredText("Ehrenmann?", 2.0, {
       value: [255, 255, 255, 1.0],
     }),
-    buildText("Wie gehts?", 2.5, 40, 40, { value: [200, 200, 200, 1.0] }),
+    // buildText("Wie gehts?", 2.5, 40, 40, { value: [200, 200, 200, 1.0] }),
     buildRect(0.6, { value: [30, 30, 30, 1.0] }),
     buildRect(0.4, { value: [20, 20, 20, 1.0] }),
     buildRect(0.2, { value: [10, 10, 10, 1.0] }),
@@ -286,7 +286,7 @@ export const EXAMPLE_ANIMATED_ENTITIES_2: Array<
   buildText("Kleine Dumpfkopf!", 1.0, 80, -30, {
     value: [255, 255, 255, 1.0],
   }),
-  buildText("Wie gehts?", 1.5, 40, 30, { value: [255, 255, 255, 1.0] }),
+  // buildText("Wie gehts?", 1.5, 40, 30, { value: [255, 255, 255, 1.0] }),
   buildRect(0.8, { value: [40, 40, 40, 1.0] }),
   buildRect(0.6, { value: [30, 30, 30, 1.0] }),
   buildRect(0.4, { value: [20, 20, 20, 1.0] }),

app/src/primitives/Entities.ts

@@ -14,7 +14,7 @@ export const Transform = z.object({
 });
 
 export const Cache = z.object({
-  valid: z.boolean().optional().default(false),
+  valid: z.boolean().optional().default(true),
 });
 
 export const BaseEntity = z.object({