From 60c8bb587747a9770e2c8fe012a6c1faffc01fb1 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Wed, 24 May 2023 20:02:44 +0200 Subject: [PATCH] refactor drawing implement shareable cache logic improve caching fix drawing bug --- app/src-tauri/tauri.conf.json | 2 +- app/src/components/Canvas/index.tsx | 139 +++-------------------- app/src/drawers/cache.ts | 35 ++++++ app/src/drawers/draw.ts | 165 ++++++++++++++++++++++++++-- app/src/drawers/staggered-text.ts | 23 ++-- app/src/drawers/text.ts | 9 +- app/src/example.ts | 6 +- app/src/primitives/Entities.ts | 2 +- 8 files changed, 231 insertions(+), 150 deletions(-) create mode 100644 app/src/drawers/cache.ts diff --git a/app/src-tauri/tauri.conf.json b/app/src-tauri/tauri.conf.json index 0451de0..4ae1b5e 100644 --- a/app/src-tauri/tauri.conf.json +++ b/app/src-tauri/tauri.conf.json @@ -7,7 +7,7 @@ "withGlobalTauri": false }, "package": { - "productName": "tempblade-creator", + "productName": "tempblade Creator", "version": "0.1.0" }, "tauri": { diff --git a/app/src/components/Canvas/index.tsx b/app/src/components/Canvas/index.tsx index 0fb1f0d..578e1ca 100644 --- a/app/src/components/Canvas/index.tsx +++ b/app/src/components/Canvas/index.tsx @@ -1,4 +1,4 @@ -import { FC } from "react"; +import { FC, useMemo } from "react"; import { useEffect, useRef, useState } from "react"; import { invoke } from "@tauri-apps/api"; import { useTimelineStore } from "stores/timeline.store"; @@ -16,6 +16,7 @@ import drawStaggeredText, { calculateLetters, } from "drawers/staggered-text"; import useMap from "hooks/useMap"; +import { Drawer } from "drawers/draw"; type CanvasProps = {}; @@ -28,13 +29,7 @@ function typedArrayToBuffer(array: Uint8Array): ArrayBuffer { const CanvasComponent: FC = () => { const canvas = useRef(null); - - const [loading, setLoading] = useState(true); - const [canvasKit, setCanvasKit] = useState(); - const [fontData, setFontData] = useState(); - const surface = useRef(); - const staggeredTextCache = useMap(); - const isLocked = useRef(false); + const [didInit, setDidInit] = useState(false); const renderState = useRenderStateStore((store) => store.renderState); const { fps, size, duration } = useTimelineStore((store) => ({ fps: store.fps, @@ -46,126 +41,24 @@ const CanvasComponent: FC = () => { updateEntityById: store.updateEntityById, })); + const drawer = useMemo(() => new Drawer(), []); + useEffect(() => { - InitCanvasKit({ - locateFile: (file) => - "https://unpkg.com/canvaskit-wasm@latest/bin/" + file, - }).then((CanvasKit) => { - setCanvasKit(CanvasKit); - - /* fetch("https://storage.googleapis.com/skia-cdn/misc/Roboto-Regular.ttf") - .then((response) => response.arrayBuffer()) - .then((arrayBuffer) => { - setLoading(false); - setFontData(arrayBuffer); - }); */ - - invoke("get_system_font", { fontName: "Helvetica-Bold" }).then((data) => { - console.log(data); - - if (Array.isArray(data)) { - const u8 = new Uint8Array(data as any); - const buffer = typedArrayToBuffer(u8); - setFontData(buffer); - setLoading(false); - - if (canvas.current) { - const CSurface = CanvasKit.MakeWebGLCanvasSurface(canvas.current); - if (CSurface) { - surface.current = CSurface; - } - } - } - }); - }); + if (canvas.current && !didInit) { + drawer + .init(canvas.current) + .then(() => { + setDidInit(true); + }) + .catch((e) => console.error(e)); + } }, []); useEffect(() => { - //console.time("calculation"); - const parsedEntities = AnimatedEntities.parse(entities); - - if (!loading && !isLocked.current) { - isLocked.current = true; - invoke("calculate_timeline_entities_at_frame", { - timeline: { - entities: parsedEntities, - render_state: renderState, - fps, - size, - duration, - }, - }).then((data) => { - const entitiesResult = Entities.safeParse(data); - - if (canvasKit && canvas.current && surface.current && fontData) { - surface.current.flush(); - - surface.current.requestAnimationFrame((skCanvas) => { - skCanvas.clear(canvasKit.WHITE); - if (entitiesResult.success) { - const entities = entitiesResult.data; - - entities.reverse().forEach((entity) => { - switch (entity.type) { - case EntityType.Enum.Rect: - drawRect(canvasKit, skCanvas, entity); - break; - case EntityType.Enum.Ellipse: - drawEllipse(canvasKit, skCanvas, entity); - break; - case EntityType.Enum.Text: - drawText(canvasKit, skCanvas, entity, fontData); - break; - case EntityType.Enum.StaggeredText: - { - let cache: StaggeredTextCache; - if (!entity.cache.valid) { - const _cache = staggeredTextCache[0].get(entity.id); - - if (_cache !== undefined) { - canvasKit.Free(_cache.glyphs); - } - - cache = calculateLetters(canvasKit, entity, fontData); - - staggeredTextCache[1].set(entity.id, cache); - updateEntityById(entity.id, { cache: { valid: true } }); - } else { - const _cache = staggeredTextCache[0].get(entity.id); - if (_cache) { - cache = _cache; - } else { - cache = calculateLetters(canvasKit, entity, fontData); - } - } - - drawStaggeredText( - canvasKit, - skCanvas, - entity, - cache.font, - cache.letterMeasures, - cache.metrics - ); - } - - break; - default: - break; - } - - isLocked.current = false; - }); - } else { - isLocked.current = false; - console.log(entitiesResult.error); - } - }); - } - //console.timeEnd("draw"); - }); + if (didInit) { + drawer.update(entities); } - }, [entities, loading, renderState.curr_frame]); + }, [entities, renderState.curr_frame, didInit]); return (
diff --git a/app/src/drawers/cache.ts b/app/src/drawers/cache.ts new file mode 100644 index 0000000..2798fed --- /dev/null +++ b/app/src/drawers/cache.ts @@ -0,0 +1,35 @@ +import { BaseEntity } from "primitives/Entities"; +import { z } from "zod"; + +export interface EntityCache { + build: () => T; + get: () => T | undefined; + set: (id: string, cache: T) => void; + cleanup: (cache: T) => void; +} + +export function handleEntityCache< + E extends z.output, + C, + EC extends EntityCache +>(entity: E, cache: EC): C { + const cached = cache.get(); + + if (!entity.cache.valid) { + console.log("Invalid cache"); + if (cached) { + cache.cleanup(cached); + } + return cache.build(); + } else { + if (!cached) { + const nextCache = cache.build(); + + cache.set(entity.id, nextCache); + + return nextCache; + } else { + return cached; + } + } +} diff --git a/app/src/drawers/draw.ts b/app/src/drawers/draw.ts index 4ffda0e..54bf8f8 100644 --- a/app/src/drawers/draw.ts +++ b/app/src/drawers/draw.ts @@ -1,49 +1,115 @@ import { invoke } from "@tauri-apps/api"; -import InitCanvasKit, { CanvasKit } from "canvaskit-wasm"; +import InitCanvasKit, { Canvas, CanvasKit, Surface } from "canvaskit-wasm"; import { AnimatedEntities } from "primitives/AnimatedEntities"; -import { Entities } from "primitives/Entities"; +import { Entities, EntityType, StaggeredText } from "primitives/Entities"; import { useRenderStateStore } from "stores/render-state.store"; import { useTimelineStore } from "stores/timeline.store"; import { z } from "zod"; -import { StaggeredTextCache } from "./staggered-text"; +import drawStaggeredText, { + StaggeredTextCache, + StaggeredTextEntityCache, + calculateLetters, +} from "./staggered-text"; +import drawText from "./text"; +import drawEllipse from "./ellipse"; +import drawRect from "./rect"; +import { useEntitiesStore } from "stores/entities.store"; +import { handleEntityCache } from "./cache"; + +function typedArrayToBuffer(array: Uint8Array): ArrayBuffer { + return array.buffer.slice( + array.byteOffset, + array.byteLength + array.byteOffset + ); +} /** * - * TODO Add dependency logic for e.g. dynamically loading fonts, images etc. + * TODO Add more sophisticated dependency logic for e.g. dynamically loading fonts, images etc. */ export class Drawer { - private readonly didLoad: boolean; + private didLoad: boolean; private entities: z.output | undefined; private ckDidLoad: boolean; private dependenciesDidLoad: boolean; + drawCount: number; private CanvasKit: CanvasKit | undefined; cache: { staggeredText: Map }; + surface: Surface | undefined; + fontData: ArrayBuffer | undefined; + raf: number | undefined; + isLocked: boolean; constructor() { this.entities = undefined; this.CanvasKit = undefined; this.ckDidLoad = false; this.dependenciesDidLoad = false; + this.drawCount = 0; + this.surface = undefined; + this.fontData = undefined; this.cache = { staggeredText: new Map(), }; + this.isLocked = false; + this.raf = undefined; this.didLoad = this.ckDidLoad && this.dependenciesDidLoad; } - init() { - InitCanvasKit({ + async init(canvas: HTMLCanvasElement) { + await this.loadCanvasKit(canvas); + await this.loadDependencies(false); + + this.didLoad = this.ckDidLoad && this.dependenciesDidLoad; + } + + async loadCanvasKit(canvas: HTMLCanvasElement) { + await InitCanvasKit({ locateFile: (file) => "https://unpkg.com/canvaskit-wasm@latest/bin/" + file, }).then((CanvasKit) => { - this.CanvasKit = CanvasKit; + if (canvas) { + const CSurface = CanvasKit.MakeWebGLCanvasSurface(canvas); + if (CSurface) { + this.CanvasKit = CanvasKit; + this.surface = CSurface; + this.ckDidLoad = true; + } + } }); } + async loadDependencies(remote: boolean) { + if (remote) { + await fetch( + "https://storage.googleapis.com/skia-cdn/misc/Roboto-Regular.ttf" + ) + .then((response) => response.arrayBuffer()) + .then((arrayBuffer) => { + this.fontData = arrayBuffer; + this.dependenciesDidLoad = true; + }); + } else { + await invoke("get_system_font", { fontName: "Helvetica-Bold" }).then( + (data) => { + if (Array.isArray(data)) { + const u8 = new Uint8Array(data as any); + const buffer = typedArrayToBuffer(u8); + this.fontData = buffer; + this.dependenciesDidLoad = true; + } + } + ); + } + } + /** * Updates the entities based on the input */ update(animatedEntities: z.input) { + console.time("calculate"); + const parsedAnimatedEntities = AnimatedEntities.parse(animatedEntities); if (this.didLoad) { @@ -59,9 +125,17 @@ export class Drawer { duration, }, }).then((data) => { + console.timeEnd("calculate"); const parsedEntities = Entities.safeParse(data); if (parsedEntities.success) { this.entities = parsedEntities.data; + + const isCached = this.entities.reduce( + (prev, curr) => prev && curr.cache.valid, + true + ); + + this.requestRedraw(!isCached); } else { console.error(parsedEntities.error); } @@ -69,8 +143,79 @@ export class Drawer { } } - draw() { - if (this.didLoad) { + requestRedraw(rebuild: boolean) { + if (this.didLoad && this.surface) { + if (rebuild && this.raf !== undefined) { + cancelAnimationFrame(this.raf); + this.surface.flush(); + this.raf = this.surface.requestAnimationFrame((canvas) => + this.draw(canvas) + ); + } else { + this.surface.flush(); + this.raf = this.surface.requestAnimationFrame((canvas) => + this.draw(canvas) + ); + } + } + } + + draw(canvas: Canvas) { + if (this.CanvasKit && this.entities && this.fontData && !this.isLocked) { + this.isLocked = true; + console.time("draw"); + const CanvasKit = this.CanvasKit; + const fontData = this.fontData; + + canvas.clear(CanvasKit.WHITE); + + this.drawCount++; + + [...this.entities].reverse().forEach((entity) => { + switch (entity.type) { + case EntityType.Enum.Rect: + drawRect(CanvasKit, canvas, entity); + break; + case EntityType.Enum.Ellipse: + drawEllipse(CanvasKit, canvas, entity); + break; + case EntityType.Enum.Text: + drawText(CanvasKit, canvas, entity, fontData); + break; + case EntityType.Enum.StaggeredText: + { + const cache = handleEntityCache< + z.output, + StaggeredTextCache, + StaggeredTextEntityCache + >(entity, { + build: () => { + const cache = calculateLetters(CanvasKit, entity, fontData); + useEntitiesStore + .getState() + .updateEntityById(entity.id, { cache: { valid: true } }); + + return cache; + }, + get: () => this.cache.staggeredText.get(entity.id), + set: (id, cache) => this.cache.staggeredText.set(id, cache), + cleanup: (cache) => { + cache.font.delete(); + cache.typeface.delete(); + CanvasKit.Free(cache.glyphs); + }, + }); + + drawStaggeredText(CanvasKit, canvas, entity, cache); + } + + break; + default: + break; + } + }); + this.isLocked = false; + console.timeEnd("draw"); } } } diff --git a/app/src/drawers/staggered-text.ts b/app/src/drawers/staggered-text.ts index 5195864..4841f90 100644 --- a/app/src/drawers/staggered-text.ts +++ b/app/src/drawers/staggered-text.ts @@ -10,6 +10,7 @@ import { import { StaggeredText } from "primitives/Entities"; import { z } from "zod"; import { buildPaintStyle } from "./paint"; +import { EntityCache } from "./cache"; export type StaggeredTextCache = { letterMeasures: Array; @@ -19,6 +20,8 @@ export type StaggeredTextCache = { glyphs: MallocObj; }; +export type StaggeredTextEntityCache = EntityCache; + function getUniqueCharacters(str: string): string { const uniqueCharacters: string[] = []; @@ -55,12 +58,9 @@ function measureLetters( currentWidth = 0; } - const glyph = glyphArr.subarray(i, i + 1) as unknown as MallocObj; - measuredLetters.push({ bounds: nextGlyph, line: currentLine, - glyph, offset: { x: currentWidth - nextGlyphWidth, }, @@ -89,7 +89,6 @@ type LetterMeasures = { x: number; }; line: number; - glyph: MallocObj; bounds: LetterBounds; }; @@ -98,8 +97,6 @@ export function calculateLetters( entity: z.output, fontData: ArrayBuffer ): StaggeredTextCache { - console.log("Called"); - const typeface = CanvasKit.Typeface.MakeFreeTypeFaceFromData( fontData ) as Typeface; @@ -110,7 +107,7 @@ export function calculateLetters( const glyphIDs = font.getGlyphIDs(entity.text); - font.setLinearMetrics(true); + // font.setLinearMetrics(true); font.setSubpixel(true); font.setHinting(CanvasKit.FontHinting.None); @@ -193,20 +190,22 @@ export default function drawStaggeredText( CanvasKit: CanvasKit, canvas: Canvas, entity: z.output, - font: Font, - measuredLetters: Array, - metrics: FontMetrics + cache: StaggeredTextCache ) { const paint = new CanvasKit.Paint(); + const { letterMeasures: measuredLetters, font, glyphs, metrics } = cache; + buildPaintStyle(CanvasKit, paint, entity.letter.paint); // Draw all those runs. for (let i = 0; i < measuredLetters.length; i++) { const measuredLetter = measuredLetters[i]; + const glyph = glyphs.subarray(i, i + 1); + const blob = CanvasKit.TextBlob.MakeFromGlyphs( - measuredLetters[i].glyph as unknown as Array, + glyph as unknown as Array, font ); if (blob) { @@ -273,6 +272,8 @@ export default function drawStaggeredText( canvas.drawTextBlob(blob, entityOrigin[0], entityOrigin[1], paint); canvas.restore(); + + blob.delete(); } } } diff --git a/app/src/drawers/text.ts b/app/src/drawers/text.ts index f3e3922..fae9638 100644 --- a/app/src/drawers/text.ts +++ b/app/src/drawers/text.ts @@ -1,7 +1,14 @@ -import { Canvas, CanvasKit } from "canvaskit-wasm"; +import { Canvas, CanvasKit, Font } from "canvaskit-wasm"; import { TextEntity } from "primitives/Entities"; import { convertToFloat } from "@tempblade/common"; import { z } from "zod"; +import { EntityCache } from "./cache"; + +export type TextCache = { + font: Font; +}; + +export type TextEntityCache = EntityCache; export default function drawText( CanvasKit: CanvasKit, diff --git a/app/src/example.ts b/app/src/example.ts index e9e091b..01fc7ad 100644 --- a/app/src/example.ts +++ b/app/src/example.ts @@ -186,7 +186,7 @@ function buildStaggeredText( return { type: "StaggeredText", text, - cache: {}, + cache: { valid: false }, id: uuid(), origin: staticAnimatedVec2(1280 / 2, 720 / 2), transform: { @@ -273,7 +273,7 @@ export const EXAMPLE_ANIMATED_ENTITIES: Array> = buildStaggeredText("Ehrenmann?", 2.0, { value: [255, 255, 255, 1.0], }), - buildText("Wie gehts?", 2.5, 40, 40, { value: [200, 200, 200, 1.0] }), + // buildText("Wie gehts?", 2.5, 40, 40, { value: [200, 200, 200, 1.0] }), buildRect(0.6, { value: [30, 30, 30, 1.0] }), buildRect(0.4, { value: [20, 20, 20, 1.0] }), buildRect(0.2, { value: [10, 10, 10, 1.0] }), @@ -286,7 +286,7 @@ export const EXAMPLE_ANIMATED_ENTITIES_2: Array< buildText("Kleine Dumpfkopf!", 1.0, 80, -30, { value: [255, 255, 255, 1.0], }), - buildText("Wie gehts?", 1.5, 40, 30, { value: [255, 255, 255, 1.0] }), + // buildText("Wie gehts?", 1.5, 40, 30, { value: [255, 255, 255, 1.0] }), buildRect(0.8, { value: [40, 40, 40, 1.0] }), buildRect(0.6, { value: [30, 30, 30, 1.0] }), buildRect(0.4, { value: [20, 20, 20, 1.0] }), diff --git a/app/src/primitives/Entities.ts b/app/src/primitives/Entities.ts index 34dca68..64ff0f1 100644 --- a/app/src/primitives/Entities.ts +++ b/app/src/primitives/Entities.ts @@ -14,7 +14,7 @@ export const Transform = z.object({ }); export const Cache = z.object({ - valid: z.boolean().optional().default(false), + valid: z.boolean().optional().default(true), }); export const BaseEntity = z.object({