diff --git a/backend/goglbackend/images.go b/backend/goglbackend/images.go
index fc6dc8c..0c5885e 100644
--- a/backend/goglbackend/images.go
+++ b/backend/goglbackend/images.go
@@ -3,7 +3,6 @@ package goglbackend
 import (
 	"errors"
 	"image"
-	"runtime"
 	"unsafe"
 
 	"github.com/tfriedel6/canvas/backend/backendbase"
@@ -39,12 +38,6 @@ func (b *GoGLBackend) LoadImage(src image.Image) (backendbase.Image, error) {
 	}
 	img.b = b
 
-	runtime.SetFinalizer(img, func(img *Image) {
-		b.glChan <- func() {
-			gl.DeleteTextures(1, &img.tex)
-		}
-	})
-
 	return img, nil
 }
 
diff --git a/backend/xmobilebackend/images.go b/backend/xmobilebackend/images.go
index 541143a..bdc957a 100755
--- a/backend/xmobilebackend/images.go
+++ b/backend/xmobilebackend/images.go
@@ -3,7 +3,6 @@ package xmobilebackend
 import (
 	"errors"
 	"image"
-	"runtime"
 	"unsafe"
 
 	"github.com/tfriedel6/canvas/backend/backendbase"
@@ -39,12 +38,6 @@ func (b *XMobileBackend) LoadImage(src image.Image) (backendbase.Image, error) {
 	}
 	img.b = b
 
-	runtime.SetFinalizer(img, func(img *Image) {
-		b.glChan <- func() {
-			b.glctx.DeleteTexture(img.tex)
-		}
-	})
-
 	return img, nil
 }
 
diff --git a/canvas.go b/canvas.go
index 043fc05..8863b5e 100644
--- a/canvas.go
+++ b/canvas.go
@@ -5,11 +5,13 @@ package canvas
 import (
 	"image"
 	"image/color"
-	"sort"
+	"math"
+	"time"
 
 	"github.com/golang/freetype/truetype"
 	"github.com/tfriedel6/canvas/backend/backendbase"
 	"golang.org/x/image/font"
+	"golang.org/x/image/math/fixed"
 )
 
 //go:generate go run make_shaders.go
@@ -27,7 +29,7 @@ type Canvas struct {
 
 	images   map[interface{}]*Image
 	fonts    map[interface{}]*Font
-	fontCtxs map[fontKey]*frContext
+	fontCtxs map[fontKey]*frCache
 
 	shadowBuf [][2]float64
 }
@@ -37,7 +39,7 @@ type drawState struct {
 	fill          drawStyle
 	stroke        drawStyle
 	font          *Font
-	fontSize      float64
+	fontSize      fixed.Int26_6
 	fontMetrics   font.Metrics
 	textAlign     textAlign
 	textBaseline  textBaseline
@@ -127,7 +129,7 @@ var Performance = struct {
 	// CacheSize is only approximate
 	CacheSize int
 }{
-	CacheSize: 16_000_000,
+	CacheSize: 128_000_000,
 }
 
 // New creates a new canvas with the given viewport coordinates.
@@ -140,6 +142,7 @@ func New(backend backendbase.Backend) *Canvas {
 		stateStack: make([]drawState, 0, 20),
 		images:     make(map[interface{}]*Image),
 		fonts:      make(map[interface{}]*Font),
+		fontCtxs:   make(map[fontKey]*frCache),
 	}
 	cv.state.lineWidth = 1
 	cv.state.lineAlpha = 1
@@ -287,31 +290,12 @@ func (cv *Canvas) SetLineWidth(width float64) {
 // with the LoadFont function, a filename for a font to load (which will be
 // cached), or nil, in which case the first loaded font will be used
 func (cv *Canvas) SetFont(src interface{}, size float64) {
+	cv.state.fontSize = fixed.Int26_6(math.Round(size * 64))
 	if src == nil {
 		cv.state.font = defaultFont
 	} else {
 		cv.state.font = cv.getFont(src)
-		// switch v := src.(type) {
-		// case *Font:
-		// 	cv.state.font = v
-		// case *truetype.Font:
-		// 	cv.state.font = &Font{font: v}
-		// case string:
-		// 	if f, ok := fonts[v]; ok {
-		// 		cv.state.font = f
-		// 	} else {
-		// 		f, err := cv.LoadFont(v)
-		// 		if err != nil {
-		// 			fmt.Fprintf(os.Stderr, "Error loading font %s: %v\n", v, err)
-		// 			fonts[v] = nil
-		// 		} else {
-		// 			fonts[v] = f
-		// 			cv.state.font = f
-		// 		}
-		// 	}
-		// }
 	}
-	cv.state.fontSize = size
 
 	fontFace := truetype.NewFace(cv.state.font.font, &truetype.Options{Size: size})
 	cv.state.fontMetrics = fontFace.Metrics()
@@ -487,28 +471,42 @@ func (cv *Canvas) IsPointInStroke(x, y float64) bool {
 	return false
 }
 
-func (cv *Canvas) reduceCache(keepSize int) {
+func (cv *Canvas) reduceCache(keepSize, rec int) {
+	if rec > 100 {
+		return
+	}
+
 	var total int
-	for _, img := range cv.images {
+	oldest := time.Now()
+	var oldestFontKey fontKey
+	var oldestImageKey interface{}
+	for src, img := range cv.images {
 		w, h := img.img.Size()
 		total += w * h * 4
+		if img.lastUsed.Before(oldest) {
+			oldest = img.lastUsed
+			oldestImageKey = src
+		}
+	}
+	for key, frctx := range cv.fontCtxs {
+		total += frctx.ctx.cacheSize()
+		if frctx.lastUsed.Before(oldest) {
+			oldest = frctx.lastUsed
+			oldestFontKey = key
+			oldestImageKey = nil
+		}
 	}
 	if total <= keepSize {
 		return
 	}
-	list := make([]*Image, 0, len(cv.images))
-	for _, img := range cv.images {
-		list = append(list, img)
-	}
-	sort.Slice(list, func(i, j int) bool {
-		return list[i].lastUsed.Before(list[j].lastUsed)
-	})
-	pos := 0
-	for total > keepSize {
-		img := list[pos]
-		pos++
-		delete(cv.images, img.src)
-		w, h := img.img.Size()
-		total -= w * h * 4
+
+	if oldestImageKey != nil {
+		cv.images[oldestImageKey].Delete()
+		delete(cv.images, oldestImageKey)
+	} else {
+		cv.fontCtxs[oldestFontKey].ctx = nil
+		delete(cv.fontCtxs, oldestFontKey)
 	}
+
+	cv.reduceCache(keepSize, rec+1)
 }
diff --git a/freetype.go b/freetype.go
index 42e8120..833192a 100644
--- a/freetype.go
+++ b/freetype.go
@@ -44,11 +44,9 @@ type frContext struct {
 	glyphBuf truetype.GlyphBuf
 	// dst and src are the destination and source images for drawing.
 	dst draw.Image
-	// fontSize and dpi are used to calculate scale. scale is the number of
-	// 26.6 fixed point units in 1 em. hinting is the hinting policy.
-	fontSize, dpi float64
-	scale         fixed.Int26_6
-	hinting       font.Hinting
+
+	fontSize fixed.Int26_6
+	hinting  font.Hinting
 	// cache is the glyph cache.
 	cache [nGlyphs * nXFractions * nYFractions]cacheEntry
 }
@@ -132,7 +130,7 @@ func (c *frContext) drawContour(ps []truetype.Point, dx, dy fixed.Int26_6) {
 // The 26.6 fixed point arguments fx and fy must be in the range [0, 1).
 func (c *frContext) rasterize(glyph truetype.Index, fx, fy fixed.Int26_6) (fixed.Int26_6, *image.Alpha, image.Point, error) {
 
-	if err := c.glyphBuf.Load(c.f, c.scale, glyph, c.hinting); err != nil {
+	if err := c.glyphBuf.Load(c.f, c.fontSize, glyph, c.hinting); err != nil {
 		return 0, nil, image.Point{}, err
 	}
 	// Calculate the integer-pixel bounds for the glyph.
@@ -190,14 +188,14 @@ func (c *frContext) glyph(glyph truetype.Index, p fixed.Point26_6) (fixed.Int26_
 }
 
 func (c *frContext) glyphAdvance(glyph truetype.Index) (fixed.Int26_6, error) {
-	if err := c.glyphBuf.Load(c.f, c.scale, glyph, c.hinting); err != nil {
+	if err := c.glyphBuf.Load(c.f, c.fontSize, glyph, c.hinting); err != nil {
 		return 0, err
 	}
 	return c.glyphBuf.AdvanceWidth, nil
 }
 
 func (c *frContext) glyphMeasure(glyph truetype.Index, p fixed.Point26_6) (fixed.Int26_6, image.Rectangle, error) {
-	if err := c.glyphBuf.Load(c.f, c.scale, glyph, c.hinting); err != nil {
+	if err := c.glyphBuf.Load(c.f, c.fontSize, glyph, c.hinting); err != nil {
 		return 0, image.Rectangle{}, err
 	}
 
@@ -221,15 +219,12 @@ func (c *frContext) glyphBounds(glyph truetype.Index, p fixed.Point26_6) (image.
 
 const maxInt = int(^uint(0) >> 1)
 
-// recalc recalculates scale and bounds values from the font size, screen
-// resolution and font metrics, and invalidates the glyph cache.
 func (c *frContext) recalc() {
-	c.scale = fixed.Int26_6(c.fontSize * c.dpi * (64.0 / 72.0))
 	if c.f == nil {
 		c.r.SetBounds(0, 0)
 	} else {
 		// Set the rasterizer's bounds to be big enough to handle the largest glyph.
-		b := c.f.Bounds(c.scale)
+		b := c.f.Bounds(c.fontSize)
 		xmin := +int(b.Min.X) >> 6
 		ymin := -int(b.Max.Y) >> 6
 		xmax := +int(b.Max.X+63) >> 6
@@ -241,16 +236,7 @@ func (c *frContext) recalc() {
 	}
 }
 
-// SetDPI sets the screen resolution in dots per inch.
-func (c *frContext) setDPI(dpi float64) {
-	if c.dpi == dpi {
-		return
-	}
-	c.dpi = dpi
-	c.recalc()
-}
-
-// SetFont sets the font used to draw text.
+// setFont sets the font used to draw text.
 func (c *frContext) setFont(f *truetype.Font) {
 	if c.f == f {
 		return
@@ -259,8 +245,8 @@ func (c *frContext) setFont(f *truetype.Font) {
 	c.recalc()
 }
 
-// SetFontSize sets the font size in points (as in "a 12 point font").
-func (c *frContext) setFontSize(fontSize float64) {
+// setFontSize sets the font size in points (as in "a 12 point font").
+func (c *frContext) setFontSize(fontSize fixed.Int26_6) {
 	if c.fontSize == fontSize {
 		return
 	}
@@ -268,7 +254,7 @@ func (c *frContext) setFontSize(fontSize float64) {
 	c.recalc()
 }
 
-// SetHinting sets the hinting policy.
+// setHinting sets the hinting policy.
 func (c *frContext) setHinting(hinting font.Hinting) {
 	c.hinting = hinting
 	for i := range c.cache {
@@ -276,19 +262,32 @@ func (c *frContext) setHinting(hinting font.Hinting) {
 	}
 }
 
-// SetDst sets the destination image for draw operations.
+// setDst sets the destination image for draw operations.
 func (c *frContext) setDst(dst draw.Image) {
 	c.dst = dst
 }
 
+func (c *frContext) cacheSize() int {
+	if c.f == nil {
+		return 0
+	}
+
+	b := c.f.Bounds(c.fontSize)
+	xmin := +int(b.Min.X) >> 6
+	ymin := -int(b.Max.Y) >> 6
+	xmax := +int(b.Max.X+63) >> 6
+	ymax := -int(b.Min.Y-63) >> 6
+	w := xmax - xmin
+	h := ymax - ymin
+	return w * h * len(c.cache)
+}
+
 // TODO(nigeltao): implement Context.SetGamma.
 
 // NewContext creates a new Context.
 func newFRContext() *frContext {
 	return &frContext{
 		r:        raster.NewRasterizer(0, 0),
-		fontSize: 12,
-		dpi:      72,
-		scale:    12 << 6,
+		fontSize: fixed.I(12),
 	}
 }
diff --git a/images.go b/images.go
index a3c0a9c..3b84b66 100644
--- a/images.go
+++ b/images.go
@@ -26,16 +26,22 @@ type Image struct {
 // string. If you want the canvas package to load the image, make sure you
 // import the required format packages
 func (cv *Canvas) LoadImage(src interface{}) (*Image, error) {
+	var reload *Image
 	if img, ok := src.(*Image); ok {
-		img.lastUsed = time.Now()
-		return img, nil
+		if img.deleted {
+			reload = img
+			src = img.src
+		} else {
+			img.lastUsed = time.Now()
+			return img, nil
+		}
 	} else if _, ok := src.([]byte); !ok {
 		if img, ok := cv.images[src]; ok {
 			img.lastUsed = time.Now()
 			return img, nil
 		}
 	}
-	cv.reduceCache(Performance.CacheSize)
+	cv.reduceCache(Performance.CacheSize, 0)
 	var srcImg image.Image
 	switch v := src.(type) {
 	case image.Image:
@@ -66,6 +72,10 @@ func (cv *Canvas) LoadImage(src interface{}) (*Image, error) {
 		return nil, err
 	}
 	cvimg := &Image{cv: cv, img: backendImg, lastUsed: time.Now(), src: src}
+	if reload != nil {
+		*reload = *cvimg
+		return reload, nil
+	}
 	if _, ok := src.([]byte); !ok {
 		cv.images[src] = cvimg
 	}
@@ -101,8 +111,6 @@ func (cv *Canvas) getImage(src interface{}) *Image {
 		default:
 			fmt.Fprintf(os.Stderr, "Failed to load image: %v\n", err)
 		}
-	} else {
-		cv.images[src] = img
 	}
 	return img
 }
@@ -116,12 +124,14 @@ func (img *Image) Height() int { return img.img.Height() }
 // Size returns the width and height of the image
 func (img *Image) Size() (int, int) { return img.img.Size() }
 
-// Delete deletes the image from memory. Any draw calls with a deleted image
-// will not do anything
+// Delete deletes the image from memory
 func (img *Image) Delete() {
-	img.img.Delete()
-	img.img = nil
+	if img == nil || img.deleted {
+		return
+	}
 	img.deleted = true
+	img.img.Delete()
+	delete(img.cv.images, img.src)
 }
 
 // Replace replaces the image with the new one
@@ -147,8 +157,7 @@ func (img *Image) Replace(src interface{}) error {
 // source coordinates
 func (cv *Canvas) DrawImage(image interface{}, coords ...float64) {
 	img := cv.getImage(image)
-
-	if img == nil || img.deleted {
+	if img == nil {
 		return
 	}
 
diff --git a/text.go b/text.go
index 01611c5..a800500 100644
--- a/text.go
+++ b/text.go
@@ -7,6 +7,7 @@ import (
 	"image/draw"
 	"io/ioutil"
 	"os"
+	"time"
 
 	"github.com/golang/freetype"
 	"github.com/golang/freetype/truetype"
@@ -14,8 +15,6 @@ import (
 	"golang.org/x/image/math/fixed"
 )
 
-var fontRenderingContext = newFRContext()
-
 // Font is a loaded font that can be passed to the
 // SetFont method
 type Font struct {
@@ -27,9 +26,9 @@ type fontKey struct {
 	size fixed.Int26_6
 }
 
-type fontCache struct {
-	font  *Font
-	frctx *frContext
+type frCache struct {
+	ctx      *frContext
+	lastUsed time.Time
 }
 
 var zeroes [alphaTexSize]byte
@@ -92,20 +91,32 @@ func (cv *Canvas) getFont(src interface{}) *Font {
 	return f
 }
 
-// func (cv *Canvas) getFRContext(src interface{}, size float64) *Font {
+func (cv *Canvas) getFRContext(font *Font, size fixed.Int26_6) *frContext {
+	k := fontKey{font: font, size: size}
+	if frctx, ok := cv.fontCtxs[k]; ok {
+		frctx.lastUsed = time.Now()
+		return frctx.ctx
+	}
 
-// }
+	cv.reduceCache(Performance.CacheSize, 0)
+	frctx := newFRContext()
+	frctx.fontSize = k.size
+	frctx.f = k.font.font
+	frctx.recalc()
+
+	cv.fontCtxs[k] = &frCache{ctx: frctx, lastUsed: time.Now()}
+
+	return frctx
+}
 
 // FillText draws the given string at the given coordinates
 // using the currently set font and font height
 func (cv *Canvas) FillText(str string, x, y float64) {
-	if cv.state.font == nil {
+	if cv.state.font.font == nil {
 		return
 	}
 
-	frc := fontRenderingContext
-	frc.setFont(cv.state.font.font)
-	frc.setFontSize(cv.state.fontSize)
+	frc := cv.getFRContext(cv.state.font, cv.state.fontSize)
 	fnt := cv.state.font.font
 
 	curX := x
@@ -127,7 +138,7 @@ func (cv *Canvas) FillText(str string, x, y float64) {
 			continue
 		}
 		if hasPrev {
-			kern := fnt.Kern(frc.scale, prev, idx)
+			kern := fnt.Kern(frc.fontSize, prev, idx)
 			if frc.hinting != font.HintingNone {
 				kern = (kern + 32) &^ 63
 			}
@@ -207,7 +218,7 @@ func (cv *Canvas) FillText(str string, x, y float64) {
 			continue
 		}
 		if hasPrev {
-			kern := fnt.Kern(frc.scale, prev, idx)
+			kern := fnt.Kern(frc.fontSize, prev, idx)
 			if frc.hinting != font.HintingNone {
 				kern = (kern + 32) &^ 63
 			}
@@ -267,9 +278,7 @@ func (cv *Canvas) StrokeText(str string, x, y float64) {
 		return
 	}
 
-	frc := fontRenderingContext
-	frc.setFont(cv.state.font.font)
-	frc.setFontSize(float64(cv.state.fontSize))
+	frc := cv.getFRContext(cv.state.font, cv.state.fontSize)
 	fnt := cv.state.font.font
 
 	prevPath := cv.path
@@ -296,7 +305,7 @@ func (cv *Canvas) StrokeText(str string, x, y float64) {
 
 func (cv *Canvas) runePath(rn rune, pos vec) {
 	gb := &truetype.GlyphBuf{}
-	gb.Load(cv.state.font.font, fixed.Int26_6(cv.state.fontSize*64), cv.state.font.font.Index(rn), font.HintingNone)
+	gb.Load(cv.state.font.font, cv.state.fontSize, cv.state.font.font.Index(rn), font.HintingNone)
 
 	from := 0
 	for _, to := range gb.Ends {
@@ -376,9 +385,7 @@ func (cv *Canvas) MeasureText(str string) TextMetrics {
 		return TextMetrics{}
 	}
 
-	frc := fontRenderingContext
-	frc.setFont(cv.state.font.font)
-	frc.setFontSize(float64(cv.state.fontSize))
+	frc := cv.getFRContext(cv.state.font, cv.state.fontSize)
 	fnt := cv.state.font.font
 
 	var p fixed.Point26_6
@@ -394,7 +401,7 @@ func (cv *Canvas) MeasureText(str string) TextMetrics {
 			continue
 		}
 		if hasPrev {
-			kern := fnt.Kern(frc.scale, prev, idx)
+			kern := fnt.Kern(frc.fontSize, prev, idx)
 			if frc.hinting != font.HintingNone {
 				kern = (kern + 32) &^ 63
 			}