Import of the watch repository from Pebble

tools/font/README.md

@@ -0,0 +1,15 @@
+Pebble Font Renderer Script
+===========================
+
+These Python scripts take TrueType font files, renders a set of glyps and outputs them into .h files in the appropriate structure for consumption by Pebble's text rendering routines.
+
+Requirements:
+-------------
+* freetype library
+* freetype-py binding
+
+http://code.google.com/p/freetype-py/
+
+**Mac OS X and freetype-py**: the freetype binding works with the Freetype library that ships with Mac OS X (/usr/X11/lib/libfreetype.dylib), but you need to patch setup.py using this diff file:
+
+https://gist.github.com/3345193

tools/font/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

tools/font/dump_font.py

@@ -0,0 +1,314 @@
+#!/usr/bin/env python
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import sys
+import struct
+import array
+import pprint
+import math
+import json
+import itertools
+from PIL import Image
+
+FONT_VERSION_1 = 1
+FONT_VERSION_2 = 2
+FONT_VERSION_3 = 3
+# Features
+FEATURE_OFFSET_16 = 0x01
+FEATURE_RLE4 = 0x02
+GLYPH_MD_STRUCT = 'BBbbb'
+
+def dec_and_hex(i):
+    return "0x{:x} ({:d})".format(i, i)
+
+
+def grouper(n, iterable, fillvalue=None):
+    """grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"""
+    args = [iter(iterable)] * n
+    return itertools.izip_longest(fillvalue=fillvalue, *args)
+
+
+def get_glyph(features, tbl, offset_bytes):
+    bitmap_offset_bytes = offset_bytes + struct.calcsize(GLYPH_MD_STRUCT)
+    header = tbl[offset_bytes: bitmap_offset_bytes]
+    (width, height, left, top, adv) = struct.unpack(GLYPH_MD_STRUCT, header)
+
+    if (features & FEATURE_RLE4):
+        bitmap_length_bytes = (height + 1) / 2  # RLE4 stores 2 rle_units per byte
+    else:
+        bitmap_length_bytes = ((height * width) + 7) / 8
+
+    bitmap_length_bytes_word_aligned = ((bitmap_length_bytes + 3) / 4) * 4
+    data = tbl[bitmap_offset_bytes: bitmap_offset_bytes + bitmap_length_bytes_word_aligned]
+    return {
+        'offset_bytes' : dec_and_hex(offset_bytes),
+        'top' : top,
+        'left' : left,
+        'height' : height,
+        'width' : width,
+        'advance' : adv,
+        'bitmap' : data
+        }, header
+
+def hasher(codepoint, table_size):
+    return (codepoint % table_size)
+
+def print_hash_table(hash_table):
+    print "index\tcount\toffset"
+    for idx, sz, off in hash_table:
+        print "%d\t%d\t%s" % (idx, sz, str(dec_and_hex(off)))
+
+
+def print_glyph(features, glyph_table, offset, raw, show_image):
+
+    def decompress_glyph_RLE4(rle_stream, num_units, width):
+        bitmap = []
+
+        for b in array.array('B', rle_stream):
+            # There are two RLE4 units per byte. LSNibble first.
+            for i in range(2):
+                # Handle the padding by skipping whatever remains
+                if (num_units == 0):
+                    break
+                num_units -= 1
+
+                # Each unit is <xyyy> where x is the symbol and yyy is (length - 1)
+                length = (b & 0x7) + 1
+                symbol = 1 if ((b >> 3) & 1) == 1 else 0
+                bitmap.extend([symbol]*length)
+                b >>= 4
+
+        # Correct the height, now that we know the decompressed size
+        height = len(bitmap)/width
+
+        return bitmap, height
+
+    def glyph_bitmap_to_bitlist(g):
+        height = g['height']
+
+        if not g['bitmap']:
+            return None, height
+
+        if (features & FEATURE_RLE4):
+            b, height = decompress_glyph_RLE4(g['bitmap'], height, g['width'])
+        else:
+            b = []
+            for w in array.array('I', g['bitmap']):
+                b.extend(((w & (1 << bit)) != 0 for bit in xrange(0, 32)))
+
+        return b, height
+
+    def draw_glyph_image(bitlist, width, height):
+        img = Image.new('RGB', (width, height), "black")
+        pixels = img.load()
+        for i in range(img.size[0]):
+            for j in range(img.size[1]):
+                pixels[i, j] = (0, 0, 0) if bitlist[j * width + i] else (255, 255, 255)
+        img.show()
+
+    def draw_glyph_ascii(bitlist, width, height):
+        for y in xrange(height):
+            for x in xrange(width):
+                if bitlist[y * width + x]:
+                    sys.stdout.write('X')
+                else:
+                    sys.stdout.write(' ')
+            print
+
+    def draw_glyph_raw(header, bitlist, width, height):
+        # Header:
+        for byte in header:
+            print '{:02x}'.format(ord(byte)),
+        print ' - ',
+        # Repack the glyph data. This is required because the glyph may have been compressed
+        for byte in xrange(height * width / 8):
+            w = 0
+            for bit in range(8):
+                if bitlist[byte * 8 + bit]:
+                    w |= 1 << bit
+            print '{:02x}'.format(w),
+        print
+
+    g, header = get_glyph(features, glyph_table, offset)
+
+    # FEATURE_RLE4 uses the height field for # RLE Units!
+    bitlist, height = glyph_bitmap_to_bitlist(g)
+
+    if raw:
+        header_offset = offset + struct.calcsize(GLYPH_MD_STRUCT)
+        draw_glyph_raw(header, bitlist, g['width'], height)
+    else:
+        output = []
+        output.append("offset bytes: {}".format(g['offset_bytes']))
+        output.append("top: {}".format(g['top']))
+        output.append("left: {}".format(g['left']))
+        output.append("height: {}".format(height))
+        output.append("width: {}".format(g['width']))
+        output.append("advance: {}".format(g['advance']))
+        output.append("bitmap:")
+        print '\n'.join(output)
+        print
+
+        if bitlist:
+            if show_image:
+                draw_glyph_image(bitlist, g['width'], height)
+            else:
+                draw_glyph_ascii(bitlist, g['width'], height)
+
+
+# Allow extended codepoint encoding for 'narrow Python builds'
+def my_unichr(i):
+    try:
+        return unichr(i)
+    except ValueError:
+        return struct.pack('i', i).decode('utf-32')
+
+
+# Attempt to print unicode characters. Do the best we can when redirecting output.
+# (See PYTHONIOENCODING)
+def print_glyph_header(codepoint, offset, raw=False):
+    if raw:
+        print '{:08X}:'.format(codepoint),
+    else:
+        print
+        print u'{}\t({})\t{}'.format(dec_and_hex(codepoint), my_unichr(codepoint),
+                                     dec_and_hex(offset)).encode('utf-8', 'replace')
+        print
+
+
+def main(pfo_path, show_hash_table, offset_table, glyph, all_glyphs, show_image, raw):
+    with open(pfo_path, 'rb') as f:
+        font = f.read()
+
+    # Assume version 3 to start
+    version = 3
+    font_md_format = ['', '<BBHH', '<BBHHBB', '<BBHHBBBB']
+    font_md_size = struct.calcsize(font_md_format[version])
+    (version, max_height, num_glyphs, wildcard_cp, table_size, cp_bytes,
+     struct_size, features) = struct.unpack(font_md_format[version], font[:font_md_size])
+    if version == 3:
+        pass
+    elif version == 2:
+        # Set the defaults
+        font_md_size = struct.calcsize(font_md_format[version])
+        features = 0
+    else:
+        raise Exception('Error: Unexpected font file version {}'.format(version))
+
+    # Build up the offset entry struct
+    offset_table_format = '<'
+    offset_table_format += 'L' if cp_bytes == 4 else 'H'
+    offset_table_format += 'H' if features & FEATURE_OFFSET_16 else 'L'
+    offset_entry_size = struct.calcsize(offset_table_format)
+
+    hash_entry_format = '<BBH'
+    hash_entry_size = struct.calcsize(hash_entry_format)
+
+    def hash_iterator(tbl, num):
+        for i in xrange(0, num * hash_entry_size, hash_entry_size):
+            yield struct.unpack(hash_entry_format, tbl[i:i + hash_entry_size])
+
+    def offset_iterator(tbl, num):
+        for i in xrange(0, num * offset_entry_size, offset_entry_size):
+            yield struct.unpack(offset_table_format, tbl[i:i + offset_entry_size])
+
+    hash_table = [(a,b,c) for (a,b,c) in hash_iterator(font[font_md_size:], table_size)]
+    offset_tables_start = font_md_size + hash_entry_size * table_size
+    glyph_table_start = offset_tables_start +  offset_entry_size * (num_glyphs)
+    glyph_table = font[glyph_table_start:]
+
+    if not raw:
+        print 'Font info'
+        pprint.pprint(
+            {'version': version,
+             'max_height': max_height,
+             'num_glyphs': num_glyphs,
+             'offset_table_size': num_glyphs * offset_entry_size,
+             'glyph_table_start': dec_and_hex(glyph_table_start),
+             'wildcard_cp': dec_and_hex(wildcard_cp),
+             'codepoint bytes': cp_bytes,
+             'hash_table_size': table_size,
+             'font_header_size': font_md_size,
+             'features': features,
+             'features - offset size': 16 if (features & FEATURE_OFFSET_16) else 32,
+             'features - RLE4': True if (features & FEATURE_RLE4) else False})
+
+        print
+        print 'Hash Table start:   {}'.format(font_md_size)
+        print 'Offset Table start: {}'.format(offset_tables_start)
+        print 'Glyph Table start:  {}'.format(glyph_table_start)
+        print '--------------------------'
+
+    if all_glyphs:
+        for _,sz,off in hash_table:
+          off_table = dict([x for x in offset_iterator(font[(offset_tables_start + off):], sz)])
+          for k,v in sorted(off_table.items()):
+            print_glyph_header(k, v, raw)
+            print_glyph(features, glyph_table, v, raw, False)
+            if not raw:
+                print
+                print
+        return
+
+    if show_hash_table:
+        print
+        print_hash_table(hash_table)
+
+    if offset_table:
+        offset_table = int(offset_table)
+        _,sz,off = hash_table[offset_table]
+        if not raw:
+            print 'Offset Table {} offset: {}'.format(offset_table, offset_tables_start + off)
+        off_table = dict([x for x in offset_iterator(font[(offset_tables_start + off):], sz)])
+        for k,v in sorted(off_table.items()):
+            print_glyph_header(k, v, raw)
+            print_glyph(features, glyph_table, v, raw, show_image)
+            if not raw:
+                print
+                print
+
+    if glyph:
+        codepoint = int(glyph, 16)
+        glyph_hash = hasher(codepoint, table_size)
+        _,sz,off = hash_table[glyph_hash]
+        off_table = dict([x for x in offset_iterator(font[(offset_tables_start + off):], sz)])
+        glyph_off = 0
+        for (cp, off) in off_table.items():
+            if cp == codepoint:
+                glyph_off = off_table[codepoint]
+                break
+        else:
+            print "{} not in font".format(codepoint)
+            return
+        print_glyph_header(codepoint, glyph_off, raw)
+        print_glyph(features, glyph_table, glyph_off, raw, show_image)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-o', '--offsets', help='print hash table')
+    parser.add_argument('-t', '--hashes', action='store_true', help='print hash table')
+    parser.add_argument('-g', '--glyph', help='draw glyph in external image viewer')
+    parser.add_argument('-G', '--ascii_glyph', help='draw glyph in ASCII')
+    parser.add_argument('-a', '--all_glyphs', action='store_true', help='print all glyphs')
+    parser.add_argument('-r', '--raw', action='store_true')
+    parser.add_argument('pebble_font')
+    args = parser.parse_args()
+
+    codepoint = args.ascii_glyph if args.ascii_glyph else args.glyph
+
+    main(args.pebble_font, args.hashes, args.offsets, codepoint, args.all_glyphs,
+         args.ascii_glyph is None, args.raw)

tools/font/emit_codepoints.py

@@ -0,0 +1,82 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import freetype
+import argparse
+import os, sys, re
+from math import ceil
+
+MIN_CODEPOINT = 0x20
+MAX_CODEPOINT = 0xffff
+# Set a codepoint that the font doesn't know how to render
+# The watch will use this glyph as the wildcard character
+WILDCARD_CODEPOINT = 0x3456
+
+class Font:
+    def __init__(self, ttf_path):
+        self.version = 1
+        self.ttf_path = ttf_path
+        # Get the font's size from the filename:
+        self.basename = os.path.basename(self.ttf_path)
+        m = re.search('([0-9]+)', self.basename)
+        if m == None:
+            sys.stderr.write('Font {0}: no size found in file name...\n'.format(filename))
+            return
+        self.max_height = int(m.group(0))
+        self.face = freetype.Face(self.ttf_path)
+        self.face.set_pixel_sizes(0, self.max_height)
+        self.name = self.face.family_name + "_" + self.face.style_name
+        self.wildcard_codepoint = WILDCARD_CODEPOINT
+        self.number_of_glyphs = 0
+        return
+
+    def is_supported_glyph(self, codepoint):
+        return (self.face.get_char_index(codepoint) > 0 or (codepoint == unichr(self.wildcard_codepoint)))
+
+    def emit_codepoints(self):
+        to_file = os.path.splitext(self.ttf_path)[0] + '.codepoints'
+        f = open(to_file, 'wb')
+        for codepoint in xrange(MIN_CODEPOINT, MAX_CODEPOINT + 1):
+            self.face.load_char(unichr(codepoint))
+            if self.is_supported_glyph(codepoint):
+                print>>f,"U+%08d" % (codepoint,)
+        f.close()
+        return
+
+    def emit_codepoints_as_utf8(self):
+        to_file = os.path.splitext(self.ttf_path)[0] + '.utf8'
+        f = open(to_file, 'wb')
+        for codepoint in xrange(MIN_CODEPOINT, MAX_CODEPOINT + 1):
+            self.face.load_char(unichr(codepoint))
+            if self.is_supported_glyph(codepoint):
+                f.write(unichr(codepoint).encode('utf-8'))
+        f.close()
+        return
+
+def main():
+    font_directory = "ttf"
+    font_paths = []
+    for _, _, filenames in os.walk(font_directory):
+        for filename in filenames:
+            if os.path.splitext(filename)[1] == '.ttf':
+                font_paths.append(os.path.join(font_directory, filename))
+
+    for font_path in font_paths:
+        f = Font(font_path)
+        f.emit_codepoints()
+        f.emit_codepoints_as_utf8()
+    return
+
+if __name__ == "__main__":
+    main()

tools/font/fontgen.py

@@ -0,0 +1,471 @@
+#!/usr/bin/env python
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import freetype
+import os
+import re
+import struct
+import sys
+import itertools
+import json
+from math import ceil
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
+import generate_c_byte_array
+
+# Font v3 -- https://pebbletechnology.atlassian.net/wiki/display/DEV/Pebble+Resource+Pack+Format
+#   FontInfo
+#       (uint8_t)  version                           - v1
+#       (uint8_t)  max_height                        - v1
+#       (uint16_t) number_of_glyphs                  - v1
+#       (uint16_t) wildcard_codepoint                - v1
+#       (uint8_t)  hash_table_size                   - v2
+#       (uint8_t)  codepoint_bytes                   - v2
+#       (uint8_t)  size                              - v3  # Save the size of FontInfo for sanity
+#       (uint8_t)  features                          - v3
+#
+#   font_info_struct_size is the size of the FontInfo structure. This makes extending this structure
+#   in the future far simpler.
+#
+#   'features' is a bitmap defined as follows:
+#       0: offset table offsets uint32 if 0, uint16 if 1
+#       1: glyphs are bitmapped if 0, RLE4 encoded if 1
+#     2-7: reserved
+#
+#   (uint32_t) hash_table[]
+#       glyph_tables are found in the resource image by converting a codepoint into an offset from
+#       the start of the resource. This conversion is implemented as a hash where collisions are
+#       resolved by separate chaining. Each entry in the hash table is as follows:
+#                  (uint8_t) hash value
+#                  (uint8_t) offset_table_size
+#                  (uint16_t) offset
+#       A codepoint is converted into a hash value by the hash function -- this value is a direct
+#       index into the hash table array. 'offset' is the location of the correct offset_table list
+#       from the start of offset_tables, and offset_table_size gives the number of glyph_tables in
+#       the list (i.e., the number of codepoints that hash to the same value).
+#
+#   (uint32_t) offset_tables[][]
+#       this list of tables contains offsets into the glyph_table for the codepoint.
+#       each offset is counted in 32-bit blocks from the start of glyph_table.
+#       packed:     (codepoint_bytes [uint16_t | uint32_t]) codepoint
+#                   (features[0] [uint16_t | uint32_t]) offset
+#
+#   (uint32_t) glyph_table[]
+#       [0]: the 32-bit block for offset 0 is used to indicate that a glyph is not supported
+#       then for each glyph:
+#       [offset + 0]  packed:   (int_8) offset_top
+#                               (int_8) offset_left,
+#                              (uint_8) bitmap_height,       NB: in v3, if RLE4 compressed, this
+#                                                                field is contains the number of
+#                                                                RLE4 units.
+#                              (uint_8) bitmap_width (LSB)
+#
+#       [offset + 1]           (int_8) horizontal_advance
+#                              (24 bits) zero padding
+#       [offset + 2] bitmap data (unaligned rows of bits), padded with 0's at
+#       the end to make the bitmap data as a whole use multiples of 32-bit
+#       blocks
+
+MIN_CODEPOINT = 0x20
+MAX_2_BYTES_CODEPOINT = 0xffff
+MAX_EXTENDED_CODEPOINT = 0x10ffff
+FONT_VERSION_1 = 1
+FONT_VERSION_2 = 2
+FONT_VERSION_3 = 3
+# Set a codepoint that the font doesn't know how to render
+# The watch will use this glyph as the wildcard character
+WILDCARD_CODEPOINT = 0x25AF # White vertical rectangle
+ELLIPSIS_CODEPOINT = 0x2026
+# Features
+FEATURE_OFFSET_16 = 0x01
+FEATURE_RLE4 = 0x02
+
+
+HASH_TABLE_SIZE = 255
+OFFSET_TABLE_MAX_SIZE = 128
+MAX_GLYPHS_EXTENDED = HASH_TABLE_SIZE * OFFSET_TABLE_MAX_SIZE
+MAX_GLYPHS = 256
+
+def grouper(n, iterable, fillvalue=None):
+    """grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"""
+    args = [iter(iterable)] * n
+    return itertools.izip_longest(fillvalue=fillvalue, *args)
+
+def hasher(codepoint, num_glyphs):
+    return (codepoint % num_glyphs)
+
+def bits(x):
+    data = []
+    for i in range(8):
+        data.insert(0, int((x & 1) == 1))
+        x = x >> 1
+    return data
+
+class Font:
+    def __init__(self, ttf_path, height, max_glyphs, max_glyph_size, legacy):
+        self.version = FONT_VERSION_3
+        self.ttf_path = ttf_path
+        self.max_height = int(height)
+        self.legacy = legacy
+        self.face = freetype.Face(self.ttf_path)
+        self.face.set_pixel_sizes(0, self.max_height)
+        self.name = self.face.family_name + "_" + self.face.style_name
+        self.wildcard_codepoint = WILDCARD_CODEPOINT
+        self.number_of_glyphs = 0
+        self.table_size = HASH_TABLE_SIZE
+        self.tracking_adjust = 0
+        self.regex = None
+        self.codepoints = range(MIN_CODEPOINT, MAX_EXTENDED_CODEPOINT)
+        self.codepoint_bytes = 2
+        self.max_glyphs = max_glyphs
+        self.max_glyph_size = max_glyph_size
+        self.glyph_table = []
+        self.hash_table = [0] * self.table_size
+        self.offset_tables = [[] for i in range(self.table_size)]
+        self.offset_size_bytes = 4
+        self.features = 0
+
+        self.glyph_header = ''.join((
+            '<',  # little_endian
+            'B',  # bitmap_width
+            'B',  # bitmap_height
+            'b',  # offset_left
+            'b',  # offset_top
+            'b'   # horizontal_advance
+            ))
+
+    def set_compression(self, engine):
+        if self.version != FONT_VERSION_3:
+            raise Exception("Compression being set but version != 3 ({})". format(self.version))
+        if engine == 'RLE4':
+            self.features |= FEATURE_RLE4
+        else:
+            raise Exception("Unsupported compression engine: '{}'. Font {}".format(engine,
+                            self.ttf_path))
+
+    def set_version(self, version):
+        self.version = version
+
+    def set_tracking_adjust(self, adjust):
+        self.tracking_adjust = adjust
+
+    def set_regex_filter(self, regex_string):
+        if regex_string != ".*":
+            try:
+                self.regex = re.compile(unicode(regex_string, 'utf8'), re.UNICODE)
+            except Exception, e:
+                raise Exception("Supplied filter argument was not a valid regular expression."
+                                "Font: {}".format(self.ttf_path))
+        else:
+            self.regex = None
+
+    def set_codepoint_list(self, list_path):
+        codepoints_file = open(list_path)
+        codepoints_json = json.load(codepoints_file)
+        self.codepoints = [int(cp) for cp in codepoints_json["codepoints"]]
+
+    def is_supported_glyph(self, codepoint):
+        return (self.face.get_char_index(codepoint) > 0 or
+                (codepoint == unichr(self.wildcard_codepoint)))
+
+    def compress_glyph_RLE4(self, bitmap):
+        # This Run Length Compression scheme works by converting runs of identical symbols to the
+        # symbol and the length of the run. The length of each run of symbols is limited to
+        # [1..2**(RLElen-1)]. For RLE4, the length is 3 bits (0-7), or 1-8 consecutive symbols.
+        # For example: 11110111 is compressed to 1*4, 0*1, 1*3. or [(1, 4), (0, 1), (1, 3)]
+
+        RLE_LEN = 2**(4-1)  # TODO possibly make this a parameter.
+        # It would likely be a good idea to look into the 'bitstream' package for lengths that won't
+        # easily fit into a byte/short/int.
+
+        # First, generate a list of tuples (bit, count).
+        unit_list = [(name, len(list(group))) for name, group in itertools.groupby(bitmap)]
+
+        # Second, generate a list of RLE tuples where count <= RLE_LEN. This intermediate step will
+        # make it much easier to implement the binary stream packer below.
+        rle_unit_list = []
+        for name, length in unit_list:
+            while length > 0:
+                unit_len = min(length, RLE_LEN)
+                rle_unit_list.append((name, unit_len))
+                length -= unit_len
+
+        # Note that num_units does not include the padding added below.
+        num_units = len(rle_unit_list)
+
+        # If the list is odd, add a padding unit
+        if (num_units % 2) == 1:
+            rle_unit_list.append((0, 1))
+
+        # Now pack the tuples into a binary stream. We can't pack nibbles, so join two
+        glyph_packed = []
+        it = iter(rle_unit_list)
+        for name, length in it:
+            name2, length2 = next(it)
+            packed_byte = name << 3 | (length - 1) | name2 << 7 | (length2 - 1) << 4
+            glyph_packed.append(struct.pack('<B', packed_byte))
+
+        # Pad out to the nearest 4 bytes
+        while (len(glyph_packed) % 4) > 0:
+            glyph_packed.append(struct.pack('<B', 0))
+
+        return (glyph_packed, num_units)
+
+    # Make sure that we will be able to decompress the glyph in-place
+    def check_decompress_glyph_RLE4(self, glyph_packed, width, rle_units):
+        # The glyph buffer before decoding is arranged as follows:
+        #  [ <header> | <free space> | <encoded glyph> ]
+        # Make sure that we can decode the encoded glyph to end up with the following arrangement:
+        #  [ <header> |       <decoded glyph>          ]
+        # without overwriting the unprocessed encoded glyph in the process
+
+        header_size = struct.calcsize(self.glyph_header)
+        dst_ptr = header_size
+        src_ptr = self.max_glyph_size - len(glyph_packed)
+
+        def glyph_packed_iterator(tbl, num):
+            for i in xrange(0, num):
+                yield struct.unpack('<B', tbl[i])[0]
+
+        # Generate glyph buffer. Ignore the header
+        bitmap = [0] * self.max_glyph_size
+        bitmap[-len(glyph_packed):] = glyph_packed_iterator(glyph_packed, len(glyph_packed))
+
+        out_num_bits = 0
+        out = 0
+        total_length = 0
+        while rle_units > 0:
+            if src_ptr >= self.max_glyph_size:
+                raise Exception("Error: input stream too large for buffer. Font {}".
+                                format(self.ttf_path))
+
+            unit_pair = bitmap[src_ptr]
+            src_ptr += 1
+            for i in range(min(rle_units, 2)):
+                colour = (unit_pair >> 3) & 1
+                length = (unit_pair & 0x07) + 1
+                total_length += length
+
+                if colour:
+                    # Generate the bitpattern 111...
+                    add = (1 << length) - 1
+                    out |= (add << out_num_bits)
+                out_num_bits += length
+
+                if out_num_bits >= 8:
+                    if dst_ptr >= src_ptr:
+                        raise Exception("Error: unable to RLE4 decode in place! Overrun. Font {}".
+                                        format(self.ttf_path))
+                    if dst_ptr >= self.max_glyph_size:
+                        raise Exception("Error: output bitmap too large for buffer. Font {}".
+                                        format(self.ttf_path))
+                    bitmap[dst_ptr] = (out & 0xFF)
+                    dst_ptr += 1
+                    out >>= 8
+                    out_num_bits -= 8
+
+                unit_pair >>= 4
+                rle_units -= 1
+
+        while out_num_bits > 0:
+            bitmap[dst_ptr] = (out & 0xFF)
+            dst_ptr += 1
+            out >>= 8
+            out_num_bits -= 8
+
+        # Success! We can in-place decode this glyph
+        return True
+
+    def glyph_bits(self, codepoint, gindex):
+        flags = (freetype.FT_LOAD_RENDER if self.legacy else
+            freetype.FT_LOAD_RENDER | freetype.FT_LOAD_MONOCHROME | freetype.FT_LOAD_TARGET_MONO)
+        self.face.load_glyph(gindex, flags)
+        # Font metrics
+        bitmap = self.face.glyph.bitmap
+        advance = self.face.glyph.advance.x / 64     # Convert 26.6 fixed float format to px
+        advance += self.tracking_adjust
+        width = bitmap.width
+        height = bitmap.rows
+        left = self.face.glyph.bitmap_left
+        bottom = self.max_height - self.face.glyph.bitmap_top
+        pixel_mode = self.face.glyph.bitmap.pixel_mode
+
+        glyph_packed = []
+        if height and width:
+            glyph_bitmap = []
+            if pixel_mode == 1:  # monochrome font, 1 bit per pixel
+                for i in range(bitmap.rows):
+                    row = []
+                    for j in range(bitmap.pitch):
+                        row.extend(bits(bitmap.buffer[i*bitmap.pitch+j]))
+                    glyph_bitmap.extend(row[:bitmap.width])
+            elif pixel_mode == 2:  # grey font, 255 bits per pixel
+                for val in bitmap.buffer:
+                    glyph_bitmap.extend([1 if val > 127 else 0])
+            else:
+                # freetype-py should never give us a value not in (1,2)
+                raise Exception("Unsupported pixel mode: {}. Font {}".
+                                format(pixel_mode, self.ttf_path))
+
+            if (self.features & FEATURE_RLE4):
+                # HACK WARNING: override the height with the number of RLE4 units.
+                glyph_packed, height = self.compress_glyph_RLE4(glyph_bitmap)
+                if height > 255:
+                    raise Exception("Unable to RLE4 compress -- more than 255 units required"
+                                    "({}). Font {}".format(height, self.ttf_path))
+                # Check that we can in-place decompress. Will raise an exception if not.
+                self.check_decompress_glyph_RLE4(glyph_packed, width, height)
+            else:
+                for word in grouper(32, glyph_bitmap, 0):
+                    w = 0
+                    for index, bit in enumerate(word):
+                        w |= bit << index
+                    glyph_packed.append(struct.pack('<I', w))
+
+                # Confirm that we're smaller than the cache size
+                size = ((width * height) + (8 - 1)) / 8
+                if size > self.max_glyph_size:
+                    raise Exception("Glyph too large! codepoint {}: {} > {}. Font {}".
+                                    format(codepoint, size, self.max_glyph_size, self.ttf_path))
+
+        glyph_header = struct.pack(self.glyph_header, width, height, left, bottom, advance)
+
+        return glyph_header + ''.join(glyph_packed)
+
+    def fontinfo_bits(self):
+        if self.version == FONT_VERSION_2:
+            s = struct.Struct('<BBHHBB')
+            return s.pack(self.version,
+                          self.max_height,
+                          self.number_of_glyphs,
+                          self.wildcard_codepoint,
+                          self.table_size,
+                          self.codepoint_bytes)
+        else:
+            s = struct.Struct('<BBHHBBBB')
+            return s.pack(self.version,
+                          self.max_height,
+                          self.number_of_glyphs,
+                          self.wildcard_codepoint,
+                          self.table_size,
+                          self.codepoint_bytes,
+                          s.size,
+                          self.features)
+
+
+    def build_tables(self):
+        def build_hash_table(bucket_sizes):
+            acc = 0
+            for i in range(self.table_size):
+                bucket_size = bucket_sizes[i]
+                self.hash_table[i] = (struct.pack('<BBH', i, bucket_size, acc))
+                acc += bucket_size * (self.offset_size_bytes + self.codepoint_bytes)
+
+        def build_offset_tables(glyph_entries):
+            offset_table_format = '<'
+            offset_table_format += 'L' if self.codepoint_bytes == 4 else 'H'
+            offset_table_format += 'L' if self.offset_size_bytes == 4 else 'H'
+
+            bucket_sizes = [0] * self.table_size
+            for entry in glyph_entries:
+                codepoint, offset = entry
+                glyph_hash = hasher(codepoint, self.table_size)
+                self.offset_tables[glyph_hash].append(
+                        struct.pack(offset_table_format, codepoint, offset))
+                bucket_sizes[glyph_hash] = bucket_sizes[glyph_hash] + 1
+                if bucket_sizes[glyph_hash] > OFFSET_TABLE_MAX_SIZE:
+                    print "error: %d > 127" % bucket_sizes[glyph_hash]
+            return bucket_sizes
+
+        def add_glyph(codepoint, next_offset, gindex, glyph_indices_lookup):
+            offset = next_offset
+            if gindex not in glyph_indices_lookup:
+                glyph_bits = self.glyph_bits(codepoint, gindex)
+                glyph_indices_lookup[gindex] = offset
+                self.glyph_table.append(glyph_bits)
+                next_offset += len(glyph_bits)
+            else:
+                offset = glyph_indices_lookup[gindex]
+
+            if (codepoint > MAX_2_BYTES_CODEPOINT):
+                self.codepoint_bytes = 4
+
+            self.number_of_glyphs += 1
+            return offset, next_offset, glyph_indices_lookup
+
+        def codepoint_is_in_subset(codepoint):
+           if (codepoint not in (WILDCARD_CODEPOINT, ELLIPSIS_CODEPOINT)):
+              if self.regex is not None:
+                  if self.regex.match(unichr(codepoint)) is None:
+                      return False
+              if codepoint not in self.codepoints:
+                 return False
+           return True
+
+        glyph_entries = []
+        # MJZ: The 0th offset of the glyph table is 32-bits of
+        # padding, no idea why.
+        self.glyph_table.append(struct.pack('<I', 0))
+        self.number_of_glyphs = 0
+        glyph_indices_lookup = dict()
+        next_offset = 4
+        codepoint, gindex = self.face.get_first_char()
+
+        # add wildcard_glyph
+        offset, next_offset, glyph_indices_lookup = add_glyph(WILDCARD_CODEPOINT, next_offset, 0,
+                                                              glyph_indices_lookup)
+        glyph_entries.append((WILDCARD_CODEPOINT, offset))
+
+        while gindex:
+            # Hard limit on the number of glyphs in a font
+            if (self.number_of_glyphs > self.max_glyphs):
+                break
+
+            if (codepoint is WILDCARD_CODEPOINT):
+                raise Exception('Wildcard codepoint is used for something else in this font.'
+                                'Font {}'.format(self.ttf_path))
+
+            if (gindex is 0):
+                raise Exception('0 index is reused by a non wildcard glyph. Font {}'.
+                                format(self.ttf_path))
+
+            if (codepoint_is_in_subset(codepoint)):
+                offset, next_offset, glyph_indices_lookup = add_glyph(codepoint, next_offset,
+                                                                      gindex, glyph_indices_lookup)
+                glyph_entries.append((codepoint, offset))
+
+            codepoint, gindex = self.face.get_next_char(codepoint, gindex)
+
+        # Decide if we need 2 byte or 4 byte offsets
+        glyph_data_bytes = sum(len(glyph) for glyph in self.glyph_table)
+        if self.version == FONT_VERSION_3 and glyph_data_bytes < 65536:
+            self.features |= FEATURE_OFFSET_16
+            self.offset_size_bytes = 2
+
+        # Make sure the entries are sorted by codepoint
+        sorted_entries = sorted(glyph_entries, key=lambda entry: entry[0])
+        hash_bucket_sizes = build_offset_tables(sorted_entries)
+        build_hash_table(hash_bucket_sizes)
+
+    def bitstring(self):
+        btstr = self.fontinfo_bits()
+        btstr += ''.join(self.hash_table)
+        for table in self.offset_tables:
+            btstr += ''.join(table)
+        btstr += ''.join(self.glyph_table)
+
+        return btstr