From 22fba5e109eb472b3097bc01629dbc9c84c5d58a Mon Sep 17 00:00:00 2001 From: DraftingDreamer <264591489+DraftingDreamer@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:57:35 +0800 Subject: [PATCH] Add CJK coverage per writing system Records how much of each East Asian writing system a font covers, as a count per tier in a new "cjk" object in fonts.json. The Han tiers are the character sets national standards define, enumerated by decoding their byte ranges with the matching stdlib codec, so this adds no dependency beyond the fonttools already in requirements.txt. Hangul and the kana are complete Unicode blocks, which for them is the practical set. cjk_verify.py re-derives the same sets from Unicode's own Unihan data (UAX #38 kIRG_*Source) and compares, so the hardcoded byte ranges can be checked rather than taken on trust. It is a one-off tool, not part of the build. Refs #231 --- README.md | 64 ++++++++++++++ cjk.py | 221 ++++++++++++++++++++++++++++++++++++++++++++++ cjk_verify.py | 177 +++++++++++++++++++++++++++++++++++++ fonts-schema.json | 18 ++++ info.py | 21 +++++ 5 files changed, 501 insertions(+) create mode 100644 cjk.py create mode 100644 cjk_verify.py diff --git a/README.md b/README.md index 0c28a75e..2a068590 100755 --- a/README.md +++ b/README.md @@ -35,6 +35,70 @@ The glyphs, characters and languages data can be added to the JSON by running th python3 info.py --name yourfont ``` +### CJK coverage + +`info.py` also records how much of each East Asian writing system a font covers, +in a `cjk` object. This is measured separately from the `languages` data because +Hyperglot treats a language as covered only at 100%, which for CJK is +misleading: M PLUS 1 Code has every kanji and every hiragana on Hyperglot's +Japanese list and all but two of its katakana — the two being rare marks almost +nobody types — so it reports no Japanese support at all. + +Each tier is stored as a **count**, not a percentage, matching how `languages` +works — the totals live in the front end, so a filter threshold can be changed +without re-running the script over every font. + +| key | what it counts | total | standard | +|---|---|---|---| +| `gb2312-1` | everyday Simplified Chinese hanzi | 3,755 | GB/T 2312-1980 level 1 | +| `gb2312-2` | rarer hanzi: names, places, classical | 3,008 | GB/T 2312-1980 level 2 | +| `big5-1` | everyday Traditional Chinese hanzi | 5,401 | Big5 common | +| `big5-2` | rarer hanzi | 7,652 | Big5 less common | +| `jis0208-1` | everyday Japanese kanji | 2,965 | JIS X 0208 level 1 | +| `jis0208-2` | rarer kanji | 3,390 | JIS X 0208 level 2 | +| `hanja` | Korean hanja | 4,888 | KS X 1001:2004 | +| `hangul` | modern Hangul syllables | 11,172 | U+AC00..U+D7A3 | +| `hiragana` | hiragana | 86 | U+3041..U+3096 | +| `katakana` | katakana | 90 | U+30A1..U+30FA | + +Hiragana and katakana are counted separately because fonts ship one without the +other: Cartograph has all 90 katakana and no hiragana, which a combined number +would report as a meaningless 51%. + +#### Where these numbers come from + +The four Han tiers are not Unicode blocks — "CJK Unified Ideographs" and its +extensions run to about 100,000 code points, most of which nobody types, so +measuring against them makes even Unifont look like it covers 20% of CJK. +Instead each tier is the set of characters a national standard defines, which +`cjk.py` enumerates by decoding that standard's byte ranges with the matching +codec from Python's standard library. No extra dependency and no data file to +keep up to date. + +The byte ranges are hardcoded, so `cjk_verify.py` proves they are right rather +than asking anyone to take them on trust. Unihan records, for every ideograph, +which national standards it came from +([UAX #38](https://www.unicode.org/reports/tr38/) `kIRG_*Source`), so the sets +can be rebuilt from Unicode's own data and compared: + +```sh +python3 cjk_verify.py # downloads ~8MB from unicode.org +``` + +Last run against UCD 17.0 (Unihan of 2025-08-18): + +| tier | vs Unihan | result | +|---|---|---| +| GB/T 2312 | `G0`, 6,763 | identical | +| JIS X 0208 | `J0`, 6,356 | Unihan has one extra, 仝 (U+4EDD), which sits in JIS X 0208's symbol rows rather than the two kanji levels | +| KS X 1001 | `K0`, 4,888 | same count; the two pick different compatibility code points for 郎/郞 and 隸/隷 | +| Big5 | `T1`+`T2`, 13,064 | 13,049 in common. Unihan has no Big5 tag — `T1`/`T2` are CNS 11643 planes 1 and 2, a different standard covering nearly the same characters | + +A count says whether a font *can* set text in a language, not whether it does so +well. Han characters are shared between these standards but drawn to different +regional conventions, and only the `cmap` is inspected here, so a font can cover +Big5 with glyphs drawn to Japanese conventions. + ### Development - Running `make` installs dependencies, lints, validates `fonts.json` (against [fonts-schema.json](https://github.com/braver/programmingfonts/blob/gh-pages/fonts-schema.json) and `validate.js`), and builds the stylesheet. diff --git a/cjk.py b/cjk.py new file mode 100644 index 00000000..fc07874b --- /dev/null +++ b/cjk.py @@ -0,0 +1,221 @@ +import json +from collections import namedtuple +from os import path + +from fontTools.ttLib import TTFont + +''' +Measure CJK coverage: how much of each East Asian writing system a font +actually covers. See fonts-schema.json for the shape this writes, and the +README for where the numbers come from. + +Requires: +fonttools - https://github.com/fonttools/fonttools +brotli (for woff files) - https://github.com/google/brotli + +Both are already in requirements.txt; this module adds no new dependency. +Run cjk_verify.py to re-derive the byte ranges below from Unicode's own data. +''' + + +''' +Each tier is one national standard, enumerated by decoding the byte ranges that +standard defines with the matching stdlib codec, and keeping the ideographs. + +expected: the character count the standard specifies. It is checked when the +sets are built, so a codec change in a future Python release fails loudly +instead of silently shifting every percentage on the site. These same numbers +are the maximums in fonts-schema.json; change one and you must change both. + +irg: the source tag Unicode itself uses for the same standard in Unihan +(UAX #38, https://www.unicode.org/reports/tr38/). cjk_verify.py checks these +sets against that data, so nothing here rests on trusting this comment. + +Han tiers are split into level 1 (everyday text) and level 2 (names, place +names, classical text): a font with all of level 1 and none of level 2 is +perfectly usable for reading, and a single number would hide that. +''' +Tier = namedtuple('Tier', 'key name codec highs lows expected irg') + +EUC_LOW = [(0xA1, 0xFE)] +BIG5_LOW = [(0x40, 0x7E), (0xA1, 0xFE)] + +TIERS = [ + Tier('gb2312-1', 'GB/T 2312-1980 level 1', 'gb2312', [(0xB0, 0xD7)], EUC_LOW, 3755, 'G0'), + Tier('gb2312-2', 'GB/T 2312-1980 level 2', 'gb2312', [(0xD8, 0xF7)], EUC_LOW, 3008, 'G0'), + Tier('big5-1', 'Big5 common', 'big5', [(0xA4, 0xC6)], BIG5_LOW, 5401, 'T1'), + Tier('big5-2', 'Big5 less common', 'big5', [(0xC9, 0xF9)], BIG5_LOW, 7652, 'T2'), + Tier('jis0208-1', 'JIS X 0208 level 1', 'euc_jp', [(0xB0, 0xCF)], EUC_LOW, 2965, 'J0'), + Tier('jis0208-2', 'JIS X 0208 level 2', 'euc_jp', [(0xD0, 0xF4)], EUC_LOW, 3390, 'J0'), + Tier('hanja', 'KS X 1001:2004 hanja', 'euc_kr', [(0xCA, 0xFD)], EUC_LOW, 4888, 'K0'), +] + +''' +Syllabaries and Hangul are complete blocks rather than a selection out of a +larger standard, so the Unicode block IS the practical set: every one of these +is reachable from a keyboard. + +Hiragana and katakana stay separate because fonts really do ship one without +the other -- Cartograph has all 90 katakana and no hiragana -- and averaging +the two hides exactly that. +''' +Block = namedtuple('Block', 'key name ranges') + +BLOCKS = [ + Block('hangul', 'modern Hangul syllables', [(0xAC00, 0xD7A3)]), + Block('hiragana', 'Hiragana', [(0x3041, 0x3096)]), + Block('katakana', 'Katakana', [(0x30A1, 0x30FA)]), +] + + +def is_ideograph(cp): + ''' + Han ranges the legacy codecs above can produce: unified, Ext A, + compatibility and Ext B. Ext C and later are deliberately absent -- no + legacy codec encodes them. + ''' + return (0x3400 <= cp <= 0x4DBF or 0x4E00 <= cp <= 0x9FFF + or 0xF900 <= cp <= 0xFAFF or 0x20000 <= cp <= 0x2A6DF) + + +def decode_tier(codec, high_ranges, low_ranges): + '''Every ideograph reachable in these byte ranges of a legacy codec.''' + found = set() + for high_start, high_end in high_ranges: + for high in range(high_start, high_end + 1): + for low_start, low_end in low_ranges: + for low in range(low_start, low_end + 1): + try: + char = bytes((high, low)).decode(codec) + except UnicodeDecodeError: + continue + if len(char) == 1 and is_ideograph(ord(char)): + found.add(ord(char)) + return found + + +def expand_ranges(ranges): + '''Every code point in a list of inclusive (first, last) pairs.''' + found = set() + for start, end in ranges: + found.update(range(start, end + 1)) + return found + + +def build_sets(): + '''{key: set of code points}, each validated against its standard.''' + sets = {} + for tier in TIERS: + chars = decode_tier(tier.codec, tier.highs, tier.lows) + if len(chars) != tier.expected: + raise SystemExit( + '%s: the %s codec gives %d ideographs, the standard defines %d. ' + "Python's codec tables changed; check the byte ranges in cjk.py " + 'before trusting any coverage number.' + % (tier.name, tier.codec, len(chars), tier.expected) + ) + sets[tier.key] = chars + for block in BLOCKS: + sets[block.key] = expand_ranges(block.ranges) + return sets + + +# the order tiers are reported in +KEYS = [tier.key for tier in TIERS] + [block.key for block in BLOCKS] + +NAMES = dict([(tier.key, tier.name) for tier in TIERS] + + [(block.key, block.name) for block in BLOCKS]) + +# How many characters each tier holds, so the front end can turn the counts +# this writes into percentages, the way lang_count already does for languages. +# These are also the maximums in fonts-schema.json -- keep the two in step. +TOTALS = dict([(tier.key, tier.expected) for tier in TIERS] + + [(block.key, len(expand_ranges(block.ranges))) for block in BLOCKS]) + +SETS = build_sets() + + +def coverage(font_file): + ''' + {tier key: how many of that tier's characters the font encodes}, leaving out + tiers it has none of. Empty for a font with no CJK at all. + + Counts rather than percentages, to match how "languages" already stores its + data: the totals live in the front end, so a filter threshold can be changed + there without re-running this over every font. + ''' + font = TTFont(font_file, lazy=True) + try: + # getBestCmap() returns None for a font with no Unicode cmap subtable + encoded = set(font.getBestCmap() or {}) + finally: + font.close() + + covered = {} + for key in KEYS: + count = len(encoded & SETS[key]) + if count: + covered[key] = count + return covered + + +def report(font_name, covered): + '''Print one font's coverage, for running this file directly.''' + print('') + print('---------- ' + font_name + ' ----------') + if not covered: + print('no CJK coverage') + return + for key in KEYS: + if key in covered: + print('%-26s %6d / %-6d %5.1f%%' + % (NAMES[key], covered[key], TOTALS[key], + 100 * covered[key] / TOTALS[key])) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser(prog='cjk') + parser.add_argument('--name') + parser.add_argument('--json', action='store_true', + help='print the fonts.json fragment instead of a report') + args = parser.parse_args() + + with open('fonts.json', encoding='utf-8') as user_file: + data = json.load(user_file) + + fragments = {} + checked = 0 + for key in data: + if args.name and key != args.name: + continue + + # the same lookup info.py does; kept here so this file can be run alone + font_file = None + for ext in ['.ttf', '.otf', '.woff', '.woff2']: + candidate = path.join('.', 'fonts', 'resources', key, key + ext) + if path.isfile(candidate): + font_file = candidate + break + + if font_file is None: + continue + + checked += 1 + covered = coverage(font_file) + if covered: + fragments[key] = {'cjk': covered} + if not args.json: + report(key, covered) + + # without this a typo in --name looks exactly like "this font has no CJK" + if args.name and not checked: + raise SystemExit( + "no font file for '%s' under fonts/resources -- check the spelling " + 'against fonts.json, and that the font ships with this repo.' + % args.name + ) + + if args.json: + print(json.dumps(fragments, indent=4, ensure_ascii=False)) diff --git a/cjk_verify.py b/cjk_verify.py new file mode 100644 index 00000000..9dcb2606 --- /dev/null +++ b/cjk_verify.py @@ -0,0 +1,177 @@ +import io +import re +import sys +import urllib.request +import zipfile +from collections import defaultdict + +import cjk + +''' +Check the character sets in cjk.py against Unicode's own data. + +cjk.py builds each tier by decoding the byte ranges a national standard +defines. Those ranges are hardcoded, so this script proves they are right +rather than asking anyone to take them on faith: Unihan records, for every +ideograph, which national standards it came from (UAX #38 kIRG_*Source), and +those two independent routes should agree. + +This is a one-off check, not part of the build -- it downloads ~8MB from +unicode.org. Nothing in the site depends on it. + + python3 cjk_verify.py # download the current Unihan + python3 cjk_verify.py Unihan.zip # or use a local copy + +Last run against Unihan from UCD 17.0 (2025-08-18); see README for the result. +''' + +UNIHAN_URL = 'https://www.unicode.org/Public/UCD/latest/ucd/Unihan.zip' + +''' +How each comparison may legitimately differ: + +identical the two sets must match exactly +unihan_extra everything we claim must be in Unihan, which may hold more +same_size same number of characters, but not necessarily the same + code points +different_standards not the same standard at all; report the overlap and + assert nothing +''' +EXPECTATIONS = ('identical', 'unihan_extra', 'same_size', 'different_standards') + +# Which Unihan source tag corresponds to each tier, and whether the two are +# expected to be identical. Where they are not, the reason is recorded here and +# checked below, so an unexplained difference still shows up as a failure. +COMPARISONS = [ + { + 'tiers': ['gb2312-1', 'gb2312-2'], + 'tag': 'G0', + 'standard': 'GB/T 2312-1980', + 'expect': 'identical', + 'note': '', + }, + { + 'tiers': ['jis0208-1', 'jis0208-2'], + 'tag': 'J0', + 'standard': 'JIS X 0208-1990', + 'expect': 'unihan_extra', + 'note': ('Unihan lists every ideograph in JIS X 0208 including the ones ' + 'outside the two kanji levels; our tiers are levels 1-2 only.'), + }, + { + 'tiers': ['hanja'], + 'tag': 'K0', + 'standard': 'KS X 1001:2004', + 'expect': 'same_size', + 'note': ('Same count, but the two disagree on which compatibility code ' + 'point stands for a couple of hanja.'), + }, + { + 'tiers': ['big5-1', 'big5-2'], + 'tag': 'T1+T2', + 'standard': 'Big5 vs CNS 11643', + 'expect': 'different_standards', + 'note': ('Unihan has no Big5 tag: T1/T2 are CNS 11643 planes 1-2, a ' + 'different (national) standard covering nearly the same ' + 'characters. Overlap is reported instead of equality.'), + }, +] + + +def load_unihan(source): + '''{source tag: set of code points} from Unihan_IRGSources.txt.''' + if source: + raw = open(source, 'rb').read() + else: + print('downloading ' + UNIHAN_URL) + raw = urllib.request.urlopen(UNIHAN_URL).read() + print(' %d bytes' % len(raw)) + + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + text = archive.read('Unihan_IRGSources.txt').decode('utf-8') + + tags = defaultdict(set) + pattern = re.compile(r'^U\+([0-9A-F]+)\tkIRG_[A-Z]Source\t([A-Z0-9]+)-') + for line in text.splitlines(): + found = pattern.match(line) + if found: + tags[found.group(2)].add(int(found.group(1), 16)) + return tags + + +def describe(points, limit=6): + listed = sorted(points)[:limit] + shown = ' '.join('U+%04X %s' % (cp, chr(cp)) for cp in listed) + return shown + (' ...' if len(points) > limit else '') + + +def check(comparison, tags): + ours = set() + for tier in comparison['tiers']: + ours |= cjk.SETS[tier] + + if comparison['tag'] == 'T1+T2': + theirs = tags['T1'] | tags['T2'] + else: + theirs = tags[comparison['tag']] + + print('') + print('%s (%s)' % (comparison['standard'], ' + '.join(comparison['tiers']))) + print(' cjk.py: %6d characters' % len(ours)) + print(' Unihan: %6d characters [%s]' % (len(theirs), comparison['tag'])) + + only_ours = ours - theirs + only_theirs = theirs - ours + expect = comparison['expect'] + + if expect == 'identical': + passed = not only_ours and not only_theirs + elif expect == 'unihan_extra': + # every character we claim must be in Unihan; Unihan may hold more + passed = not only_ours + elif expect == 'same_size': + passed = len(ours) == len(theirs) + elif expect == 'different_standards': + # no equality to assert between two standards, just report the overlap + passed = True + else: + # never fall through to "passed": a typo here would silently disable + # the very check this script exists to perform + raise SystemExit( + "unknown expect value '%s' for %s. Valid values: %s" + % (expect, comparison['standard'], ', '.join(EXPECTATIONS)) + ) + + if only_ours: + print(' only in cjk.py: %d %s' % (len(only_ours), describe(only_ours))) + if only_theirs: + print(' only in Unihan: %d %s' % (len(only_theirs), describe(only_theirs))) + if expect == 'different_standards': + print(' in both: %d' % len(ours & theirs)) + if comparison['note']: + print(' expected: ' + comparison['note']) + + print(' => ' + ('OK' if passed else 'UNEXPECTED DIFFERENCE')) + return passed + + +def main(): + tags = load_unihan(sys.argv[1] if len(sys.argv) > 1 else None) + + print('') + print('Tier sizes built by cjk.py (already checked against each standard)') + for key in cjk.KEYS: + print(' %-26s %6d' % (cjk.NAMES[key], cjk.TOTALS[key])) + + results = [check(comparison, tags) for comparison in COMPARISONS] + + print('') + if all(results): + print('All comparisons matched expectations.') + return 0 + print('Some comparison differed unexpectedly -- see above.') + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/fonts-schema.json b/fonts-schema.json index e58c479f..62da6144 100644 --- a/fonts-schema.json +++ b/fonts-schema.json @@ -94,6 +94,24 @@ "patternProperties": { "^[a-zA-Z]+$": { "type": "integer" } } + }, + "cjk": { + "$comment": "coverage per writing system, written by cjk.py; the maximums are that script's tier totals -- keep the two in step", + "type": "object", + "properties": { + "gb2312-1": { "type": "integer", "minimum": 1, "maximum": 3755 }, + "gb2312-2": { "type": "integer", "minimum": 1, "maximum": 3008 }, + "big5-1": { "type": "integer", "minimum": 1, "maximum": 5401 }, + "big5-2": { "type": "integer", "minimum": 1, "maximum": 7652 }, + "jis0208-1": { "type": "integer", "minimum": 1, "maximum": 2965 }, + "jis0208-2": { "type": "integer", "minimum": 1, "maximum": 3390 }, + "hanja": { "type": "integer", "minimum": 1, "maximum": 4888 }, + "hangul": { "type": "integer", "minimum": 1, "maximum": 11172 }, + "hiragana": { "type": "integer", "minimum": 1, "maximum": 86 }, + "katakana": { "type": "integer", "minimum": 1, "maximum": 90 } + }, + "additionalProperties": false, + "minProperties": 1 } }, "additionalProperties": false, diff --git a/info.py b/info.py index 6eaeb147..8fac67f0 100644 --- a/info.py +++ b/info.py @@ -4,11 +4,17 @@ from hyperglot.checker import FontChecker from fontTools.ttLib import TTFont, woff2 +import cjk + ''' List various metadata about each font. Requires: fonttools - https://github.com/fonttools/fonttools hyperglot - https://github.com/rosettatype/hyperglot brotli (for woff files) - https://github.com/google/brotli + +CJK coverage comes from cjk.py in this repo, which needs nothing beyond +fonttools. Run `python3 cjk.py` on its own to see the numbers per writing +system rather than just the counts written here. ''' # optional --name foo arguments @@ -113,6 +119,21 @@ print(font['maxp'].numGlyphs) data[key]['glyphs'] = int(font['maxp'].numGlyphs) + ''' + How much of each East Asian writing system this font covers, as a count + per tier. Hyperglot is all-or-nothing per language, which for CJK means + a font missing a couple of rare marks reports no coverage at all, so + these are measured separately. fontTools reads woff2 directly, so unlike + the language check below this needs no decompressed copy. + ''' + covered = cjk.coverage(font_file) + if covered: + print('cjk:', covered) + data[key]['cjk'] = covered + elif 'cjk' in data[key]: + # the font was replaced by one without CJK; don't leave stale data + del data[key]['cjk'] + ''' name table: https://learn.microsoft.com/en-us/typography/opentype/spec/name