diff --git a/README.md b/README.md index 0c28a75..2a06859 100755 --- a/README.md +++ b/README.md @@ -35,6 +35,70 @@ The glyphs, characters and languages data can be added to the JSON by running th python3 info.py --name yourfont ``` +### CJK coverage + +`info.py` also records how much of each East Asian writing system a font covers, +in a `cjk` object. This is measured separately from the `languages` data because +Hyperglot treats a language as covered only at 100%, which for CJK is +misleading: M PLUS 1 Code has every kanji and every hiragana on Hyperglot's +Japanese list and all but two of its katakana — the two being rare marks almost +nobody types — so it reports no Japanese support at all. + +Each tier is stored as a **count**, not a percentage, matching how `languages` +works — the totals live in the front end, so a filter threshold can be changed +without re-running the script over every font. + +| key | what it counts | total | standard | +|---|---|---|---| +| `gb2312-1` | everyday Simplified Chinese hanzi | 3,755 | GB/T 2312-1980 level 1 | +| `gb2312-2` | rarer hanzi: names, places, classical | 3,008 | GB/T 2312-1980 level 2 | +| `big5-1` | everyday Traditional Chinese hanzi | 5,401 | Big5 common | +| `big5-2` | rarer hanzi | 7,652 | Big5 less common | +| `jis0208-1` | everyday Japanese kanji | 2,965 | JIS X 0208 level 1 | +| `jis0208-2` | rarer kanji | 3,390 | JIS X 0208 level 2 | +| `hanja` | Korean hanja | 4,888 | KS X 1001:2004 | +| `hangul` | modern Hangul syllables | 11,172 | U+AC00..U+D7A3 | +| `hiragana` | hiragana | 86 | U+3041..U+3096 | +| `katakana` | katakana | 90 | U+30A1..U+30FA | + +Hiragana and katakana are counted separately because fonts ship one without the +other: Cartograph has all 90 katakana and no hiragana, which a combined number +would report as a meaningless 51%. + +#### Where these numbers come from + +The four Han tiers are not Unicode blocks — "CJK Unified Ideographs" and its +extensions run to about 100,000 code points, most of which nobody types, so +measuring against them makes even Unifont look like it covers 20% of CJK. +Instead each tier is the set of characters a national standard defines, which +`cjk.py` enumerates by decoding that standard's byte ranges with the matching +codec from Python's standard library. No extra dependency and no data file to +keep up to date. + +The byte ranges are hardcoded, so `cjk_verify.py` proves they are right rather +than asking anyone to take them on trust. Unihan records, for every ideograph, +which national standards it came from +([UAX #38](https://www.unicode.org/reports/tr38/) `kIRG_*Source`), so the sets +can be rebuilt from Unicode's own data and compared: + +```sh +python3 cjk_verify.py # downloads ~8MB from unicode.org +``` + +Last run against UCD 17.0 (Unihan of 2025-08-18): + +| tier | vs Unihan | result | +|---|---|---| +| GB/T 2312 | `G0`, 6,763 | identical | +| JIS X 0208 | `J0`, 6,356 | Unihan has one extra, 仝 (U+4EDD), which sits in JIS X 0208's symbol rows rather than the two kanji levels | +| KS X 1001 | `K0`, 4,888 | same count; the two pick different compatibility code points for 郎/郞 and 隸/隷 | +| Big5 | `T1`+`T2`, 13,064 | 13,049 in common. Unihan has no Big5 tag — `T1`/`T2` are CNS 11643 planes 1 and 2, a different standard covering nearly the same characters | + +A count says whether a font *can* set text in a language, not whether it does so +well. Han characters are shared between these standards but drawn to different +regional conventions, and only the `cmap` is inspected here, so a font can cover +Big5 with glyphs drawn to Japanese conventions. + ### Development - Running `make` installs dependencies, lints, validates `fonts.json` (against [fonts-schema.json](https://github.com/braver/programmingfonts/blob/gh-pages/fonts-schema.json) and `validate.js`), and builds the stylesheet. diff --git a/cjk.py b/cjk.py new file mode 100644 index 0000000..fc07874 --- /dev/null +++ b/cjk.py @@ -0,0 +1,221 @@ +import json +from collections import namedtuple +from os import path + +from fontTools.ttLib import TTFont + +''' +Measure CJK coverage: how much of each East Asian writing system a font +actually covers. See fonts-schema.json for the shape this writes, and the +README for where the numbers come from. + +Requires: +fonttools - https://github.com/fonttools/fonttools +brotli (for woff files) - https://github.com/google/brotli + +Both are already in requirements.txt; this module adds no new dependency. +Run cjk_verify.py to re-derive the byte ranges below from Unicode's own data. +''' + + +''' +Each tier is one national standard, enumerated by decoding the byte ranges that +standard defines with the matching stdlib codec, and keeping the ideographs. + +expected: the character count the standard specifies. It is checked when the +sets are built, so a codec change in a future Python release fails loudly +instead of silently shifting every percentage on the site. These same numbers +are the maximums in fonts-schema.json; change one and you must change both. + +irg: the source tag Unicode itself uses for the same standard in Unihan +(UAX #38, https://www.unicode.org/reports/tr38/). cjk_verify.py checks these +sets against that data, so nothing here rests on trusting this comment. + +Han tiers are split into level 1 (everyday text) and level 2 (names, place +names, classical text): a font with all of level 1 and none of level 2 is +perfectly usable for reading, and a single number would hide that. +''' +Tier = namedtuple('Tier', 'key name codec highs lows expected irg') + +EUC_LOW = [(0xA1, 0xFE)] +BIG5_LOW = [(0x40, 0x7E), (0xA1, 0xFE)] + +TIERS = [ + Tier('gb2312-1', 'GB/T 2312-1980 level 1', 'gb2312', [(0xB0, 0xD7)], EUC_LOW, 3755, 'G0'), + Tier('gb2312-2', 'GB/T 2312-1980 level 2', 'gb2312', [(0xD8, 0xF7)], EUC_LOW, 3008, 'G0'), + Tier('big5-1', 'Big5 common', 'big5', [(0xA4, 0xC6)], BIG5_LOW, 5401, 'T1'), + Tier('big5-2', 'Big5 less common', 'big5', [(0xC9, 0xF9)], BIG5_LOW, 7652, 'T2'), + Tier('jis0208-1', 'JIS X 0208 level 1', 'euc_jp', [(0xB0, 0xCF)], EUC_LOW, 2965, 'J0'), + Tier('jis0208-2', 'JIS X 0208 level 2', 'euc_jp', [(0xD0, 0xF4)], EUC_LOW, 3390, 'J0'), + Tier('hanja', 'KS X 1001:2004 hanja', 'euc_kr', [(0xCA, 0xFD)], EUC_LOW, 4888, 'K0'), +] + +''' +Syllabaries and Hangul are complete blocks rather than a selection out of a +larger standard, so the Unicode block IS the practical set: every one of these +is reachable from a keyboard. + +Hiragana and katakana stay separate because fonts really do ship one without +the other -- Cartograph has all 90 katakana and no hiragana -- and averaging +the two hides exactly that. +''' +Block = namedtuple('Block', 'key name ranges') + +BLOCKS = [ + Block('hangul', 'modern Hangul syllables', [(0xAC00, 0xD7A3)]), + Block('hiragana', 'Hiragana', [(0x3041, 0x3096)]), + Block('katakana', 'Katakana', [(0x30A1, 0x30FA)]), +] + + +def is_ideograph(cp): + ''' + Han ranges the legacy codecs above can produce: unified, Ext A, + compatibility and Ext B. Ext C and later are deliberately absent -- no + legacy codec encodes them. + ''' + return (0x3400 <= cp <= 0x4DBF or 0x4E00 <= cp <= 0x9FFF + or 0xF900 <= cp <= 0xFAFF or 0x20000 <= cp <= 0x2A6DF) + + +def decode_tier(codec, high_ranges, low_ranges): + '''Every ideograph reachable in these byte ranges of a legacy codec.''' + found = set() + for high_start, high_end in high_ranges: + for high in range(high_start, high_end + 1): + for low_start, low_end in low_ranges: + for low in range(low_start, low_end + 1): + try: + char = bytes((high, low)).decode(codec) + except UnicodeDecodeError: + continue + if len(char) == 1 and is_ideograph(ord(char)): + found.add(ord(char)) + return found + + +def expand_ranges(ranges): + '''Every code point in a list of inclusive (first, last) pairs.''' + found = set() + for start, end in ranges: + found.update(range(start, end + 1)) + return found + + +def build_sets(): + '''{key: set of code points}, each validated against its standard.''' + sets = {} + for tier in TIERS: + chars = decode_tier(tier.codec, tier.highs, tier.lows) + if len(chars) != tier.expected: + raise SystemExit( + '%s: the %s codec gives %d ideographs, the standard defines %d. ' + "Python's codec tables changed; check the byte ranges in cjk.py " + 'before trusting any coverage number.' + % (tier.name, tier.codec, len(chars), tier.expected) + ) + sets[tier.key] = chars + for block in BLOCKS: + sets[block.key] = expand_ranges(block.ranges) + return sets + + +# the order tiers are reported in +KEYS = [tier.key for tier in TIERS] + [block.key for block in BLOCKS] + +NAMES = dict([(tier.key, tier.name) for tier in TIERS] + + [(block.key, block.name) for block in BLOCKS]) + +# How many characters each tier holds, so the front end can turn the counts +# this writes into percentages, the way lang_count already does for languages. +# These are also the maximums in fonts-schema.json -- keep the two in step. +TOTALS = dict([(tier.key, tier.expected) for tier in TIERS] + + [(block.key, len(expand_ranges(block.ranges))) for block in BLOCKS]) + +SETS = build_sets() + + +def coverage(font_file): + ''' + {tier key: how many of that tier's characters the font encodes}, leaving out + tiers it has none of. Empty for a font with no CJK at all. + + Counts rather than percentages, to match how "languages" already stores its + data: the totals live in the front end, so a filter threshold can be changed + there without re-running this over every font. + ''' + font = TTFont(font_file, lazy=True) + try: + # getBestCmap() returns None for a font with no Unicode cmap subtable + encoded = set(font.getBestCmap() or {}) + finally: + font.close() + + covered = {} + for key in KEYS: + count = len(encoded & SETS[key]) + if count: + covered[key] = count + return covered + + +def report(font_name, covered): + '''Print one font's coverage, for running this file directly.''' + print('') + print('---------- ' + font_name + ' ----------') + if not covered: + print('no CJK coverage') + return + for key in KEYS: + if key in covered: + print('%-26s %6d / %-6d %5.1f%%' + % (NAMES[key], covered[key], TOTALS[key], + 100 * covered[key] / TOTALS[key])) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser(prog='cjk') + parser.add_argument('--name') + parser.add_argument('--json', action='store_true', + help='print the fonts.json fragment instead of a report') + args = parser.parse_args() + + with open('fonts.json', encoding='utf-8') as user_file: + data = json.load(user_file) + + fragments = {} + checked = 0 + for key in data: + if args.name and key != args.name: + continue + + # the same lookup info.py does; kept here so this file can be run alone + font_file = None + for ext in ['.ttf', '.otf', '.woff', '.woff2']: + candidate = path.join('.', 'fonts', 'resources', key, key + ext) + if path.isfile(candidate): + font_file = candidate + break + + if font_file is None: + continue + + checked += 1 + covered = coverage(font_file) + if covered: + fragments[key] = {'cjk': covered} + if not args.json: + report(key, covered) + + # without this a typo in --name looks exactly like "this font has no CJK" + if args.name and not checked: + raise SystemExit( + "no font file for '%s' under fonts/resources -- check the spelling " + 'against fonts.json, and that the font ships with this repo.' + % args.name + ) + + if args.json: + print(json.dumps(fragments, indent=4, ensure_ascii=False)) diff --git a/cjk_verify.py b/cjk_verify.py new file mode 100644 index 0000000..9dcb260 --- /dev/null +++ b/cjk_verify.py @@ -0,0 +1,177 @@ +import io +import re +import sys +import urllib.request +import zipfile +from collections import defaultdict + +import cjk + +''' +Check the character sets in cjk.py against Unicode's own data. + +cjk.py builds each tier by decoding the byte ranges a national standard +defines. Those ranges are hardcoded, so this script proves they are right +rather than asking anyone to take them on faith: Unihan records, for every +ideograph, which national standards it came from (UAX #38 kIRG_*Source), and +those two independent routes should agree. + +This is a one-off check, not part of the build -- it downloads ~8MB from +unicode.org. Nothing in the site depends on it. + + python3 cjk_verify.py # download the current Unihan + python3 cjk_verify.py Unihan.zip # or use a local copy + +Last run against Unihan from UCD 17.0 (2025-08-18); see README for the result. +''' + +UNIHAN_URL = 'https://www.unicode.org/Public/UCD/latest/ucd/Unihan.zip' + +''' +How each comparison may legitimately differ: + +identical the two sets must match exactly +unihan_extra everything we claim must be in Unihan, which may hold more +same_size same number of characters, but not necessarily the same + code points +different_standards not the same standard at all; report the overlap and + assert nothing +''' +EXPECTATIONS = ('identical', 'unihan_extra', 'same_size', 'different_standards') + +# Which Unihan source tag corresponds to each tier, and whether the two are +# expected to be identical. Where they are not, the reason is recorded here and +# checked below, so an unexplained difference still shows up as a failure. +COMPARISONS = [ + { + 'tiers': ['gb2312-1', 'gb2312-2'], + 'tag': 'G0', + 'standard': 'GB/T 2312-1980', + 'expect': 'identical', + 'note': '', + }, + { + 'tiers': ['jis0208-1', 'jis0208-2'], + 'tag': 'J0', + 'standard': 'JIS X 0208-1990', + 'expect': 'unihan_extra', + 'note': ('Unihan lists every ideograph in JIS X 0208 including the ones ' + 'outside the two kanji levels; our tiers are levels 1-2 only.'), + }, + { + 'tiers': ['hanja'], + 'tag': 'K0', + 'standard': 'KS X 1001:2004', + 'expect': 'same_size', + 'note': ('Same count, but the two disagree on which compatibility code ' + 'point stands for a couple of hanja.'), + }, + { + 'tiers': ['big5-1', 'big5-2'], + 'tag': 'T1+T2', + 'standard': 'Big5 vs CNS 11643', + 'expect': 'different_standards', + 'note': ('Unihan has no Big5 tag: T1/T2 are CNS 11643 planes 1-2, a ' + 'different (national) standard covering nearly the same ' + 'characters. Overlap is reported instead of equality.'), + }, +] + + +def load_unihan(source): + '''{source tag: set of code points} from Unihan_IRGSources.txt.''' + if source: + raw = open(source, 'rb').read() + else: + print('downloading ' + UNIHAN_URL) + raw = urllib.request.urlopen(UNIHAN_URL).read() + print(' %d bytes' % len(raw)) + + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + text = archive.read('Unihan_IRGSources.txt').decode('utf-8') + + tags = defaultdict(set) + pattern = re.compile(r'^U\+([0-9A-F]+)\tkIRG_[A-Z]Source\t([A-Z0-9]+)-') + for line in text.splitlines(): + found = pattern.match(line) + if found: + tags[found.group(2)].add(int(found.group(1), 16)) + return tags + + +def describe(points, limit=6): + listed = sorted(points)[:limit] + shown = ' '.join('U+%04X %s' % (cp, chr(cp)) for cp in listed) + return shown + (' ...' if len(points) > limit else '') + + +def check(comparison, tags): + ours = set() + for tier in comparison['tiers']: + ours |= cjk.SETS[tier] + + if comparison['tag'] == 'T1+T2': + theirs = tags['T1'] | tags['T2'] + else: + theirs = tags[comparison['tag']] + + print('') + print('%s (%s)' % (comparison['standard'], ' + '.join(comparison['tiers']))) + print(' cjk.py: %6d characters' % len(ours)) + print(' Unihan: %6d characters [%s]' % (len(theirs), comparison['tag'])) + + only_ours = ours - theirs + only_theirs = theirs - ours + expect = comparison['expect'] + + if expect == 'identical': + passed = not only_ours and not only_theirs + elif expect == 'unihan_extra': + # every character we claim must be in Unihan; Unihan may hold more + passed = not only_ours + elif expect == 'same_size': + passed = len(ours) == len(theirs) + elif expect == 'different_standards': + # no equality to assert between two standards, just report the overlap + passed = True + else: + # never fall through to "passed": a typo here would silently disable + # the very check this script exists to perform + raise SystemExit( + "unknown expect value '%s' for %s. Valid values: %s" + % (expect, comparison['standard'], ', '.join(EXPECTATIONS)) + ) + + if only_ours: + print(' only in cjk.py: %d %s' % (len(only_ours), describe(only_ours))) + if only_theirs: + print(' only in Unihan: %d %s' % (len(only_theirs), describe(only_theirs))) + if expect == 'different_standards': + print(' in both: %d' % len(ours & theirs)) + if comparison['note']: + print(' expected: ' + comparison['note']) + + print(' => ' + ('OK' if passed else 'UNEXPECTED DIFFERENCE')) + return passed + + +def main(): + tags = load_unihan(sys.argv[1] if len(sys.argv) > 1 else None) + + print('') + print('Tier sizes built by cjk.py (already checked against each standard)') + for key in cjk.KEYS: + print(' %-26s %6d' % (cjk.NAMES[key], cjk.TOTALS[key])) + + results = [check(comparison, tags) for comparison in COMPARISONS] + + print('') + if all(results): + print('All comparisons matched expectations.') + return 0 + print('Some comparison differed unexpectedly -- see above.') + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/fonts-schema.json b/fonts-schema.json index e58c479..62da614 100644 --- a/fonts-schema.json +++ b/fonts-schema.json @@ -94,6 +94,24 @@ "patternProperties": { "^[a-zA-Z]+$": { "type": "integer" } } + }, + "cjk": { + "$comment": "coverage per writing system, written by cjk.py; the maximums are that script's tier totals -- keep the two in step", + "type": "object", + "properties": { + "gb2312-1": { "type": "integer", "minimum": 1, "maximum": 3755 }, + "gb2312-2": { "type": "integer", "minimum": 1, "maximum": 3008 }, + "big5-1": { "type": "integer", "minimum": 1, "maximum": 5401 }, + "big5-2": { "type": "integer", "minimum": 1, "maximum": 7652 }, + "jis0208-1": { "type": "integer", "minimum": 1, "maximum": 2965 }, + "jis0208-2": { "type": "integer", "minimum": 1, "maximum": 3390 }, + "hanja": { "type": "integer", "minimum": 1, "maximum": 4888 }, + "hangul": { "type": "integer", "minimum": 1, "maximum": 11172 }, + "hiragana": { "type": "integer", "minimum": 1, "maximum": 86 }, + "katakana": { "type": "integer", "minimum": 1, "maximum": 90 } + }, + "additionalProperties": false, + "minProperties": 1 } }, "additionalProperties": false, diff --git a/info.py b/info.py index 6eaeb14..8fac67f 100644 --- a/info.py +++ b/info.py @@ -4,11 +4,17 @@ from hyperglot.checker import FontChecker from fontTools.ttLib import TTFont, woff2 +import cjk + ''' List various metadata about each font. Requires: fonttools - https://github.com/fonttools/fonttools hyperglot - https://github.com/rosettatype/hyperglot brotli (for woff files) - https://github.com/google/brotli + +CJK coverage comes from cjk.py in this repo, which needs nothing beyond +fonttools. Run `python3 cjk.py` on its own to see the numbers per writing +system rather than just the counts written here. ''' # optional --name foo arguments @@ -113,6 +119,21 @@ print(font['maxp'].numGlyphs) data[key]['glyphs'] = int(font['maxp'].numGlyphs) + ''' + How much of each East Asian writing system this font covers, as a count + per tier. Hyperglot is all-or-nothing per language, which for CJK means + a font missing a couple of rare marks reports no coverage at all, so + these are measured separately. fontTools reads woff2 directly, so unlike + the language check below this needs no decompressed copy. + ''' + covered = cjk.coverage(font_file) + if covered: + print('cjk:', covered) + data[key]['cjk'] = covered + elif 'cjk' in data[key]: + # the font was replaced by one without CJK; don't leave stale data + del data[key]['cjk'] + ''' name table: https://learn.microsoft.com/en-us/typography/opentype/spec/name