| #!/usr/bin/env python |
| # -*- coding: utf-8 -*- |
| # |
| # This Source Code Form is subject to the terms of the Mozilla Public |
| # License, v. 2.0. If a copy of the MPL was not distributed with this |
| # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| |
| """ Usage: make_intl_data.py [language-subtag-registry.txt] |
| |
| This script extracts information about mappings between deprecated and |
| current BCP 47 language tags from the IANA Language Subtag Registry and |
| converts it to JavaScript object definitions in IntlData.js. The definitions |
| are used in Intl.js. |
| |
| The IANA Language Subtag Registry is imported from |
| http://www.iana.org/assignments/language-subtag-registry |
| and uses the syntax specified in |
| http://tools.ietf.org/html/rfc5646#section-3 |
| """ |
| |
| def readRegistryRecord(registry): |
| """ Yields the records of the IANA Language Subtag Registry as dictionaries. """ |
| record = {} |
| for line in registry: |
| line = line.strip() |
| if line == "": |
| continue |
| if line == "%%": |
| yield record |
| record = {} |
| else: |
| if ":" in line: |
| key, value = line.split(":", 1) |
| key, value = key.strip(), value.strip() |
| record[key] = value |
| else: |
| # continuation line |
| record[key] += " " + line |
| if record: |
| yield record |
| return |
| |
| |
| def readRegistry(registry): |
| """ Reads IANA Language Subtag Registry and extracts information for Intl.js. |
| |
| Information extracted: |
| - langTagMappings: mappings from complete language tags to preferred |
| complete language tags |
| - langSubtagMappings: mappings from subtags to preferred subtags |
| - extlangMappings: mappings from extlang subtags to preferred subtags, |
| with prefix to be removed |
| Returns these three mappings as dictionaries, along with the registry's |
| file date. |
| |
| We also check that mappings for language subtags don't affect extlang |
| subtags and vice versa, so that CanonicalizeLanguageTag doesn't have |
| to separate them for processing. Region codes are separated by case, |
| and script codes by length, so they're unproblematic. |
| """ |
| langTagMappings = {} |
| langSubtagMappings = {} |
| extlangMappings = {} |
| languageSubtags = set() |
| extlangSubtags = set() |
| |
| for record in readRegistryRecord(registry): |
| if "File-Date" in record: |
| fileDate = record["File-Date"] |
| continue |
| |
| if record["Type"] == "grandfathered": |
| # Grandfathered tags don't use standard syntax, so |
| # CanonicalizeLanguageTag expects the mapping table to provide |
| # the final form for all. |
| # For langTagMappings, keys must be in lower case; values in |
| # the case used in the registry. |
| tag = record["Tag"] |
| if "Preferred-Value" in record: |
| langTagMappings[tag.lower()] = record["Preferred-Value"] |
| else: |
| langTagMappings[tag.lower()] = tag |
| elif record["Type"] == "redundant": |
| # For langTagMappings, keys must be in lower case; values in |
| # the case used in the registry. |
| if "Preferred-Value" in record: |
| langTagMappings[record["Tag"].lower()] = record["Preferred-Value"] |
| elif record["Type"] in ("language", "script", "region", "variant"): |
| # For langSubtagMappings, keys and values must be in the case used |
| # in the registry. |
| subtag = record["Subtag"] |
| if record["Type"] == "language": |
| languageSubtags.add(subtag) |
| if "Preferred-Value" in record: |
| if subtag == "heploc": |
| # The entry for heploc is unique in its complexity; handle |
| # it as special case below. |
| continue |
| if "Prefix" in record: |
| # This might indicate another heploc-like complex case. |
| raise Exception("Please evaluate: subtag mapping with prefix value.") |
| langSubtagMappings[subtag] = record["Preferred-Value"] |
| elif record["Type"] == "extlang": |
| # For extlangMappings, keys must be in the case used in the |
| # registry; values are records with the preferred value and the |
| # prefix to be removed. |
| subtag = record["Subtag"] |
| extlangSubtags.add(subtag) |
| if "Preferred-Value" in record: |
| preferred = record["Preferred-Value"] |
| prefix = record["Prefix"] |
| extlangMappings[subtag] = {"preferred": preferred, "prefix": prefix} |
| else: |
| # No other types are allowed by |
| # http://tools.ietf.org/html/rfc5646#section-3.1.3 |
| assert False, "Unrecognized Type: {0}".format(record["Type"]) |
| |
| # Check that mappings for language subtags and extlang subtags don't affect |
| # each other. |
| for lang in languageSubtags: |
| if lang in extlangMappings and extlangMappings[lang]["preferred"] != lang: |
| raise Exception("Conflict: lang with extlang mapping: " + lang) |
| for extlang in extlangSubtags: |
| if extlang in langSubtagMappings: |
| raise Exception("Conflict: extlang with lang mapping: " + extlang) |
| |
| # Special case for heploc. |
| langTagMappings["ja-latn-hepburn-heploc"] = "ja-Latn-alalc97" |
| |
| return {"fileDate": fileDate, |
| "langTagMappings": langTagMappings, |
| "langSubtagMappings": langSubtagMappings, |
| "extlangMappings": extlangMappings} |
| |
| |
| def writeMappingsVar(intlData, dict, name, description, fileDate, url): |
| """ Writes a variable definition with a mapping table to file intlData. |
| |
| Writes the contents of dictionary dict to file intlData with the given |
| variable name and a comment with description, fileDate, and URL. |
| """ |
| intlData.write("\n") |
| intlData.write("// {0}.\n".format(description)) |
| intlData.write("// Derived from IANA Language Subtag Registry, file date {0}.\n".format(fileDate)) |
| intlData.write("// {0}\n".format(url)) |
| intlData.write("var {0} = {{\n".format(name)) |
| keys = sorted(dict) |
| for key in keys: |
| if isinstance(dict[key], basestring): |
| value = '"{0}"'.format(dict[key]) |
| else: |
| preferred = dict[key]["preferred"] |
| prefix = dict[key]["prefix"] |
| value = '{{preferred: "{0}", prefix: "{1}"}}'.format(preferred, prefix) |
| intlData.write(' "{0}": {1},\n'.format(key, value)) |
| intlData.write("};\n") |
| |
| |
| def writeLanguageTagData(intlData, fileDate, url, langTagMappings, langSubtagMappings, extlangMappings): |
| """ Writes the language tag data to the Intl data file. """ |
| writeMappingsVar(intlData, langTagMappings, "langTagMappings", |
| "Mappings from complete tags to preferred values", fileDate, url) |
| writeMappingsVar(intlData, langSubtagMappings, "langSubtagMappings", |
| "Mappings from non-extlang subtags to preferred values", fileDate, url) |
| writeMappingsVar(intlData, extlangMappings, "extlangMappings", |
| "Mappings from extlang subtags to preferred values", fileDate, url) |
| |
| |
| if __name__ == '__main__': |
| import codecs |
| import sys |
| import urllib2 |
| |
| url = "http://www.iana.org/assignments/language-subtag-registry" |
| if len(sys.argv) > 1: |
| print("Always make sure you have the newest language-subtag-registry.txt!") |
| registry = codecs.open(sys.argv[1], "r", encoding="utf-8") |
| else: |
| print("Downloading IANA Language Subtag Registry...") |
| reader = urllib2.urlopen(url) |
| text = reader.read().decode("utf-8") |
| reader.close() |
| registry = codecs.open("language-subtag-registry.txt", "w+", encoding="utf-8") |
| registry.write(text) |
| registry.seek(0) |
| |
| print("Processing IANA Language Subtag Registry...") |
| data = readRegistry(registry) |
| fileDate = data["fileDate"] |
| langTagMappings = data["langTagMappings"] |
| langSubtagMappings = data["langSubtagMappings"] |
| extlangMappings = data["extlangMappings"] |
| registry.close() |
| |
| print("Writing Intl data...") |
| intlData = codecs.open("IntlData.js", "w", encoding="utf-8") |
| intlData.write("// Generated by make_intl_data.py. DO NOT EDIT.\n") |
| writeLanguageTagData(intlData, fileDate, url, langTagMappings, langSubtagMappings, extlangMappings) |
| intlData.close() |
