--- /dev/null
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Collation table generator.
+// Data read from the web.
+
+package main
+
+import (
+ "archive/zip"
+ "bufio"
+ "bytes"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "os"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+ "unicode/utf8"
+
+ "golang.org/x/text/collate"
+ "golang.org/x/text/collate/build"
+ "golang.org/x/text/internal/colltab"
+ "golang.org/x/text/internal/gen"
+ "golang.org/x/text/language"
+ "golang.org/x/text/unicode/cldr"
+)
+
+var (
+ test = flag.Bool("test", false,
+ "test existing tables; can be used to compare web data with package data.")
+ short = flag.Bool("short", false, `Use "short" alternatives, when available.`)
+ draft = flag.Bool("draft", false, `Use draft versions, when available.`)
+ tags = flag.String("tags", "", "build tags to be included after +build directive")
+ pkg = flag.String("package", "collate",
+ "the name of the package in which the generated file is to be included")
+
+ tables = flagStringSetAllowAll("tables", "collate", "collate,chars",
+ "comma-spearated list of tables to generate.")
+ exclude = flagStringSet("exclude", "zh2", "",
+ "comma-separated list of languages to exclude.")
+ include = flagStringSet("include", "", "",
+ "comma-separated list of languages to include. Include trumps exclude.")
+ // TODO: Not included: unihan gb2312han zhuyin big5han (for size reasons)
+ // TODO: Not included: traditional (buggy for Bengali)
+ types = flagStringSetAllowAll("types", "standard,phonebook,phonetic,reformed,pinyin,stroke", "",
+ "comma-separated list of types that should be included.")
+)
+
+// stringSet implements an ordered set based on a list. It implements flag.Value
+// to allow a set to be specified as a comma-separated list.
+type stringSet struct {
+ s []string
+ allowed *stringSet
+ dirty bool // needs compaction if true
+ all bool
+ allowAll bool
+}
+
+func flagStringSet(name, def, allowed, usage string) *stringSet {
+ ss := &stringSet{}
+ if allowed != "" {
+ usage += fmt.Sprintf(" (allowed values: any of %s)", allowed)
+ ss.allowed = &stringSet{}
+ failOnError(ss.allowed.Set(allowed))
+ }
+ ss.Set(def)
+ flag.Var(ss, name, usage)
+ return ss
+}
+
+func flagStringSetAllowAll(name, def, allowed, usage string) *stringSet {
+ ss := &stringSet{allowAll: true}
+ if allowed == "" {
+ flag.Var(ss, name, usage+fmt.Sprintf(` Use "all" to select all.`))
+ } else {
+ ss.allowed = &stringSet{}
+ failOnError(ss.allowed.Set(allowed))
+ flag.Var(ss, name, usage+fmt.Sprintf(` (allowed values: "all" or any of %s)`, allowed))
+ }
+ ss.Set(def)
+ return ss
+}
+
+func (ss stringSet) Len() int {
+ return len(ss.s)
+}
+
+func (ss stringSet) String() string {
+ return strings.Join(ss.s, ",")
+}
+
+func (ss *stringSet) Set(s string) error {
+ if ss.allowAll && s == "all" {
+ ss.s = nil
+ ss.all = true
+ return nil
+ }
+ ss.s = ss.s[:0]
+ for _, s := range strings.Split(s, ",") {
+ if s := strings.TrimSpace(s); s != "" {
+ if ss.allowed != nil && !ss.allowed.contains(s) {
+ return fmt.Errorf("unsupported value %q; must be one of %s", s, ss.allowed)
+ }
+ ss.add(s)
+ }
+ }
+ ss.compact()
+ return nil
+}
+
+func (ss *stringSet) add(s string) {
+ ss.s = append(ss.s, s)
+ ss.dirty = true
+}
+
+func (ss *stringSet) values() []string {
+ ss.compact()
+ return ss.s
+}
+
+func (ss *stringSet) contains(s string) bool {
+ if ss.all {
+ return true
+ }
+ for _, v := range ss.s {
+ if v == s {
+ return true
+ }
+ }
+ return false
+}
+
+func (ss *stringSet) compact() {
+ if !ss.dirty {
+ return
+ }
+ a := ss.s
+ sort.Strings(a)
+ k := 0
+ for i := 1; i < len(a); i++ {
+ if a[k] != a[i] {
+ a[k+1] = a[i]
+ k++
+ }
+ }
+ ss.s = a[:k+1]
+ ss.dirty = false
+}
+
+func skipLang(l string) bool {
+ if include.Len() > 0 {
+ return !include.contains(l)
+ }
+ return exclude.contains(l)
+}
+
+// altInclude returns a list of alternatives (for the LDML alt attribute)
+// in order of preference. An empty string in this list indicates the
+// default entry.
+func altInclude() []string {
+ l := []string{}
+ if *short {
+ l = append(l, "short")
+ }
+ l = append(l, "")
+ // TODO: handle draft using cldr.SetDraftLevel
+ if *draft {
+ l = append(l, "proposed")
+ }
+ return l
+}
+
+func failOnError(e error) {
+ if e != nil {
+ log.Panic(e)
+ }
+}
+
+func openArchive() *zip.Reader {
+ f := gen.OpenCLDRCoreZip()
+ buffer, err := ioutil.ReadAll(f)
+ f.Close()
+ failOnError(err)
+ archive, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
+ failOnError(err)
+ return archive
+}
+
+// parseUCA parses a Default Unicode Collation Element Table of the format
+// specified in https://www.unicode.org/reports/tr10/#File_Format.
+// It returns the variable top.
+func parseUCA(builder *build.Builder) {
+ var r io.ReadCloser
+ var err error
+ for _, f := range openArchive().File {
+ if strings.HasSuffix(f.Name, "allkeys_CLDR.txt") {
+ r, err = f.Open()
+ }
+ }
+ if r == nil {
+ log.Fatal("File allkeys_CLDR.txt not found in archive.")
+ }
+ failOnError(err)
+ defer r.Close()
+ scanner := bufio.NewScanner(r)
+ colelem := regexp.MustCompile(`\[([.*])([0-9A-F.]+)\]`)
+ for i := 1; scanner.Scan(); i++ {
+ line := scanner.Text()
+ if len(line) == 0 || line[0] == '#' {
+ continue
+ }
+ if line[0] == '@' {
+ // parse properties
+ switch {
+ case strings.HasPrefix(line[1:], "version "):
+ a := strings.Split(line[1:], " ")
+ if a[1] != gen.UnicodeVersion() {
+ log.Fatalf("incompatible version %s; want %s", a[1], gen.UnicodeVersion())
+ }
+ case strings.HasPrefix(line[1:], "backwards "):
+ log.Fatalf("%d: unsupported option backwards", i)
+ default:
+ log.Printf("%d: unknown option %s", i, line[1:])
+ }
+ } else {
+ // parse entries
+ part := strings.Split(line, " ; ")
+ if len(part) != 2 {
+ log.Fatalf("%d: production rule without ';': %v", i, line)
+ }
+ lhs := []rune{}
+ for _, v := range strings.Split(part[0], " ") {
+ if v == "" {
+ continue
+ }
+ lhs = append(lhs, rune(convHex(i, v)))
+ }
+ var n int
+ var vars []int
+ rhs := [][]int{}
+ for i, m := range colelem.FindAllStringSubmatch(part[1], -1) {
+ n += len(m[0])
+ elem := []int{}
+ for _, h := range strings.Split(m[2], ".") {
+ elem = append(elem, convHex(i, h))
+ }
+ if m[1] == "*" {
+ vars = append(vars, i)
+ }
+ rhs = append(rhs, elem)
+ }
+ if len(part[1]) < n+3 || part[1][n+1] != '#' {
+ log.Fatalf("%d: expected comment; found %s", i, part[1][n:])
+ }
+ if *test {
+ testInput.add(string(lhs))
+ }
+ failOnError(builder.Add(lhs, rhs, vars))
+ }
+ }
+ if scanner.Err() != nil {
+ log.Fatal(scanner.Err())
+ }
+}
+
+func convHex(line int, s string) int {
+ r, e := strconv.ParseInt(s, 16, 32)
+ if e != nil {
+ log.Fatalf("%d: %v", line, e)
+ }
+ return int(r)
+}
+
+var testInput = stringSet{}
+
+var charRe = regexp.MustCompile(`&#x([0-9A-F]*);`)
+var tagRe = regexp.MustCompile(`<([a-z_]*) */>`)
+
+var mainLocales = []string{}
+
+// charsets holds a list of exemplar characters per category.
+type charSets map[string][]string
+
+func (p charSets) fprint(w io.Writer) {
+ fmt.Fprintln(w, "[exN]string{")
+ for i, k := range []string{"", "contractions", "punctuation", "auxiliary", "currencySymbol", "index"} {
+ if set := p[k]; len(set) != 0 {
+ fmt.Fprintf(w, "\t\t%d: %q,\n", i, strings.Join(set, " "))
+ }
+ }
+ fmt.Fprintln(w, "\t},")
+}
+
+var localeChars = make(map[string]charSets)
+
+const exemplarHeader = `
+type exemplarType int
+const (
+ exCharacters exemplarType = iota
+ exContractions
+ exPunctuation
+ exAuxiliary
+ exCurrency
+ exIndex
+ exN
+)
+`
+
+func printExemplarCharacters(w io.Writer) {
+ fmt.Fprintln(w, exemplarHeader)
+ fmt.Fprintln(w, "var exemplarCharacters = map[string][exN]string{")
+ for _, loc := range mainLocales {
+ fmt.Fprintf(w, "\t%q: ", loc)
+ localeChars[loc].fprint(w)
+ }
+ fmt.Fprintln(w, "}")
+}
+
+func decodeCLDR(d *cldr.Decoder) *cldr.CLDR {
+ r := gen.OpenCLDRCoreZip()
+ data, err := d.DecodeZip(r)
+ failOnError(err)
+ return data
+}
+
+// parseMain parses XML files in the main directory of the CLDR core.zip file.
+func parseMain() {
+ d := &cldr.Decoder{}
+ d.SetDirFilter("main")
+ d.SetSectionFilter("characters")
+ data := decodeCLDR(d)
+ for _, loc := range data.Locales() {
+ x := data.RawLDML(loc)
+ if skipLang(x.Identity.Language.Type) {
+ continue
+ }
+ if x.Characters != nil {
+ x, _ = data.LDML(loc)
+ loc = language.Make(loc).String()
+ for _, ec := range x.Characters.ExemplarCharacters {
+ if ec.Draft != "" {
+ continue
+ }
+ if _, ok := localeChars[loc]; !ok {
+ mainLocales = append(mainLocales, loc)
+ localeChars[loc] = make(charSets)
+ }
+ localeChars[loc][ec.Type] = parseCharacters(ec.Data())
+ }
+ }
+ }
+}
+
+func parseCharacters(chars string) []string {
+ parseSingle := func(s string) (r rune, tail string, escaped bool) {
+ if s[0] == '\\' {
+ return rune(s[1]), s[2:], true
+ }
+ r, sz := utf8.DecodeRuneInString(s)
+ return r, s[sz:], false
+ }
+ chars = strings.TrimSpace(chars)
+ if n := len(chars) - 1; chars[n] == ']' && chars[0] == '[' {
+ chars = chars[1:n]
+ }
+ list := []string{}
+ var r, last, end rune
+ for len(chars) > 0 {
+ if chars[0] == '{' { // character sequence
+ buf := []rune{}
+ for chars = chars[1:]; len(chars) > 0; {
+ r, chars, _ = parseSingle(chars)
+ if r == '}' {
+ break
+ }
+ if r == ' ' {
+ log.Fatalf("space not supported in sequence %q", chars)
+ }
+ buf = append(buf, r)
+ }
+ list = append(list, string(buf))
+ last = 0
+ } else { // single character
+ escaped := false
+ r, chars, escaped = parseSingle(chars)
+ if r != ' ' {
+ if r == '-' && !escaped {
+ if last == 0 {
+ log.Fatal("'-' should be preceded by a character")
+ }
+ end, chars, _ = parseSingle(chars)
+ for ; last <= end; last++ {
+ list = append(list, string(last))
+ }
+ last = 0
+ } else {
+ list = append(list, string(r))
+ last = r
+ }
+ }
+ }
+ }
+ return list
+}
+
+var fileRe = regexp.MustCompile(`.*/collation/(.*)\.xml`)
+
+// typeMap translates legacy type keys to their BCP47 equivalent.
+var typeMap = map[string]string{
+ "phonebook": "phonebk",
+ "traditional": "trad",
+}
+
+// parseCollation parses XML files in the collation directory of the CLDR core.zip file.
+func parseCollation(b *build.Builder) {
+ d := &cldr.Decoder{}
+ d.SetDirFilter("collation")
+ data := decodeCLDR(d)
+ for _, loc := range data.Locales() {
+ x, err := data.LDML(loc)
+ failOnError(err)
+ if skipLang(x.Identity.Language.Type) {
+ continue
+ }
+ cs := x.Collations.Collation
+ sl := cldr.MakeSlice(&cs)
+ if len(types.s) == 0 {
+ sl.SelectAnyOf("type", x.Collations.Default())
+ } else if !types.all {
+ sl.SelectAnyOf("type", types.s...)
+ }
+ sl.SelectOnePerGroup("alt", altInclude())
+
+ for _, c := range cs {
+ id, err := language.Parse(loc)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "invalid locale: %q", err)
+ continue
+ }
+ // Support both old- and new-style defaults.
+ d := c.Type
+ if x.Collations.DefaultCollation == nil {
+ d = x.Collations.Default()
+ } else {
+ d = x.Collations.DefaultCollation.Data()
+ }
+ // We assume tables are being built either for search or collation,
+ // but not both. For search the default is always "search".
+ if d != c.Type && c.Type != "search" {
+ typ := c.Type
+ if len(c.Type) > 8 {
+ typ = typeMap[c.Type]
+ }
+ id, err = id.SetTypeForKey("co", typ)
+ failOnError(err)
+ }
+ t := b.Tailoring(id)
+ c.Process(processor{t})
+ }
+ }
+}
+
+type processor struct {
+ t *build.Tailoring
+}
+
+func (p processor) Reset(anchor string, before int) (err error) {
+ if before != 0 {
+ err = p.t.SetAnchorBefore(anchor)
+ } else {
+ err = p.t.SetAnchor(anchor)
+ }
+ failOnError(err)
+ return nil
+}
+
+func (p processor) Insert(level int, str, context, extend string) error {
+ str = context + str
+ if *test {
+ testInput.add(str)
+ }
+ // TODO: mimic bug in old maketables: remove.
+ err := p.t.Insert(colltab.Level(level-1), str, context+extend)
+ failOnError(err)
+ return nil
+}
+
+func (p processor) Index(id string) {
+}
+
+func testCollator(c *collate.Collator) {
+ c0 := collate.New(language.Und)
+
+ // iterator over all characters for all locales and check
+ // whether Key is equal.
+ buf := collate.Buffer{}
+
+ // Add all common and not too uncommon runes to the test set.
+ for i := rune(0); i < 0x30000; i++ {
+ testInput.add(string(i))
+ }
+ for i := rune(0xE0000); i < 0xF0000; i++ {
+ testInput.add(string(i))
+ }
+ for _, str := range testInput.values() {
+ k0 := c0.KeyFromString(&buf, str)
+ k := c.KeyFromString(&buf, str)
+ if !bytes.Equal(k0, k) {
+ failOnError(fmt.Errorf("test:%U: keys differ (%x vs %x)", []rune(str), k0, k))
+ }
+ buf.Reset()
+ }
+ fmt.Println("PASS")
+}
+
+func main() {
+ gen.Init()
+ b := build.NewBuilder()
+ parseUCA(b)
+ if tables.contains("chars") {
+ parseMain()
+ }
+ parseCollation(b)
+
+ c, err := b.Build()
+ failOnError(err)
+
+ if *test {
+ testCollator(collate.NewFromTable(c))
+ } else {
+ w := &bytes.Buffer{}
+
+ gen.WriteUnicodeVersion(w)
+ gen.WriteCLDRVersion(w)
+
+ if tables.contains("collate") {
+ _, err = b.Print(w)
+ failOnError(err)
+ }
+ if tables.contains("chars") {
+ printExemplarCharacters(w)
+ }
+ gen.WriteGoFile("tables.go", *pkg, w.Bytes())
+ }
+}
Code Review / barometer.git / blobdiff