1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Package gen contains common code for the various code generation tools in the
6 // text repository. Its usage ensures consistency between tools.
8 // This package defines command line flags that are common to most generation
9 // tools. The flags allow for specifying specific Unicode and CLDR versions
10 // in the public Unicode data repository (https://www.unicode.org/Public).
12 // A local Unicode data mirror can be set through the flag -local or the
13 // environment variable UNICODE_DIR. The former takes precedence. The local
14 // directory should follow the same structure as the public repository.
16 // IANA data can also optionally be mirrored by putting it in the iana directory
17 // rooted at the top of the local mirror. Beware, though, that IANA data is not
18 // versioned. So it is up to the developer to use the right version.
19 package gen // import "golang.org/x/text/internal/gen"
39 "golang.org/x/text/unicode/cldr"
43 url = flag.String("url",
44 "https://www.unicode.org/Public",
45 "URL of Unicode database directory")
46 iana = flag.String("iana",
47 "http://www.iana.org",
48 "URL of the IANA repository")
49 unicodeVersion = flag.String("unicode",
50 getEnv("UNICODE_VERSION", unicode.Version),
51 "unicode version to use")
52 cldrVersion = flag.String("cldr",
53 getEnv("CLDR_VERSION", cldr.Version),
54 "cldr version to use")
57 func getEnv(name, def string) string {
58 if v := os.Getenv(name); v != "" {
64 // Init performs common initialization for a gen command. It parses the flags
65 // and sets up the standard logging parameters.
68 log.SetFlags(log.Lshortfile)
72 const header = `// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
76 // UnicodeVersion reports the requested Unicode version.
77 func UnicodeVersion() string {
78 return *unicodeVersion
81 // CLDRVersion reports the requested CLDR version.
82 func CLDRVersion() string {
86 var tags = []struct{ version, buildTags string }{
88 {"10.0.0", "go1.10,!go1.13"},
92 // buildTags reports the build tags used for the current Unicode version.
93 func buildTags() string {
95 for _, e := range tags {
100 log.Fatalf("Unknown build tags for Unicode version %q.", v)
104 // IsLocal reports whether data files are available locally.
105 func IsLocal() bool {
106 dir, err := localReadmeFile()
110 if _, err = os.Stat(dir); err != nil {
116 // OpenUCDFile opens the requested UCD file. The file is specified relative to
117 // the public Unicode root directory. It will call log.Fatal if there are any
119 func OpenUCDFile(file string) io.ReadCloser {
120 return openUnicode(path.Join(*unicodeVersion, "ucd", file))
123 // OpenCLDRCoreZip opens the CLDR core zip file. It will call log.Fatal if there
125 func OpenCLDRCoreZip() io.ReadCloser {
126 return OpenUnicodeFile("cldr", *cldrVersion, "core.zip")
129 // OpenUnicodeFile opens the requested file of the requested category from the
130 // root of the Unicode data archive. The file is specified relative to the
131 // public Unicode root directory. If version is "", it will use the default
132 // Unicode version. It will call log.Fatal if there are any errors.
133 func OpenUnicodeFile(category, version, file string) io.ReadCloser {
135 version = UnicodeVersion()
137 return openUnicode(path.Join(category, version, file))
140 // OpenIANAFile opens the requested IANA file. The file is specified relative
141 // to the IANA root, which is typically either http://www.iana.org or the
142 // iana directory in the local mirror. It will call log.Fatal if there are any
144 func OpenIANAFile(path string) io.ReadCloser {
145 return Open(*iana, "iana", path)
153 const permissions = 0755
155 func localReadmeFile() (string, error) {
156 p, err := build.Import("golang.org/x/text", "", build.FindOnly)
158 return "", fmt.Errorf("Could not locate package: %v", err)
160 return filepath.Join(p.Dir, "DATA", "README"), nil
163 func getLocalDir() string {
165 defer dirMutex.Unlock()
167 readme, err := localReadmeFile()
171 dir := filepath.Dir(readme)
172 if _, err := os.Stat(readme); err != nil {
173 if err := os.MkdirAll(dir, permissions); err != nil {
174 log.Fatalf("Could not create directory: %v", err)
176 ioutil.WriteFile(readme, []byte(readmeTxt), permissions)
181 const readmeTxt = `Generated by golang.org/x/text/internal/gen. DO NOT EDIT.
183 This directory contains downloaded files used to generate the various tables
184 in the golang.org/x/text subrepo.
186 Note that the language subtag repo (iana/assignments/language-subtag-registry)
187 and all other times in the iana subdirectory are not versioned and will need
188 to be periodically manually updated. The easiest way to do this is to remove
189 the entire iana directory. This is mostly of concern when updating the language
193 // Open opens subdir/path if a local directory is specified and the file exists,
194 // where subdir is a directory relative to the local root, or fetches it from
195 // urlRoot/path otherwise. It will call log.Fatal if there are any errors.
196 func Open(urlRoot, subdir, path string) io.ReadCloser {
197 file := filepath.Join(getLocalDir(), subdir, filepath.FromSlash(path))
198 return open(file, urlRoot, path)
201 func openUnicode(path string) io.ReadCloser {
202 file := filepath.Join(getLocalDir(), filepath.FromSlash(path))
203 return open(file, *url, path)
206 // TODO: automatically periodically update non-versioned files.
208 func open(file, urlRoot, path string) io.ReadCloser {
209 if f, err := os.Open(file); err == nil {
212 r := get(urlRoot, path)
214 b, err := ioutil.ReadAll(r)
216 log.Fatalf("Could not download file: %v", err)
218 os.MkdirAll(filepath.Dir(file), permissions)
219 if err := ioutil.WriteFile(file, b, permissions); err != nil {
220 log.Fatalf("Could not create file: %v", err)
222 return ioutil.NopCloser(bytes.NewReader(b))
225 func get(root, path string) io.ReadCloser {
226 url := root + "/" + path
227 fmt.Printf("Fetching %s...", url)
228 defer fmt.Println(" done.")
229 resp, err := http.Get(url)
231 log.Fatalf("HTTP GET: %v", err)
233 if resp.StatusCode != 200 {
234 log.Fatalf("Bad GET status for %q: %q", url, resp.Status)
239 // TODO: use Write*Version in all applicable packages.
241 // WriteUnicodeVersion writes a constant for the Unicode version from which the
242 // tables are generated.
243 func WriteUnicodeVersion(w io.Writer) {
244 fmt.Fprintf(w, "// UnicodeVersion is the Unicode version from which the tables in this package are derived.\n")
245 fmt.Fprintf(w, "const UnicodeVersion = %q\n\n", UnicodeVersion())
248 // WriteCLDRVersion writes a constant for the CLDR version from which the
249 // tables are generated.
250 func WriteCLDRVersion(w io.Writer) {
251 fmt.Fprintf(w, "// CLDRVersion is the CLDR version from which the tables in this package are derived.\n")
252 fmt.Fprintf(w, "const CLDRVersion = %q\n\n", CLDRVersion())
255 // WriteGoFile prepends a standard file comment and package statement to the
256 // given bytes, applies gofmt, and writes them to a file with the given name.
257 // It will call log.Fatal if there are any errors.
258 func WriteGoFile(filename, pkg string, b []byte) {
259 w, err := os.Create(filename)
261 log.Fatalf("Could not create file %s: %v", filename, err)
264 if _, err = WriteGo(w, pkg, "", b); err != nil {
265 log.Fatalf("Error writing file %s: %v", filename, err)
269 func fileToPattern(filename string) string {
271 if strings.HasSuffix(filename, "_test.go") {
274 prefix := filename[:len(filename)-len(suffix)]
275 return fmt.Sprint(prefix, "%s", suffix)
278 func updateBuildTags(pattern string) {
279 for _, t := range tags {
280 oldFile := fmt.Sprintf(pattern, t.version)
281 b, err := ioutil.ReadFile(oldFile)
285 build := fmt.Sprintf("// +build %s", t.buildTags)
286 b = regexp.MustCompile(`// \+build .*`).ReplaceAll(b, []byte(build))
287 err = ioutil.WriteFile(oldFile, b, 0644)
294 // WriteVersionedGoFile prepends a standard file comment, adds build tags to
295 // version the file for the current Unicode version, and package statement to
296 // the given bytes, applies gofmt, and writes them to a file with the given
297 // name. It will call log.Fatal if there are any errors.
298 func WriteVersionedGoFile(filename, pkg string, b []byte) {
299 pattern := fileToPattern(filename)
300 updateBuildTags(pattern)
301 filename = fmt.Sprintf(pattern, UnicodeVersion())
303 w, err := os.Create(filename)
305 log.Fatalf("Could not create file %s: %v", filename, err)
308 if _, err = WriteGo(w, pkg, buildTags(), b); err != nil {
309 log.Fatalf("Error writing file %s: %v", filename, err)
313 // WriteGo prepends a standard file comment and package statement to the given
314 // bytes, applies gofmt, and writes them to w.
315 func WriteGo(w io.Writer, pkg, tags string, b []byte) (n int, err error) {
316 src := []byte(header)
318 src = append(src, fmt.Sprintf("// +build %s\n\n", tags)...)
320 src = append(src, fmt.Sprintf("package %s\n\n", pkg)...)
321 src = append(src, b...)
322 formatted, err := format.Source(src)
324 // Print the generated code even in case of an error so that the
325 // returned error can be meaningfully interpreted.
329 return w.Write(formatted)
332 // Repackage rewrites a Go file from belonging to package main to belonging to
333 // the given package.
334 func Repackage(inFile, outFile, pkg string) {
335 src, err := ioutil.ReadFile(inFile)
337 log.Fatalf("reading %s: %v", inFile, err)
339 const toDelete = "package main\n\n"
340 i := bytes.Index(src, []byte(toDelete))
342 log.Fatalf("Could not find %q in %s.", toDelete, inFile)
345 w.Write(src[i+len(toDelete):])
346 WriteGoFile(outFile, pkg, w.Bytes())