From 1492043a155839cb863210d4f564be3fa640c0d9 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 8 Oct 2022 11:41:06 +0800 Subject: [PATCH] update Signed-off-by: Weizhen Wang --- BUILD.bazel | 27 +++++ WORKSPACE | 2 + gen_breaktest.go | 213 -------------------------------------- gen_properties.go | 256 ---------------------------------------------- 4 files changed, 29 insertions(+), 469 deletions(-) create mode 100644 BUILD.bazel create mode 100644 WORKSPACE delete mode 100644 gen_breaktest.go delete mode 100644 gen_properties.go diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 0000000..a1e5c89 --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "uniseg", + srcs = [ + "doc.go", + "eastasianwidth.go", + "emojipresentation.go", + "grapheme.go", + "graphemeproperties.go", + "graphemerules.go", + "line.go", + "lineproperties.go", + "linerules.go", + "properties.go", + "sentence.go", + "sentenceproperties.go", + "sentencerules.go", + "step.go", + "width.go", + "word.go", + "wordproperties.go", + "wordrules.go", + ], + importpath = "github.com/rivo/uniseg", + visibility = ["//visibility:public"], +) diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..d596273 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,2 @@ +# DO NOT EDIT: automatically generated WORKSPACE file for go_repository rule +workspace(name = "com_github_rivo_uniseg") diff --git a/gen_breaktest.go b/gen_breaktest.go deleted file mode 100644 index e613c4c..0000000 --- a/gen_breaktest.go +++ /dev/null @@ -1,213 +0,0 @@ -//go:build generate - -// This program generates a Go containing a slice of test cases based on the -// Unicode Character Database auxiliary data files. The command line arguments -// are as follows: -// -// 1. The name of the Unicode data file (just the filename, without extension). -// 2. The name of the locally generated Go file. -// 3. The name of the slice containing the test cases. -// 4. The name of the generator, for logging purposes. -// -//go:generate go run gen_breaktest.go GraphemeBreakTest graphemebreak_test.go graphemeBreakTestCases graphemes -//go:generate go run gen_breaktest.go WordBreakTest wordbreak_test.go wordBreakTestCases words -//go:generate go run gen_breaktest.go SentenceBreakTest sentencebreak_test.go sentenceBreakTestCases sentences -//go:generate go run gen_breaktest.go LineBreakTest linebreak_test.go lineBreakTestCases lines - -package main - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "go/format" - "io/ioutil" - "log" - "net/http" - "os" - "time" -) - -// We want to test against a specific version rather than the latest. When the -// package is upgraded to a new version, change these to generate new tests. -const ( - testCaseURL = `https://www.unicode.org/Public/14.0.0/ucd/auxiliary/%s.txt` -) - -func main() { - if len(os.Args) < 5 { - fmt.Println("Not enough arguments, see code for details") - os.Exit(1) - } - - log.SetPrefix("gen_breaktest (" + os.Args[4] + "): ") - log.SetFlags(0) - - // Read text of testcases and parse into Go source code. - src, err := parse(fmt.Sprintf(testCaseURL, os.Args[1])) - if err != nil { - log.Fatal(err) - } - - // Format the Go code. - formatted, err := format.Source(src) - if err != nil { - log.Fatalln("gofmt:", err) - } - - // Write it out. - log.Print("Writing to ", os.Args[2]) - if err := ioutil.WriteFile(os.Args[2], formatted, 0644); err != nil { - log.Fatal(err) - } -} - -// parse reads a break text file, either from a local file or from a URL. It -// parses the file data into Go source code representing the test cases. -func parse(url string) ([]byte, error) { - log.Printf("Parsing %s", url) - res, err := http.Get(url) - if err != nil { - return nil, err - } - body := res.Body - defer body.Close() - - buf := new(bytes.Buffer) - buf.Grow(120 << 10) - buf.WriteString(`package uniseg - -// Code generated via go generate from gen_breaktest.go. DO NOT EDIT. - -// ` + os.Args[3] + ` are Grapheme testcases taken from -// ` + url + ` -// on ` + time.Now().Format("January 2, 2006") + `. See -// https://www.unicode.org/license.html for the Unicode license agreement. -var ` + os.Args[3] + ` = []testCase { -`) - - sc := bufio.NewScanner(body) - num := 1 - var line []byte - original := make([]byte, 0, 64) - expected := make([]byte, 0, 64) - for sc.Scan() { - num++ - line = sc.Bytes() - if len(line) == 0 || line[0] == '#' { - continue - } - var comment []byte - if i := bytes.IndexByte(line, '#'); i >= 0 { - comment = bytes.TrimSpace(line[i+1:]) - line = bytes.TrimSpace(line[:i]) - } - original, expected, err := parseRuneSequence(line, original[:0], expected[:0]) - if err != nil { - return nil, fmt.Errorf(`line %d: %v: %q`, num, err, line) - } - fmt.Fprintf(buf, "\t{original: \"%s\", expected: %s}, // %s\n", original, expected, comment) - } - if err := sc.Err(); err != nil { - return nil, err - } - - // Check for final "# EOF", useful check if we're streaming via HTTP - if !bytes.Equal(line, []byte("# EOF")) { - return nil, fmt.Errorf(`line %d: exected "# EOF" as final line, got %q`, num, line) - } - buf.WriteString("}\n") - return buf.Bytes(), nil -} - -// Used by parseRuneSequence to match input via bytes.HasPrefix. -var ( - prefixBreak = []byte("÷ ") - prefixDontBreak = []byte("× ") - breakOk = []byte("÷") - breakNo = []byte("×") -) - -// parseRuneSequence parses a rune + breaking opportunity sequence from b -// and appends the Go code for testcase.original to orig -// and appends the Go code for testcase.expected to exp. -// It retuns the new orig and exp slices. -// -// E.g. for the input b="÷ 0020 × 0308 ÷ 1F1E6 ÷" -// it will append -// "\u0020\u0308\U0001F1E6" -// and "[][]rune{{0x0020,0x0308},{0x1F1E6},}" -// to orig and exp respectively. -// -// The formatting of exp is expected to be cleaned up by gofmt or format.Source. -// Note we explicitly require the sequence to start with ÷ and we implicitly -// require it to end with ÷. -func parseRuneSequence(b, orig, exp []byte) ([]byte, []byte, error) { - // Check for and remove first ÷ or ×. - if !bytes.HasPrefix(b, prefixBreak) && !bytes.HasPrefix(b, prefixDontBreak) { - return nil, nil, errors.New("expected ÷ or × as first character") - } - if bytes.HasPrefix(b, prefixBreak) { - b = b[len(prefixBreak):] - } else { - b = b[len(prefixDontBreak):] - } - - boundary := true - exp = append(exp, "[][]rune{"...) - for len(b) > 0 { - if boundary { - exp = append(exp, '{') - } - exp = append(exp, "0x"...) - // Find end of hex digits. - var i int - for i = 0; i < len(b) && b[i] != ' '; i++ { - if d := b[i]; ('0' <= d || d <= '9') || - ('A' <= d || d <= 'F') || - ('a' <= d || d <= 'f') { - continue - } - return nil, nil, errors.New("bad hex digit") - } - switch i { - case 4: - orig = append(orig, "\\u"...) - case 5: - orig = append(orig, "\\U000"...) - default: - return nil, nil, errors.New("unsupport code point hex length") - } - orig = append(orig, b[:i]...) - exp = append(exp, b[:i]...) - b = b[i:] - - // Check for space between hex and ÷ or ×. - if len(b) < 1 || b[0] != ' ' { - return nil, nil, errors.New("bad input") - } - b = b[1:] - - // Check for next boundary. - switch { - case bytes.HasPrefix(b, breakOk): - boundary = true - b = b[len(breakOk):] - case bytes.HasPrefix(b, breakNo): - boundary = false - b = b[len(breakNo):] - default: - return nil, nil, errors.New("missing ÷ or ×") - } - if boundary { - exp = append(exp, '}') - } - exp = append(exp, ',') - if len(b) > 0 && b[0] == ' ' { - b = b[1:] - } - } - exp = append(exp, '}') - return orig, exp, nil -} diff --git a/gen_properties.go b/gen_properties.go deleted file mode 100644 index 999d5ef..0000000 --- a/gen_properties.go +++ /dev/null @@ -1,256 +0,0 @@ -//go:build generate - -// This program generates a property file in Go file from Unicode Character -// Database auxiliary data files. The command line arguments are as follows: -// -// 1. The name of the Unicode data file (just the filename, without extension). -// Can be "-" (to skip) if the emoji flag is included. -// 2. The name of the locally generated Go file. -// 3. The name of the slice mapping code points to properties. -// 4. The name of the generator, for logging purposes. -// 5. (Optional) Flags, comma-separated. The following flags are available: -// - "emojis=": include the specified emoji properties (e.g. -// "Extended_Pictographic"). -// - "gencat": include general category properties. -// -//go:generate go run gen_properties.go auxiliary/GraphemeBreakProperty graphemeproperties.go graphemeCodePoints graphemes emojis=Extended_Pictographic -//go:generate go run gen_properties.go auxiliary/WordBreakProperty wordproperties.go workBreakCodePoints words emojis=Extended_Pictographic -//go:generate go run gen_properties.go auxiliary/SentenceBreakProperty sentenceproperties.go sentenceBreakCodePoints sentences -//go:generate go run gen_properties.go LineBreak lineproperties.go lineBreakCodePoints lines gencat -//go:generate go run gen_properties.go EastAsianWidth eastasianwidth.go eastAsianWidth eastasianwidth -//go:generate go run gen_properties.go - emojipresentation.go emojiPresentation emojipresentation emojis=Emoji_Presentation -package main - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "go/format" - "io/ioutil" - "log" - "net/http" - "os" - "regexp" - "sort" - "strconv" - "strings" - "time" -) - -// We want to test against a specific version rather than the latest. When the -// package is upgraded to a new version, change these to generate new tests. -const ( - propertyURL = `https://www.unicode.org/Public/14.0.0/ucd/%s.txt` - emojiURL = `https://unicode.org/Public/14.0.0/ucd/emoji/emoji-data.txt` -) - -// The regular expression for a line containing a code point range property. -var propertyPattern = regexp.MustCompile(`^([0-9A-F]{4,6})(\.\.([0-9A-F]{4,6}))?\s*;\s*([A-Za-z0-9_]+)\s*#\s(.+)$`) - -func main() { - if len(os.Args) < 5 { - fmt.Println("Not enough arguments, see code for details") - os.Exit(1) - } - - log.SetPrefix("gen_properties (" + os.Args[4] + "): ") - log.SetFlags(0) - - // Parse flags. - flags := make(map[string]string) - if len(os.Args) >= 6 { - for _, flag := range strings.Split(os.Args[5], ",") { - flagFields := strings.Split(flag, "=") - if len(flagFields) == 1 { - flags[flagFields[0]] = "yes" - } else { - flags[flagFields[0]] = flagFields[1] - } - } - } - - // Parse the text file and generate Go source code from it. - _, includeGeneralCategory := flags["gencat"] - var mainURL string - if os.Args[1] != "-" { - mainURL = fmt.Sprintf(propertyURL, os.Args[1]) - } - src, err := parse(mainURL, flags["emojis"], includeGeneralCategory) - if err != nil { - log.Fatal(err) - } - - // Format the Go code. - formatted, err := format.Source([]byte(src)) - if err != nil { - log.Fatal("gofmt:", err) - } - - // Save it to the (local) target file. - log.Print("Writing to ", os.Args[2]) - if err := ioutil.WriteFile(os.Args[2], formatted, 0644); err != nil { - log.Fatal(err) - } -} - -// parse parses the Unicode Properties text files located at the given URLs and -// returns their equivalent Go source code to be used in the uniseg package. If -// "emojiProperty" is not an empty string, emoji code points for that emoji -// property (e.g. "Extended_Pictographic") will be included. In those cases, you -// may pass an empty "propertyURL" to skip parsing the main properties file. If -// "includeGeneralCategory" is true, the Unicode General Category property will -// be extracted from the comments and included in the output. -func parse(propertyURL, emojiProperty string, includeGeneralCategory bool) (string, error) { - if propertyURL == "" && emojiProperty == "" { - return "", errors.New("no properties to parse") - } - - // Temporary buffer to hold properties. - var properties [][4]string - - // Open the first URL. - if propertyURL != "" { - log.Printf("Parsing %s", propertyURL) - res, err := http.Get(propertyURL) - if err != nil { - return "", err - } - in1 := res.Body - defer in1.Close() - - // Parse it. - scanner := bufio.NewScanner(in1) - num := 0 - for scanner.Scan() { - num++ - line := strings.TrimSpace(scanner.Text()) - - // Skip comments and empty lines. - if strings.HasPrefix(line, "#") || line == "" { - continue - } - - // Everything else must be a code point range, a property and a comment. - from, to, property, comment, err := parseProperty(line) - if err != nil { - return "", fmt.Errorf("%s line %d: %v", os.Args[4], num, err) - } - properties = append(properties, [4]string{from, to, property, comment}) - } - if err := scanner.Err(); err != nil { - return "", err - } - } - - // Open the second URL. - if emojiProperty != "" { - log.Printf("Parsing %s", emojiURL) - res, err := http.Get(emojiURL) - if err != nil { - return "", err - } - in2 := res.Body - defer in2.Close() - - // Parse it. - scanner := bufio.NewScanner(in2) - num := 0 - for scanner.Scan() { - num++ - line := scanner.Text() - - // Skip comments, empty lines, and everything not containing - // "Extended_Pictographic". - if strings.HasPrefix(line, "#") || line == "" || !strings.Contains(line, emojiProperty) { - continue - } - - // Everything else must be a code point range, a property and a comment. - from, to, property, comment, err := parseProperty(line) - if err != nil { - return "", fmt.Errorf("emojis line %d: %v", num, err) - } - properties = append(properties, [4]string{from, to, property, comment}) - } - if err := scanner.Err(); err != nil { - return "", err - } - } - - // Sort properties. - sort.Slice(properties, func(i, j int) bool { - left, _ := strconv.ParseUint(properties[i][0], 16, 64) - right, _ := strconv.ParseUint(properties[j][0], 16, 64) - return left < right - }) - - // Header. - var ( - buf bytes.Buffer - emojiComment string - ) - columns := 3 - if includeGeneralCategory { - columns = 4 - } - if emojiURL != "" { - emojiComment = ` -// and -// ` + emojiURL + ` -// ("Extended_Pictographic" only)` - } - buf.WriteString(`package uniseg - -// Code generated via go generate from gen_properties.go. DO NOT EDIT. - -// ` + os.Args[3] + ` are taken from -// ` + propertyURL + emojiComment + ` -// on ` + time.Now().Format("January 2, 2006") + `. See https://www.unicode.org/license.html for the Unicode -// license agreement. -var ` + os.Args[3] + ` = [][` + strconv.Itoa(columns) + `]int{ - `) - - // Properties. - for _, prop := range properties { - if includeGeneralCategory { - generalCategory := "gc" + prop[3][:2] - if generalCategory == "gcL&" { - generalCategory = "gcLC" - } - prop[3] = prop[3][3:] - fmt.Fprintf(&buf, "{0x%s,0x%s,%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), generalCategory, prop[3]) - } else { - fmt.Fprintf(&buf, "{0x%s,0x%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), prop[3]) - } - } - - // Tail. - buf.WriteString("}") - - return buf.String(), nil -} - -// parseProperty parses a line of the Unicode properties text file containing a -// property for a code point range and returns it along with its comment. -func parseProperty(line string) (from, to, property, comment string, err error) { - fields := propertyPattern.FindStringSubmatch(line) - if fields == nil { - err = errors.New("no property found") - return - } - from = fields[1] - to = fields[3] - if to == "" { - to = from - } - property = fields[4] - comment = fields[5] - return -} - -// translateProperty translates a property name as used in the Unicode data file -// to a variable used in the Go code. -func translateProperty(prefix, property string) string { - return prefix + strings.ReplaceAll(property, "_", "") -} -- 2.38.0