Initial commit

vendor/golang.org/x/text/encoding/encoding.go

@@ -0,0 +1,335 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package encoding defines an interface for character encodings, such as Shift
+// JIS and Windows 1252, that can convert to and from UTF-8.
+//
+// Encoding implementations are provided in other packages, such as
+// golang.org/x/text/encoding/charmap and
+// golang.org/x/text/encoding/japanese.
+package encoding // import "golang.org/x/text/encoding"
+
+import (
+	"errors"
+	"io"
+	"strconv"
+	"unicode/utf8"
+
+	"golang.org/x/text/encoding/internal/identifier"
+	"golang.org/x/text/transform"
+)
+
+// TODO:
+// - There seems to be some inconsistency in when decoders return errors
+//   and when not. Also documentation seems to suggest they shouldn't return
+//   errors at all (except for UTF-16).
+// - Encoders seem to rely on or at least benefit from the input being in NFC
+//   normal form. Perhaps add an example how users could prepare their output.
+
+// Encoding is a character set encoding that can be transformed to and from
+// UTF-8.
+type Encoding interface {
+	// NewDecoder returns a Decoder.
+	NewDecoder() *Decoder
+
+	// NewEncoder returns an Encoder.
+	NewEncoder() *Encoder
+}
+
+// A Decoder converts bytes to UTF-8. It implements transform.Transformer.
+//
+// Transforming source bytes that are not of that encoding will not result in an
+// error per se. Each byte that cannot be transcoded will be represented in the
+// output by the UTF-8 encoding of '\uFFFD', the replacement rune.
+type Decoder struct {
+	transform.Transformer
+
+	// This forces external creators of Decoders to use names in struct
+	// initializers, allowing for future extendibility without having to break
+	// code.
+	_ struct{}
+}
+
+// Bytes converts the given encoded bytes to UTF-8. It returns the converted
+// bytes or nil, err if any error occurred.
+func (d *Decoder) Bytes(b []byte) ([]byte, error) {
+	b, _, err := transform.Bytes(d, b)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+// String converts the given encoded string to UTF-8. It returns the converted
+// string or "", err if any error occurred.
+func (d *Decoder) String(s string) (string, error) {
+	s, _, err := transform.String(d, s)
+	if err != nil {
+		return "", err
+	}
+	return s, nil
+}
+
+// Reader wraps another Reader to decode its bytes.
+//
+// The Decoder may not be used for any other operation as long as the returned
+// Reader is in use.
+func (d *Decoder) Reader(r io.Reader) io.Reader {
+	return transform.NewReader(r, d)
+}
+
+// An Encoder converts bytes from UTF-8. It implements transform.Transformer.
+//
+// Each rune that cannot be transcoded will result in an error. In this case,
+// the transform will consume all source byte up to, not including the offending
+// rune. Transforming source bytes that are not valid UTF-8 will be replaced by
+// `\uFFFD`. To return early with an error instead, use transform.Chain to
+// preprocess the data with a UTF8Validator.
+type Encoder struct {
+	transform.Transformer
+
+	// This forces external creators of Encoders to use names in struct
+	// initializers, allowing for future extendibility without having to break
+	// code.
+	_ struct{}
+}
+
+// Bytes converts bytes from UTF-8. It returns the converted bytes or nil, err if
+// any error occurred.
+func (e *Encoder) Bytes(b []byte) ([]byte, error) {
+	b, _, err := transform.Bytes(e, b)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+// String converts a string from UTF-8. It returns the converted string or
+// "", err if any error occurred.
+func (e *Encoder) String(s string) (string, error) {
+	s, _, err := transform.String(e, s)
+	if err != nil {
+		return "", err
+	}
+	return s, nil
+}
+
+// Writer wraps another Writer to encode its UTF-8 output.
+//
+// The Encoder may not be used for any other operation as long as the returned
+// Writer is in use.
+func (e *Encoder) Writer(w io.Writer) io.Writer {
+	return transform.NewWriter(w, e)
+}
+
+// ASCIISub is the ASCII substitute character, as recommended by
+// http://unicode.org/reports/tr36/#Text_Comparison
+const ASCIISub = '\x1a'
+
+// Nop is the nop encoding. Its transformed bytes are the same as the source
+// bytes; it does not replace invalid UTF-8 sequences.
+var Nop Encoding = nop{}
+
+type nop struct{}
+
+func (nop) NewDecoder() *Decoder {
+	return &Decoder{Transformer: transform.Nop}
+}
+func (nop) NewEncoder() *Encoder {
+	return &Encoder{Transformer: transform.Nop}
+}
+
+// Replacement is the replacement encoding. Decoding from the replacement
+// encoding yields a single '\uFFFD' replacement rune. Encoding from UTF-8 to
+// the replacement encoding yields the same as the source bytes except that
+// invalid UTF-8 is converted to '\uFFFD'.
+//
+// It is defined at http://encoding.spec.whatwg.org/#replacement
+var Replacement Encoding = replacement{}
+
+type replacement struct{}
+
+func (replacement) NewDecoder() *Decoder {
+	return &Decoder{Transformer: replacementDecoder{}}
+}
+
+func (replacement) NewEncoder() *Encoder {
+	return &Encoder{Transformer: replacementEncoder{}}
+}
+
+func (replacement) ID() (mib identifier.MIB, other string) {
+	return identifier.Replacement, ""
+}
+
+type replacementDecoder struct{ transform.NopResetter }
+
+func (replacementDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	if len(dst) < 3 {
+		return 0, 0, transform.ErrShortDst
+	}
+	if atEOF {
+		const fffd = "\ufffd"
+		dst[0] = fffd[0]
+		dst[1] = fffd[1]
+		dst[2] = fffd[2]
+		nDst = 3
+	}
+	return nDst, len(src), nil
+}
+
+type replacementEncoder struct{ transform.NopResetter }
+
+func (replacementEncoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	r, size := rune(0), 0
+
+	for ; nSrc < len(src); nSrc += size {
+		r = rune(src[nSrc])
+
+		// Decode a 1-byte rune.
+		if r < utf8.RuneSelf {
+			size = 1
+
+		} else {
+			// Decode a multi-byte rune.
+			r, size = utf8.DecodeRune(src[nSrc:])
+			if size == 1 {
+				// All valid runes of size 1 (those below utf8.RuneSelf) were
+				// handled above. We have invalid UTF-8 or we haven't seen the
+				// full character yet.
+				if !atEOF && !utf8.FullRune(src[nSrc:]) {
+					err = transform.ErrShortSrc
+					break
+				}
+				r = '\ufffd'
+			}
+		}
+
+		if nDst+utf8.RuneLen(r) > len(dst) {
+			err = transform.ErrShortDst
+			break
+		}
+		nDst += utf8.EncodeRune(dst[nDst:], r)
+	}
+	return nDst, nSrc, err
+}
+
+// HTMLEscapeUnsupported wraps encoders to replace source runes outside the
+// repertoire of the destination encoding with HTML escape sequences.
+//
+// This wrapper exists to comply to URL and HTML forms requiring a
+// non-terminating legacy encoder. The produced sequences may lead to data
+// loss as they are indistinguishable from legitimate input. To avoid this
+// issue, use UTF-8 encodings whenever possible.
+func HTMLEscapeUnsupported(e *Encoder) *Encoder {
+	return &Encoder{Transformer: &errorHandler{e, errorToHTML}}
+}
+
+// ReplaceUnsupported wraps encoders to replace source runes outside the
+// repertoire of the destination encoding with an encoding-specific
+// replacement.
+//
+// This wrapper is only provided for backwards compatibility and legacy
+// handling. Its use is strongly discouraged. Use UTF-8 whenever possible.
+func ReplaceUnsupported(e *Encoder) *Encoder {
+	return &Encoder{Transformer: &errorHandler{e, errorToReplacement}}
+}
+
+type errorHandler struct {
+	*Encoder
+	handler func(dst []byte, r rune, err repertoireError) (n int, ok bool)
+}
+
+// TODO: consider making this error public in some form.
+type repertoireError interface {
+	Replacement() byte
+}
+
+func (h errorHandler) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	nDst, nSrc, err = h.Transformer.Transform(dst, src, atEOF)
+	for err != nil {
+		rerr, ok := err.(repertoireError)
+		if !ok {
+			return nDst, nSrc, err
+		}
+		r, sz := utf8.DecodeRune(src[nSrc:])
+		n, ok := h.handler(dst[nDst:], r, rerr)
+		if !ok {
+			return nDst, nSrc, transform.ErrShortDst
+		}
+		err = nil
+		nDst += n
+		if nSrc += sz; nSrc < len(src) {
+			var dn, sn int
+			dn, sn, err = h.Transformer.Transform(dst[nDst:], src[nSrc:], atEOF)
+			nDst += dn
+			nSrc += sn
+		}
+	}
+	return nDst, nSrc, err
+}
+
+func errorToHTML(dst []byte, r rune, err repertoireError) (n int, ok bool) {
+	buf := [8]byte{}
+	b := strconv.AppendUint(buf[:0], uint64(r), 10)
+	if n = len(b) + len("&#;"); n >= len(dst) {
+		return 0, false
+	}
+	dst[0] = '&'
+	dst[1] = '#'
+	dst[copy(dst[2:], b)+2] = ';'
+	return n, true
+}
+
+func errorToReplacement(dst []byte, r rune, err repertoireError) (n int, ok bool) {
+	if len(dst) == 0 {
+		return 0, false
+	}
+	dst[0] = err.Replacement()
+	return 1, true
+}
+
+// ErrInvalidUTF8 means that a transformer encountered invalid UTF-8.
+var ErrInvalidUTF8 = errors.New("encoding: invalid UTF-8")
+
+// UTF8Validator is a transformer that returns ErrInvalidUTF8 on the first
+// input byte that is not valid UTF-8.
+var UTF8Validator transform.Transformer = utf8Validator{}
+
+type utf8Validator struct{ transform.NopResetter }
+
+func (utf8Validator) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	n := len(src)
+	if n > len(dst) {
+		n = len(dst)
+	}
+	for i := 0; i < n; {
+		if c := src[i]; c < utf8.RuneSelf {
+			dst[i] = c
+			i++
+			continue
+		}
+		_, size := utf8.DecodeRune(src[i:])
+		if size == 1 {
+			// All valid runes of size 1 (those below utf8.RuneSelf) were
+			// handled above. We have invalid UTF-8 or we haven't seen the
+			// full character yet.
+			err = ErrInvalidUTF8
+			if !atEOF && !utf8.FullRune(src[i:]) {
+				err = transform.ErrShortSrc
+			}
+			return i, i, err
+		}
+		if i+size > len(dst) {
+			return i, i, transform.ErrShortDst
+		}
+		for ; size > 0; size-- {
+			dst[i] = src[i]
+			i++
+		}
+	}
+	if len(src) > len(dst) {
+		err = transform.ErrShortDst
+	}
+	return n, n, err
+}

vendor/golang.org/x/text/encoding/internal/identifier/gen.go

@@ -0,0 +1,137 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"log"
+	"strings"
+
+	"golang.org/x/text/internal/gen"
+)
+
+type registry struct {
+	XMLName  xml.Name `xml:"registry"`
+	Updated  string   `xml:"updated"`
+	Registry []struct {
+		ID     string `xml:"id,attr"`
+		Record []struct {
+			Name string `xml:"name"`
+			Xref []struct {
+				Type string `xml:"type,attr"`
+				Data string `xml:"data,attr"`
+			} `xml:"xref"`
+			Desc struct {
+				Data string `xml:",innerxml"`
+				// Any []struct {
+				// 	Data string `xml:",chardata"`
+				// } `xml:",any"`
+				// Data string `xml:",chardata"`
+			} `xml:"description,"`
+			MIB   string   `xml:"value"`
+			Alias []string `xml:"alias"`
+			MIME  string   `xml:"preferred_alias"`
+		} `xml:"record"`
+	} `xml:"registry"`
+}
+
+func main() {
+	r := gen.OpenIANAFile("assignments/character-sets/character-sets.xml")
+	reg := &registry{}
+	if err := xml.NewDecoder(r).Decode(&reg); err != nil && err != io.EOF {
+		log.Fatalf("Error decoding charset registry: %v", err)
+	}
+	if len(reg.Registry) == 0 || reg.Registry[0].ID != "character-sets-1" {
+		log.Fatalf("Unexpected ID %s", reg.Registry[0].ID)
+	}
+
+	w := &bytes.Buffer{}
+	fmt.Fprintf(w, "const (\n")
+	for _, rec := range reg.Registry[0].Record {
+		constName := ""
+		for _, a := range rec.Alias {
+			if strings.HasPrefix(a, "cs") && strings.IndexByte(a, '-') == -1 {
+				// Some of the constant definitions have comments in them. Strip those.
+				constName = strings.Title(strings.SplitN(a[2:], "\n", 2)[0])
+			}
+		}
+		if constName == "" {
+			switch rec.MIB {
+			case "2085":
+				constName = "HZGB2312" // Not listed as alias for some reason.
+			default:
+				log.Fatalf("No cs alias defined for %s.", rec.MIB)
+			}
+		}
+		if rec.MIME != "" {
+			rec.MIME = fmt.Sprintf(" (MIME: %s)", rec.MIME)
+		}
+		fmt.Fprintf(w, "// %s is the MIB identifier with IANA name %s%s.\n//\n", constName, rec.Name, rec.MIME)
+		if len(rec.Desc.Data) > 0 {
+			fmt.Fprint(w, "// ")
+			d := xml.NewDecoder(strings.NewReader(rec.Desc.Data))
+			inElem := true
+			attr := ""
+			for {
+				t, err := d.Token()
+				if err != nil {
+					if err != io.EOF {
+						log.Fatal(err)
+					}
+					break
+				}
+				switch x := t.(type) {
+				case xml.CharData:
+					attr = "" // Don't need attribute info.
+					a := bytes.Split([]byte(x), []byte("\n"))
+					for i, b := range a {
+						if b = bytes.TrimSpace(b); len(b) != 0 {
+							if !inElem && i > 0 {
+								fmt.Fprint(w, "\n// ")
+							}
+							inElem = false
+							fmt.Fprintf(w, "%s ", string(b))
+						}
+					}
+				case xml.StartElement:
+					if x.Name.Local == "xref" {
+						inElem = true
+						use := false
+						for _, a := range x.Attr {
+							if a.Name.Local == "type" {
+								use = use || a.Value != "person"
+							}
+							if a.Name.Local == "data" && use {
+								attr = a.Value + " "
+							}
+						}
+					}
+				case xml.EndElement:
+					inElem = false
+					fmt.Fprint(w, attr)
+				}
+			}
+			fmt.Fprint(w, "\n")
+		}
+		for _, x := range rec.Xref {
+			switch x.Type {
+			case "rfc":
+				fmt.Fprintf(w, "// Reference: %s\n", strings.ToUpper(x.Data))
+			case "uri":
+				fmt.Fprintf(w, "// Reference: %s\n", x.Data)
+			}
+		}
+		fmt.Fprintf(w, "%s MIB = %s\n", constName, rec.MIB)
+		fmt.Fprintln(w)
+	}
+	fmt.Fprintln(w, ")")
+
+	gen.WriteGoFile("mib.go", "identifier", w.Bytes())
+}

vendor/golang.org/x/text/encoding/internal/identifier/identifier.go

@@ -0,0 +1,81 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run gen.go
+
+// Package identifier defines the contract between implementations of Encoding
+// and Index by defining identifiers that uniquely identify standardized coded
+// character sets (CCS) and character encoding schemes (CES), which we will
+// together refer to as encodings, for which Encoding implementations provide
+// converters to and from UTF-8. This package is typically only of concern to
+// implementers of Indexes and Encodings.
+//
+// One part of the identifier is the MIB code, which is defined by IANA and
+// uniquely identifies a CCS or CES. Each code is associated with data that
+// references authorities, official documentation as well as aliases and MIME
+// names.
+//
+// Not all CESs are covered by the IANA registry. The "other" string that is
+// returned by ID can be used to identify other character sets or versions of
+// existing ones.
+//
+// It is recommended that each package that provides a set of Encodings provide
+// the All and Common variables to reference all supported encodings and
+// commonly used subset. This allows Index implementations to include all
+// available encodings without explicitly referencing or knowing about them.
+package identifier
+
+// Note: this package is internal, but could be made public if there is a need
+// for writing third-party Indexes and Encodings.
+
+// References:
+// - http://source.icu-project.org/repos/icu/icu/trunk/source/data/mappings/convrtrs.txt
+// - http://www.iana.org/assignments/character-sets/character-sets.xhtml
+// - http://www.iana.org/assignments/ianacharset-mib/ianacharset-mib
+// - http://www.ietf.org/rfc/rfc2978.txt
+// - http://www.unicode.org/reports/tr22/
+// - http://www.w3.org/TR/encoding/
+// - https://encoding.spec.whatwg.org/
+// - https://encoding.spec.whatwg.org/encodings.json
+// - https://tools.ietf.org/html/rfc6657#section-5
+
+// Interface can be implemented by Encodings to define the CCS or CES for which
+// it implements conversions.
+type Interface interface {
+	// ID returns an encoding identifier. Exactly one of the mib and other
+	// values should be non-zero.
+	//
+	// In the usual case it is only necessary to indicate the MIB code. The
+	// other string can be used to specify encodings for which there is no MIB,
+	// such as "x-mac-dingbat".
+	//
+	// The other string may only contain the characters a-z, A-Z, 0-9, - and _.
+	ID() (mib MIB, other string)
+
+	// NOTE: the restrictions on the encoding are to allow extending the syntax
+	// with additional information such as versions, vendors and other variants.
+}
+
+// A MIB identifies an encoding. It is derived from the IANA MIB codes and adds
+// some identifiers for some encodings that are not covered by the IANA
+// standard.
+//
+// See http://www.iana.org/assignments/ianacharset-mib.
+type MIB uint16
+
+// These additional MIB types are not defined in IANA. They are added because
+// they are common and defined within the text repo.
+const (
+	// Unofficial marks the start of encodings not registered by IANA.
+	Unofficial MIB = 10000 + iota
+
+	// Replacement is the WhatWG replacement encoding.
+	Replacement
+
+	// XUserDefined is the code for x-user-defined.
+	XUserDefined
+
+	// MacintoshCyrillic is the code for x-mac-cyrillic.
+	MacintoshCyrillic
+)