258 lines
6.1 KiB
Go
258 lines
6.1 KiB
Go
|
// Copyright 2013 The Go Authors. All rights reserved.
|
||
|
|
// Use of this source code is governed by a BSD-style
|
||
|
|
// license that can be found in the LICENSE file.
|
||
|
|
|
||
|
|
// Package charset provides common text encodings for HTML documents.
|
||
|
|
//
|
||
|
|
// The mapping from encoding labels to encodings is defined at
|
||
|
|
// https://encoding.spec.whatwg.org/.
|
||
|
|
package charset // import "golang.org/x/net/html/charset"
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bytes"
|
||
|
|
"fmt"
|
||
|
|
"io"
|
||
|
|
"mime"
|
||
|
|
"strings"
|
||
|
|
"unicode/utf8"
|
||
|
|
|
||
|
|
"golang.org/x/net/html"
|
||
|
|
"golang.org/x/text/encoding"
|
||
|
|
"golang.org/x/text/encoding/charmap"
|
||
|
|
"golang.org/x/text/encoding/htmlindex"
|
||
|
|
"golang.org/x/text/transform"
|
||
|
|
)
|
||
|
|
|
||
|
|
// Lookup returns the encoding with the specified label, and its canonical
|
||
|
|
// name. It returns nil and the empty string if label is not one of the
|
||
|
|
// standard encodings for HTML. Matching is case-insensitive and ignores
|
||
|
|
// leading and trailing whitespace. Encoders will use HTML escape sequences for
|
||
|
|
// runes that are not supported by the character set.
|
||
|
|
func Lookup(label string) (e encoding.Encoding, name string) {
|
||
|
|
e, err := htmlindex.Get(label)
|
||
|
|
if err != nil {
|
||
|
|
return nil, ""
|
||
|
|
}
|
||
|
|
name, _ = htmlindex.Name(e)
|
||
|
|
return &htmlEncoding{e}, name
|
||
|
|
}
|
||
|
|
|
||
|
|
type htmlEncoding struct{ encoding.Encoding }
|
||
|
|
|
||
|
|
func (h *htmlEncoding) NewEncoder() *encoding.Encoder {
|
||
|
|
// HTML requires a non-terminating legacy encoder. We use HTML escapes to
|
||
|
|
// substitute unsupported code points.
|
||
|
|
return encoding.HTMLEscapeUnsupported(h.Encoding.NewEncoder())
|
||
|
|
}
|
||
|
|
|
||
|
|
// DetermineEncoding determines the encoding of an HTML document by examining
|
||
|
|
// up to the first 1024 bytes of content and the declared Content-Type.
|
||
|
|
//
|
||
|
|
// See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding
|
||
|
|
func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) {
|
||
|
|
if len(content) > 1024 {
|
||
|
|
content = content[:1024]
|
||
|
|
}
|
||
|
|
|
||
|
|
for _, b := range boms {
|
||
|
|
if bytes.HasPrefix(content, b.bom) {
|
||
|
|
e, name = Lookup(b.enc)
|
||
|
|
return e, name, true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if _, params, err := mime.ParseMediaType(contentType); err == nil {
|
||
|
|
if cs, ok := params["charset"]; ok {
|
||
|
|
if e, name = Lookup(cs); e != nil {
|
||
|
|
return e, name, true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(content) > 0 {
|
||
|
|
e, name = prescan(content)
|
||
|
|
if e != nil {
|
||
|
|
return e, name, false
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Try to detect UTF-8.
|
||
|
|
// First eliminate any partial rune at the end.
|
||
|
|
for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- {
|
||
|
|
b := content[i]
|
||
|
|
if b < 0x80 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
if utf8.RuneStart(b) {
|
||
|
|
content = content[:i]
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
hasHighBit := false
|
||
|
|
for _, c := range content {
|
||
|
|
if c >= 0x80 {
|
||
|
|
hasHighBit = true
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if hasHighBit && utf8.Valid(content) {
|
||
|
|
return encoding.Nop, "utf-8", false
|
||
|
|
}
|
||
|
|
|
||
|
|
// TODO: change default depending on user's locale?
|
||
|
|
return charmap.Windows1252, "windows-1252", false
|
||
|
|
}
|
||
|
|
|
||
|
|
// NewReader returns an io.Reader that converts the content of r to UTF-8.
|
||
|
|
// It calls DetermineEncoding to find out what r's encoding is.
|
||
|
|
func NewReader(r io.Reader, contentType string) (io.Reader, error) {
|
||
|
|
preview := make([]byte, 1024)
|
||
|
|
n, err := io.ReadFull(r, preview)
|
||
|
|
switch {
|
||
|
|
case err == io.ErrUnexpectedEOF:
|
||
|
|
preview = preview[:n]
|
||
|
|
r = bytes.NewReader(preview)
|
||
|
|
case err != nil:
|
||
|
|
return nil, err
|
||
|
|
default:
|
||
|
|
r = io.MultiReader(bytes.NewReader(preview), r)
|
||
|
|
}
|
||
|
|
|
||
|
|
if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop {
|
||
|
|
r = transform.NewReader(r, e.NewDecoder())
|
||
|
|
}
|
||
|
|
return r, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// NewReaderLabel returns a reader that converts from the specified charset to
|
||
|
|
// UTF-8. It uses Lookup to find the encoding that corresponds to label, and
|
||
|
|
// returns an error if Lookup returns nil. It is suitable for use as
|
||
|
|
// encoding/xml.Decoder's CharsetReader function.
|
||
|
|
func NewReaderLabel(label string, input io.Reader) (io.Reader, error) {
|
||
|
|
e, _ := Lookup(label)
|
||
|
|
if e == nil {
|
||
|
|
return nil, fmt.Errorf("unsupported charset: %q", label)
|
||
|
|
}
|
||
|
|
return transform.NewReader(input, e.NewDecoder()), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func prescan(content []byte) (e encoding.Encoding, name string) {
|
||
|
|
z := html.NewTokenizer(bytes.NewReader(content))
|
||
|
|
for {
|
||
|
|
switch z.Next() {
|
||
|
|
case html.ErrorToken:
|
||
|
|
return nil, ""
|
||
|
|
|
||
|
|
case html.StartTagToken, html.SelfClosingTagToken:
|
||
|
|
tagName, hasAttr := z.TagName()
|
||
|
|
if !bytes.Equal(tagName, []byte("meta")) {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
attrList := make(map[string]bool)
|
||
|
|
gotPragma := false
|
||
|
|
|
||
|
|
const (
|
||
|
|
dontKnow = iota
|
||
|
|
doNeedPragma
|
||
|
|
doNotNeedPragma
|
||
|
|
)
|
||
|
|
needPragma := dontKnow
|
||
|
|
|
||
|
|
name = ""
|
||
|
|
e = nil
|
||
|
|
for hasAttr {
|
||
|
|
var key, val []byte
|
||
|
|
key, val, hasAttr = z.TagAttr()
|
||
|
|
ks := string(key)
|
||
|
|
if attrList[ks] {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
attrList[ks] = true
|
||
|
|
for i, c := range val {
|
||
|
|
if 'A' <= c && c <= 'Z' {
|
||
|
|
val[i] = c + 0x20
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
switch ks {
|
||
|
|
case "http-equiv":
|
||
|
|
if bytes.Equal(val, []byte("content-type")) {
|
||
|
|
gotPragma = true
|
||
|
|
}
|
||
|
|
|
||
|
|
case "content":
|
||
|
|
if e == nil {
|
||
|
|
name = fromMetaElement(string(val))
|
||
|
|
if name != "" {
|
||
|
|
e, name = Lookup(name)
|
||
|
|
if e != nil {
|
||
|
|
needPragma = doNeedPragma
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
case "charset":
|
||
|
|
e, name = Lookup(string(val))
|
||
|
|
needPragma = doNotNeedPragma
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
|
||
|
|
if strings.HasPrefix(name, "utf-16") {
|
||
|
|
name = "utf-8"
|
||
|
|
e = encoding.Nop
|
||
|
|
}
|
||
|
|
|
||
|
|
if e != nil {
|
||
|
|
return e, name
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func fromMetaElement(s string) string {
|
||
|
|
for s != "" {
|
||
|
|
csLoc := strings.Index(s, "charset")
|
||
|
|
if csLoc == -1 {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
s = s[csLoc+len("charset"):]
|
||
|
|
s = strings.TrimLeft(s, " \t\n\f\r")
|
||
|
|
if !strings.HasPrefix(s, "=") {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
s = s[1:]
|
||
|
|
s = strings.TrimLeft(s, " \t\n\f\r")
|
||
|
|
if s == "" {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
if q := s[0]; q == '"' || q == '\'' {
|
||
|
|
s = s[1:]
|
||
|
|
closeQuote := strings.IndexRune(s, rune(q))
|
||
|
|
if closeQuote == -1 {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
return s[:closeQuote]
|
||
|
|
}
|
||
|
|
|
||
|
|
end := strings.IndexAny(s, "; \t\n\f\r")
|
||
|
|
if end == -1 {
|
||
|
|
end = len(s)
|
||
|
|
}
|
||
|
|
return s[:end]
|
||
|
|
}
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
|
||
|
|
var boms = []struct {
|
||
|
|
bom []byte
|
||
|
|
enc string
|
||
|
|
}{
|
||
|
|
{[]byte{0xfe, 0xff}, "utf-16be"},
|
||
|
|
{[]byte{0xff, 0xfe}, "utf-16le"},
|
||
|
|
{[]byte{0xef, 0xbb, 0xbf}, "utf-8"},
|
||
|
|
}
|