diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 88b7e712aa7..84c64e44d16 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -20,8 +20,8 @@ }, { "ImportPath": "github.com/Sirupsen/logrus", - "Comment": "v0.7.3-2-g26709e2", - "Rev": "26709e2714106fb8ad40b773b711ebce25b78914" + "Comment": "v0.8.2-2-g6ba91e2", + "Rev": "6ba91e24c498b49d0363c723e9e2ab2b5b8fd012" }, { "ImportPath": "github.com/bren2010/proquint", @@ -37,11 +37,11 @@ }, { "ImportPath": "github.com/cenkalti/backoff", - "Rev": "9831e1e25c874e0a0601b6dc43641071414eec7a" + "Rev": "6c45d6bc1e78d94431dff8fc28a99f20bafa355a" }, { "ImportPath": "github.com/cheggaaa/pb", - "Rev": "e8c7cc515bfde3e267957a3b110080ceed51354e" + "Rev": "d7729fd7ec1372c15b83db39834bf842bf2d69fb" }, { "ImportPath": "github.com/codahale/hdrhistogram", @@ -57,11 +57,11 @@ }, { "ImportPath": "github.com/crowdmob/goamz/aws", - "Rev": "82345796204222aa56be89cf930c316b1297f906" + "Rev": "3a06871fe9fc0281ca90f3a7d97258d042ed64c0" }, { "ImportPath": "github.com/crowdmob/goamz/s3", - "Rev": "82345796204222aa56be89cf930c316b1297f906" + "Rev": "3a06871fe9fc0281ca90f3a7d97258d042ed64c0" }, { "ImportPath": "github.com/cryptix/mdns", @@ -77,20 +77,20 @@ }, { "ImportPath": "github.com/fd/go-nat", - "Rev": "50e7633d5f27d81490026a13e5b92d2e42d8c6bb" + "Rev": "dcaf50131e4810440bed2cbb6f7f32c4f4cc95dd" }, { "ImportPath": "github.com/fzzy/radix/redis", - "Comment": "v0.5.1", - "Rev": "27a863cdffdb0998d13e1e11992b18489aeeaa25" + "Comment": "v0.5.6", + "Rev": "031cc11e9800a2626ee2ae629655a922b630a07d" }, { "ImportPath": "github.com/gogo/protobuf/io", - "Rev": "0ac967c269268f1af7d9bcc7927ccc9a589b2b36" + "Rev": "b9e369e8ffb6773efc654ea13594566404314ee1" }, { "ImportPath": "github.com/gogo/protobuf/proto", - "Rev": "0ac967c269268f1af7d9bcc7927ccc9a589b2b36" + "Rev": "b9e369e8ffb6773efc654ea13594566404314ee1" }, { "ImportPath": "github.com/h2so5/utp", @@ -98,28 +98,32 @@ }, { "ImportPath": "github.com/hashicorp/golang-lru", - "Rev": "253b2dc1ca8bae42c3b5b6e53dd2eab1a7551116" + "Rev": "995efda3e073b6946b175ed93901d729ad47466a" }, { "ImportPath": "github.com/hashicorp/yamux", - "Rev": "9feabe6854fadca1abec9cd3bd2a613fe9a34000" + "Rev": "b2e55852ddaf823a85c67f798080eb7d08acd71d" }, { "ImportPath": "github.com/howeyc/fsnotify", - "Comment": "v0.9.0-11-g6b1ef89", - "Rev": "6b1ef893dc11e0447abda6da20a5203481878dda" + "Comment": "v0.9.0-14-g4894fe7", + "Rev": "4894fe7efedeeef21891033e1cce3b23b9af7ad2" }, { "ImportPath": "github.com/huin/goupnp", - "Rev": "223008361153d7d434c1f0ac990cd3fcae6931f5" + "Rev": "c57ae84388ab59076fd547f1abeab71c2edb0a21" }, { "ImportPath": "github.com/inconshreveable/go-update", "Rev": "68f5725818189545231c1fd8694793d45f2fc529" }, + { + "ImportPath": "github.com/jackpal/gateway", + "Rev": "192609c58b8985e645cbe82ddcb28a4362ca0fdc" + }, { "ImportPath": "github.com/jackpal/go-nat-pmp", - "Rev": "a45aa3d54aef73b504e15eb71bea0e5565b5e6e1" + "Rev": "46523a463303c6ede3ddfe45bde1c7ed52ebaacd" }, { "ImportPath": "github.com/jbenet/go-base58", @@ -169,7 +173,7 @@ }, { "ImportPath": "github.com/jbenet/go-peerstream", - "Rev": "8d52ed2801410a2af995b4e87660272d11c8a9a4" + "Rev": "675a5da7e3500d73c2edc84565d6c46b540ad1b4" }, { "ImportPath": "github.com/jbenet/go-random", @@ -193,7 +197,7 @@ }, { "ImportPath": "github.com/kardianos/osext", - "Rev": "8fef92e41e22a70e700a96b29f066cda30ea24ef" + "Rev": "6e7f843663477789fac7c02def0d0909e969b4e5" }, { "ImportPath": "github.com/kr/binarydist", @@ -201,7 +205,7 @@ }, { "ImportPath": "github.com/miekg/dns", - "Rev": "82ffc45b1f84ff71bd1cebed8b210118ce3d181e" + "Rev": "bb1103f648f811d2018d4bedcb2d4b2bce34a0f1" }, { "ImportPath": "github.com/mitchellh/go-homedir", @@ -222,7 +226,7 @@ }, { "ImportPath": "github.com/syndtr/goleveldb/leveldb", - "Rev": "4875955338b0a434238a31165cb87255ab6e9e4a" + "Rev": "315fcfb05d4d46d4354b313d146ef688dda272a9" }, { "ImportPath": "github.com/syndtr/gosnappy/snappy", @@ -238,31 +242,43 @@ }, { "ImportPath": "github.com/whyrusleeping/iptb", - "Rev": "3970c95a864f1a40037f796ff596607ce8ae43be" + "Rev": "fa9bbc437fae1c3a9410e7f1bc3dd02f0449279a" }, { "ImportPath": "golang.org/x/crypto/blowfish", - "Rev": "c84e1f8e3a7e322d497cd16c0e8a13c7e127baf3" + "Rev": "ce6bda69189e9f4ff278a5e181691cd695f753ae" }, { "ImportPath": "golang.org/x/crypto/sha3", - "Rev": "c84e1f8e3a7e322d497cd16c0e8a13c7e127baf3" + "Rev": "ce6bda69189e9f4ff278a5e181691cd695f753ae" }, { "ImportPath": "golang.org/x/net/context", - "Rev": "ff8eb9a34a5cbb9941ffc6f84a19a8014c2646ad" + "Rev": "589db58a47224e5786650dac2677b9c302bab6c2" + }, + { + "ImportPath": "golang.org/x/net/html", + "Rev": "589db58a47224e5786650dac2677b9c302bab6c2" }, { "ImportPath": "golang.org/x/net/internal/iana", - "Rev": "ff8eb9a34a5cbb9941ffc6f84a19a8014c2646ad" + "Rev": "589db58a47224e5786650dac2677b9c302bab6c2" }, { "ImportPath": "golang.org/x/net/ipv4", - "Rev": "ff8eb9a34a5cbb9941ffc6f84a19a8014c2646ad" + "Rev": "589db58a47224e5786650dac2677b9c302bab6c2" }, { "ImportPath": "golang.org/x/net/ipv6", - "Rev": "ff8eb9a34a5cbb9941ffc6f84a19a8014c2646ad" + "Rev": "589db58a47224e5786650dac2677b9c302bab6c2" + }, + { + "ImportPath": "golang.org/x/text/encoding", + "Rev": "c93e7c9fff19fb9139b5ab04ce041833add0134e" + }, + { + "ImportPath": "golang.org/x/text/transform", + "Rev": "c93e7c9fff19fb9139b5ab04ce041833add0134e" }, { "ImportPath": "gopkg.in/fsnotify.v1", @@ -271,8 +287,8 @@ }, { "ImportPath": "gopkg.in/natefinch/lumberjack.v2", - "Comment": "v1.0-12-gd28785c", - "Rev": "d28785c2f27cd682d872df46ccd8232843629f54" + "Comment": "v1.0-15-g588a21f", + "Rev": "588a21fb0fa0ebdfde42670fa214576b6f0f22df" }, { "ImportPath": "gopkg.in/tomb.v1", diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/CHANGELOG.md b/Godeps/_workspace/src/github.com/Sirupsen/logrus/CHANGELOG.md index eb72bff93b7..49c5506a3e8 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/CHANGELOG.md +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/CHANGELOG.md @@ -1,3 +1,17 @@ +# 0.8.2 + +logrus: fix more Fatal family functions + +# 0.8.1 + +logrus: fix not exiting on `Fatalf` and `Fatalln` + +# 0.8.0 + +logrus: defaults to stderr instead of stdout +hooks/sentry: add special field for `*http.Request` +formatter/text: ignore Windows for colors + # 0.7.3 formatter/\*: allow configuration of timestamp layout diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/README.md b/Godeps/_workspace/src/github.com/Sirupsen/logrus/README.md index d55f9092479..c963c7d4d75 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/README.md +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/README.md @@ -32,7 +32,7 @@ ocean","size":10,"time":"2014-03-10 19:57:38.562264131 -0400 EDT"} "time":"2014-03-10 19:57:38.562543128 -0400 EDT"} ``` -With the default `log.Formatter = new(logrus.TextFormatter)` when a TTY is not +With the default `log.Formatter = new(&log.TextFormatter{})` when a TTY is not attached, the output is compatible with the [logfmt](http://godoc.org/github.com/kr/logfmt) format: @@ -183,7 +183,7 @@ Logrus comes with [built-in hooks](hooks/). Add those, or your custom hook, in import ( log "github.com/Sirupsen/logrus" "github.com/Sirupsen/logrus/hooks/airbrake" - "github.com/Sirupsen/logrus/hooks/syslog" + logrus_syslog "github.com/Sirupsen/logrus/hooks/syslog" "log/syslog" ) @@ -211,6 +211,7 @@ func init() { | [Slackrus](https://github.com/johntdyer/slackrus) | Hook for Slack chat. | | [Journalhook](https://github.com/wercker/journalhook) | Hook for logging to `systemd-journald` | | [Graylog](https://github.com/gemnasium/logrus-hooks/tree/master/graylog) | Hook for logging to [Graylog](http://graylog2.org/) | +| [Raygun](https://github.com/squirkle/logrus-raygun-hook) | Hook for logging to [Raygun.io](http://raygun.io/) | #### Level logging @@ -269,7 +270,7 @@ init() { log.SetFormatter(logrus.JSONFormatter) } else { // The TextFormatter is default, you don't actually have to do this. - log.SetFormatter(logrus.TextFormatter) + log.SetFormatter(&log.TextFormatter{}) } } ``` diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/entry.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/entry.go index 17fe6f707bc..699ea035cce 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/entry.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/entry.go @@ -188,6 +188,7 @@ func (entry *Entry) Fatalf(format string, args ...interface{}) { if entry.Logger.Level >= FatalLevel { entry.Fatal(fmt.Sprintf(format, args...)) } + os.Exit(1) } func (entry *Entry) Panicf(format string, args ...interface{}) { @@ -234,6 +235,7 @@ func (entry *Entry) Fatalln(args ...interface{}) { if entry.Logger.Level >= FatalLevel { entry.Fatal(entry.sprintlnn(args...)) } + os.Exit(1) } func (entry *Entry) Panicln(args ...interface{}) { diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/README.md b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/README.md index 19e58bb4572..4e1c1476f5b 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/README.md +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/README.md @@ -34,12 +34,13 @@ func main() { ## Special fields Some logrus fields have a special meaning in this hook, -these are server_name and logger. +these are `server_name`, `logger` and `http_request`. When logs are sent to sentry these fields are treated differently. -- server_name (also known as hostname) is the name of the server which +- `server_name` (also known as hostname) is the name of the server which is logging the event (hostname.example.com) -- logger is the part of the application which is logging the event. +- `logger` is the part of the application which is logging the event. In go this usually means setting it to the name of the package. +- `http_request` is the in-coming request(*http.Request). The detailed request data are sent to Sentry. ## Timeout diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/sentry.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/sentry.go index bb3b1e9e9dd..9810c5a9d92 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/sentry.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/sentry.go @@ -2,6 +2,7 @@ package logrus_sentry import ( "fmt" + "net/http" "time" "github.com/getsentry/raven-go" @@ -36,6 +37,22 @@ func getAndDel(d logrus.Fields, key string) (string, bool) { return val, true } +func getAndDelRequest(d logrus.Fields, key string) (*http.Request, bool) { + var ( + ok bool + v interface{} + req *http.Request + ) + if v, ok = d[key]; !ok { + return nil, false + } + if req, ok = v.(*http.Request); !ok || req == nil { + return nil, false + } + delete(d, key) + return req, true +} + // SentryHook delivers logs to a sentry server. type SentryHook struct { // Timeout sets the time to wait for a delivery error from the sentry server. @@ -61,7 +78,7 @@ func NewSentryHook(DSN string, levels []logrus.Level) (*SentryHook, error) { // Called when an event should be sent to sentry // Special fields that sentry uses to give more information to the server // are extracted from entry.Data (if they are found) -// These fields are: logger and server_name +// These fields are: logger, server_name and http_request func (hook *SentryHook) Fire(entry *logrus.Entry) error { packet := &raven.Packet{ Message: entry.Message, @@ -78,6 +95,9 @@ func (hook *SentryHook) Fire(entry *logrus.Entry) error { if serverName, ok := getAndDel(d, "server_name"); ok { packet.ServerName = serverName } + if req, ok := getAndDelRequest(d, "http_request"); ok { + packet.Interfaces = append(packet.Interfaces, raven.NewHttp(req)) + } packet.Extra = map[string]interface{}(d) _, errCh := hook.client.Capture(packet, nil) diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/sentry_test.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/sentry_test.go index dff1a06f4a1..dc6f98b5340 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/sentry_test.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/sentry_test.go @@ -61,9 +61,12 @@ func TestSpecialFields(t *testing.T) { t.Fatal(err.Error()) } logger.Hooks.Add(hook) + + req, _ := http.NewRequest("GET", "url", nil) logger.WithFields(logrus.Fields{ - "server_name": server_name, - "logger": logger_name, + "server_name": server_name, + "logger": logger_name, + "http_request": req, }).Error(message) packet := <-pch diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter.go index dcc4f1d9fd7..2ad6dc5cf4f 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter.go @@ -24,11 +24,12 @@ func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) { } prefixFieldClashes(data) - if f.TimestampFormat == "" { - f.TimestampFormat = DefaultTimestampFormat + timestampFormat := f.TimestampFormat + if timestampFormat == "" { + timestampFormat = DefaultTimestampFormat } - data["time"] = entry.Time.Format(f.TimestampFormat) + data["time"] = entry.Time.Format(timestampFormat) data["msg"] = entry.Message data["level"] = entry.Level.String() diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/logger.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/logger.go index da928a37509..292b0b2c070 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/logger.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/logger.go @@ -44,7 +44,7 @@ type Logger struct { // It's recommended to make this a global instance called `log`. func New() *Logger { return &Logger{ - Out: os.Stdout, + Out: os.Stderr, Formatter: new(TextFormatter), Hooks: make(levelHooks), Level: InfoLevel, @@ -102,6 +102,7 @@ func (logger *Logger) Fatalf(format string, args ...interface{}) { if logger.Level >= FatalLevel { NewEntry(logger).Fatalf(format, args...) } + os.Exit(1) } func (logger *Logger) Panicf(format string, args ...interface{}) { @@ -148,6 +149,7 @@ func (logger *Logger) Fatal(args ...interface{}) { if logger.Level >= FatalLevel { NewEntry(logger).Fatal(args...) } + os.Exit(1) } func (logger *Logger) Panic(args ...interface{}) { @@ -194,6 +196,7 @@ func (logger *Logger) Fatalln(args ...interface{}) { if logger.Level >= FatalLevel { NewEntry(logger).Fatalln(args...) } + os.Exit(1) } func (logger *Logger) Panicln(args ...interface{}) { diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter.go index 612417ff9c8..4ed90e1e52d 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter.go @@ -3,6 +3,7 @@ package logrus import ( "bytes" "fmt" + "runtime" "sort" "strings" "time" @@ -69,7 +70,8 @@ func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { prefixFieldClashes(entry.Data) - isColored := (f.ForceColors || isTerminal) && !f.DisableColors + isColorTerminal := isTerminal && (runtime.GOOS != "windows") + isColored := (f.ForceColors || isColorTerminal) && !f.DisableColors if f.TimestampFormat == "" { f.TimestampFormat = DefaultTimestampFormat diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential.go b/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential.go index e81e9c67a93..59ddd734bc3 100644 --- a/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential.go +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential.go @@ -74,7 +74,7 @@ const ( // NewExponentialBackOff creates an instance of ExponentialBackOff using default values. func NewExponentialBackOff() *ExponentialBackOff { - return &ExponentialBackOff{ + b := &ExponentialBackOff{ InitialInterval: DefaultInitialInterval, RandomizationFactor: DefaultRandomizationFactor, Multiplier: DefaultMultiplier, @@ -82,6 +82,8 @@ func NewExponentialBackOff() *ExponentialBackOff { MaxElapsedTime: DefaultMaxElapsedTime, Clock: SystemClock, } + b.Reset() + return b } type systemClock struct{} diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential_test.go b/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential_test.go index 2af22b8bd23..339c534baba 100644 --- a/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential_test.go +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential_test.go @@ -77,9 +77,6 @@ func TestGetElapsedTime(t *testing.T) { func TestMaxElapsedTime(t *testing.T) { var exp = NewExponentialBackOff() exp.Clock = &TestClock{start: time.Time{}.Add(10000 * time.Second)} - if exp.NextBackOff() != Stop { - t.Error("error2") - } // Change the currentElapsedTime to be 0 ensuring that the elapsed time will be greater // than the max elapsed time. exp.startTime = time.Time{} diff --git a/Godeps/_workspace/src/github.com/cheggaaa/pb/README.md b/Godeps/_workspace/src/github.com/cheggaaa/pb/README.md index 9a6f6b41fa4..af5c4bd50f3 100644 --- a/Godeps/_workspace/src/github.com/cheggaaa/pb/README.md +++ b/Godeps/_workspace/src/github.com/cheggaaa/pb/README.md @@ -58,10 +58,10 @@ bar.ShowTimeLeft = true bar.ShowSpeed = true // sets the width of the progress bar -bar.SetWith(80) +bar.SetWidth(80) // sets the width of the progress bar, but if terminal size smaller will be ignored -bar.SetMaxWith(80) +bar.SetMaxWidth(80) // convert output to readable format (like KB, MB) bar.SetUnits(pb.U_BYTES) diff --git a/Godeps/_workspace/src/github.com/cheggaaa/pb/format.go b/Godeps/_workspace/src/github.com/cheggaaa/pb/format.go index e024e36d40c..1dd210be4b7 100644 --- a/Godeps/_workspace/src/github.com/cheggaaa/pb/format.go +++ b/Godeps/_workspace/src/github.com/cheggaaa/pb/format.go @@ -6,21 +6,24 @@ import ( "strings" ) +type Units int + const ( // By default, without type handle - U_NO = 0 + U_NO Units = iota // Handle as b, Kb, Mb, etc - U_BYTES = 1 + U_BYTES ) // Format integer -func Format(i int64, units int) string { +func Format(i int64, units Units) string { switch units { case U_BYTES: return FormatBytes(i) + default: + // by default just convert to string + return strconv.FormatInt(i, 10) } - // by default just convert to string - return strconv.Itoa(int(i)) } // Convert bytes to human readable string. Like a 2 MB, 64.2 KB, 52 B diff --git a/Godeps/_workspace/src/github.com/cheggaaa/pb/pb.go b/Godeps/_workspace/src/github.com/cheggaaa/pb/pb.go index 01c715720ed..104bd4a60e2 100644 --- a/Godeps/_workspace/src/github.com/cheggaaa/pb/pb.go +++ b/Godeps/_workspace/src/github.com/cheggaaa/pb/pb.go @@ -5,6 +5,7 @@ import ( "io" "math" "strings" + "sync" "sync/atomic" "time" ) @@ -24,13 +25,13 @@ var ( ) // Create new progress bar object -func New(total int) (pb *ProgressBar) { +func New(total int) *ProgressBar { return New64(int64(total)) } // Create new progress bar object uding int64 as total -func New64(total int64) (pb *ProgressBar) { - pb = &ProgressBar{ +func New64(total int64) *ProgressBar { + pb := &ProgressBar{ Total: total, RefreshRate: DEFAULT_REFRESH_RATE, ShowPercent: true, @@ -38,18 +39,17 @@ func New64(total int64) (pb *ProgressBar) { ShowBar: true, ShowTimeLeft: true, ShowFinalTime: true, + Units: U_NO, ManualUpdate: false, + isFinish: make(chan struct{}), currentValue: -1, } - pb.Format(FORMAT) - return + return pb.Format(FORMAT) } // Create new object and start -func StartNew(total int) (pb *ProgressBar) { - pb = New(total) - pb.Start() - return +func StartNew(total int) *ProgressBar { + return New(total).Start() } // Callback for custom output @@ -71,13 +71,16 @@ type ProgressBar struct { Output io.Writer Callback Callback NotPrint bool - Units int + Units Units Width int ForceWidth bool ManualUpdate bool - isFinish bool + finishOnce sync.Once //Guards isFinish + isFinish chan struct{} + startTime time.Time + startValue int64 currentValue int64 prefix, postfix string @@ -90,16 +93,18 @@ type ProgressBar struct { } // Start print -func (pb *ProgressBar) Start() { +func (pb *ProgressBar) Start() *ProgressBar { pb.startTime = time.Now() + pb.startValue = pb.current if pb.Total == 0 { pb.ShowBar = false pb.ShowTimeLeft = false pb.ShowPercent = false - } + } if !pb.ManualUpdate { go pb.writer() } + return pb } // Increment current value @@ -108,8 +113,14 @@ func (pb *ProgressBar) Increment() int { } // Set current value -func (pb *ProgressBar) Set(current int) { - atomic.StoreInt64(&pb.current, int64(current)) +func (pb *ProgressBar) Set(current int) *ProgressBar { + return pb.Set64(int64(current)) +} + +// Set64 sets the current value as int64 +func (pb *ProgressBar) Set64(current int64) *ProgressBar { + atomic.StoreInt64(&pb.current, current) + return pb } // Add to current value @@ -122,75 +133,69 @@ func (pb *ProgressBar) Add64(add int64) int64 { } // Set prefix string -func (pb *ProgressBar) Prefix(prefix string) (bar *ProgressBar) { +func (pb *ProgressBar) Prefix(prefix string) *ProgressBar { pb.prefix = prefix return pb } // Set postfix string -func (pb *ProgressBar) Postfix(postfix string) (bar *ProgressBar) { +func (pb *ProgressBar) Postfix(postfix string) *ProgressBar { pb.postfix = postfix return pb } // Set custom format for bar // Example: bar.Format("[=>_]") -func (pb *ProgressBar) Format(format string) (bar *ProgressBar) { - bar = pb +func (pb *ProgressBar) Format(format string) *ProgressBar { formatEntries := strings.Split(format, "") - if len(formatEntries) != 5 { - return + if len(formatEntries) == 5 { + pb.BarStart = formatEntries[0] + pb.BarEnd = formatEntries[4] + pb.Empty = formatEntries[3] + pb.Current = formatEntries[1] + pb.CurrentN = formatEntries[2] } - pb.BarStart = formatEntries[0] - pb.BarEnd = formatEntries[4] - pb.Empty = formatEntries[3] - pb.Current = formatEntries[1] - pb.CurrentN = formatEntries[2] - return + return pb } // Set bar refresh rate -func (pb *ProgressBar) SetRefreshRate(rate time.Duration) (bar *ProgressBar) { - bar = pb +func (pb *ProgressBar) SetRefreshRate(rate time.Duration) *ProgressBar { pb.RefreshRate = rate - return + return pb } // Set units // bar.SetUnits(U_NO) - by default // bar.SetUnits(U_BYTES) - for Mb, Kb, etc -func (pb *ProgressBar) SetUnits(units int) (bar *ProgressBar) { - bar = pb - switch units { - case U_NO, U_BYTES: - pb.Units = units - } - return +func (pb *ProgressBar) SetUnits(units Units) *ProgressBar { + pb.Units = units + return pb } // Set max width, if width is bigger than terminal width, will be ignored -func (pb *ProgressBar) SetMaxWidth(width int) (bar *ProgressBar) { - bar = pb +func (pb *ProgressBar) SetMaxWidth(width int) *ProgressBar { pb.Width = width pb.ForceWidth = false - return + return pb } // Set bar width -func (pb *ProgressBar) SetWidth(width int) (bar *ProgressBar) { - bar = pb +func (pb *ProgressBar) SetWidth(width int) *ProgressBar { pb.Width = width pb.ForceWidth = true - return + return pb } // End print func (pb *ProgressBar) Finish() { - pb.isFinish = true - pb.write(atomic.LoadInt64(&pb.current)) - if !pb.NotPrint { - fmt.Println() - } + //Protect multiple calls + pb.finishOnce.Do(func() { + close(pb.isFinish) + pb.write(atomic.LoadInt64(&pb.current)) + if !pb.NotPrint { + fmt.Println() + } + }) } // End print and write string 'str' @@ -228,7 +233,7 @@ func (pb *ProgressBar) write(current int64) { percent := float64(current) / (float64(pb.Total) / float64(100)) percentBox = fmt.Sprintf(" %#.02f %% ", percent) } - + // counters if pb.ShowCounters { if pb.Total > 0 { @@ -240,22 +245,26 @@ func (pb *ProgressBar) write(current int64) { // time left fromStart := time.Now().Sub(pb.startTime) - if pb.isFinish { + currentFromStart := current - pb.startValue + select { + case <-pb.isFinish: if pb.ShowFinalTime { left := (fromStart / time.Second) * time.Second timeLeftBox = left.String() } - } else if pb.ShowTimeLeft && current > 0 { - perEntry := fromStart / time.Duration(current) - left := time.Duration(pb.Total-current) * perEntry - left = (left / time.Second) * time.Second - timeLeftBox = left.String() + default: + if pb.ShowTimeLeft && currentFromStart > 0 { + perEntry := fromStart / time.Duration(currentFromStart) + left := time.Duration(pb.Total-currentFromStart) * perEntry + left = (left / time.Second) * time.Second + timeLeftBox = left.String() + } } // speed - if pb.ShowSpeed && current > 0 { + if pb.ShowSpeed && currentFromStart > 0 { fromStart := time.Now().Sub(pb.startTime) - speed := float64(current) / (float64(fromStart) / float64(time.Second)) + speed := float64(currentFromStart) / (float64(fromStart) / float64(time.Second)) speedBox = Format(int64(speed), pb.Units) + "/s " } @@ -324,12 +333,14 @@ func (pb *ProgressBar) Update() { // Internal loop for writing progressbar func (pb *ProgressBar) writer() { + pb.Update() for { - if pb.isFinish { - break + select { + case <-pb.isFinish: + return + case <-time.After(pb.RefreshRate): + pb.Update() } - pb.Update() - time.Sleep(pb.RefreshRate) } } diff --git a/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_nix.go b/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_nix.go index 75f9ba31469..5db4e523fc4 100644 --- a/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_nix.go +++ b/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_nix.go @@ -1,4 +1,4 @@ -// +build linux darwin freebsd openbsd +// +build linux darwin freebsd netbsd openbsd package pb diff --git a/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_test.go b/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_test.go index 75783d63089..dfe394fd4bb 100644 --- a/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_test.go +++ b/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_test.go @@ -28,3 +28,10 @@ func Test_Width(t *testing.T) { bar.Increment() bar.Finish() } + +func Test_MultipleFinish(t *testing.T) { + bar := New(5000) + bar.Add(2000) + bar.Finish() + bar.Finish() +} diff --git a/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_x.go b/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_x.go index b630e592f11..dd5f906e186 100644 --- a/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_x.go +++ b/Godeps/_workspace/src/github.com/cheggaaa/pb/pb_x.go @@ -1,4 +1,4 @@ -// +build linux darwin freebsd openbsd solaris +// +build linux darwin freebsd netbsd openbsd solaris package pb diff --git a/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/regions.go b/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/regions.go index 97e12e1dcf4..4e39069ef60 100644 --- a/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/regions.go +++ b/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/regions.go @@ -63,7 +63,7 @@ var USWest = Region{ ServiceInfo{"https://monitoring.us-west-1.amazonaws.com", V2Signature}, "https://autoscaling.us-west-1.amazonaws.com", ServiceInfo{"https://rds.us-west-1.amazonaws.com", V2Signature}, - "", + "https://kinesis.us-west-1.amazonaws.com", "https://sts.amazonaws.com", "https://cloudformation.us-west-1.amazonaws.com", "https://elasticache.us-west-1.amazonaws.com", diff --git a/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3.go b/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3.go index c4800330317..0f8607d37fc 100644 --- a/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3.go +++ b/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3.go @@ -806,7 +806,7 @@ func (b *Bucket) SignedURLWithMethod(method, path string, expires time.Time, par if err != nil { panic(err) } - if b.S3.Auth.Token() != "" { + if b.S3.Auth.Token() != "" && b.S3.Signature == aws.V2Signature { return u.String() + "&x-amz-security-token=" + url.QueryEscape(req.headers["X-Amz-Security-Token"][0]) } else { return u.String() @@ -1041,8 +1041,10 @@ func (s3 *S3) prepare(req *request) error { } } - if s3.Auth.Token() != "" { + if s3.Signature == aws.V2Signature && s3.Auth.Token() != "" { req.headers["X-Amz-Security-Token"] = []string{s3.Auth.Token()} + } else if s3.Auth.Token() != "" { + req.params.Set("X-Amz-Security-Token", s3.Auth.Token()) } if s3.Signature == aws.V2Signature { @@ -1233,6 +1235,19 @@ func shouldRetry(err error) bool { case "read", "write": return true } + case *url.Error: + // url.Error can be returned either by net/url if a URL cannot be + // parsed, or by net/http if the response is closed before the headers + // are received or parsed correctly. In that later case, e.Op is set to + // the HTTP method name with the first letter uppercased. We don't want + // to retry on POST operations, since those are not idempotent, all the + // other ones should be safe to retry. + switch e.Op { + case "Get", "Put", "Delete", "Head": + return shouldRetry(e.Err) + default: + return false + } case *Error: switch e.Code { case "InternalError", "NoSuchUpload", "NoSuchBucket": diff --git a/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3test/server.go b/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3test/server.go index 9b9b3c09c70..d2a520ece89 100644 --- a/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3test/server.go +++ b/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3test/server.go @@ -432,11 +432,74 @@ func (r bucketResource) put(a *action) interface{} { return nil } -func (bucketResource) post(a *action) interface{} { - fatalf(400, "Method", "bucket POST method not available") +func (r bucketResource) post(a *action) interface{} { + if _, multiDel := a.req.URL.Query()["delete"]; multiDel { + return r.multiDel(a) + } + + fatalf(400, "Method", "bucket operation not supported") return nil } +func (b bucketResource) multiDel(a *action) interface{} { + type multiDelRequestObject struct { + Key string + VersionId string + } + + type multiDelRequest struct { + Quiet bool + Object []*multiDelRequestObject + } + + type multiDelDelete struct { + XMLName struct{} `xml:"Deleted"` + Key string + } + + type multiDelError struct { + XMLName struct{} `xml:"Error"` + Key string + Code string + Message string + } + + type multiDelResult struct { + XMLName struct{} `xml:"DeleteResult"` + Deleted []*multiDelDelete + Error []*multiDelError + } + + req := &multiDelRequest{} + + if err := xml.NewDecoder(a.req.Body).Decode(req); err != nil { + fatalf(400, "InvalidRequest", err.Error()) + } + + res := &multiDelResult{ + Deleted: []*multiDelDelete{}, + Error: []*multiDelError{}, + } + + for _, o := range req.Object { + if _, exists := b.bucket.objects[o.Key]; exists { + delete(b.bucket.objects, o.Key) + + res.Deleted = append(res.Deleted, &multiDelDelete{ + Key: o.Key, + }) + } else { + res.Error = append(res.Error, &multiDelError{ + Key: o.Key, + Code: "AccessDenied", + Message: "Access Denied", + }) + } + } + + return res +} + // validBucketName returns whether name is a valid bucket name. // Here are the rules, from: // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/BucketRestrictions.html diff --git a/Godeps/_workspace/src/github.com/fd/go-nat/natpmp.go b/Godeps/_workspace/src/github.com/fd/go-nat/natpmp.go index 9d5fb675f77..541dacb4ceb 100644 --- a/Godeps/_workspace/src/github.com/fd/go-nat/natpmp.go +++ b/Godeps/_workspace/src/github.com/fd/go-nat/natpmp.go @@ -4,82 +4,19 @@ import ( "net" "time" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jackpal/gateway" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jackpal/go-nat-pmp" ) var ( - _ NAT = (*natpmp_NAT)(nil) + _ NAT = (*natpmpNAT)(nil) ) -func natpmp_PotentialGateways() ([]net.IP, error) { - _, ipNet_10, err := net.ParseCIDR("10.0.0.0/8") - if err != nil { - panic(err) - } - - _, ipNet_172_16, err := net.ParseCIDR("172.16.0.0/12") - if err != nil { - panic(err) - } - - _, ipNet_192_168, err := net.ParseCIDR("192.168.0.0/16") - if err != nil { - panic(err) - } - - ifaces, err := net.Interfaces() - if err != nil { - return nil, err - } - - var ips []net.IP - - for _, iface := range ifaces { - addrs, err := iface.Addrs() - if err != nil { - return nil, err - } - - for _, addr := range addrs { - switch x := addr.(type) { - case *net.IPNet: - var ipNet *net.IPNet - if ipNet_10.Contains(x.IP) { - ipNet = ipNet_10 - } else if ipNet_172_16.Contains(x.IP) { - ipNet = ipNet_172_16 - } else if ipNet_192_168.Contains(x.IP) { - ipNet = ipNet_192_168 - } - - if ipNet != nil { - ip := x.IP.Mask(x.Mask) - ip = ip.To4() - if ip != nil { - ip[3] = ip[3] | 0x01 - ips = append(ips, ip) - } - } - } - } - } - - if len(ips) == 0 { - return nil, ErrNoNATFound - } - - return ips, nil -} - func discoverNATPMP() <-chan NAT { - ips, err := natpmp_PotentialGateways() - if err != nil { - return nil - } - - res := make(chan NAT, len(ips)) + res := make(chan NAT, 1) - for _, ip := range ips { + ip, err := gateway.DiscoverGateway() + if err == nil { go discoverNATPMPWithAddr(res, ip) } @@ -93,20 +30,20 @@ func discoverNATPMPWithAddr(c chan NAT, ip net.IP) { return } - c <- &natpmp_NAT{client, ip, make(map[int]int)} + c <- &natpmpNAT{client, ip, make(map[int]int)} } -type natpmp_NAT struct { +type natpmpNAT struct { c *natpmp.Client gateway net.IP ports map[int]int } -func (n *natpmp_NAT) GetDeviceAddress() (addr net.IP, err error) { +func (n *natpmpNAT) GetDeviceAddress() (addr net.IP, err error) { return n.gateway, nil } -func (n *natpmp_NAT) GetInternalAddress() (addr net.IP, err error) { +func (n *natpmpNAT) GetInternalAddress() (addr net.IP, err error) { ifaces, err := net.Interfaces() if err != nil { return nil, err @@ -131,7 +68,7 @@ func (n *natpmp_NAT) GetInternalAddress() (addr net.IP, err error) { return nil, ErrNoInternalAddress } -func (n *natpmp_NAT) GetExternalAddress() (addr net.IP, err error) { +func (n *natpmpNAT) GetExternalAddress() (addr net.IP, err error) { res, err := n.c.GetExternalAddress() if err != nil { return nil, err @@ -141,7 +78,7 @@ func (n *natpmp_NAT) GetExternalAddress() (addr net.IP, err error) { return net.IPv4(d[0], d[1], d[2], d[3]), nil } -func (n *natpmp_NAT) AddPortMapping(protocol string, internalPort int, description string, timeout time.Duration) (int, error) { +func (n *natpmpNAT) AddPortMapping(protocol string, internalPort int, description string, timeout time.Duration) (int, error) { var ( err error ) @@ -168,11 +105,11 @@ func (n *natpmp_NAT) AddPortMapping(protocol string, internalPort int, descripti return 0, err } -func (n *natpmp_NAT) DeletePortMapping(protocol string, internalPort int) (err error) { +func (n *natpmpNAT) DeletePortMapping(protocol string, internalPort int) (err error) { delete(n.ports, internalPort) return nil } -func (n *natpmp_NAT) Type() string { +func (n *natpmpNAT) Type() string { return "NAT-PMP" } diff --git a/Godeps/_workspace/src/github.com/fzzy/radix/redis/client.go b/Godeps/_workspace/src/github.com/fzzy/radix/redis/client.go index 21a3dace887..2f1592a627c 100644 --- a/Godeps/_workspace/src/github.com/fzzy/radix/redis/client.go +++ b/Godeps/_workspace/src/github.com/fzzy/radix/redis/client.go @@ -30,6 +30,7 @@ type Client struct { reader *bufio.Reader pending []*request completed []*Reply + writeBuf []byte } // request describes a client's request to the redis server @@ -42,7 +43,7 @@ type request struct { // used as the read/write timeout when communicating with redis func DialTimeout(network, addr string, timeout time.Duration) (*Client, error) { // establish a connection - conn, err := net.Dial(network, addr) + conn, err := net.DialTimeout(network, addr, timeout) if err != nil { return nil, err } @@ -51,6 +52,7 @@ func DialTimeout(network, addr string, timeout time.Duration) (*Client, error) { c.Conn = conn c.timeout = timeout c.reader = bufio.NewReaderSize(conn, bufSize) + c.writeBuf = make([]byte, 0, 1024) return c, nil } @@ -152,7 +154,9 @@ func (c *Client) writeRequest(requests ...*request) error { req := make([]interface{}, 0, len(requests[i].args)+1) req = append(req, requests[i].cmd) req = append(req, requests[i].args...) - err := resp.WriteArbitraryAsFlattenedStrings(c.Conn, req) + buf := resp.AppendArbitraryAsFlattenedStrings(c.writeBuf[:0], req) + + _, err := c.Conn.Write(buf) if err != nil { c.Close() return err diff --git a/Godeps/_workspace/src/github.com/fzzy/radix/redis/client_test.go b/Godeps/_workspace/src/github.com/fzzy/radix/redis/client_test.go index afbe830a01b..3e2f59815da 100644 --- a/Godeps/_workspace/src/github.com/fzzy/radix/redis/client_test.go +++ b/Godeps/_workspace/src/github.com/fzzy/radix/redis/client_test.go @@ -3,9 +3,10 @@ package redis import ( "bufio" "bytes" - "github.com/stretchr/testify/assert" . "testing" "time" + + "github.com/stretchr/testify/assert" ) func dial(t *T) *Client { diff --git a/Godeps/_workspace/src/github.com/fzzy/radix/redis/reply.go b/Godeps/_workspace/src/github.com/fzzy/radix/redis/reply.go index c4ad7c2d3b0..7341da284b3 100644 --- a/Godeps/_workspace/src/github.com/fzzy/radix/redis/reply.go +++ b/Godeps/_workspace/src/github.com/fzzy/radix/redis/reply.go @@ -124,6 +124,29 @@ func (r *Reply) Int() (int, error) { return int(i64), nil } +// Float64 returns the reply value as a float64 or an error, +// if the reply type is not BulkReply or the reply type +// BulkReply could not be parsed to an float64. +func (r *Reply) Float64() (float64, error) { + if r.Type == ErrorReply { + return 0, r.Err + } + if r.Type == BulkReply { + s, err := r.Str() + if err != nil { + return 0, err + } + f64, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, errors.New("failed to parse float value from string value") + } else { + return f64, nil + } + } + + return 0, errors.New("float value is not available for this reply type") +} + // Bool returns false, if the reply value equals to 0 or "0", otherwise true; or // an error, if the reply type is not IntegerReply or BulkReply. func (r *Reply) Bool() (bool, error) { diff --git a/Godeps/_workspace/src/github.com/fzzy/radix/redis/reply_test.go b/Godeps/_workspace/src/github.com/fzzy/radix/redis/reply_test.go index 0002966abd9..1601271964b 100644 --- a/Godeps/_workspace/src/github.com/fzzy/radix/redis/reply_test.go +++ b/Godeps/_workspace/src/github.com/fzzy/radix/redis/reply_test.go @@ -1,8 +1,9 @@ package redis import ( - "github.com/stretchr/testify/assert" . "testing" + + "github.com/stretchr/testify/assert" ) func TestStr(t *T) { @@ -80,6 +81,25 @@ func TestBool(t *T) { assert.NotNil(t, err) } +func TestFloat64(t *T) { + r := &Reply{Type: ErrorReply, Err: LoadingError} + _, err := r.Float64() + assert.Equal(t, LoadingError, err) + + r = &Reply{Type: IntegerReply, int: 5} + _, err = r.Float64() + assert.NotNil(t, err) + + r = &Reply{Type: BulkReply, buf: []byte("5.1")} + b, err := r.Float64() + assert.Nil(t, err) + assert.Equal(t, float64(5.1), b) + + r = &Reply{Type: BulkReply, buf: []byte("foo")} + _, err = r.Float64() + assert.NotNil(t, err) +} + func TestList(t *T) { r := &Reply{Type: MultiReply} r.Elems = make([]*Reply, 3) diff --git a/Godeps/_workspace/src/github.com/fzzy/radix/redis/resp/resp.go b/Godeps/_workspace/src/github.com/fzzy/radix/redis/resp/resp.go index 5afac97cade..54d08c13823 100644 --- a/Godeps/_workspace/src/github.com/fzzy/radix/redis/resp/resp.go +++ b/Godeps/_workspace/src/github.com/fzzy/radix/redis/resp/resp.go @@ -29,12 +29,12 @@ const ( Nil ) -const ( - simpleStrPrefix = '+' - errPrefix = '-' - intPrefix = ':' - bulkStrPrefix = '$' - arrayPrefix = '*' +var ( + simpleStrPrefix = []byte{'+'} + errPrefix = []byte{'-'} + intPrefix = []byte{':'} + bulkStrPrefix = []byte{'$'} + arrayPrefix = []byte{'*'} ) // Parse errors @@ -65,7 +65,7 @@ func NewMessage(b []byte) (*Message, error) { // resp.WriteArbitrary(w, []interface{}{bar, baz}) // func NewSimpleString(s string) *Message { - b := append(make([]byte, 0, len(s) + 3), '+') + b := append(make([]byte, 0, len(s)+3), '+') b = append(b, []byte(s)...) b = append(b, '\r', '\n') return &Message{ @@ -88,15 +88,15 @@ func bufioReadMessage(r *bufio.Reader) (*Message, error) { return nil, err } switch b[0] { - case simpleStrPrefix: + case simpleStrPrefix[0]: return readSimpleStr(r) - case errPrefix: + case errPrefix[0]: return readError(r) - case intPrefix: + case intPrefix[0]: return readInt(r) - case bulkStrPrefix: + case bulkStrPrefix[0]: return readBulkStr(r) - case arrayPrefix: + case arrayPrefix[0]: return readArray(r) default: return nil, badType @@ -248,6 +248,14 @@ func (m *Message) Array() ([]*Message, error) { return nil, badType } +func writeBytesHelper(w io.Writer, b []byte, lastErr error) error { + if lastErr != nil { + return lastErr + } + _, err := w.Write(b) + return err +} + // WriteMessage takes in the given Message and writes its encoded form to the // given io.Writer func WriteMessage(w io.Writer, m *Message) error { @@ -255,22 +263,54 @@ func WriteMessage(w io.Writer, m *Message) error { return err } +// AppendArbitrary takes in any primitive golang value, or Message, and appends +// its encoded form to the given buffer, inferring types where appropriate. It +// then returns the appended buffer +func AppendArbitrary(buf []byte, m interface{}) []byte { + return appendArb(buf, m, false, false) +} + // WriteArbitrary takes in any primitive golang value, or Message, and writes // its encoded form to the given io.Writer, inferring types where appropriate. func WriteArbitrary(w io.Writer, m interface{}) error { - b := format(m, false) - _, err := w.Write(b) + buf := AppendArbitrary(make([]byte, 0, 1024), m) + _, err := w.Write(buf) return err } +// AppendArbitraryAsString is similar to AppendArbitraryAsFlattenedString except +// that it won't flatten any embedded arrays. +func AppendArbitraryAsStrings(buf []byte, m interface{}) []byte { + return appendArb(buf, m, true, false) +} + // WriteArbitraryAsString is similar to WriteArbitraryAsFlattenedString except // that it won't flatten any embedded arrays. func WriteArbitraryAsString(w io.Writer, m interface{}) error { - b := format(m, true) - _, err := w.Write(b) + buf := AppendArbitraryAsStrings(make([]byte, 0, 1024), m) + _, err := w.Write(buf) return err } +// AppendArbitraryAsFlattenedStrings is similar to AppendArbitrary except that +// it will encode all types except Array as a BulkStr, converting the argument +// into a string first as necessary. It will also flatten any embedded arrays +// into a single long array. This is useful because commands to a redis server +// must be given as an array of bulk strings. If the argument isn't already in a +// slice or map it will be wrapped so that it is written as an Array of size +// one. +// +// Note that if a Message type is found it will *not* be encoded to a BulkStr, +// but will simply be passed through as whatever type it already represents. +func AppendArbitraryAsFlattenedStrings(buf []byte, m interface{}) []byte { + fl := flattenedLength(m) + buf = append(buf, arrayPrefix...) + buf = strconv.AppendInt(buf, int64(fl), 10) + buf = append(buf, delim...) + + return appendArb(buf, m, true, true) +} + // WriteArbitraryAsFlattenedStrings is similar to WriteArbitrary except that it // will encode all types except Array as a BulkStr, converting the argument into // a string first as necessary. It will also flatten any embedded arrays into a @@ -281,76 +321,85 @@ func WriteArbitraryAsString(w io.Writer, m interface{}) error { // Note that if a Message type is found it will *not* be encoded to a BulkStr, // but will simply be passed through as whatever type it already represents. func WriteArbitraryAsFlattenedStrings(w io.Writer, m interface{}) error { - fm := flatten(m) - return WriteArbitraryAsString(w, fm) + buf := AppendArbitraryAsFlattenedStrings(make([]byte, 0, 1024), m) + _, err := w.Write(buf) + return err } -func format(m interface{}, forceString bool) []byte { +func appendArb(buf []byte, m interface{}, forceString, flattened bool) []byte { switch mt := m.(type) { case []byte: - return formatStr(mt) + return appendStr(buf, mt) case string: - return formatStr([]byte(mt)) + return appendStr(buf, []byte(mt)) case bool: if mt { - return formatStr([]byte("1")) + return appendStr(buf, []byte("1")) } else { - return formatStr([]byte("0")) + return appendStr(buf, []byte("0")) } case nil: if forceString { - return formatStr([]byte{}) + return appendStr(buf, []byte{}) } else { - return formatNil() + return appendNil(buf) } case int: - return formatInt(int64(mt), forceString) + return appendInt(buf, int64(mt), forceString) case int8: - return formatInt(int64(mt), forceString) + return appendInt(buf, int64(mt), forceString) case int16: - return formatInt(int64(mt), forceString) + return appendInt(buf, int64(mt), forceString) case int32: - return formatInt(int64(mt), forceString) + return appendInt(buf, int64(mt), forceString) case int64: - return formatInt(mt, forceString) + return appendInt(buf, mt, forceString) case uint: - return formatInt(int64(mt), forceString) + return appendInt(buf, int64(mt), forceString) case uint8: - return formatInt(int64(mt), forceString) + return appendInt(buf, int64(mt), forceString) case uint16: - return formatInt(int64(mt), forceString) + return appendInt(buf, int64(mt), forceString) case uint32: - return formatInt(int64(mt), forceString) + return appendInt(buf, int64(mt), forceString) case uint64: - return formatInt(int64(mt), forceString) + return appendInt(buf, int64(mt), forceString) case float32: ft := strconv.FormatFloat(float64(mt), 'f', -1, 32) - return formatStr([]byte(ft)) + return appendStr(buf, []byte(ft)) case float64: ft := strconv.FormatFloat(mt, 'f', -1, 64) - return formatStr([]byte(ft)) + return appendStr(buf, []byte(ft)) case error: if forceString { - return formatStr([]byte(mt.Error())) + return appendStr(buf, []byte(mt.Error())) } else { - return formatErr(mt) + return appendErr(buf, mt) } + // For the following cases, where we are writing an array, we only append the + // array header (a new array) if flattened is false, otherwise we just append + // each element inline and assume the array header has already been written + // We duplicate the below code here a bit, since this is the common case and // it'd be better to not get the reflect package involved here case []interface{}: l := len(mt) - b := make([]byte, 0, l*1024) - b = append(b, '*') - b = append(b, []byte(strconv.Itoa(l))...) - b = append(b, []byte("\r\n")...) + + if !flattened { + buf = append(buf, arrayPrefix...) + buf = strconv.AppendInt(buf, int64(l), 10) + buf = append(buf, delim...) + } + for i := 0; i < l; i++ { - b = append(b, format(mt[i], forceString)...) + buf = appendArb(buf, mt[i], forceString, flattened) } - return b + return buf case *Message: - return mt.raw + buf = append(buf, mt.raw...) + return buf default: // Fallback to reflect-based. @@ -358,109 +407,119 @@ func format(m interface{}, forceString bool) []byte { case reflect.Slice: rm := reflect.ValueOf(mt) l := rm.Len() - b := make([]byte, 0, l*1024) - b = append(b, '*') - b = append(b, []byte(strconv.Itoa(l))...) - b = append(b, []byte("\r\n")...) + + if !flattened { + buf = append(buf, arrayPrefix...) + buf = strconv.AppendInt(buf, int64(l), 10) + buf = append(buf, delim...) + } + for i := 0; i < l; i++ { vv := rm.Index(i).Interface() - b = append(b, format(vv, forceString)...) + buf = appendArb(buf, vv, forceString, flattened) } + return buf - return b case reflect.Map: rm := reflect.ValueOf(mt) l := rm.Len() * 2 - b := make([]byte, 0, l*1024) - b = append(b, '*') - b = append(b, []byte(strconv.Itoa(l))...) - b = append(b, []byte("\r\n")...) + + if !flattened { + buf = append(buf, arrayPrefix...) + buf = strconv.AppendInt(buf, int64(l), 10) + buf = append(buf, delim...) + } + keys := rm.MapKeys() for _, k := range keys { kv := k.Interface() vv := rm.MapIndex(k).Interface() - b = append(b, format(kv, forceString)...) - b = append(b, format(vv, forceString)...) + buf = appendArb(buf, kv, forceString, flattened) + buf = appendArb(buf, vv, forceString, flattened) } - return b + return buf + default: - return formatStr([]byte(fmt.Sprint(m))) + return appendStr(buf, []byte(fmt.Sprint(m))) } } } var typeOfBytes = reflect.TypeOf([]byte(nil)) -func flatten(m interface{}) []interface{} { +func flattenedLength(m interface{}) int { t := reflect.TypeOf(m) // If it's a byte-slice we don't want to flatten if t == typeOfBytes { - return []interface{}{m} + return 1 } + total := 0 + switch t.Kind() { case reflect.Slice: rm := reflect.ValueOf(m) l := rm.Len() - ret := make([]interface{}, 0, l) for i := 0; i < l; i++ { - ret = append(ret, flatten(rm.Index(i).Interface())...) + total += flattenedLength(rm.Index(i).Interface()) } - return ret case reflect.Map: rm := reflect.ValueOf(m) - l := rm.Len() * 2 keys := rm.MapKeys() - ret := make([]interface{}, 0, l) for _, k := range keys { kv := k.Interface() vv := rm.MapIndex(k).Interface() - ret = append(ret, flatten(kv)...) - ret = append(ret, flatten(vv)...) + total += flattenedLength(kv) + total += flattenedLength(vv) } - return ret default: - return []interface{}{m} + total++ } + + return total } -func formatStr(b []byte) []byte { - l := strconv.Itoa(len(b)) - bs := make([]byte, 0, len(l)+len(b)+5) - bs = append(bs, bulkStrPrefix) - bs = append(bs, []byte(l)...) - bs = append(bs, delim...) - bs = append(bs, b...) - bs = append(bs, delim...) - return bs +func appendStr(buf []byte, b []byte) []byte { + buf = append(buf, bulkStrPrefix...) + buf = strconv.AppendInt(buf, int64(len(b)), 10) + buf = append(buf, delim...) + buf = append(buf, b...) + buf = append(buf, delim...) + return buf } -func formatErr(ierr error) []byte { - ierrstr := []byte(ierr.Error()) - bs := make([]byte, 0, len(ierrstr)+3) - bs = append(bs, errPrefix) - bs = append(bs, ierrstr...) - bs = append(bs, delim...) - return bs +func appendErr(buf []byte, ierr error) []byte { + buf = append(buf, errPrefix...) + buf = append(buf, []byte(ierr.Error())...) + buf = append(buf, delim...) + return buf } -func formatInt(i int64, forceString bool) []byte { - istr := strconv.FormatInt(i, 10) - if forceString { - return formatStr([]byte(istr)) +func appendInt(buf []byte, i int64, forceString bool) []byte { + if !forceString { + buf = append(buf, intPrefix...) + } else { + // Really want to avoid alloating a new []byte. So I write the int to + // the buf for the sole purpose of getting its length as a string, and + // even though it'll be immediately overwritten right after and + // AppendInt will be called again. This isn't great. + tmpBuf := strconv.AppendInt(buf, i, 10) + + buf = append(buf, bulkStrPrefix...) + buf = strconv.AppendInt(buf, int64(len(tmpBuf)-len(buf)+1), 10) + buf = append(buf, delim...) } - bs := make([]byte, 0, len(istr)+3) - bs = append(bs, intPrefix) - bs = append(bs, istr...) - bs = append(bs, delim...) - return bs + + buf = strconv.AppendInt(buf, i, 10) + buf = append(buf, delim...) + return buf } var nilFormatted = []byte("$-1\r\n") -func formatNil() []byte { - return nilFormatted +func appendNil(buf []byte) []byte { + return append(buf, nilFormatted...) } diff --git a/Godeps/_workspace/src/github.com/fzzy/radix/redis/resp/resp_test.go b/Godeps/_workspace/src/github.com/fzzy/radix/redis/resp/resp_test.go index 20112c0b9ea..6c5ca14fa33 100644 --- a/Godeps/_workspace/src/github.com/fzzy/radix/redis/resp/resp_test.go +++ b/Godeps/_workspace/src/github.com/fzzy/radix/redis/resp/resp_test.go @@ -3,8 +3,9 @@ package resp import ( "bytes" "errors" - "github.com/stretchr/testify/assert" . "testing" + + "github.com/stretchr/testify/assert" ) func TestRead(t *T) { diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode_gogo.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode_gogo.go index 66bc0bd535e..b414a3ce72c 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode_gogo.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode_gogo.go @@ -40,6 +40,10 @@ import ( "reflect" ) +func NewRequiredNotSetError(field string) *RequiredNotSetError { + return &RequiredNotSetError{field} +} + type Sizer interface { Size() int } diff --git a/Godeps/_workspace/src/github.com/hashicorp/golang-lru/README.md b/Godeps/_workspace/src/github.com/hashicorp/golang-lru/README.md index 37fa54d4265..33e58cfaf97 100644 --- a/Godeps/_workspace/src/github.com/hashicorp/golang-lru/README.md +++ b/Godeps/_workspace/src/github.com/hashicorp/golang-lru/README.md @@ -20,6 +20,6 @@ for i := 0; i < 256; i++ { l.Add(i, nil) } if l.Len() != 128 { - panic("bad len: %v", l.Len()) + panic(fmt.Sprintf("bad len: %v", l.Len())) } ``` diff --git a/Godeps/_workspace/src/github.com/hashicorp/golang-lru/lru.go b/Godeps/_workspace/src/github.com/hashicorp/golang-lru/lru.go index 174767c541d..5f1e8a1af35 100644 --- a/Godeps/_workspace/src/github.com/hashicorp/golang-lru/lru.go +++ b/Godeps/_workspace/src/github.com/hashicorp/golang-lru/lru.go @@ -14,7 +14,8 @@ type Cache struct { size int evictList *list.List items map[interface{}]*list.Element - lock sync.Mutex + lock sync.RWMutex + onEvicted func(key interface{}, value interface{}) } // entry is used to hold a value in the evictList @@ -25,6 +26,10 @@ type entry struct { // New creates an LRU of the given size func New(size int) (*Cache, error) { + return NewWithEvict(size, nil) +} + +func NewWithEvict(size int, onEvicted func(key interface{}, value interface{})) (*Cache, error) { if size <= 0 { return nil, errors.New("Must provide a positive size") } @@ -32,6 +37,7 @@ func New(size int) (*Cache, error) { size: size, evictList: list.New(), items: make(map[interface{}]*list.Element, size), + onEvicted: onEvicted, } return c, nil } @@ -40,12 +46,19 @@ func New(size int) (*Cache, error) { func (c *Cache) Purge() { c.lock.Lock() defer c.lock.Unlock() + + if c.onEvicted != nil { + for k, v := range c.items { + c.onEvicted(k, v.Value.(*entry).value) + } + } + c.evictList = list.New() c.items = make(map[interface{}]*list.Element, c.size) } -// Add adds a value to the cache. -func (c *Cache) Add(key, value interface{}) { +// Add adds a value to the cache. Returns true if an eviction occured. +func (c *Cache) Add(key, value interface{}) bool { c.lock.Lock() defer c.lock.Unlock() @@ -53,7 +66,7 @@ func (c *Cache) Add(key, value interface{}) { if ent, ok := c.items[key]; ok { c.evictList.MoveToFront(ent) ent.Value.(*entry).value = value - return + return false } // Add new item @@ -61,10 +74,12 @@ func (c *Cache) Add(key, value interface{}) { entry := c.evictList.PushFront(ent) c.items[key] = entry + evict := c.evictList.Len() > c.size // Verify size not exceeded - if c.evictList.Len() > c.size { + if evict { c.removeOldest() } + return evict } // Get looks up a key's value from the cache. @@ -79,6 +94,27 @@ func (c *Cache) Get(key interface{}) (value interface{}, ok bool) { return } +// Check if a key is in the cache, without updating the recent-ness or deleting it for being stale. +func (c *Cache) Contains(key interface{}) (ok bool) { + c.lock.RLock() + defer c.lock.RUnlock() + + _, ok = c.items[key] + return ok +} + +// Returns the key value (or undefined if not found) without updating the "recently used"-ness of the key. +// (If you find yourself using this a lot, you might be using the wrong sort of data structure, but there are some use cases where it's handy.) +func (c *Cache) Peek(key interface{}) (value interface{}, ok bool) { + c.lock.RLock() + defer c.lock.RUnlock() + + if ent, ok := c.items[key]; ok { + return ent.Value.(*entry).value, true + } + return nil, ok +} + // Remove removes the provided key from the cache. func (c *Cache) Remove(key interface{}) { c.lock.Lock() @@ -96,21 +132,30 @@ func (c *Cache) RemoveOldest() { c.removeOldest() } -// Keys returns a slice of the keys in the cache. +// Keys returns a slice of the keys in the cache, from oldest to newest. func (c *Cache) Keys() []interface{} { - c.lock.Lock() - defer c.lock.Unlock() + c.lock.RLock() + defer c.lock.RUnlock() keys := make([]interface{}, len(c.items)) + ent := c.evictList.Back() i := 0 - for k := range c.items { - keys[i] = k + for ent != nil { + keys[i] = ent.Value.(*entry).key + ent = ent.Prev() i++ } return keys } +// Len returns the number of items in the cache. +func (c *Cache) Len() int { + c.lock.RLock() + defer c.lock.RUnlock() + return c.evictList.Len() +} + // removeOldest removes the oldest item from the cache. func (c *Cache) removeOldest() { ent := c.evictList.Back() @@ -124,11 +169,7 @@ func (c *Cache) removeElement(e *list.Element) { c.evictList.Remove(e) kv := e.Value.(*entry) delete(c.items, kv.key) -} - -// Len returns the number of items in the cache. -func (c *Cache) Len() int { - c.lock.Lock() - defer c.lock.Unlock() - return c.evictList.Len() + if c.onEvicted != nil { + c.onEvicted(kv.key, kv.value) + } } diff --git a/Godeps/_workspace/src/github.com/hashicorp/golang-lru/lru_test.go b/Godeps/_workspace/src/github.com/hashicorp/golang-lru/lru_test.go index 626b95fbe66..b676cfd9db1 100644 --- a/Godeps/_workspace/src/github.com/hashicorp/golang-lru/lru_test.go +++ b/Godeps/_workspace/src/github.com/hashicorp/golang-lru/lru_test.go @@ -3,18 +3,31 @@ package lru import "testing" func TestLRU(t *testing.T) { - l, err := New(128) + evictCounter := 0 + onEvicted := func(k interface{}, v interface{}) { + if k != v { + t.Fatalf("Evict values not equal (%v!=%v)", k, v) + } + evictCounter += 1 + } + l, err := NewWithEvict(128, onEvicted) if err != nil { t.Fatalf("err: %v", err) } + for i := 0; i < 256; i++ { l.Add(i, i) } if l.Len() != 128 { t.Fatalf("bad len: %v", l.Len()) } - for _, k := range l.Keys() { - if v, ok := l.Get(k); !ok || v != k { + + if evictCounter != 128 { + t.Fatalf("bad evict count: %v", evictCounter) + } + + for i, k := range l.Keys() { + if v, ok := l.Get(k); !ok || v != k || v != i+128 { t.Fatalf("bad key: %v", k) } } @@ -38,6 +51,14 @@ func TestLRU(t *testing.T) { } } + l.Get(192) // expect 192 to be last key in l.Keys() + + for i, k := range l.Keys() { + if (i < 63 && k != i+193) || (i == 63 && k != 192) { + t.Fatalf("out of order key: %v", k) + } + } + l.Purge() if l.Len() != 0 { t.Fatalf("bad len: %v", l.Len()) @@ -46,3 +67,61 @@ func TestLRU(t *testing.T) { t.Fatalf("should contain nothing") } } + +// test that Add returns true/false if an eviction occured +func TestLRUAdd(t *testing.T) { + evictCounter := 0 + onEvicted := func(k interface{}, v interface{}) { + evictCounter += 1 + } + + l, err := NewWithEvict(1, onEvicted) + if err != nil { + t.Fatalf("err: %v", err) + } + + if l.Add(1, 1) == true || evictCounter != 0 { + t.Errorf("should not have an eviction") + } + if l.Add(2, 2) == false || evictCounter != 1 { + t.Errorf("should have an eviction") + } +} + +// test that Contains doesn't update recent-ness +func TestLRUContains(t *testing.T) { + l, err := New(2) + if err != nil { + t.Fatalf("err: %v", err) + } + + l.Add(1, 1) + l.Add(2, 2) + if !l.Contains(1) { + t.Errorf("1 should be contained") + } + + l.Add(3, 3) + if l.Contains(1) { + t.Errorf("Contains should not have updated recent-ness of 1") + } +} + +// test that Peek doesn't update recent-ness +func TestLRUPeek(t *testing.T) { + l, err := New(2) + if err != nil { + t.Fatalf("err: %v", err) + } + + l.Add(1, 1) + l.Add(2, 2) + if v, ok := l.Peek(1); !ok || v != 1 { + t.Errorf("1 should be set to 1: %v, %v", v, ok) + } + + l.Add(3, 3) + if l.Contains(1) { + t.Errorf("should not have updated recent-ness of 1") + } +} diff --git a/Godeps/_workspace/src/github.com/hashicorp/yamux/session.go b/Godeps/_workspace/src/github.com/hashicorp/yamux/session.go index 2def31e3cad..56ff91292ab 100644 --- a/Godeps/_workspace/src/github.com/hashicorp/yamux/session.go +++ b/Godeps/_workspace/src/github.com/hashicorp/yamux/session.go @@ -50,6 +50,11 @@ type Session struct { streams map[uint32]*Stream streamLock sync.Mutex + // synCh acts like a semaphore. It is sized to the AcceptBacklog which + // is assumed to be symmetric between the client and server. This allows + // the client to avoid exceeding the backlog and instead blocks the open. + synCh chan struct{} + // acceptCh is used to pass ready streams to the client acceptCh chan *Stream @@ -57,6 +62,10 @@ type Session struct { // or to send a header out directly. sendCh chan sendReady + // recvDoneCh is closed when recv() exits to avoid a race + // between stream registration and stream shutdown + recvDoneCh chan struct{} + // shutdown is used to safely close a session shutdown bool shutdownErr error @@ -81,8 +90,10 @@ func newSession(config *Config, conn io.ReadWriteCloser, client bool) *Session { bufRead: bufio.NewReader(conn), pings: make(map[uint32]chan struct{}), streams: make(map[uint32]*Stream), + synCh: make(chan struct{}, config.AcceptBacklog), acceptCh: make(chan *Stream, config.AcceptBacklog), sendCh: make(chan sendReady, 64), + recvDoneCh: make(chan struct{}), shutdownCh: make(chan struct{}), } if client { @@ -108,6 +119,14 @@ func (s *Session) IsClosed() bool { } } +// NumStreams returns the number of currently open streams +func (s *Session) NumStreams() int { + s.streamLock.Lock() + num := len(s.streams) + s.streamLock.Unlock() + return num +} + // Open is used to create a new stream as a net.Conn func (s *Session) Open() (net.Conn, error) { return s.OpenStream() @@ -122,6 +141,13 @@ func (s *Session) OpenStream() (*Stream, error) { return nil, ErrRemoteGoAway } + // Block if we have too many inflight SYNs + select { + case s.synCh <- struct{}{}: + case <-s.shutdownCh: + return nil, ErrSessionShutdown + } + GET_ID: // Get and ID, and check for stream exhaustion id := atomic.LoadUint32(&s.nextStreamID) @@ -139,7 +165,10 @@ GET_ID: s.streamLock.Unlock() // Send the window update to create - return stream, stream.sendWindowUpdate() + if err := stream.sendWindowUpdate(); err != nil { + return nil, err + } + return stream, nil } // Accept is used to block until the next available stream @@ -153,7 +182,10 @@ func (s *Session) Accept() (net.Conn, error) { func (s *Session) AcceptStream() (*Stream, error) { select { case stream := <-s.acceptCh: - return stream, stream.sendWindowUpdate() + if err := stream.sendWindowUpdate(); err != nil { + return nil, err + } + return stream, nil case <-s.shutdownCh: return nil, s.shutdownErr } @@ -174,6 +206,7 @@ func (s *Session) Close() error { } close(s.shutdownCh) s.conn.Close() + <-s.recvDoneCh s.streamLock.Lock() defer s.streamLock.Unlock() @@ -325,6 +358,14 @@ func (s *Session) send() { // recv is a long running goroutine that accepts new data func (s *Session) recv() { + if err := s.recvLoop(); err != nil { + s.exitErr(err) + } +} + +// recvLoop continues to receive data until a fatal error is encountered +func (s *Session) recvLoop() error { + defer close(s.recvDoneCh) hdr := header(make([]byte, headerSize)) var handler func(header) error for { @@ -333,15 +374,13 @@ func (s *Session) recv() { if err != io.EOF && !strings.Contains(err.Error(), "closed") && !strings.Contains(err.Error(), "reset by peer") { s.logger.Printf("[ERR] yamux: Failed to read header: %v", err) } - s.exitErr(err) - return + return err } // Verify the version if hdr.Version() != protoVersion { s.logger.Printf("[ERR] yamux: Invalid protocol version: %d", hdr.Version()) - s.exitErr(ErrInvalidVersion) - return + return ErrInvalidVersion } // Switch on the type @@ -355,14 +394,12 @@ func (s *Session) recv() { case typePing: handler = s.handlePing default: - s.exitErr(ErrInvalidMsgType) - return + return ErrInvalidMsgType } // Invoke the handler if err := handler(hdr); err != nil { - s.exitErr(err) - return + return err } } } @@ -503,3 +540,13 @@ func (s *Session) closeStream(id uint32) { delete(s.streams, id) s.streamLock.Unlock() } + +// establishStream is used to mark a stream that was in the +// SYN Sent state as established. +func (s *Session) establishStream() { + select { + case <-s.synCh: + default: + panic("established stream without inflight syn") + } +} diff --git a/Godeps/_workspace/src/github.com/hashicorp/yamux/session_test.go b/Godeps/_workspace/src/github.com/hashicorp/yamux/session_test.go index 245269f74e0..85d0ffabc40 100644 --- a/Godeps/_workspace/src/github.com/hashicorp/yamux/session_test.go +++ b/Godeps/_workspace/src/github.com/hashicorp/yamux/session_test.go @@ -76,6 +76,13 @@ func TestAccept(t *testing.T) { defer client.Close() defer server.Close() + if client.NumStreams() != 0 { + t.Fatalf("bad") + } + if server.NumStreams() != 0 { + t.Fatalf("bad") + } + wg := &sync.WaitGroup{} wg.Add(4) @@ -163,6 +170,10 @@ func TestSendData_Small(t *testing.T) { t.Fatalf("err: %v", err) } + if server.NumStreams() != 1 { + t.Fatalf("bad") + } + buf := make([]byte, 4) for i := 0; i < 1000; i++ { n, err := stream.Read(buf) @@ -189,6 +200,10 @@ func TestSendData_Small(t *testing.T) { t.Fatalf("err: %v", err) } + if client.NumStreams() != 1 { + t.Fatalf("bad") + } + for i := 0; i < 1000; i++ { n, err := stream.Write([]byte("test")) if err != nil { @@ -214,6 +229,13 @@ func TestSendData_Small(t *testing.T) { case <-time.After(time.Second): panic("timeout") } + + if client.NumStreams() != 0 { + t.Fatalf("bad") + } + if server.NumStreams() != 0 { + t.Fatalf("bad") + } } func TestSendData_Large(t *testing.T) { @@ -576,21 +598,26 @@ func TestBacklogExceeded(t *testing.T) { } } - // Exceed the backlog! - stream, err := client.Open() - if err != nil { - t.Fatalf("err: %v", err) - } - defer stream.Close() + // Attempt to open a new stream + errCh := make(chan error, 1) + go func() { + _, err := client.Open() + errCh <- err + }() - if _, err := stream.Write([]byte("foo")); err != nil { - t.Fatalf("err: %v", err) - } + // Shutdown the server + go func() { + time.Sleep(10 * time.Millisecond) + server.Close() + }() - buf := make([]byte, 4) - stream.SetReadDeadline(time.Now().Add(50 * time.Millisecond)) - if _, err := stream.Read(buf); err != ErrConnectionReset { - t.Fatalf("err: %v", err) + select { + case err := <-errCh: + if err == nil { + t.Fatalf("open should fail") + } + case <-time.After(time.Second): + t.Fatalf("timeout") } } @@ -727,3 +754,33 @@ func TestSendData_VeryLarge(t *testing.T) { panic("timeout") } } + +func TestBacklogExceeded_Accept(t *testing.T) { + client, server := testClientServer() + defer client.Close() + defer server.Close() + + max := 5 * client.config.AcceptBacklog + go func() { + for i := 0; i < max; i++ { + stream, err := server.Accept() + if err != nil { + t.Fatalf("err: %v", err) + } + defer stream.Close() + } + }() + + // Fill the backlog + for i := 0; i < max; i++ { + stream, err := client.Open() + if err != nil { + t.Fatalf("err: %v", err) + } + defer stream.Close() + + if _, err := stream.Write([]byte("foo")); err != nil { + t.Fatalf("err: %v", err) + } + } +} diff --git a/Godeps/_workspace/src/github.com/hashicorp/yamux/stream.go b/Godeps/_workspace/src/github.com/hashicorp/yamux/stream.go index b842dad2159..ad5b0eb1709 100644 --- a/Godeps/_workspace/src/github.com/hashicorp/yamux/stream.go +++ b/Godeps/_workspace/src/github.com/hashicorp/yamux/stream.go @@ -86,6 +86,8 @@ func (s *Stream) Read(b []byte) (n int, err error) { START: s.stateLock.Lock() switch s.state { + case streamLocalClose: + fallthrough case streamRemoteClose: fallthrough case streamClosed: @@ -265,6 +267,7 @@ func (s *Stream) sendClose() error { // Close is used to close the stream func (s *Stream) Close() error { + closeStream := false s.stateLock.Lock() switch s.state { // Opened means we need to signal a close @@ -279,7 +282,7 @@ func (s *Stream) Close() error { case streamLocalClose: case streamRemoteClose: s.state = streamClosed - s.session.closeStream(s.id) + closeStream = true goto SEND_CLOSE case streamClosed: @@ -293,6 +296,9 @@ SEND_CLOSE: s.stateLock.Unlock() s.sendClose() s.notifyWaiting() + if closeStream { + s.session.closeStream(s.id) + } return nil } @@ -307,14 +313,23 @@ func (s *Stream) forceClose() { // processFlags is used to update the state of the stream // based on set flags, if any. Lock must be held func (s *Stream) processFlags(flags uint16) error { + // Close the stream without holding the state lock + closeStream := false + defer func() { + if closeStream { + s.session.closeStream(s.id) + } + }() + s.stateLock.Lock() defer s.stateLock.Unlock() if flags&flagACK == flagACK { if s.state == streamSYNSent { s.state = streamEstablished } - - } else if flags&flagFIN == flagFIN { + s.session.establishStream() + } + if flags&flagFIN == flagFIN { switch s.state { case streamSYNSent: fallthrough @@ -325,15 +340,19 @@ func (s *Stream) processFlags(flags uint16) error { s.notifyWaiting() case streamLocalClose: s.state = streamClosed - s.session.closeStream(s.id) + closeStream = true s.notifyWaiting() default: s.session.logger.Printf("[ERR] yamux: unexpected FIN flag in state %d", s.state) return ErrUnexpectedFlag } - } else if flags&flagRST == flagRST { + } + if flags&flagRST == flagRST { + if s.state == streamSYNSent { + s.session.establishStream() + } s.state = streamReset - s.session.closeStream(s.id) + closeStream = true s.notifyWaiting() } return nil diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_bsd.go b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_bsd.go index e6ffd7e5b5e..40c84768bc3 100644 --- a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_bsd.go +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_bsd.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build freebsd openbsd netbsd darwin +// +build freebsd openbsd netbsd dragonfly darwin package fsnotify diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_bsd.go b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_bsd.go index 37ea998d09e..f728a6ffd73 100644 --- a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_bsd.go +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_bsd.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build freebsd openbsd netbsd +// +build freebsd openbsd netbsd dragonfly package fsnotify diff --git a/Godeps/_workspace/src/github.com/huin/goupnp/goupnp.go b/Godeps/_workspace/src/github.com/huin/goupnp/goupnp.go index 0d4845a693d..fee92a5b498 100644 --- a/Godeps/_workspace/src/github.com/huin/goupnp/goupnp.go +++ b/Godeps/_workspace/src/github.com/huin/goupnp/goupnp.go @@ -22,6 +22,7 @@ import ( "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/huin/goupnp/httpu" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/huin/goupnp/ssdp" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/html/charset" ) // ContextError is an error that wraps an error with some context information. @@ -104,6 +105,7 @@ func requestXml(url string, defaultSpace string, doc interface{}) error { decoder := xml.NewDecoder(resp.Body) decoder.DefaultSpace = defaultSpace + decoder.CharsetReader = charset.NewReaderLabel return decoder.Decode(doc) } diff --git a/Godeps/_workspace/src/github.com/huin/goupnp/soap/soap.go b/Godeps/_workspace/src/github.com/huin/goupnp/soap/soap.go index 33cbe283c7b..815610734cf 100644 --- a/Godeps/_workspace/src/github.com/huin/goupnp/soap/soap.go +++ b/Godeps/_workspace/src/github.com/huin/goupnp/soap/soap.go @@ -42,7 +42,7 @@ func (client *SOAPClient) PerformAction(actionNamespace, actionName string, inAc Method: "POST", URL: &client.EndpointURL, Header: http.Header{ - "SOAPACTION": []string{actionNamespace + "#" + actionName}, + "SOAPACTION": []string{`"` + actionNamespace + "#" + actionName + `"`}, "CONTENT-TYPE": []string{"text/xml; charset=\"utf-8\""}, }, Body: ioutil.NopCloser(bytes.NewBuffer(requestBytes)), diff --git a/Godeps/_workspace/src/github.com/jackpal/gateway/LICENSE b/Godeps/_workspace/src/github.com/jackpal/gateway/LICENSE new file mode 100644 index 00000000000..c9efac32e02 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jackpal/gateway/LICENSE @@ -0,0 +1,27 @@ +// Copyright (c) 2010 Jack Palevich. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/jackpal/gateway/README.md b/Godeps/_workspace/src/github.com/jackpal/gateway/README.md new file mode 100644 index 00000000000..49cc0b85c5a --- /dev/null +++ b/Godeps/_workspace/src/github.com/jackpal/gateway/README.md @@ -0,0 +1,7 @@ +# gateway + +A very simple library for discovering the IP address of the local LAN gateway. + +Provides implementations for Linux, OS X (Darwin) and Windows. + +Pull requests for other OSs happily considered! diff --git a/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_darwin.go b/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_darwin.go new file mode 100644 index 00000000000..fc6ef68d954 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_darwin.go @@ -0,0 +1,40 @@ +package gateway + +import ( + "bytes" + "io/ioutil" + "net" + "os/exec" +) + +func DiscoverGateway() (ip net.IP, err error) { + routeCmd := exec.Command("route", "-n", "get", "0.0.0.0") + stdOut, err := routeCmd.StdoutPipe() + if err != nil { + return + } + if err = routeCmd.Start(); err != nil { + return + } + output, err := ioutil.ReadAll(stdOut) + if err != nil { + return + } + + // Darwin route out format is always like this: + // route to: default + // destination: default + // mask: default + // gateway: 192.168.1.1 + outputLines := bytes.Split(output, []byte("\n")) + for _, line := range outputLines { + if bytes.Contains(line, []byte("gateway:")) { + gatewayFields := bytes.Fields(line) + ip = net.ParseIP(string(gatewayFields[1])) + break + } + } + + err = routeCmd.Wait() + return +} diff --git a/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_linux.go b/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_linux.go new file mode 100644 index 00000000000..333bde1dcb1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_linux.go @@ -0,0 +1,75 @@ +package gateway + +import ( + "bytes" + "io/ioutil" + "net" + "os/exec" +) + +func discoverGatewayUsingIp() (ip net.IP, err error) { + routeCmd := exec.Command("ip", "route", "show") + stdOut, err := routeCmd.StdoutPipe() + if err != nil { + return + } + if err = routeCmd.Start(); err != nil { + return + } + output, err := ioutil.ReadAll(stdOut) + if err != nil { + return + } + + // Linux 'ip route show' format looks like this: + // default via 192.168.178.1 dev wlp3s0 metric 303 + // 192.168.178.0/24 dev wlp3s0 proto kernel scope link src 192.168.178.76 metric 303 + outputLines := bytes.Split(output, []byte("\n")) + for _, line := range outputLines { + if bytes.Contains(line, []byte("default")) { + ipFields := bytes.Fields(line) + ip = net.ParseIP(string(ipFields[2])) + break + } + } + err = routeCmd.Wait() + return +} + +func discoverGatewayUsingRoute() (ip net.IP, err error) { + routeCmd := exec.Command("route", "-n") + stdOut, err := routeCmd.StdoutPipe() + if err != nil { + return + } + if err = routeCmd.Start(); err != nil { + return + } + output, err := ioutil.ReadAll(stdOut) + if err != nil { + return + } + + // Linux route out format is always like this: + // Kernel IP routing table + // Destination Gateway Genmask Flags Metric Ref Use Iface + // 0.0.0.0 192.168.1.1 0.0.0.0 UG 0 0 0 eth0 + outputLines := bytes.Split(output, []byte("\n")) + for _, line := range outputLines { + if bytes.Contains(line, []byte("0.0.0.0")) { + ipFields := bytes.Fields(line) + ip = net.ParseIP(string(ipFields[1])) + break + } + } + err = routeCmd.Wait() + return +} + +func DiscoverGateway() (ip net.IP, err error) { + ip, err = discoverGatewayUsingRoute() + if err != nil { + ip, err = discoverGatewayUsingIp() + } + return +} diff --git a/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_test.go b/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_test.go new file mode 100644 index 00000000000..b127b02dc02 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_test.go @@ -0,0 +1,10 @@ +package gateway + +import "testing" + +func TestGateway(t *testing.T) { + ip, err := DiscoverGateway() + if err != nil { + t.Errorf("DiscoverGateway() = %v,%v", ip, err) + } +} diff --git a/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_unimplemented.go b/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_unimplemented.go new file mode 100644 index 00000000000..35042b910d6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_unimplemented.go @@ -0,0 +1,14 @@ +// +build !darwin,!linux,!windows + +package gateway + +import ( + "fmt" + "net" + "runtime" +) + +func DiscoverGateway() (ip net.IP, err error) { + err = fmt.Errorf("DiscoverGateway not implemented for OS %s", runtime.GOOS) + return +} diff --git a/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_windows.go b/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_windows.go new file mode 100644 index 00000000000..282c8f68524 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jackpal/gateway/gateway_windows.go @@ -0,0 +1,43 @@ +package gateway + +import ( + "bytes" + "io/ioutil" + "net" + "os/exec" +) + +func DiscoverGateway() (ip net.IP, err error) { + routeCmd := exec.Command("route", "print", "0.0.0.0") + stdOut, err := routeCmd.StdoutPipe() + if err != nil { + return + } + if err = routeCmd.Start(); err != nil { + return + } + output, err := ioutil.ReadAll(stdOut) + if err != nil { + return + } + + // Windows route output format is always like this: + // =========================================================================== + // Active Routes: + // Network Destination Netmask Gateway Interface Metric + // 0.0.0.0 0.0.0.0 192.168.1.1 192.168.1.100 20 + // =========================================================================== + // I'm trying to pick the active route, + // then jump 2 lines and pick the third IP + // Not using regex because output is quite standard from Windows XP to 8 (NEEDS TESTING) + outputLines := bytes.Split(output, []byte("\n")) + for idx, line := range outputLines { + if bytes.Contains(line, []byte("Active Routes:")) { + ipFields := bytes.Fields(outputLines[idx+2]) + ip = net.ParseIP(string(ipFields[2])) + break + } + } + err = routeCmd.Wait() + return +} diff --git a/Godeps/_workspace/src/github.com/jackpal/go-nat-pmp/natpmp.go b/Godeps/_workspace/src/github.com/jackpal/go-nat-pmp/natpmp.go index 8ce4e8342e2..c5ee542e6ba 100644 --- a/Godeps/_workspace/src/github.com/jackpal/go-nat-pmp/natpmp.go +++ b/Godeps/_workspace/src/github.com/jackpal/go-nat-pmp/natpmp.go @@ -2,6 +2,7 @@ package natpmp import ( "fmt" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jackpal/gateway" "log" "net" "time" @@ -36,6 +37,17 @@ func NewClient(gateway net.IP) (nat *Client) { return &Client{gateway} } +// Create a NAT-PMP client for the NAT-PMP server at the default gateway. +func NewClientForDefaultGateway() (nat *Client, err error) { + var g net.IP + g, err = gateway.DiscoverGateway() + if err != nil { + return + } + nat = NewClient(g) + return +} + // Results of the NAT-PMP GetExternalAddress operation type GetExternalAddressResult struct { SecondsSinceStartOfEpoc uint32 diff --git a/Godeps/_workspace/src/github.com/jackpal/go-nat-pmp/natpmp_test.go b/Godeps/_workspace/src/github.com/jackpal/go-nat-pmp/natpmp_test.go new file mode 100644 index 00000000000..28d28a22a45 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jackpal/go-nat-pmp/natpmp_test.go @@ -0,0 +1,13 @@ +package natpmp + +import ( + "testing" +) + +func TestNatPMP(t *testing.T) { + client, err := NewClientForDefaultGateway() + if err != nil { + t.Errorf("NewClientForDefaultGateway() = %v,%v", client, err) + return + } +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-peerstream/listener.go b/Godeps/_workspace/src/github.com/jbenet/go-peerstream/listener.go index 2d2dd8ebc22..ed7f9869711 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-peerstream/listener.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-peerstream/listener.go @@ -9,7 +9,7 @@ import ( ) // AcceptConcurrency is how many connections can simultaneously be -// in process of being accepted. Handshakes can sometimes occurr as +// in process of being accepted. Handshakes can sometimes occur as // part of this process, so it may take some time. It is imporant to // rate limit lest a malicious influx of connections would cause our // node to consume all its resources accepting new connections. diff --git a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/impl_unix.go b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/impl_unix.go index c58303f343b..822de21ab36 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/impl_unix.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/impl_unix.go @@ -115,9 +115,7 @@ func dial(dialer net.Dialer, netw, addr string) (c net.Conn, err error) { // here we just try again 3 times. for i := 0; i < 3; i++ { if !deadline.IsZero() && deadline.Before(time.Now()) { - if err == nil { - err = errTimeout - } + err = errTimeout break } @@ -357,6 +355,6 @@ var errTimeout = &timeoutError{} type timeoutError struct{} -func (e *timeoutError) Error() string { return "i/o timeout (reuseport)" } +func (e *timeoutError) Error() string { return "i/o timeout" } func (e *timeoutError) Timeout() bool { return true } func (e *timeoutError) Temporary() bool { return true } diff --git a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/error.go b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/error.go index 28b26a9d2d4..c13fb81327e 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/error.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/error.go @@ -4,6 +4,6 @@ var errTimeout = &timeoutError{} type timeoutError struct{} -func (e *timeoutError) Error() string { return "i/o timeout (reuseport poll)" } +func (e *timeoutError) Error() string { return "i/o timeout" } func (e *timeoutError) Timeout() bool { return true } func (e *timeoutError) Temporary() bool { return true } diff --git a/Godeps/_workspace/src/github.com/kardianos/osext/README.md b/Godeps/_workspace/src/github.com/kardianos/osext/README.md index 820e1ecb544..61350baba5b 100644 --- a/Godeps/_workspace/src/github.com/kardianos/osext/README.md +++ b/Godeps/_workspace/src/github.com/kardianos/osext/README.md @@ -4,7 +4,9 @@ There is sometimes utility in finding the current executable file that is running. This can be used for upgrading the current executable -or finding resources located relative to the executable file. +or finding resources located relative to the executable file. Both +working directory and the os.Args[0] value are arbitrary and cannot +be relied on; os.Args[0] can be "faked". Multi-platform and supports: * Linux diff --git a/Godeps/_workspace/src/github.com/kardianos/osext/osext.go b/Godeps/_workspace/src/github.com/kardianos/osext/osext.go index 4ed4b9aa334..7bef46f03eb 100644 --- a/Godeps/_workspace/src/github.com/kardianos/osext/osext.go +++ b/Godeps/_workspace/src/github.com/kardianos/osext/osext.go @@ -16,12 +16,12 @@ func Executable() (string, error) { } // Returns same path as Executable, returns just the folder -// path. Excludes the executable name. +// path. Excludes the executable name and any trailing slash. func ExecutableFolder() (string, error) { p, err := Executable() if err != nil { return "", err } - folder, _ := filepath.Split(p) - return folder, nil + + return filepath.Dir(p), nil } diff --git a/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go b/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go index 5aafa3af2d2..77ccc28e9e9 100644 --- a/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go +++ b/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go @@ -24,6 +24,29 @@ const ( executableEnvValueDelete = "delete" ) +func TestPrintExecutable(t *testing.T) { + ef, err := Executable() + if err != nil { + t.Fatalf("Executable failed: %v", err) + } + t.Log("Executable:", ef) +} +func TestPrintExecutableFolder(t *testing.T) { + ef, err := ExecutableFolder() + if err != nil { + t.Fatalf("ExecutableFolder failed: %v", err) + } + t.Log("Executable Folder:", ef) +} +func TestExecutableFolder(t *testing.T) { + ef, err := ExecutableFolder() + if err != nil { + t.Fatalf("ExecutableFolder failed: %v", err) + } + if ef[len(ef)-1] == filepath.Separator { + t.Fatal("ExecutableFolder ends with a trailing slash.") + } +} func TestExecutableMatch(t *testing.T) { ep, err := Executable() if err != nil { diff --git a/Godeps/_workspace/src/github.com/miekg/dns/client.go b/Godeps/_workspace/src/github.com/miekg/dns/client.go index cdab4432a63..64516e165f5 100644 --- a/Godeps/_workspace/src/github.com/miekg/dns/client.go +++ b/Godeps/_workspace/src/github.com/miekg/dns/client.go @@ -66,6 +66,9 @@ func Exchange(m *Msg, a string) (r *Msg, err error) { return nil, err } r, err = co.ReadMsg() + if err == nil && r.Id != m.Id { + err = ErrId + } return r, err } @@ -86,6 +89,9 @@ func ExchangeConn(c net.Conn, m *Msg) (r *Msg, err error) { return nil, err } r, err = co.ReadMsg() + if err == nil && r.Id != m.Id { + err = ErrId + } return r, err } @@ -122,31 +128,39 @@ func (c *Client) Exchange(m *Msg, a string) (r *Msg, rtt time.Duration, err erro return r, rtt, nil } -func (c *Client) exchange(m *Msg, a string) (r *Msg, rtt time.Duration, err error) { - timeout := dnsTimeout - var co *Conn +func (c *Client) dialTimeout() time.Duration { if c.DialTimeout != 0 { - timeout = c.DialTimeout + return c.DialTimeout + } + return dnsTimeout +} + +func (c *Client) readTimeout() time.Duration { + if c.ReadTimeout != 0 { + return c.ReadTimeout + } + return dnsTimeout +} + +func (c *Client) writeTimeout() time.Duration { + if c.WriteTimeout != 0 { + return c.WriteTimeout } + return dnsTimeout +} + +func (c *Client) exchange(m *Msg, a string) (r *Msg, rtt time.Duration, err error) { + var co *Conn if c.Net == "" { - co, err = DialTimeout("udp", a, timeout) + co, err = DialTimeout("udp", a, c.dialTimeout()) } else { - co, err = DialTimeout(c.Net, a, timeout) + co, err = DialTimeout(c.Net, a, c.dialTimeout()) } if err != nil { return nil, 0, err } - timeout = dnsTimeout - if c.ReadTimeout != 0 { - timeout = c.ReadTimeout - } - co.SetReadDeadline(time.Now().Add(timeout)) - timeout = dnsTimeout - if c.WriteTimeout != 0 { - timeout = c.WriteTimeout - } - co.SetWriteDeadline(time.Now().Add(timeout)) defer co.Close() + opt := m.IsEdns0() // If EDNS0 is used use that for size. if opt != nil && opt.UDPSize() >= MinMsgSize { @@ -156,11 +170,18 @@ func (c *Client) exchange(m *Msg, a string) (r *Msg, rtt time.Duration, err erro if opt == nil && c.UDPSize >= MinMsgSize { co.UDPSize = c.UDPSize } + + co.SetReadDeadline(time.Now().Add(c.readTimeout())) + co.SetWriteDeadline(time.Now().Add(c.writeTimeout())) + co.TsigSecret = c.TsigSecret if err = co.WriteMsg(m); err != nil { return nil, 0, err } r, err = co.ReadMsg() + if err == nil && r.Id != m.Id { + err = ErrId + } return r, co.rtt, err } diff --git a/Godeps/_workspace/src/github.com/miekg/dns/client_test.go b/Godeps/_workspace/src/github.com/miekg/dns/client_test.go index d0294f9734c..8a70c7eac4f 100644 --- a/Godeps/_workspace/src/github.com/miekg/dns/client_test.go +++ b/Godeps/_workspace/src/github.com/miekg/dns/client_test.go @@ -37,6 +37,29 @@ func TestClientSync(t *testing.T) { } } +func TestClientSyncBadId(t *testing.T) { + HandleFunc("miek.nl.", HelloServerBadId) + defer HandleRemove("miek.nl.") + + s, addrstr, err := RunLocalUDPServer("127.0.0.1:0") + if err != nil { + t.Fatalf("Unable to run test server: %v", err) + } + defer s.Shutdown() + + m := new(Msg) + m.SetQuestion("miek.nl.", TypeSOA) + + c := new(Client) + if _, _, err := c.Exchange(m, addrstr); err != ErrId { + t.Errorf("did not find a bad Id") + } + // And now with plain Exchange(). + if _, err := Exchange(m, addrstr); err != ErrId { + t.Errorf("did not find a bad Id") + } +} + func TestClientEDNS0(t *testing.T) { HandleFunc("miek.nl.", HelloServer) defer HandleRemove("miek.nl.") diff --git a/Godeps/_workspace/src/github.com/miekg/dns/dnssec_keyscan.go b/Godeps/_workspace/src/github.com/miekg/dns/dnssec_keyscan.go index d9c0d9b5ca2..ac84fbcb4f4 100644 --- a/Godeps/_workspace/src/github.com/miekg/dns/dnssec_keyscan.go +++ b/Godeps/_workspace/src/github.com/miekg/dns/dnssec_keyscan.go @@ -6,6 +6,7 @@ import ( "crypto/rsa" "io" "math/big" + "strconv" "strings" ) @@ -34,8 +35,12 @@ func (k *DNSKEY) ReadPrivateKey(q io.Reader, file string) (PrivateKey, error) { return nil, ErrPrivKey } // TODO(mg): check if the pubkey matches the private key - switch m["algorithm"] { - case "3 (DSA)": + algo, err := strconv.Atoi(strings.SplitN(m["algorithm"], " ", 2)[0]) + if err != nil { + return nil, ErrPrivKey + } + switch uint8(algo) { + case DSA: priv, e := readPrivateKeyDSA(m) if e != nil { return nil, e @@ -46,15 +51,15 @@ func (k *DNSKEY) ReadPrivateKey(q io.Reader, file string) (PrivateKey, error) { } priv.PublicKey = *pub return (*DSAPrivateKey)(priv), e - case "1 (RSAMD5)": + case RSAMD5: fallthrough - case "5 (RSASHA1)": + case RSASHA1: fallthrough - case "7 (RSASHA1NSEC3SHA1)": + case RSASHA1NSEC3SHA1: fallthrough - case "8 (RSASHA256)": + case RSASHA256: fallthrough - case "10 (RSASHA512)": + case RSASHA512: priv, e := readPrivateKeyRSA(m) if e != nil { return nil, e @@ -65,11 +70,11 @@ func (k *DNSKEY) ReadPrivateKey(q io.Reader, file string) (PrivateKey, error) { } priv.PublicKey = *pub return (*RSAPrivateKey)(priv), e - case "12 (ECC-GOST)": + case ECCGOST: return nil, ErrPrivKey - case "13 (ECDSAP256SHA256)": + case ECDSAP256SHA256: fallthrough - case "14 (ECDSAP384SHA384)": + case ECDSAP384SHA384: priv, e := readPrivateKeyECDSA(m) if e != nil { return nil, e diff --git a/Godeps/_workspace/src/github.com/miekg/dns/parse_test.go b/Godeps/_workspace/src/github.com/miekg/dns/parse_test.go index c04a9764ad2..b0d02f4574f 100644 --- a/Godeps/_workspace/src/github.com/miekg/dns/parse_test.go +++ b/Godeps/_workspace/src/github.com/miekg/dns/parse_test.go @@ -548,6 +548,9 @@ a.example.com. IN A 127.0.0.1 8db7._openpgpkey.example.com. IN OPENPGPKEY mQCNAzIG $ORIGIN a.example.com. test IN A 127.0.0.1 + IN SSHFP 1 2 ( + BC6533CDC95A79078A39A56EA7635984ED655318ADA9 + B6159E30723665DA95BB ) $ORIGIN b.example.com. test IN CNAME test.a.example.com. ` @@ -904,8 +907,9 @@ func TestILNP(t *testing.T) { func TestNsapGposEidNimloc(t *testing.T) { dt := map[string]string{ - "foo.bar.com. IN NSAP 21 47000580ffff000000321099991111222233334444": "foo.bar.com.\t3600\tIN\tNSAP\t21 47000580ffff000000321099991111222233334444", - "host.school.de IN NSAP 17 39276f3100111100002222333344449876": "host.school.de.\t3600\tIN\tNSAP\t17 39276f3100111100002222333344449876", + "foo.bar.com. IN NSAP 21 47000580ffff000000321099991111222233334444": "foo.bar.com.\t3600\tIN\tNSAP\t0x47000580ffff000000321099991111222233334444", + "foo.bar.com. IN NSAP 0x47000580ffff000000321099991111222233334444": "foo.bar.com.\t3600\tIN\tNSAP\t0x47000580ffff000000321099991111222233334444", + "host.school.de IN NSAP 17 39276f3100111100002222333344449876": "host.school.de.\t3600\tIN\tNSAP\t0x39276f3100111100002222333344449876", "444433332222111199990123000000ff. NSAP-PTR foo.bar.com.": "444433332222111199990123000000ff.\t3600\tIN\tNSAP-PTR\tfoo.bar.com.", "lillee. IN GPOS -32.6882 116.8652 10.0": "lillee.\t3600\tIN\tGPOS\t-32.6882 116.8652 10.0", "hinault. IN GPOS -22.6882 116.8652 250.0": "hinault.\t3600\tIN\tGPOS\t-22.6882 116.8652 250.0", @@ -1376,3 +1380,78 @@ func TestParseIPSECKEY(t *testing.T) { i++ } } + +func TestParseTokenOverflow(t *testing.T) { + _, err := NewRR("_443._tcp.example.org. IN TLSA 0 0 0 308205e8308204d0a00302010202100411de8f53b462f6a5a861b712ec6b59300d06092a864886f70d01010b05003070310b300906035504061302555331153013060355040a130c446967694365727420496e6331193017060355040b13107777772e64696769636572742e636f6d312f302d06035504031326446967694365727420534841322048696768204173737572616e636520536572766572204341301e170d3134313130363030303030305a170d3135313131333132303030305a3081a5310b3009060355040613025553311330110603550408130a43616c69666f726e6961311430120603550407130b4c6f7320416e67656c6573313c303a060355040a1333496e7465726e657420436f72706f726174696f6e20666f722041737369676e6564204e616d657320616e64204e756d6265727331133011060355040b130a546563686e6f6c6f6779311830160603550403130f7777772e6578616d706c652e6f726730820122300d06092a864886f70d01010105000382010f003082010a02820101009e663f52a3d18cb67cdfed547408a4e47e4036538988da2798da3b6655f7240d693ed1cb3fe6d6ad3a9e657ff6efa86b83b0cad24e5d31ff2bf70ec3b78b213f1b4bf61bdc669cbbc07d67154128ca92a9b3cbb4213a836fb823ddd4d7cc04918314d25f06086fa9970ba17e357cca9b458c27eb71760ab95e3f9bc898ae89050ae4d09ba2f7e4259d9ff1e072a6971b18355a8b9e53670c3d5dbdbd283f93a764e71b3a4140ca0746090c08510e2e21078d7d07844bf9c03865b531a0bf2ee766bc401f6451c5a1e6f6fb5d5c1d6a97a0abe91ae8b02e89241e07353909ccd5b41c46de207c06801e08f20713603827f2ae3e68cf15ef881d7e0608f70742e30203010001a382024630820242301f0603551d230418301680145168ff90af0207753cccd9656462a212b859723b301d0603551d0e04160414b000a7f422e9b1ce216117c4c46e7164c8e60c553081810603551d11047a3078820f7777772e6578616d706c652e6f7267820b6578616d706c652e636f6d820b6578616d706c652e656475820b6578616d706c652e6e6574820b6578616d706c652e6f7267820f7777772e6578616d706c652e636f6d820f7777772e6578616d706c652e656475820f7777772e6578616d706c652e6e6574300e0603551d0f0101ff0404030205a0301d0603551d250416301406082b0601050507030106082b0601050507030230750603551d1f046e306c3034a032a030862e687474703a2f2f63726c332e64696769636572742e636f6d2f736861322d68612d7365727665722d67332e63726c3034a032a030862e687474703a2f2f63726c342e64696769636572742e636f6d2f736861322d68612d7365727665722d67332e63726c30420603551d20043b3039303706096086480186fd6c0101302a302806082b06010505070201161c68747470733a2f2f7777772e64696769636572742e636f6d2f43505330818306082b0601050507010104773075302406082b060105050730018618687474703a2f2f6f6373702e64696769636572742e636f6d304d06082b060105050730028641687474703a2f2f636163657274732e64696769636572742e636f6d2f446967694365727453484132486967684173737572616e636553657276657243412e637274300c0603551d130101ff04023000300d06092a864886f70d01010b050003820101005eac2124dedb3978a86ff3608406acb542d3cb54cb83facd63aec88144d6a1bf15dbf1f215c4a73e241e582365cba9ea50dd306541653b3513af1a0756c1b2720e8d112b34fb67181efad9c4609bdc670fb025fa6e6d42188161b026cf3089a08369c2f3609fc84bcc3479140c1922ede430ca8dbac2b2a3cdacb305ba15dc7361c4c3a5e6daa99cb446cb221b28078a7a944efba70d96f31ac143d959bccd2fd50e30c325ea2624fb6b6dbe9344dbcf133bfbd5b4e892d635dbf31596451672c6b65ba5ac9b3cddea92b35dab1065cae3c8cb6bb450a62ea2f72ea7c6bdc7b65fa09b012392543734083c7687d243f8d0375304d99ccd2e148966a8637a6797") + if err == nil { + t.Fatalf("token overflow should return an error") + } + t.Logf("err: %s\n", err) +} + +func TestParseTLSA(t *testing.T) { + lt := []string{ + "_443._tcp.example.org.\t3600\tIN\tTLSA\t1 1 1 c22be239f483c08957bc106219cc2d3ac1a308dfbbdd0a365f17b9351234cf00", + "_443._tcp.example.org.\t3600\tIN\tTLSA\t2 1 2 4e85f45179e9cd6e0e68e2eb5be2e85ec9b92d91c609caf3ef0315213e3f92ece92c38397a607214de95c7fadc0ad0f1c604a469a0387959745032c0d51492f3", + "_443._tcp.example.org.\t3600\tIN\tTLSA\t3 0 2 69ec8d2277360b215d0cd956b0e2747108dff34b27d461a41c800629e38ee6c2d1230cc9e8e36711330adc6766e6ff7c5fbb37f106f248337c1a20ad682888d2", + } + for _, o := range lt { + rr, err := NewRR(o) + if err != nil { + t.Error("failed to parse RR: ", err) + continue + } + if rr.String() != o { + t.Errorf("`%s' should be equal to\n`%s', but is `%s'", o, o, rr.String()) + } else { + t.Logf("RR is OK: `%s'", rr.String()) + } + } +} + +func TestParseSSHFP(t *testing.T) { + lt := []string{ + "test.example.org.\t300\tSSHFP\t1 2 (\n" + + "\t\t\t\t\tBC6533CDC95A79078A39A56EA7635984ED655318ADA9\n" + + "\t\t\t\t\tB6159E30723665DA95BB )", + "test.example.org.\t300\tSSHFP\t1 2 ( BC6533CDC 95A79078A39A56EA7635984ED655318AD A9B6159E3072366 5DA95BB )", + } + result := "test.example.org.\t300\tIN\tSSHFP\t1 2 BC6533CDC95A79078A39A56EA7635984ED655318ADA9B6159E30723665DA95BB" + for _, o := range lt { + rr, err := NewRR(o) + if err != nil { + t.Error("failed to parse RR: ", err) + continue + } + if rr.String() != result { + t.Errorf("`%s' should be equal to\n\n`%s', but is \n`%s'", o, result, rr.String()) + } else { + t.Logf("RR is OK: `%s'", rr.String()) + } + } +} + +func TestParseHINFO(t *testing.T) { + dt := map[string]string{ + "example.net. HINFO A B": "example.net. 3600 IN HINFO \"A\" \"B\"", + "example.net. HINFO \"A\" \"B\"": "example.net. 3600 IN HINFO \"A\" \"B\"", + "example.net. HINFO A B C D E F": "example.net. 3600 IN HINFO \"A\" \"B C D E F\"", + "example.net. HINFO AB": "example.net. 3600 IN HINFO \"AB\" \"\"", + // "example.net. HINFO PC-Intel-700mhz \"Redhat Linux 7.1\"": "example.net. 3600 IN HINFO \"PC-Intel-700mhz\" \"Redhat Linux 7.1\"", + // This one is recommended in Pro Bind book http://www.zytrax.com/books/dns/ch8/hinfo.html + // but effectively, even Bind would replace it to correctly formed text when you AXFR + // TODO: remove this set of comments or figure support for quoted/unquoted combinations in endingToTxtSlice function + } + for i, o := range dt { + rr, err := NewRR(i) + if err != nil { + t.Error("failed to parse RR: ", err) + continue + } + if rr.String() != o { + t.Errorf("`%s' should be equal to\n`%s', but is `%s'", i, o, rr.String()) + } else { + t.Logf("RR is OK: `%s'", rr.String()) + } + } +} diff --git a/Godeps/_workspace/src/github.com/miekg/dns/server_test.go b/Godeps/_workspace/src/github.com/miekg/dns/server_test.go index c2422b1f798..dff0fb528b5 100644 --- a/Godeps/_workspace/src/github.com/miekg/dns/server_test.go +++ b/Godeps/_workspace/src/github.com/miekg/dns/server_test.go @@ -17,6 +17,16 @@ func HelloServer(w ResponseWriter, req *Msg) { w.WriteMsg(m) } +func HelloServerBadId(w ResponseWriter, req *Msg) { + m := new(Msg) + m.SetReply(req) + m.Id += 1 + + m.Extra = make([]RR, 1) + m.Extra[0] = &TXT{Hdr: RR_Header{Name: m.Question[0].Name, Rrtype: TypeTXT, Class: ClassINET, Ttl: 0}, Txt: []string{"Hello world"}} + w.WriteMsg(m) +} + func AnotherHelloServer(w ResponseWriter, req *Msg) { m := new(Msg) m.SetReply(req) diff --git a/Godeps/_workspace/src/github.com/miekg/dns/types.go b/Godeps/_workspace/src/github.com/miekg/dns/types.go index 40cc0a4f404..bd95fd9a7d9 100644 --- a/Godeps/_workspace/src/github.com/miekg/dns/types.go +++ b/Godeps/_workspace/src/github.com/miekg/dns/types.go @@ -255,8 +255,10 @@ type HINFO struct { func (rr *HINFO) Header() *RR_Header { return &rr.Hdr } func (rr *HINFO) copy() RR { return &HINFO{*rr.Hdr.copyHeader(), rr.Cpu, rr.Os} } -func (rr *HINFO) String() string { return rr.Hdr.String() + rr.Cpu + " " + rr.Os } -func (rr *HINFO) len() int { return rr.Hdr.len() + len(rr.Cpu) + len(rr.Os) } +func (rr *HINFO) String() string { + return rr.Hdr.String() + sprintTxt([]string{rr.Cpu, rr.Os}) +} +func (rr *HINFO) len() int { return rr.Hdr.len() + len(rr.Cpu) + len(rr.Os) } type MB struct { Hdr RR_Header @@ -1146,14 +1148,13 @@ func (rr *RKEY) String() string { } type NSAP struct { - Hdr RR_Header - Length uint8 - Nsap string + Hdr RR_Header + Nsap string } func (rr *NSAP) Header() *RR_Header { return &rr.Hdr } -func (rr *NSAP) copy() RR { return &NSAP{*rr.Hdr.copyHeader(), rr.Length, rr.Nsap} } -func (rr *NSAP) String() string { return rr.Hdr.String() + strconv.Itoa(int(rr.Length)) + " " + rr.Nsap } +func (rr *NSAP) copy() RR { return &NSAP{*rr.Hdr.copyHeader(), rr.Nsap} } +func (rr *NSAP) String() string { return rr.Hdr.String() + "0x" + rr.Nsap } func (rr *NSAP) len() int { return rr.Hdr.len() + 1 + len(rr.Nsap) + 1 } type NSAPPTR struct { diff --git a/Godeps/_workspace/src/github.com/miekg/dns/zscan.go b/Godeps/_workspace/src/github.com/miekg/dns/zscan.go index 44622829171..06be9cc6d17 100644 --- a/Godeps/_workspace/src/github.com/miekg/dns/zscan.go +++ b/Godeps/_workspace/src/github.com/miekg/dns/zscan.go @@ -500,14 +500,14 @@ func zlexer(s *scan, c chan lex) { for err == nil { l.column = s.position.Column l.line = s.position.Line - if stri > maxTok { + if stri >= maxTok { l.token = "token length insufficient for parsing" l.err = true debug.Printf("[%+v]", l.token) c <- l return } - if comi > maxTok { + if comi >= maxTok { l.token = "comment length insufficient for parsing" l.err = true debug.Printf("[%+v]", l.token) diff --git a/Godeps/_workspace/src/github.com/miekg/dns/zscan_rr.go b/Godeps/_workspace/src/github.com/miekg/dns/zscan_rr.go index 1ffd13c4109..e5ca2615a16 100644 --- a/Godeps/_workspace/src/github.com/miekg/dns/zscan_rr.go +++ b/Godeps/_workspace/src/github.com/miekg/dns/zscan_rr.go @@ -49,6 +49,9 @@ func endingToString(c chan lex, errstr, f string) (string, *ParseError, string) s := "" l := <-c // zString for l.value != zNewline && l.value != zEOF { + if l.err { + return s, &ParseError{f, errstr, l}, "" + } switch l.value { case zString: s += l.token @@ -68,11 +71,17 @@ func endingToTxtSlice(c chan lex, errstr, f string) ([]string, *ParseError, stri quote := false l := <-c var s []string + if l.err { + return s, &ParseError{f, errstr, l}, "" + } switch l.value == zQuote { case true: // A number of quoted string s = make([]string, 0) empty := true for l.value != zNewline && l.value != zEOF { + if l.err { + return nil, &ParseError{f, errstr, l}, "" + } switch l.value { case zString: empty = false @@ -91,7 +100,7 @@ func endingToTxtSlice(c chan lex, errstr, f string) ([]string, *ParseError, stri p, i = p+255, i+255 } s = append(s, sx...) - break; + break } s = append(s, l.token) @@ -117,6 +126,9 @@ func endingToTxtSlice(c chan lex, errstr, f string) ([]string, *ParseError, stri case false: // Unquoted text record s = make([]string, 1) for l.value != zNewline && l.value != zEOF { + if l.err { + return s, &ParseError{f, errstr, l}, "" + } s[0] += l.token l = <-c } @@ -333,11 +345,24 @@ func setHINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(HINFO) rr.Hdr = h - l := <-c - rr.Cpu = l.token - <-c // zBlank - l = <-c // zString - rr.Os = l.token + chunks, e, c1 := endingToTxtSlice(c, "bad HINFO Fields", f) + if e != nil { + return nil, e, c1 + } + + if ln := len(chunks); ln == 0 { + return rr, nil, "" + } else if ln == 1 { + // Can we split it? + if out := strings.Fields(chunks[0]); len(out) > 1 { + chunks = out + } else { + chunks = append(chunks, "") + } + } + + rr.Cpu = chunks[0] + rr.Os = strings.Join(chunks[1:], " ") return rr, nil, "" } @@ -1438,9 +1463,9 @@ func setWKS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { case zString: if k, err = net.LookupPort(proto, l.token); err != nil { if i, e := strconv.Atoi(l.token); e != nil { // If a number use that - rr.BitMap = append(rr.BitMap, uint16(i)) - } else { return nil, &ParseError{f, "bad WKS BitMap", l}, "" + } else { + rr.BitMap = append(rr.BitMap, uint16(i)) } } rr.BitMap = append(rr.BitMap, uint16(k)) @@ -1473,8 +1498,11 @@ func setSSHFP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { } rr.Type = uint8(i) <-c // zBlank - l = <-c - rr.FingerPrint = l.token + s, e1, c1 := endingToString(c, "bad SSHFP Fingerprint", f) + if e1 != nil { + return nil, e1, c1 + } + rr.FingerPrint = s return rr, nil, "" } @@ -1594,21 +1622,28 @@ func setNIMLOC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { func setNSAP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(NSAP) rr.Hdr = h - l := <-c - if l.length == 0 { - return rr, nil, l.comment + chunks, e1, c1 := endingToTxtSlice(c, "bad NSAP Nsap", f) + if e1 != nil { + return nil, e1, c1 } - i, e := strconv.Atoi(l.token) - if e != nil { - return nil, &ParseError{f, "bad NSAP Length", l}, "" + // data would come as one string or multiple... Just to ignore possible + // variety let's merge things back together and split to actual "words" + s := strings.Fields(strings.Join(chunks, " ")) + if len(s) == 0 { + return rr, nil, c1 } - rr.Length = uint8(i) - <-c // zBlank - s, e1, c1 := endingToString(c, "bad NSAP Nsap", f) - if e != nil { - return nil, e1, c1 + if len(s[0]) >= 2 && s[0][0:2] == "0x" || s[0][0:2] == "0X" { + // although RFC only suggests 0x there is no clarification that X is not allowed + rr.Nsap = strings.Join(s, "")[2:] + } else { + // since we do not know what to do with this data, and, we would not use original length + // in formatting, it's moot to check correctness of the length + _, err := strconv.Atoi(s[0]) + if err != nil { + return nil, &ParseError{f, "bad NSAP Length", lex{token: s[0]}}, "" + } + rr.Nsap = strings.Join(s[1:], "") } - rr.Nsap = s return rr, nil, c1 } @@ -1764,9 +1799,10 @@ func setTLSA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { return nil, &ParseError{f, "bad TLSA MatchingType", l}, "" } rr.MatchingType = uint8(i) - s, e, c1 := endingToString(c, "bad TLSA Certificate", f) - if e != nil { - return nil, e.(*ParseError), c1 + // So this needs be e2 (i.e. different than e), because...??t + s, e2, c1 := endingToString(c, "bad TLSA Certificate", f) + if e2 != nil { + return nil, e2, c1 } rr.Certificate = s return rr, nil, c1 @@ -2153,7 +2189,7 @@ var typeToparserFunc = map[uint16]parserFunc{ TypeEUI64: parserFunc{setEUI64, false}, TypeGID: parserFunc{setGID, false}, TypeGPOS: parserFunc{setGPOS, false}, - TypeHINFO: parserFunc{setHINFO, false}, + TypeHINFO: parserFunc{setHINFO, true}, TypeHIP: parserFunc{setHIP, true}, TypeIPSECKEY: parserFunc{setIPSECKEY, true}, TypeKX: parserFunc{setKX, false}, @@ -2189,7 +2225,7 @@ var typeToparserFunc = map[uint16]parserFunc{ TypeSOA: parserFunc{setSOA, false}, TypeSPF: parserFunc{setSPF, true}, TypeSRV: parserFunc{setSRV, false}, - TypeSSHFP: parserFunc{setSSHFP, false}, + TypeSSHFP: parserFunc{setSSHFP, true}, TypeTALINK: parserFunc{setTALINK, false}, TypeTA: parserFunc{setTA, true}, TypeTLSA: parserFunc{setTLSA, true}, diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go index b117078688c..16af8554080 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go @@ -63,13 +63,14 @@ type DB struct // DB is a LevelDB database. journalAckC chan error // Compaction. - tcompCmdC chan cCmd - tcompPauseC chan chan<- struct{} - mcompCmdC chan cCmd - compErrC chan error - compPerErrC chan error - compErrSetC chan error - compStats []cStats + tcompCmdC chan cCmd + tcompPauseC chan chan<- struct{} + mcompCmdC chan cCmd + compErrC chan error + compPerErrC chan error + compErrSetC chan error + compWriteLocking bool + compStats []cStats // Close. closeW sync.WaitGroup @@ -108,28 +109,44 @@ func openDB(s *session) (*DB, error) { closeC: make(chan struct{}), } - if err := db.recoverJournal(); err != nil { - return nil, err - } + // Read-only mode. + readOnly := s.o.GetReadOnly() - // Remove any obsolete files. - if err := db.checkAndCleanFiles(); err != nil { - // Close journal. - if db.journal != nil { - db.journal.Close() - db.journalWriter.Close() + if readOnly { + // Recover journals (read-only mode). + if err := db.recoverJournalRO(); err != nil { + return nil, err } - return nil, err + } else { + // Recover journals. + if err := db.recoverJournal(); err != nil { + return nil, err + } + + // Remove any obsolete files. + if err := db.checkAndCleanFiles(); err != nil { + // Close journal. + if db.journal != nil { + db.journal.Close() + db.journalWriter.Close() + } + return nil, err + } + } // Doesn't need to be included in the wait group. go db.compactionError() go db.mpoolDrain() - db.closeW.Add(3) - go db.tCompaction() - go db.mCompaction() - go db.jWriter() + if readOnly { + db.SetReadOnly() + } else { + db.closeW.Add(3) + go db.tCompaction() + go db.mCompaction() + go db.jWriter() + } s.logf("db@open done T·%v", time.Since(start)) @@ -275,7 +292,7 @@ func recoverTable(s *session, o *opt.Options) error { // We will drop corrupted table. strict = o.GetStrict(opt.StrictRecovery) - rec = &sessionRecord{numLevel: o.GetNumLevel()} + rec = &sessionRecord{} bpool = util.NewBufferPool(o.GetBlockSize() + 5) ) buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) { @@ -450,132 +467,136 @@ func recoverTable(s *session, o *opt.Options) error { } func (db *DB) recoverJournal() error { - // Get all tables and sort it by file number. - journalFiles_, err := db.s.getFiles(storage.TypeJournal) + // Get all journals and sort it by file number. + allJournalFiles, err := db.s.getFiles(storage.TypeJournal) if err != nil { return err } - journalFiles := files(journalFiles_) - journalFiles.sort() + files(allJournalFiles).sort() - // Discard older journal. - prev := -1 - for i, file := range journalFiles { - if file.Num() >= db.s.stJournalNum { - if prev >= 0 { - i-- - journalFiles[i] = journalFiles[prev] - } - journalFiles = journalFiles[i:] - break - } else if file.Num() == db.s.stPrevJournalNum { - prev = i - } - } - - var jr *journal.Reader - var of storage.File - var mem *memdb.DB - batch := new(Batch) - cm := newCMem(db.s) - buf := new(util.Buffer) - // Options. - strict := db.s.o.GetStrict(opt.StrictJournal) - checksum := db.s.o.GetStrict(opt.StrictJournalChecksum) - writeBuffer := db.s.o.GetWriteBuffer() - recoverJournal := func(file storage.File) error { - db.logf("journal@recovery recovering @%d", file.Num()) - reader, err := file.Open() - if err != nil { - return err + // Journals that will be recovered. + var recJournalFiles []storage.File + for _, jf := range allJournalFiles { + if jf.Num() >= db.s.stJournalNum || jf.Num() == db.s.stPrevJournalNum { + recJournalFiles = append(recJournalFiles, jf) } - defer reader.Close() + } - // Create/reset journal reader instance. - if jr == nil { - jr = journal.NewReader(reader, dropper{db.s, file}, strict, checksum) - } else { - jr.Reset(reader, dropper{db.s, file}, strict, checksum) - } + var ( + of storage.File // Obsolete file. + rec = &sessionRecord{} + ) - // Flush memdb and remove obsolete journal file. - if of != nil { - if mem.Len() > 0 { - if err := cm.flush(mem, 0); err != nil { - return err - } - } - if err := cm.commit(file.Num(), db.seq); err != nil { + // Recover journals. + if len(recJournalFiles) > 0 { + db.logf("journal@recovery F·%d", len(recJournalFiles)) + + // Mark file number as used. + db.s.markFileNum(recJournalFiles[len(recJournalFiles)-1].Num()) + + var ( + // Options. + strict = db.s.o.GetStrict(opt.StrictJournal) + checksum = db.s.o.GetStrict(opt.StrictJournalChecksum) + writeBuffer = db.s.o.GetWriteBuffer() + + jr *journal.Reader + mdb = memdb.New(db.s.icmp, writeBuffer) + buf = &util.Buffer{} + batch = &Batch{} + ) + + for _, jf := range recJournalFiles { + db.logf("journal@recovery recovering @%d", jf.Num()) + + fr, err := jf.Open() + if err != nil { return err } - cm.reset() - of.Remove() - of = nil - } - // Replay journal to memdb. - mem.Reset() - for { - r, err := jr.Next() - if err != nil { - if err == io.EOF { - break - } - return errors.SetFile(err, file) + // Create or reset journal reader instance. + if jr == nil { + jr = journal.NewReader(fr, dropper{db.s, jf}, strict, checksum) + } else { + jr.Reset(fr, dropper{db.s, jf}, strict, checksum) } - buf.Reset() - if _, err := buf.ReadFrom(r); err != nil { - if err == io.ErrUnexpectedEOF { - // This is error returned due to corruption, with strict == false. - continue - } else { - return errors.SetFile(err, file) + // Flush memdb and remove obsolete journal file. + if of != nil { + if mdb.Len() > 0 { + if _, err := db.s.flushMemdb(rec, mdb, -1); err != nil { + fr.Close() + return err + } } - } - if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mem); err != nil { - if strict || !errors.IsCorrupted(err) { - return errors.SetFile(err, file) - } else { - db.s.logf("journal error: %v (skipped)", err) - // We won't apply sequence number as it might be corrupted. - continue + + rec.setJournalNum(jf.Num()) + rec.setSeqNum(db.seq) + if err := db.s.commit(rec); err != nil { + fr.Close() + return err } + rec.resetAddedTables() + + of.Remove() + of = nil } - // Save sequence number. - db.seq = batch.seq + uint64(batch.Len()) + // Replay journal to memdb. + mdb.Reset() + for { + r, err := jr.Next() + if err != nil { + if err == io.EOF { + break + } - // Flush it if large enough. - if mem.Size() >= writeBuffer { - if err := cm.flush(mem, 0); err != nil { - return err + fr.Close() + return errors.SetFile(err, jf) } - mem.Reset() - } - } - of = file - return nil - } + buf.Reset() + if _, err := buf.ReadFrom(r); err != nil { + if err == io.ErrUnexpectedEOF { + // This is error returned due to corruption, with strict == false. + continue + } - // Recover all journals. - if len(journalFiles) > 0 { - db.logf("journal@recovery F·%d", len(journalFiles)) + fr.Close() + return errors.SetFile(err, jf) + } + if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil { + if !strict && errors.IsCorrupted(err) { + db.s.logf("journal error: %v (skipped)", err) + // We won't apply sequence number as it might be corrupted. + continue + } + + fr.Close() + return errors.SetFile(err, jf) + } - // Mark file number as used. - db.s.markFileNum(journalFiles[len(journalFiles)-1].Num()) + // Save sequence number. + db.seq = batch.seq + uint64(batch.Len()) - mem = memdb.New(db.s.icmp, writeBuffer) - for _, file := range journalFiles { - if err := recoverJournal(file); err != nil { - return err + // Flush it if large enough. + if mdb.Size() >= writeBuffer { + if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil { + fr.Close() + return err + } + + mdb.Reset() + } } + + fr.Close() + of = jf } - // Flush the last journal. - if mem.Len() > 0 { - if err := cm.flush(mem, 0); err != nil { + // Flush the last memdb. + if mdb.Len() > 0 { + if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil { return err } } @@ -587,8 +608,10 @@ func (db *DB) recoverJournal() error { } // Commit. - if err := cm.commit(db.journalFile.Num(), db.seq); err != nil { - // Close journal. + rec.setJournalNum(db.journalFile.Num()) + rec.setSeqNum(db.seq) + if err := db.s.commit(rec); err != nil { + // Close journal on error. if db.journal != nil { db.journal.Close() db.journalWriter.Close() @@ -604,6 +627,103 @@ func (db *DB) recoverJournal() error { return nil } +func (db *DB) recoverJournalRO() error { + // Get all journals and sort it by file number. + allJournalFiles, err := db.s.getFiles(storage.TypeJournal) + if err != nil { + return err + } + files(allJournalFiles).sort() + + // Journals that will be recovered. + var recJournalFiles []storage.File + for _, jf := range allJournalFiles { + if jf.Num() >= db.s.stJournalNum || jf.Num() == db.s.stPrevJournalNum { + recJournalFiles = append(recJournalFiles, jf) + } + } + + var ( + // Options. + strict = db.s.o.GetStrict(opt.StrictJournal) + checksum = db.s.o.GetStrict(opt.StrictJournalChecksum) + writeBuffer = db.s.o.GetWriteBuffer() + + mdb = memdb.New(db.s.icmp, writeBuffer) + ) + + // Recover journals. + if len(recJournalFiles) > 0 { + db.logf("journal@recovery RO·Mode F·%d", len(recJournalFiles)) + + var ( + jr *journal.Reader + buf = &util.Buffer{} + batch = &Batch{} + ) + + for _, jf := range recJournalFiles { + db.logf("journal@recovery recovering @%d", jf.Num()) + + fr, err := jf.Open() + if err != nil { + return err + } + + // Create or reset journal reader instance. + if jr == nil { + jr = journal.NewReader(fr, dropper{db.s, jf}, strict, checksum) + } else { + jr.Reset(fr, dropper{db.s, jf}, strict, checksum) + } + + // Replay journal to memdb. + for { + r, err := jr.Next() + if err != nil { + if err == io.EOF { + break + } + + fr.Close() + return errors.SetFile(err, jf) + } + + buf.Reset() + if _, err := buf.ReadFrom(r); err != nil { + if err == io.ErrUnexpectedEOF { + // This is error returned due to corruption, with strict == false. + continue + } + + fr.Close() + return errors.SetFile(err, jf) + } + if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil { + if !strict && errors.IsCorrupted(err) { + db.s.logf("journal error: %v (skipped)", err) + // We won't apply sequence number as it might be corrupted. + continue + } + + fr.Close() + return errors.SetFile(err, jf) + } + + // Save sequence number. + db.seq = batch.seq + uint64(batch.Len()) + } + + fr.Close() + } + } + + // Set memDB. + db.mem = &memDB{db: db, DB: mdb, ref: 1} + + return nil +} + func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) { ikey := newIkey(key, seq, ktSeek) @@ -614,7 +734,7 @@ func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, er } defer m.decref() - mk, mv, me := m.mdb.Find(ikey) + mk, mv, me := m.Find(ikey) if me == nil { ukey, _, kt, kerr := parseIkey(mk) if kerr != nil { @@ -652,7 +772,7 @@ func (db *DB) has(key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err er } defer m.decref() - mk, _, me := m.mdb.Find(ikey) + mk, _, me := m.Find(ikey) if me == nil { ukey, _, kt, kerr := parseIkey(mk) if kerr != nil { @@ -784,7 +904,7 @@ func (db *DB) GetProperty(name string) (value string, err error) { const prefix = "leveldb." if !strings.HasPrefix(name, prefix) { - return "", errors.New("leveldb: GetProperty: unknown property: " + name) + return "", ErrNotFound } p := name[len(prefix):] @@ -798,7 +918,7 @@ func (db *DB) GetProperty(name string) (value string, err error) { var rest string n, _ := fmt.Sscanf(p[len(numFilesPrefix):], "%d%s", &level, &rest) if n != 1 || int(level) >= db.s.o.GetNumLevel() { - err = errors.New("leveldb: GetProperty: invalid property: " + name) + err = ErrNotFound } else { value = fmt.Sprint(v.tLen(int(level))) } @@ -837,7 +957,7 @@ func (db *DB) GetProperty(name string) (value string, err error) { case p == "aliveiters": value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveIters)) default: - err = errors.New("leveldb: GetProperty: unknown property: " + name) + err = ErrNotFound } return @@ -900,6 +1020,9 @@ func (db *DB) Close() error { var err error select { case err = <-db.compErrC: + if err == ErrReadOnly { + err = nil + } default: } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go index 6bf2c4bb6cd..db9f81c525e 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go @@ -11,7 +11,6 @@ import ( "time" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors" - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt" ) @@ -62,58 +61,8 @@ func (p *cStatsStaging) stopTimer() { } } -type cMem struct { - s *session - level int - rec *sessionRecord -} - -func newCMem(s *session) *cMem { - return &cMem{s: s, rec: &sessionRecord{numLevel: s.o.GetNumLevel()}} -} - -func (c *cMem) flush(mem *memdb.DB, level int) error { - s := c.s - - // Write memdb to table. - iter := mem.NewIterator(nil) - defer iter.Release() - t, n, err := s.tops.createFrom(iter) - if err != nil { - return err - } - - // Pick level. - if level < 0 { - v := s.version() - level = v.pickLevel(t.imin.ukey(), t.imax.ukey()) - v.release() - } - c.rec.addTableFile(level, t) - - s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.imin, t.imax) - - c.level = level - return nil -} - -func (c *cMem) reset() { - c.rec = &sessionRecord{numLevel: c.s.o.GetNumLevel()} -} - -func (c *cMem) commit(journal, seq uint64) error { - c.rec.setJournalNum(journal) - c.rec.setSeqNum(seq) - - // Commit changes. - return c.s.commit(c.rec) -} - func (db *DB) compactionError() { - var ( - err error - wlocked bool - ) + var err error noerr: // No error. for { @@ -121,7 +70,7 @@ noerr: case err = <-db.compErrSetC: switch { case err == nil: - case errors.IsCorrupted(err): + case err == ErrReadOnly, errors.IsCorrupted(err): goto hasperr default: goto haserr @@ -139,7 +88,7 @@ haserr: switch { case err == nil: goto noerr - case errors.IsCorrupted(err): + case err == ErrReadOnly, errors.IsCorrupted(err): goto hasperr default: } @@ -155,9 +104,9 @@ hasperr: case db.compPerErrC <- err: case db.writeLockC <- struct{}{}: // Hold write lock, so that write won't pass-through. - wlocked = true + db.compWriteLocking = true case _, _ = <-db.closeC: - if wlocked { + if db.compWriteLocking { // We should release the lock or Close will hang. <-db.writeLockC } @@ -287,21 +236,18 @@ func (db *DB) compactionExitTransact() { } func (db *DB) memCompaction() { - mem := db.getFrozenMem() - if mem == nil { + mdb := db.getFrozenMem() + if mdb == nil { return } - defer mem.decref() - - c := newCMem(db.s) - stats := new(cStatsStaging) + defer mdb.decref() - db.logf("mem@flush N·%d S·%s", mem.mdb.Len(), shortenb(mem.mdb.Size())) + db.logf("memdb@flush N·%d S·%s", mdb.Len(), shortenb(mdb.Size())) // Don't compact empty memdb. - if mem.mdb.Len() == 0 { - db.logf("mem@flush skipping") - // drop frozen mem + if mdb.Len() == 0 { + db.logf("memdb@flush skipping") + // drop frozen memdb db.dropFrozenMem() return } @@ -317,13 +263,20 @@ func (db *DB) memCompaction() { return } - db.compactionTransactFunc("mem@flush", func(cnt *compactionTransactCounter) (err error) { + var ( + rec = &sessionRecord{} + stats = &cStatsStaging{} + flushLevel int + ) + + db.compactionTransactFunc("memdb@flush", func(cnt *compactionTransactCounter) (err error) { stats.startTimer() - defer stats.stopTimer() - return c.flush(mem.mdb, -1) + flushLevel, err = db.s.flushMemdb(rec, mdb.DB, -1) + stats.stopTimer() + return }, func() error { - for _, r := range c.rec.addedTables { - db.logf("mem@flush revert @%d", r.num) + for _, r := range rec.addedTables { + db.logf("memdb@flush revert @%d", r.num) f := db.s.getTableFile(r.num) if err := f.Remove(); err != nil { return err @@ -332,20 +285,23 @@ func (db *DB) memCompaction() { return nil }) - db.compactionTransactFunc("mem@commit", func(cnt *compactionTransactCounter) (err error) { + db.compactionTransactFunc("memdb@commit", func(cnt *compactionTransactCounter) (err error) { stats.startTimer() - defer stats.stopTimer() - return c.commit(db.journalFile.Num(), db.frozenSeq) + rec.setJournalNum(db.journalFile.Num()) + rec.setSeqNum(db.frozenSeq) + err = db.s.commit(rec) + stats.stopTimer() + return }, nil) - db.logf("mem@flush committed F·%d T·%v", len(c.rec.addedTables), stats.duration) + db.logf("memdb@flush committed F·%d T·%v", len(rec.addedTables), stats.duration) - for _, r := range c.rec.addedTables { + for _, r := range rec.addedTables { stats.write += r.size } - db.compStats[c.level].add(stats) + db.compStats[flushLevel].add(stats) - // Drop frozen mem. + // Drop frozen memdb. db.dropFrozenMem() // Resume table compaction. @@ -557,7 +513,7 @@ func (b *tableCompactionBuilder) revert() error { func (db *DB) tableCompaction(c *compaction, noTrivial bool) { defer c.release() - rec := &sessionRecord{numLevel: db.s.o.GetNumLevel()} + rec := &sessionRecord{} rec.addCompPtr(c.level, c.imax) if !noTrivial && c.trivial() { diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go index 1366bdb54c7..f62ce54a041 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go @@ -40,11 +40,11 @@ func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.It ti := v.getIterators(slice, ro) n := len(ti) + 2 i := make([]iterator.Iterator, 0, n) - emi := em.mdb.NewIterator(slice) + emi := em.NewIterator(slice) emi.SetReleaser(&memdbReleaser{m: em}) i = append(i, emi) if fm != nil { - fmi := fm.mdb.NewIterator(slice) + fmi := fm.NewIterator(slice) fmi.SetReleaser(&memdbReleaser{m: fm}) i = append(i, fmi) } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go index f642283ba89..4c92695d48f 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go @@ -15,8 +15,8 @@ import ( ) type memDB struct { - db *DB - mdb *memdb.DB + db *DB + *memdb.DB ref int32 } @@ -27,12 +27,12 @@ func (m *memDB) incref() { func (m *memDB) decref() { if ref := atomic.AddInt32(&m.ref, -1); ref == 0 { // Only put back memdb with std capacity. - if m.mdb.Capacity() == m.db.s.o.GetWriteBuffer() { - m.mdb.Reset() - m.db.mpoolPut(m.mdb) + if m.Capacity() == m.db.s.o.GetWriteBuffer() { + m.Reset() + m.db.mpoolPut(m.DB) } m.db = nil - m.mdb = nil + m.DB = nil } else if ref < 0 { panic("negative memdb ref") } @@ -126,7 +126,7 @@ func (db *DB) newMem(n int) (mem *memDB, err error) { } mem = &memDB{ db: db, - mdb: mdb, + DB: mdb, ref: 2, } db.mem = mem diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go index 2cb0a55d4af..68172adb50f 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go @@ -2445,7 +2445,7 @@ func TestDB_TableCompactionBuilder(t *testing.T) { if err != nil { t.Fatal(err) } - rec := &sessionRecord{numLevel: s.o.GetNumLevel()} + rec := &sessionRecord{} rec.addTableFile(i, tf) if err := s.commit(rec); err != nil { t.Fatal(err) @@ -2455,7 +2455,7 @@ func TestDB_TableCompactionBuilder(t *testing.T) { // Build grandparent. v := s.version() c := newCompaction(s, v, 1, append(tFiles{}, v.tables[1]...)) - rec := &sessionRecord{numLevel: s.o.GetNumLevel()} + rec := &sessionRecord{} b := &tableCompactionBuilder{ s: s, c: c, @@ -2479,7 +2479,7 @@ func TestDB_TableCompactionBuilder(t *testing.T) { // Build level-1. v = s.version() c = newCompaction(s, v, 0, append(tFiles{}, v.tables[0]...)) - rec = &sessionRecord{numLevel: s.o.GetNumLevel()} + rec = &sessionRecord{} b = &tableCompactionBuilder{ s: s, c: c, @@ -2523,7 +2523,7 @@ func TestDB_TableCompactionBuilder(t *testing.T) { // Compaction with transient error. v = s.version() c = newCompaction(s, v, 1, append(tFiles{}, v.tables[1]...)) - rec = &sessionRecord{numLevel: s.o.GetNumLevel()} + rec = &sessionRecord{} b = &tableCompactionBuilder{ s: s, c: c, @@ -2663,3 +2663,39 @@ func TestDB_IterTriggeredCompaction(t *testing.T) { func TestDB_IterTriggeredCompactionHalf(t *testing.T) { testDB_IterTriggeredCompaction(t, 2) } + +func TestDB_ReadOnly(t *testing.T) { + h := newDbHarness(t) + defer h.close() + + h.put("foo", "v1") + h.put("bar", "v2") + h.compactMem() + + h.put("xfoo", "v1") + h.put("xbar", "v2") + + t.Log("Trigger read-only") + if err := h.db.SetReadOnly(); err != nil { + h.close() + t.Fatalf("SetReadOnly error: %v", err) + } + + h.stor.SetEmuErr(storage.TypeAll, tsOpCreate, tsOpReplace, tsOpRemove, tsOpWrite, tsOpWrite, tsOpSync) + + ro := func(key, value, wantValue string) { + if err := h.db.Put([]byte(key), []byte(value), h.wo); err != ErrReadOnly { + t.Fatalf("unexpected error: %v", err) + } + h.getVal(key, wantValue) + } + + ro("foo", "vx", "v1") + + h.o.ReadOnly = true + h.reopenDB() + + ro("foo", "vx", "v1") + ro("bar", "vx", "v2") + h.assertNumKeys(4) +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go index f1c6b73270c..b85a72a2dad 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go @@ -63,24 +63,24 @@ func (db *DB) rotateMem(n int) (mem *memDB, err error) { return } -func (db *DB) flush(n int) (mem *memDB, nn int, err error) { +func (db *DB) flush(n int) (mdb *memDB, mdbFree int, err error) { delayed := false flush := func() (retry bool) { v := db.s.version() defer v.release() - mem = db.getEffectiveMem() + mdb = db.getEffectiveMem() defer func() { if retry { - mem.decref() - mem = nil + mdb.decref() + mdb = nil } }() - nn = mem.mdb.Free() + mdbFree = mdb.Free() switch { case v.tLen(0) >= db.s.o.GetWriteL0SlowdownTrigger() && !delayed: delayed = true time.Sleep(time.Millisecond) - case nn >= n: + case mdbFree >= n: return false case v.tLen(0) >= db.s.o.GetWriteL0PauseTrigger(): delayed = true @@ -90,15 +90,15 @@ func (db *DB) flush(n int) (mem *memDB, nn int, err error) { } default: // Allow memdb to grow if it has no entry. - if mem.mdb.Len() == 0 { - nn = n + if mdb.Len() == 0 { + mdbFree = n } else { - mem.decref() - mem, err = db.rotateMem(n) + mdb.decref() + mdb, err = db.rotateMem(n) if err == nil { - nn = mem.mdb.Free() + mdbFree = mdb.Free() } else { - nn = 0 + mdbFree = 0 } } return false @@ -157,18 +157,18 @@ func (db *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) { } }() - mem, memFree, err := db.flush(b.size()) + mdb, mdbFree, err := db.flush(b.size()) if err != nil { return } - defer mem.decref() + defer mdb.decref() // Calculate maximum size of the batch. m := 1 << 20 if x := b.size(); x <= 128<<10 { m = x + (128 << 10) } - m = minInt(m, memFree) + m = minInt(m, mdbFree) // Merge with other batch. drain: @@ -197,7 +197,7 @@ drain: select { case db.journalC <- b: // Write into memdb - if berr := b.memReplay(mem.mdb); berr != nil { + if berr := b.memReplay(mdb.DB); berr != nil { panic(berr) } case err = <-db.compPerErrC: @@ -211,7 +211,7 @@ drain: case err = <-db.journalAckC: if err != nil { // Revert memdb if error detected - if berr := b.revertMemReplay(mem.mdb); berr != nil { + if berr := b.revertMemReplay(mdb.DB); berr != nil { panic(berr) } return @@ -225,7 +225,7 @@ drain: if err != nil { return } - if berr := b.memReplay(mem.mdb); berr != nil { + if berr := b.memReplay(mdb.DB); berr != nil { panic(berr) } } @@ -233,7 +233,7 @@ drain: // Set last seq number. db.addSeq(uint64(b.Len())) - if b.size() >= memFree { + if b.size() >= mdbFree { db.rotateMem(0) } return @@ -290,9 +290,9 @@ func (db *DB) CompactRange(r util.Range) error { } // Check for overlaps in memdb. - mem := db.getEffectiveMem() - defer mem.decref() - if isMemOverlaps(db.s.icmp, mem.mdb, r.Start, r.Limit) { + mdb := db.getEffectiveMem() + defer mdb.decref() + if isMemOverlaps(db.s.icmp, mdb.DB, r.Start, r.Limit) { // Memdb compaction. if _, err := db.rotateMem(0); err != nil { <-db.writeLockC @@ -309,3 +309,31 @@ func (db *DB) CompactRange(r util.Range) error { // Table compaction. return db.compSendRange(db.tcompCmdC, -1, r.Start, r.Limit) } + +// SetReadOnly makes DB read-only. It will stay read-only until reopened. +func (db *DB) SetReadOnly() error { + if err := db.ok(); err != nil { + return err + } + + // Lock writer. + select { + case db.writeLockC <- struct{}{}: + db.compWriteLocking = true + case err := <-db.compPerErrC: + return err + case _, _ = <-db.closeC: + return ErrClosed + } + + // Set compaction read-only. + select { + case db.compErrSetC <- ErrReadOnly: + case perr := <-db.compPerErrC: + return perr + case _, _ = <-db.closeC: + return ErrClosed + } + + return nil +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors.go index 5d96d6d34d1..727fef91d1d 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors.go @@ -12,6 +12,7 @@ import ( var ( ErrNotFound = errors.ErrNotFound + ErrReadOnly = errors.New("leveldb: read-only mode") ErrSnapshotReleased = errors.New("leveldb: snapshot released") ErrIterReleased = errors.New("leveldb: iterator released") ErrClosed = errors.New("leveldb: closed") diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go index 4e03a7cd542..de509c42703 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go @@ -206,6 +206,7 @@ func (p *DB) randHeight() (h int) { return } +// Must hold RW-lock if prev == true, as it use shared prevNode slice. func (p *DB) findGE(key []byte, prev bool) (int, bool) { node := 0 h := p.maxHeight - 1 @@ -302,7 +303,7 @@ func (p *DB) Put(key []byte, value []byte) error { node := len(p.nodeData) p.nodeData = append(p.nodeData, kvOffset, len(key), len(value), h) for i, n := range p.prevNode[:h] { - m := n + 4 + i + m := n + nNext + i p.nodeData = append(p.nodeData, p.nodeData[m]) p.nodeData[m] = node } @@ -434,20 +435,22 @@ func (p *DB) Len() int { // Reset resets the DB to initial empty state. Allows reuse the buffer. func (p *DB) Reset() { + p.mu.Lock() p.rnd = rand.New(rand.NewSource(0xdeadbeef)) p.maxHeight = 1 p.n = 0 p.kvSize = 0 p.kvData = p.kvData[:0] - p.nodeData = p.nodeData[:4+tMaxHeight] + p.nodeData = p.nodeData[:nNext+tMaxHeight] p.nodeData[nKV] = 0 p.nodeData[nKey] = 0 p.nodeData[nVal] = 0 p.nodeData[nHeight] = tMaxHeight for n := 0; n < tMaxHeight; n++ { - p.nodeData[4+n] = 0 + p.nodeData[nNext+n] = 0 p.prevNode[n] = 0 } + p.mu.Unlock() } // New creates a new initalized in-memory key/value DB. The capacity diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go index 79ab9e748a7..bb7dd5f1a1f 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go @@ -250,6 +250,11 @@ type Options struct { // The default value (DefaultCompression) uses snappy compression. Compression Compression + // DisableBufferPool allows disable use of util.BufferPool functionality. + // + // The default value is false. + DisableBufferPool bool + // DisableBlockCache allows disable use of cache.Cache functionality on // 'sorted table' block. // @@ -321,6 +326,11 @@ type Options struct { // The default value is 500. OpenFilesCacheCapacity int + // If true then opens DB in read-only mode. + // + // The default value is false. + ReadOnly bool + // Strict defines the DB strict level. Strict Strict @@ -472,6 +482,20 @@ func (o *Options) GetCompression() Compression { return o.Compression } +func (o *Options) GetDisableBufferPool() bool { + if o == nil { + return false + } + return o.DisableBufferPool +} + +func (o *Options) GetDisableBlockCache() bool { + if o == nil { + return false + } + return o.DisableBlockCache +} + func (o *Options) GetDisableCompactionBackoff() bool { if o == nil { return false @@ -548,6 +572,13 @@ func (o *Options) GetOpenFilesCacheCapacity() int { return o.OpenFilesCacheCapacity } +func (o *Options) GetReadOnly() bool { + if o == nil { + return false + } + return o.ReadOnly +} + func (o *Options) GetStrict(strict Strict) bool { if o == nil || o.Strict == 0 { return DefaultStrict&strict != 0 diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go index 097c0f22513..5684e223c48 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go @@ -11,10 +11,8 @@ import ( "io" "os" "sync" - "sync/atomic" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors" - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage" @@ -127,11 +125,16 @@ func (s *session) recover() (err error) { return } defer reader.Close() - strict := s.o.GetStrict(opt.StrictManifest) - jr := journal.NewReader(reader, dropper{s, m}, strict, true) - staging := s.stVersion.newStaging() - rec := &sessionRecord{numLevel: s.o.GetNumLevel()} + var ( + // Options. + numLevel = s.o.GetNumLevel() + strict = s.o.GetStrict(opt.StrictManifest) + + jr = journal.NewReader(reader, dropper{s, m}, strict, true) + rec = &sessionRecord{} + staging = s.stVersion.newStaging() + ) for { var r io.Reader r, err = jr.Next() @@ -143,7 +146,7 @@ func (s *session) recover() (err error) { return errors.SetFile(err, m) } - err = rec.decode(r) + err = rec.decode(r, numLevel) if err == nil { // save compact pointers for _, r := range rec.compPtrs { @@ -206,250 +209,3 @@ func (s *session) commit(r *sessionRecord) (err error) { return } - -// Pick a compaction based on current state; need external synchronization. -func (s *session) pickCompaction() *compaction { - v := s.version() - - var level int - var t0 tFiles - if v.cScore >= 1 { - level = v.cLevel - cptr := s.stCompPtrs[level] - tables := v.tables[level] - for _, t := range tables { - if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 { - t0 = append(t0, t) - break - } - } - if len(t0) == 0 { - t0 = append(t0, tables[0]) - } - } else { - if p := atomic.LoadPointer(&v.cSeek); p != nil { - ts := (*tSet)(p) - level = ts.level - t0 = append(t0, ts.table) - } else { - v.release() - return nil - } - } - - return newCompaction(s, v, level, t0) -} - -// Create compaction from given level and range; need external synchronization. -func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction { - v := s.version() - - t0 := v.tables[level].getOverlaps(nil, s.icmp, umin, umax, level == 0) - if len(t0) == 0 { - v.release() - return nil - } - - // Avoid compacting too much in one shot in case the range is large. - // But we cannot do this for level-0 since level-0 files can overlap - // and we must not pick one file and drop another older file if the - // two files overlap. - if level > 0 { - limit := uint64(v.s.o.GetCompactionSourceLimit(level)) - total := uint64(0) - for i, t := range t0 { - total += t.size - if total >= limit { - s.logf("table@compaction limiting F·%d -> F·%d", len(t0), i+1) - t0 = t0[:i+1] - break - } - } - } - - return newCompaction(s, v, level, t0) -} - -func newCompaction(s *session, v *version, level int, t0 tFiles) *compaction { - c := &compaction{ - s: s, - v: v, - level: level, - tables: [2]tFiles{t0, nil}, - maxGPOverlaps: uint64(s.o.GetCompactionGPOverlaps(level)), - tPtrs: make([]int, s.o.GetNumLevel()), - } - c.expand() - c.save() - return c -} - -// compaction represent a compaction state. -type compaction struct { - s *session - v *version - - level int - tables [2]tFiles - maxGPOverlaps uint64 - - gp tFiles - gpi int - seenKey bool - gpOverlappedBytes uint64 - imin, imax iKey - tPtrs []int - released bool - - snapGPI int - snapSeenKey bool - snapGPOverlappedBytes uint64 - snapTPtrs []int -} - -func (c *compaction) save() { - c.snapGPI = c.gpi - c.snapSeenKey = c.seenKey - c.snapGPOverlappedBytes = c.gpOverlappedBytes - c.snapTPtrs = append(c.snapTPtrs[:0], c.tPtrs...) -} - -func (c *compaction) restore() { - c.gpi = c.snapGPI - c.seenKey = c.snapSeenKey - c.gpOverlappedBytes = c.snapGPOverlappedBytes - c.tPtrs = append(c.tPtrs[:0], c.snapTPtrs...) -} - -func (c *compaction) release() { - if !c.released { - c.released = true - c.v.release() - } -} - -// Expand compacted tables; need external synchronization. -func (c *compaction) expand() { - limit := uint64(c.s.o.GetCompactionExpandLimit(c.level)) - vt0, vt1 := c.v.tables[c.level], c.v.tables[c.level+1] - - t0, t1 := c.tables[0], c.tables[1] - imin, imax := t0.getRange(c.s.icmp) - // We expand t0 here just incase ukey hop across tables. - t0 = vt0.getOverlaps(t0, c.s.icmp, imin.ukey(), imax.ukey(), c.level == 0) - if len(t0) != len(c.tables[0]) { - imin, imax = t0.getRange(c.s.icmp) - } - t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false) - // Get entire range covered by compaction. - amin, amax := append(t0, t1...).getRange(c.s.icmp) - - // See if we can grow the number of inputs in "level" without - // changing the number of "level+1" files we pick up. - if len(t1) > 0 { - exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), c.level == 0) - if len(exp0) > len(t0) && t1.size()+exp0.size() < limit { - xmin, xmax := exp0.getRange(c.s.icmp) - exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false) - if len(exp1) == len(t1) { - c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)", - c.level, c.level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())), - len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size()))) - imin, imax = xmin, xmax - t0, t1 = exp0, exp1 - amin, amax = append(t0, t1...).getRange(c.s.icmp) - } - } - } - - // Compute the set of grandparent files that overlap this compaction - // (parent == level+1; grandparent == level+2) - if c.level+2 < c.s.o.GetNumLevel() { - c.gp = c.v.tables[c.level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false) - } - - c.tables[0], c.tables[1] = t0, t1 - c.imin, c.imax = imin, imax -} - -// Check whether compaction is trivial. -func (c *compaction) trivial() bool { - return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= c.maxGPOverlaps -} - -func (c *compaction) baseLevelForKey(ukey []byte) bool { - for level, tables := range c.v.tables[c.level+2:] { - for c.tPtrs[level] < len(tables) { - t := tables[c.tPtrs[level]] - if c.s.icmp.uCompare(ukey, t.imax.ukey()) <= 0 { - // We've advanced far enough. - if c.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 { - // Key falls in this file's range, so definitely not base level. - return false - } - break - } - c.tPtrs[level]++ - } - } - return true -} - -func (c *compaction) shouldStopBefore(ikey iKey) bool { - for ; c.gpi < len(c.gp); c.gpi++ { - gp := c.gp[c.gpi] - if c.s.icmp.Compare(ikey, gp.imax) <= 0 { - break - } - if c.seenKey { - c.gpOverlappedBytes += gp.size - } - } - c.seenKey = true - - if c.gpOverlappedBytes > c.maxGPOverlaps { - // Too much overlap for current output; start new output. - c.gpOverlappedBytes = 0 - return true - } - return false -} - -// Creates an iterator. -func (c *compaction) newIterator() iterator.Iterator { - // Creates iterator slice. - icap := len(c.tables) - if c.level == 0 { - // Special case for level-0 - icap = len(c.tables[0]) + 1 - } - its := make([]iterator.Iterator, 0, icap) - - // Options. - ro := &opt.ReadOptions{ - DontFillCache: true, - Strict: opt.StrictOverride, - } - strict := c.s.o.GetStrict(opt.StrictCompaction) - if strict { - ro.Strict |= opt.StrictReader - } - - for i, tables := range c.tables { - if len(tables) == 0 { - continue - } - - // Level-0 is not sorted and may overlaps each other. - if c.level+i == 0 { - for _, t := range tables { - its = append(its, c.s.tops.newIterator(t, nil, ro)) - } - } else { - it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict) - its = append(its, it) - } - } - - return iterator.NewMergedIterator(its, c.s.icmp, strict) -} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_compaction.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_compaction.go new file mode 100644 index 00000000000..53ad8b0cd5b --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_compaction.go @@ -0,0 +1,287 @@ +// Copyright (c) 2012, Suryandaru Triandana +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package leveldb + +import ( + "sync/atomic" + + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt" +) + +func (s *session) pickMemdbLevel(umin, umax []byte) int { + v := s.version() + defer v.release() + return v.pickMemdbLevel(umin, umax) +} + +func (s *session) flushMemdb(rec *sessionRecord, mdb *memdb.DB, level int) (level_ int, err error) { + // Create sorted table. + iter := mdb.NewIterator(nil) + defer iter.Release() + t, n, err := s.tops.createFrom(iter) + if err != nil { + return level, err + } + + // Pick level and add to record. + if level < 0 { + level = s.pickMemdbLevel(t.imin.ukey(), t.imax.ukey()) + } + rec.addTableFile(level, t) + + s.logf("memdb@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.imin, t.imax) + return level, nil +} + +// Pick a compaction based on current state; need external synchronization. +func (s *session) pickCompaction() *compaction { + v := s.version() + + var level int + var t0 tFiles + if v.cScore >= 1 { + level = v.cLevel + cptr := s.stCompPtrs[level] + tables := v.tables[level] + for _, t := range tables { + if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 { + t0 = append(t0, t) + break + } + } + if len(t0) == 0 { + t0 = append(t0, tables[0]) + } + } else { + if p := atomic.LoadPointer(&v.cSeek); p != nil { + ts := (*tSet)(p) + level = ts.level + t0 = append(t0, ts.table) + } else { + v.release() + return nil + } + } + + return newCompaction(s, v, level, t0) +} + +// Create compaction from given level and range; need external synchronization. +func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction { + v := s.version() + + t0 := v.tables[level].getOverlaps(nil, s.icmp, umin, umax, level == 0) + if len(t0) == 0 { + v.release() + return nil + } + + // Avoid compacting too much in one shot in case the range is large. + // But we cannot do this for level-0 since level-0 files can overlap + // and we must not pick one file and drop another older file if the + // two files overlap. + if level > 0 { + limit := uint64(v.s.o.GetCompactionSourceLimit(level)) + total := uint64(0) + for i, t := range t0 { + total += t.size + if total >= limit { + s.logf("table@compaction limiting F·%d -> F·%d", len(t0), i+1) + t0 = t0[:i+1] + break + } + } + } + + return newCompaction(s, v, level, t0) +} + +func newCompaction(s *session, v *version, level int, t0 tFiles) *compaction { + c := &compaction{ + s: s, + v: v, + level: level, + tables: [2]tFiles{t0, nil}, + maxGPOverlaps: uint64(s.o.GetCompactionGPOverlaps(level)), + tPtrs: make([]int, s.o.GetNumLevel()), + } + c.expand() + c.save() + return c +} + +// compaction represent a compaction state. +type compaction struct { + s *session + v *version + + level int + tables [2]tFiles + maxGPOverlaps uint64 + + gp tFiles + gpi int + seenKey bool + gpOverlappedBytes uint64 + imin, imax iKey + tPtrs []int + released bool + + snapGPI int + snapSeenKey bool + snapGPOverlappedBytes uint64 + snapTPtrs []int +} + +func (c *compaction) save() { + c.snapGPI = c.gpi + c.snapSeenKey = c.seenKey + c.snapGPOverlappedBytes = c.gpOverlappedBytes + c.snapTPtrs = append(c.snapTPtrs[:0], c.tPtrs...) +} + +func (c *compaction) restore() { + c.gpi = c.snapGPI + c.seenKey = c.snapSeenKey + c.gpOverlappedBytes = c.snapGPOverlappedBytes + c.tPtrs = append(c.tPtrs[:0], c.snapTPtrs...) +} + +func (c *compaction) release() { + if !c.released { + c.released = true + c.v.release() + } +} + +// Expand compacted tables; need external synchronization. +func (c *compaction) expand() { + limit := uint64(c.s.o.GetCompactionExpandLimit(c.level)) + vt0, vt1 := c.v.tables[c.level], c.v.tables[c.level+1] + + t0, t1 := c.tables[0], c.tables[1] + imin, imax := t0.getRange(c.s.icmp) + // We expand t0 here just incase ukey hop across tables. + t0 = vt0.getOverlaps(t0, c.s.icmp, imin.ukey(), imax.ukey(), c.level == 0) + if len(t0) != len(c.tables[0]) { + imin, imax = t0.getRange(c.s.icmp) + } + t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false) + // Get entire range covered by compaction. + amin, amax := append(t0, t1...).getRange(c.s.icmp) + + // See if we can grow the number of inputs in "level" without + // changing the number of "level+1" files we pick up. + if len(t1) > 0 { + exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), c.level == 0) + if len(exp0) > len(t0) && t1.size()+exp0.size() < limit { + xmin, xmax := exp0.getRange(c.s.icmp) + exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false) + if len(exp1) == len(t1) { + c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)", + c.level, c.level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())), + len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size()))) + imin, imax = xmin, xmax + t0, t1 = exp0, exp1 + amin, amax = append(t0, t1...).getRange(c.s.icmp) + } + } + } + + // Compute the set of grandparent files that overlap this compaction + // (parent == level+1; grandparent == level+2) + if c.level+2 < c.s.o.GetNumLevel() { + c.gp = c.v.tables[c.level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false) + } + + c.tables[0], c.tables[1] = t0, t1 + c.imin, c.imax = imin, imax +} + +// Check whether compaction is trivial. +func (c *compaction) trivial() bool { + return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= c.maxGPOverlaps +} + +func (c *compaction) baseLevelForKey(ukey []byte) bool { + for level, tables := range c.v.tables[c.level+2:] { + for c.tPtrs[level] < len(tables) { + t := tables[c.tPtrs[level]] + if c.s.icmp.uCompare(ukey, t.imax.ukey()) <= 0 { + // We've advanced far enough. + if c.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 { + // Key falls in this file's range, so definitely not base level. + return false + } + break + } + c.tPtrs[level]++ + } + } + return true +} + +func (c *compaction) shouldStopBefore(ikey iKey) bool { + for ; c.gpi < len(c.gp); c.gpi++ { + gp := c.gp[c.gpi] + if c.s.icmp.Compare(ikey, gp.imax) <= 0 { + break + } + if c.seenKey { + c.gpOverlappedBytes += gp.size + } + } + c.seenKey = true + + if c.gpOverlappedBytes > c.maxGPOverlaps { + // Too much overlap for current output; start new output. + c.gpOverlappedBytes = 0 + return true + } + return false +} + +// Creates an iterator. +func (c *compaction) newIterator() iterator.Iterator { + // Creates iterator slice. + icap := len(c.tables) + if c.level == 0 { + // Special case for level-0. + icap = len(c.tables[0]) + 1 + } + its := make([]iterator.Iterator, 0, icap) + + // Options. + ro := &opt.ReadOptions{ + DontFillCache: true, + Strict: opt.StrictOverride, + } + strict := c.s.o.GetStrict(opt.StrictCompaction) + if strict { + ro.Strict |= opt.StrictReader + } + + for i, tables := range c.tables { + if len(tables) == 0 { + continue + } + + // Level-0 is not sorted and may overlaps each other. + if c.level+i == 0 { + for _, t := range tables { + its = append(its, c.s.tops.newIterator(t, nil, ro)) + } + } else { + it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict) + its = append(its, it) + } + } + + return iterator.NewMergedIterator(its, c.s.icmp, strict) +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go index 375d49beba6..9fda3d0c087 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go @@ -52,8 +52,6 @@ type dtRecord struct { } type sessionRecord struct { - numLevel int - hasRec int comparer string journalNum uint64 @@ -230,7 +228,7 @@ func (p *sessionRecord) readBytes(field string, r byteReader) []byte { return x } -func (p *sessionRecord) readLevel(field string, r io.ByteReader) int { +func (p *sessionRecord) readLevel(field string, r io.ByteReader, numLevel int) int { if p.err != nil { return 0 } @@ -238,14 +236,14 @@ func (p *sessionRecord) readLevel(field string, r io.ByteReader) int { if p.err != nil { return 0 } - if x >= uint64(p.numLevel) { + if x >= uint64(numLevel) { p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "invalid level number"}) return 0 } return int(x) } -func (p *sessionRecord) decode(r io.Reader) error { +func (p *sessionRecord) decode(r io.Reader, numLevel int) error { br, ok := r.(byteReader) if !ok { br = bufio.NewReader(r) @@ -286,13 +284,13 @@ func (p *sessionRecord) decode(r io.Reader) error { p.setSeqNum(x) } case recCompPtr: - level := p.readLevel("comp-ptr.level", br) + level := p.readLevel("comp-ptr.level", br, numLevel) ikey := p.readBytes("comp-ptr.ikey", br) if p.err == nil { p.addCompPtr(level, iKey(ikey)) } case recAddTable: - level := p.readLevel("add-table.level", br) + level := p.readLevel("add-table.level", br, numLevel) num := p.readUvarint("add-table.num", br) size := p.readUvarint("add-table.size", br) imin := p.readBytes("add-table.imin", br) @@ -301,7 +299,7 @@ func (p *sessionRecord) decode(r io.Reader) error { p.addTable(level, num, size, imin, imax) } case recDelTable: - level := p.readLevel("del-table.level", br) + level := p.readLevel("del-table.level", br, numLevel) num := p.readUvarint("del-table.num", br) if p.err == nil { p.delTable(level, num) diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go index c83bf8a6a3f..aaaf0f9ef47 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go @@ -19,8 +19,8 @@ func decodeEncode(v *sessionRecord) (res bool, err error) { if err != nil { return } - v2 := &sessionRecord{numLevel: opt.DefaultNumLevel} - err = v.decode(b) + v2 := &sessionRecord{} + err = v.decode(b, opt.DefaultNumLevel) if err != nil { return } @@ -34,7 +34,7 @@ func decodeEncode(v *sessionRecord) (res bool, err error) { func TestSessionRecord_EncodeDecode(t *testing.T) { big := uint64(1) << 50 - v := &sessionRecord{numLevel: opt.DefaultNumLevel} + v := &sessionRecord{} i := uint64(0) test := func() { res, err := decodeEncode(v) diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go index 6e10d20e5fd..e0a155cffd5 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go @@ -180,7 +180,7 @@ func (s *session) newManifest(rec *sessionRecord, v *version) (err error) { defer v.release() } if rec == nil { - rec = &sessionRecord{numLevel: s.o.GetNumLevel()} + rec = &sessionRecord{} } s.fillRecord(rec, true) v.fillRecord(rec) diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go index 482ae83ad51..d6d75a4af50 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go @@ -42,6 +42,8 @@ type tsOp uint const ( tsOpOpen tsOp = iota tsOpCreate + tsOpReplace + tsOpRemove tsOpRead tsOpReadAt tsOpWrite @@ -241,6 +243,10 @@ func (tf tsFile) Replace(newfile storage.File) (err error) { if err != nil { return } + if tf.shouldErr(tsOpReplace) { + err = errors.New("leveldb.testStorage: emulated create error") + return + } err = tf.File.Replace(newfile.(tsFile).File) if err != nil { ts.t.Errorf("E: cannot replace file, num=%d type=%v: %v", tf.Num(), tf.Type(), err) @@ -258,6 +264,10 @@ func (tf tsFile) Remove() (err error) { if err != nil { return } + if tf.shouldErr(tsOpRemove) { + err = errors.New("leveldb.testStorage: emulated create error") + return + } err = tf.File.Remove() if err != nil { ts.t.Errorf("E: cannot remove file, num=%d type=%v: %v", tf.Num(), tf.Type(), err) diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go index 18dd9a665c0..c6b0fb9d7c0 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go @@ -441,22 +441,26 @@ func newTableOps(s *session) *tOps { var ( cacher cache.Cacher bcache *cache.Cache + bpool *util.BufferPool ) if s.o.GetOpenFilesCacheCapacity() > 0 { cacher = cache.NewLRU(s.o.GetOpenFilesCacheCapacity()) } - if !s.o.DisableBlockCache { + if !s.o.GetDisableBlockCache() { var bcacher cache.Cacher if s.o.GetBlockCacheCapacity() > 0 { bcacher = cache.NewLRU(s.o.GetBlockCacheCapacity()) } bcache = cache.NewCache(bcacher) } + if !s.o.GetDisableBufferPool() { + bpool = util.NewBufferPool(s.o.GetBlockSize() + 5) + } return &tOps{ s: s, cache: cache.NewCache(cacher), bcache: bcache, - bpool: util.NewBufferPool(s.o.GetBlockSize() + 5), + bpool: bpool, } } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go index e3cc28d2d04..ec7f76bd7fc 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go @@ -300,7 +300,7 @@ func (v *version) offsetOf(ikey iKey) (n uint64, err error) { return } -func (v *version) pickLevel(umin, umax []byte) (level int) { +func (v *version) pickMemdbLevel(umin, umax []byte) (level int) { if !v.tables[0].overlaps(v.s.icmp, umin, umax, true) { var overlaps tFiles maxLevel := v.s.o.GetMaxMemCompationLevel() diff --git a/Godeps/_workspace/src/github.com/whyrusleeping/iptb/main.go b/Godeps/_workspace/src/github.com/whyrusleeping/iptb/main.go index 461edee77e8..3a7b4cb34dd 100644 --- a/Godeps/_workspace/src/github.com/whyrusleeping/iptb/main.go +++ b/Godeps/_workspace/src/github.com/whyrusleeping/iptb/main.go @@ -12,13 +12,15 @@ import ( "os/exec" "path" "strconv" + "strings" "sync" "syscall" "time" + serial "github.com/ipfs/go-ipfs/repo/fsrepo/serialize" + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" - serial "github.com/ipfs/go-ipfs/repo/fsrepo/serialize" ) // GetNumNodes returns the number of testbed nodes configured in the testbed directory @@ -70,6 +72,7 @@ type initCfg struct { Force bool Bootstrap string PortStart int + Mdns bool } func (c *initCfg) swarmAddrForPeer(i int) string { @@ -144,6 +147,7 @@ func starBootstrap(icfg *initCfg) error { bcfg.Addresses.Swarm = []string{icfg.swarmAddrForPeer(0)} bcfg.Addresses.API = icfg.apiAddrForPeer(0) bcfg.Addresses.Gateway = "" + bcfg.Discovery.MDNS.Enabled = icfg.Mdns err = serial.WriteConfigFile(cfgpath, bcfg) if err != nil { return err @@ -156,8 +160,11 @@ func starBootstrap(icfg *initCfg) error { return err } - cfg.Bootstrap = []string{fmt.Sprintf("%s/ipfs/%s", bcfg.Addresses.Swarm[0], bcfg.Identity.PeerID)} + ba := fmt.Sprintf("%s/ipfs/%s", bcfg.Addresses.Swarm[0], bcfg.Identity.PeerID) + ba = strings.Replace(ba, "0.0.0.0", "127.0.0.1", -1) + cfg.Bootstrap = []string{ba} cfg.Addresses.Gateway = "" + cfg.Discovery.MDNS.Enabled = icfg.Mdns cfg.Addresses.Swarm = []string{ icfg.swarmAddrForPeer(i), } @@ -182,6 +189,7 @@ func clearBootstrapping(icfg *initCfg) error { cfg.Addresses.Gateway = "" cfg.Addresses.Swarm = []string{icfg.swarmAddrForPeer(i)} cfg.Addresses.API = icfg.apiAddrForPeer(i) + cfg.Discovery.MDNS.Enabled = icfg.Mdns err = serial.WriteConfigFile(cfgpath, cfg) if err != nil { return err @@ -440,6 +448,7 @@ func main() { flag.IntVar(&cfg.Count, "n", 0, "number of ipfs nodes to initialize") flag.IntVar(&cfg.PortStart, "p", 4002, "port to start allocations from") flag.BoolVar(&cfg.Force, "f", false, "force initialization (overwrite existing configs)") + flag.BoolVar(&cfg.Mdns, "mdns", false, "turn on mdns for nodes") flag.StringVar(&cfg.Bootstrap, "bootstrap", "star", "select bootstrapping style for cluster") wait := flag.Bool("wait", false, "wait for nodes to come fully online before exiting") diff --git a/Godeps/_workspace/src/golang.org/x/net/html/atom/atom.go b/Godeps/_workspace/src/golang.org/x/net/html/atom/atom.go new file mode 100644 index 00000000000..227404bdafa --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/atom/atom.go @@ -0,0 +1,78 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package atom provides integer codes (also known as atoms) for a fixed set of +// frequently occurring HTML strings: tag names and attribute keys such as "p" +// and "id". +// +// Sharing an atom's name between all elements with the same tag can result in +// fewer string allocations when tokenizing and parsing HTML. Integer +// comparisons are also generally faster than string comparisons. +// +// The value of an atom's particular code is not guaranteed to stay the same +// between versions of this package. Neither is any ordering guaranteed: +// whether atom.H1 < atom.H2 may also change. The codes are not guaranteed to +// be dense. The only guarantees are that e.g. looking up "div" will yield +// atom.Div, calling atom.Div.String will return "div", and atom.Div != 0. +package atom + +// Atom is an integer code for a string. The zero value maps to "". +type Atom uint32 + +// String returns the atom's name. +func (a Atom) String() string { + start := uint32(a >> 8) + n := uint32(a & 0xff) + if start+n > uint32(len(atomText)) { + return "" + } + return atomText[start : start+n] +} + +func (a Atom) string() string { + return atomText[a>>8 : a>>8+a&0xff] +} + +// fnv computes the FNV hash with an arbitrary starting value h. +func fnv(h uint32, s []byte) uint32 { + for i := range s { + h ^= uint32(s[i]) + h *= 16777619 + } + return h +} + +func match(s string, t []byte) bool { + for i, c := range t { + if s[i] != c { + return false + } + } + return true +} + +// Lookup returns the atom whose name is s. It returns zero if there is no +// such atom. The lookup is case sensitive. +func Lookup(s []byte) Atom { + if len(s) == 0 || len(s) > maxAtomLen { + return 0 + } + h := fnv(hash0, s) + if a := table[h&uint32(len(table)-1)]; int(a&0xff) == len(s) && match(a.string(), s) { + return a + } + if a := table[(h>>16)&uint32(len(table)-1)]; int(a&0xff) == len(s) && match(a.string(), s) { + return a + } + return 0 +} + +// String returns a string whose contents are equal to s. In that sense, it is +// equivalent to string(s) but may be more efficient. +func String(s []byte) string { + if a := Lookup(s); a != 0 { + return a.String() + } + return string(s) +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/atom/atom_test.go b/Godeps/_workspace/src/golang.org/x/net/html/atom/atom_test.go new file mode 100644 index 00000000000..6e33704dd5e --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/atom/atom_test.go @@ -0,0 +1,109 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atom + +import ( + "sort" + "testing" +) + +func TestKnown(t *testing.T) { + for _, s := range testAtomList { + if atom := Lookup([]byte(s)); atom.String() != s { + t.Errorf("Lookup(%q) = %#x (%q)", s, uint32(atom), atom.String()) + } + } +} + +func TestHits(t *testing.T) { + for _, a := range table { + if a == 0 { + continue + } + got := Lookup([]byte(a.String())) + if got != a { + t.Errorf("Lookup(%q) = %#x, want %#x", a.String(), uint32(got), uint32(a)) + } + } +} + +func TestMisses(t *testing.T) { + testCases := []string{ + "", + "\x00", + "\xff", + "A", + "DIV", + "Div", + "dIV", + "aa", + "a\x00", + "ab", + "abb", + "abbr0", + "abbr ", + " abbr", + " a", + "acceptcharset", + "acceptCharset", + "accept_charset", + "h0", + "h1h2", + "h7", + "onClick", + "λ", + // The following string has the same hash (0xa1d7fab7) as "onmouseover". + "\x00\x00\x00\x00\x00\x50\x18\xae\x38\xd0\xb7", + } + for _, tc := range testCases { + got := Lookup([]byte(tc)) + if got != 0 { + t.Errorf("Lookup(%q): got %d, want 0", tc, got) + } + } +} + +func TestForeignObject(t *testing.T) { + const ( + afo = Foreignobject + afO = ForeignObject + sfo = "foreignobject" + sfO = "foreignObject" + ) + if got := Lookup([]byte(sfo)); got != afo { + t.Errorf("Lookup(%q): got %#v, want %#v", sfo, got, afo) + } + if got := Lookup([]byte(sfO)); got != afO { + t.Errorf("Lookup(%q): got %#v, want %#v", sfO, got, afO) + } + if got := afo.String(); got != sfo { + t.Errorf("Atom(%#v).String(): got %q, want %q", afo, got, sfo) + } + if got := afO.String(); got != sfO { + t.Errorf("Atom(%#v).String(): got %q, want %q", afO, got, sfO) + } +} + +func BenchmarkLookup(b *testing.B) { + sortedTable := make([]string, 0, len(table)) + for _, a := range table { + if a != 0 { + sortedTable = append(sortedTable, a.String()) + } + } + sort.Strings(sortedTable) + + x := make([][]byte, 1000) + for i := range x { + x[i] = []byte(sortedTable[i%len(sortedTable)]) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, s := range x { + Lookup(s) + } + } +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/atom/gen.go b/Godeps/_workspace/src/golang.org/x/net/html/atom/gen.go new file mode 100644 index 00000000000..6bfa8660198 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/atom/gen.go @@ -0,0 +1,648 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This program generates table.go and table_test.go. +// Invoke as +// +// go run gen.go |gofmt >table.go +// go run gen.go -test |gofmt >table_test.go + +import ( + "flag" + "fmt" + "math/rand" + "os" + "sort" + "strings" +) + +// identifier converts s to a Go exported identifier. +// It converts "div" to "Div" and "accept-charset" to "AcceptCharset". +func identifier(s string) string { + b := make([]byte, 0, len(s)) + cap := true + for _, c := range s { + if c == '-' { + cap = true + continue + } + if cap && 'a' <= c && c <= 'z' { + c -= 'a' - 'A' + } + cap = false + b = append(b, byte(c)) + } + return string(b) +} + +var test = flag.Bool("test", false, "generate table_test.go") + +func main() { + flag.Parse() + + var all []string + all = append(all, elements...) + all = append(all, attributes...) + all = append(all, eventHandlers...) + all = append(all, extra...) + sort.Strings(all) + + if *test { + fmt.Printf("// generated by go run gen.go -test; DO NOT EDIT\n\n") + fmt.Printf("package atom\n\n") + fmt.Printf("var testAtomList = []string{\n") + for _, s := range all { + fmt.Printf("\t%q,\n", s) + } + fmt.Printf("}\n") + return + } + + // uniq - lists have dups + // compute max len too + maxLen := 0 + w := 0 + for _, s := range all { + if w == 0 || all[w-1] != s { + if maxLen < len(s) { + maxLen = len(s) + } + all[w] = s + w++ + } + } + all = all[:w] + + // Find hash that minimizes table size. + var best *table + for i := 0; i < 1000000; i++ { + if best != nil && 1<<(best.k-1) < len(all) { + break + } + h := rand.Uint32() + for k := uint(0); k <= 16; k++ { + if best != nil && k >= best.k { + break + } + var t table + if t.init(h, k, all) { + best = &t + break + } + } + } + if best == nil { + fmt.Fprintf(os.Stderr, "failed to construct string table\n") + os.Exit(1) + } + + // Lay out strings, using overlaps when possible. + layout := append([]string{}, all...) + + // Remove strings that are substrings of other strings + for changed := true; changed; { + changed = false + for i, s := range layout { + if s == "" { + continue + } + for j, t := range layout { + if i != j && t != "" && strings.Contains(s, t) { + changed = true + layout[j] = "" + } + } + } + } + + // Join strings where one suffix matches another prefix. + for { + // Find best i, j, k such that layout[i][len-k:] == layout[j][:k], + // maximizing overlap length k. + besti := -1 + bestj := -1 + bestk := 0 + for i, s := range layout { + if s == "" { + continue + } + for j, t := range layout { + if i == j { + continue + } + for k := bestk + 1; k <= len(s) && k <= len(t); k++ { + if s[len(s)-k:] == t[:k] { + besti = i + bestj = j + bestk = k + } + } + } + } + if bestk > 0 { + layout[besti] += layout[bestj][bestk:] + layout[bestj] = "" + continue + } + break + } + + text := strings.Join(layout, "") + + atom := map[string]uint32{} + for _, s := range all { + off := strings.Index(text, s) + if off < 0 { + panic("lost string " + s) + } + atom[s] = uint32(off<<8 | len(s)) + } + + // Generate the Go code. + fmt.Printf("// generated by go run gen.go; DO NOT EDIT\n\n") + fmt.Printf("package atom\n\nconst (\n") + for _, s := range all { + fmt.Printf("\t%s Atom = %#x\n", identifier(s), atom[s]) + } + fmt.Printf(")\n\n") + + fmt.Printf("const hash0 = %#x\n\n", best.h0) + fmt.Printf("const maxAtomLen = %d\n\n", maxLen) + + fmt.Printf("var table = [1<<%d]Atom{\n", best.k) + for i, s := range best.tab { + if s == "" { + continue + } + fmt.Printf("\t%#x: %#x, // %s\n", i, atom[s], s) + } + fmt.Printf("}\n") + datasize := (1 << best.k) * 4 + + fmt.Printf("const atomText =\n") + textsize := len(text) + for len(text) > 60 { + fmt.Printf("\t%q +\n", text[:60]) + text = text[60:] + } + fmt.Printf("\t%q\n\n", text) + + fmt.Fprintf(os.Stderr, "%d atoms; %d string bytes + %d tables = %d total data\n", len(all), textsize, datasize, textsize+datasize) +} + +type byLen []string + +func (x byLen) Less(i, j int) bool { return len(x[i]) > len(x[j]) } +func (x byLen) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x byLen) Len() int { return len(x) } + +// fnv computes the FNV hash with an arbitrary starting value h. +func fnv(h uint32, s string) uint32 { + for i := 0; i < len(s); i++ { + h ^= uint32(s[i]) + h *= 16777619 + } + return h +} + +// A table represents an attempt at constructing the lookup table. +// The lookup table uses cuckoo hashing, meaning that each string +// can be found in one of two positions. +type table struct { + h0 uint32 + k uint + mask uint32 + tab []string +} + +// hash returns the two hashes for s. +func (t *table) hash(s string) (h1, h2 uint32) { + h := fnv(t.h0, s) + h1 = h & t.mask + h2 = (h >> 16) & t.mask + return +} + +// init initializes the table with the given parameters. +// h0 is the initial hash value, +// k is the number of bits of hash value to use, and +// x is the list of strings to store in the table. +// init returns false if the table cannot be constructed. +func (t *table) init(h0 uint32, k uint, x []string) bool { + t.h0 = h0 + t.k = k + t.tab = make([]string, 1< len(t.tab) { + return false + } + s := t.tab[i] + h1, h2 := t.hash(s) + j := h1 + h2 - i + if t.tab[j] != "" && !t.push(j, depth+1) { + return false + } + t.tab[j] = s + return true +} + +// The lists of element names and attribute keys were taken from +// https://html.spec.whatwg.org/multipage/indices.html#index +// as of the "HTML Living Standard - Last Updated 21 February 2015" version. + +var elements = []string{ + "a", + "abbr", + "address", + "area", + "article", + "aside", + "audio", + "b", + "base", + "bdi", + "bdo", + "blockquote", + "body", + "br", + "button", + "canvas", + "caption", + "cite", + "code", + "col", + "colgroup", + "command", + "data", + "datalist", + "dd", + "del", + "details", + "dfn", + "dialog", + "div", + "dl", + "dt", + "em", + "embed", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "hgroup", + "hr", + "html", + "i", + "iframe", + "img", + "input", + "ins", + "kbd", + "keygen", + "label", + "legend", + "li", + "link", + "map", + "mark", + "menu", + "menuitem", + "meta", + "meter", + "nav", + "noscript", + "object", + "ol", + "optgroup", + "option", + "output", + "p", + "param", + "pre", + "progress", + "q", + "rp", + "rt", + "ruby", + "s", + "samp", + "script", + "section", + "select", + "small", + "source", + "span", + "strong", + "style", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "template", + "textarea", + "tfoot", + "th", + "thead", + "time", + "title", + "tr", + "track", + "u", + "ul", + "var", + "video", + "wbr", +} + +// https://html.spec.whatwg.org/multipage/indices.html#attributes-3 + +var attributes = []string{ + "abbr", + "accept", + "accept-charset", + "accesskey", + "action", + "alt", + "async", + "autocomplete", + "autofocus", + "autoplay", + "challenge", + "charset", + "checked", + "cite", + "class", + "cols", + "colspan", + "command", + "content", + "contenteditable", + "contextmenu", + "controls", + "coords", + "crossorigin", + "data", + "datetime", + "default", + "defer", + "dir", + "dirname", + "disabled", + "download", + "draggable", + "dropzone", + "enctype", + "for", + "form", + "formaction", + "formenctype", + "formmethod", + "formnovalidate", + "formtarget", + "headers", + "height", + "hidden", + "high", + "href", + "hreflang", + "http-equiv", + "icon", + "id", + "inputmode", + "ismap", + "itemid", + "itemprop", + "itemref", + "itemscope", + "itemtype", + "keytype", + "kind", + "label", + "lang", + "list", + "loop", + "low", + "manifest", + "max", + "maxlength", + "media", + "mediagroup", + "method", + "min", + "minlength", + "multiple", + "muted", + "name", + "novalidate", + "open", + "optimum", + "pattern", + "ping", + "placeholder", + "poster", + "preload", + "radiogroup", + "readonly", + "rel", + "required", + "reversed", + "rows", + "rowspan", + "sandbox", + "spellcheck", + "scope", + "scoped", + "seamless", + "selected", + "shape", + "size", + "sizes", + "sortable", + "sorted", + "span", + "src", + "srcdoc", + "srclang", + "start", + "step", + "style", + "tabindex", + "target", + "title", + "translate", + "type", + "typemustmatch", + "usemap", + "value", + "width", + "wrap", +} + +var eventHandlers = []string{ + "onabort", + "onautocomplete", + "onautocompleteerror", + "onafterprint", + "onbeforeprint", + "onbeforeunload", + "onblur", + "oncancel", + "oncanplay", + "oncanplaythrough", + "onchange", + "onclick", + "onclose", + "oncontextmenu", + "oncuechange", + "ondblclick", + "ondrag", + "ondragend", + "ondragenter", + "ondragleave", + "ondragover", + "ondragstart", + "ondrop", + "ondurationchange", + "onemptied", + "onended", + "onerror", + "onfocus", + "onhashchange", + "oninput", + "oninvalid", + "onkeydown", + "onkeypress", + "onkeyup", + "onlanguagechange", + "onload", + "onloadeddata", + "onloadedmetadata", + "onloadstart", + "onmessage", + "onmousedown", + "onmousemove", + "onmouseout", + "onmouseover", + "onmouseup", + "onmousewheel", + "onoffline", + "ononline", + "onpagehide", + "onpageshow", + "onpause", + "onplay", + "onplaying", + "onpopstate", + "onprogress", + "onratechange", + "onreset", + "onresize", + "onscroll", + "onseeked", + "onseeking", + "onselect", + "onshow", + "onsort", + "onstalled", + "onstorage", + "onsubmit", + "onsuspend", + "ontimeupdate", + "ontoggle", + "onunload", + "onvolumechange", + "onwaiting", +} + +// extra are ad-hoc values not covered by any of the lists above. +var extra = []string{ + "align", + "annotation", + "annotation-xml", + "applet", + "basefont", + "bgsound", + "big", + "blink", + "center", + "color", + "desc", + "face", + "font", + "foreignObject", // HTML is case-insensitive, but SVG-embedded-in-HTML is case-sensitive. + "foreignobject", + "frame", + "frameset", + "image", + "isindex", + "listing", + "malignmark", + "marquee", + "math", + "mglyph", + "mi", + "mn", + "mo", + "ms", + "mtext", + "nobr", + "noembed", + "noframes", + "plaintext", + "prompt", + "public", + "spacer", + "strike", + "svg", + "system", + "tt", + "xmp", +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/atom/table.go b/Godeps/_workspace/src/golang.org/x/net/html/atom/table.go new file mode 100644 index 00000000000..2605ba3102f --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/atom/table.go @@ -0,0 +1,713 @@ +// generated by go run gen.go; DO NOT EDIT + +package atom + +const ( + A Atom = 0x1 + Abbr Atom = 0x4 + Accept Atom = 0x2106 + AcceptCharset Atom = 0x210e + Accesskey Atom = 0x3309 + Action Atom = 0x1f606 + Address Atom = 0x4f307 + Align Atom = 0x1105 + Alt Atom = 0x4503 + Annotation Atom = 0x1670a + AnnotationXml Atom = 0x1670e + Applet Atom = 0x2b306 + Area Atom = 0x2fa04 + Article Atom = 0x38807 + Aside Atom = 0x8305 + Async Atom = 0x7b05 + Audio Atom = 0xa605 + Autocomplete Atom = 0x1fc0c + Autofocus Atom = 0xb309 + Autoplay Atom = 0xce08 + B Atom = 0x101 + Base Atom = 0xd604 + Basefont Atom = 0xd608 + Bdi Atom = 0x1a03 + Bdo Atom = 0xe703 + Bgsound Atom = 0x11807 + Big Atom = 0x12403 + Blink Atom = 0x12705 + Blockquote Atom = 0x12c0a + Body Atom = 0x2f04 + Br Atom = 0x202 + Button Atom = 0x13606 + Canvas Atom = 0x7f06 + Caption Atom = 0x1bb07 + Center Atom = 0x5b506 + Challenge Atom = 0x21f09 + Charset Atom = 0x2807 + Checked Atom = 0x32807 + Cite Atom = 0x3c804 + Class Atom = 0x4de05 + Code Atom = 0x14904 + Col Atom = 0x15003 + Colgroup Atom = 0x15008 + Color Atom = 0x15d05 + Cols Atom = 0x16204 + Colspan Atom = 0x16207 + Command Atom = 0x17507 + Content Atom = 0x42307 + Contenteditable Atom = 0x4230f + Contextmenu Atom = 0x3310b + Controls Atom = 0x18808 + Coords Atom = 0x19406 + Crossorigin Atom = 0x19f0b + Data Atom = 0x44a04 + Datalist Atom = 0x44a08 + Datetime Atom = 0x23c08 + Dd Atom = 0x26702 + Default Atom = 0x8607 + Defer Atom = 0x14b05 + Del Atom = 0x3ef03 + Desc Atom = 0x4db04 + Details Atom = 0x4807 + Dfn Atom = 0x6103 + Dialog Atom = 0x1b06 + Dir Atom = 0x6903 + Dirname Atom = 0x6907 + Disabled Atom = 0x10c08 + Div Atom = 0x11303 + Dl Atom = 0x11e02 + Download Atom = 0x40008 + Draggable Atom = 0x17b09 + Dropzone Atom = 0x39108 + Dt Atom = 0x50902 + Em Atom = 0x6502 + Embed Atom = 0x6505 + Enctype Atom = 0x21107 + Face Atom = 0x5b304 + Fieldset Atom = 0x1b008 + Figcaption Atom = 0x1b80a + Figure Atom = 0x1cc06 + Font Atom = 0xda04 + Footer Atom = 0x8d06 + For Atom = 0x1d803 + ForeignObject Atom = 0x1d80d + Foreignobject Atom = 0x1e50d + Form Atom = 0x1f204 + Formaction Atom = 0x1f20a + Formenctype Atom = 0x20d0b + Formmethod Atom = 0x2280a + Formnovalidate Atom = 0x2320e + Formtarget Atom = 0x2470a + Frame Atom = 0x9a05 + Frameset Atom = 0x9a08 + H1 Atom = 0x26e02 + H2 Atom = 0x29402 + H3 Atom = 0x2a702 + H4 Atom = 0x2e902 + H5 Atom = 0x2f302 + H6 Atom = 0x50b02 + Head Atom = 0x2d504 + Header Atom = 0x2d506 + Headers Atom = 0x2d507 + Height Atom = 0x25106 + Hgroup Atom = 0x25906 + Hidden Atom = 0x26506 + High Atom = 0x26b04 + Hr Atom = 0x27002 + Href Atom = 0x27004 + Hreflang Atom = 0x27008 + Html Atom = 0x25504 + HttpEquiv Atom = 0x2780a + I Atom = 0x601 + Icon Atom = 0x42204 + Id Atom = 0x8502 + Iframe Atom = 0x29606 + Image Atom = 0x29c05 + Img Atom = 0x2a103 + Input Atom = 0x3e805 + Inputmode Atom = 0x3e809 + Ins Atom = 0x1a803 + Isindex Atom = 0x2a907 + Ismap Atom = 0x2b005 + Itemid Atom = 0x33c06 + Itemprop Atom = 0x3c908 + Itemref Atom = 0x5ad07 + Itemscope Atom = 0x2b909 + Itemtype Atom = 0x2c308 + Kbd Atom = 0x1903 + Keygen Atom = 0x3906 + Keytype Atom = 0x53707 + Kind Atom = 0x10904 + Label Atom = 0xf005 + Lang Atom = 0x27404 + Legend Atom = 0x18206 + Li Atom = 0x1202 + Link Atom = 0x12804 + List Atom = 0x44e04 + Listing Atom = 0x44e07 + Loop Atom = 0xf404 + Low Atom = 0x11f03 + Malignmark Atom = 0x100a + Manifest Atom = 0x5f108 + Map Atom = 0x2b203 + Mark Atom = 0x1604 + Marquee Atom = 0x2cb07 + Math Atom = 0x2d204 + Max Atom = 0x2e103 + Maxlength Atom = 0x2e109 + Media Atom = 0x6e05 + Mediagroup Atom = 0x6e0a + Menu Atom = 0x33804 + Menuitem Atom = 0x33808 + Meta Atom = 0x45d04 + Meter Atom = 0x24205 + Method Atom = 0x22c06 + Mglyph Atom = 0x2a206 + Mi Atom = 0x2eb02 + Min Atom = 0x2eb03 + Minlength Atom = 0x2eb09 + Mn Atom = 0x23502 + Mo Atom = 0x3ed02 + Ms Atom = 0x2bc02 + Mtext Atom = 0x2f505 + Multiple Atom = 0x30308 + Muted Atom = 0x30b05 + Name Atom = 0x6c04 + Nav Atom = 0x3e03 + Nobr Atom = 0x5704 + Noembed Atom = 0x6307 + Noframes Atom = 0x9808 + Noscript Atom = 0x3d208 + Novalidate Atom = 0x2360a + Object Atom = 0x1ec06 + Ol Atom = 0xc902 + Onabort Atom = 0x13a07 + Onafterprint Atom = 0x1c00c + Onautocomplete Atom = 0x1fa0e + Onautocompleteerror Atom = 0x1fa13 + Onbeforeprint Atom = 0x6040d + Onbeforeunload Atom = 0x4e70e + Onblur Atom = 0xaa06 + Oncancel Atom = 0xe908 + Oncanplay Atom = 0x28509 + Oncanplaythrough Atom = 0x28510 + Onchange Atom = 0x3a708 + Onclick Atom = 0x31007 + Onclose Atom = 0x31707 + Oncontextmenu Atom = 0x32f0d + Oncuechange Atom = 0x3420b + Ondblclick Atom = 0x34d0a + Ondrag Atom = 0x35706 + Ondragend Atom = 0x35709 + Ondragenter Atom = 0x3600b + Ondragleave Atom = 0x36b0b + Ondragover Atom = 0x3760a + Ondragstart Atom = 0x3800b + Ondrop Atom = 0x38f06 + Ondurationchange Atom = 0x39f10 + Onemptied Atom = 0x39609 + Onended Atom = 0x3af07 + Onerror Atom = 0x3b607 + Onfocus Atom = 0x3bd07 + Onhashchange Atom = 0x3da0c + Oninput Atom = 0x3e607 + Oninvalid Atom = 0x3f209 + Onkeydown Atom = 0x3fb09 + Onkeypress Atom = 0x4080a + Onkeyup Atom = 0x41807 + Onlanguagechange Atom = 0x43210 + Onload Atom = 0x44206 + Onloadeddata Atom = 0x4420c + Onloadedmetadata Atom = 0x45510 + Onloadstart Atom = 0x46b0b + Onmessage Atom = 0x47609 + Onmousedown Atom = 0x47f0b + Onmousemove Atom = 0x48a0b + Onmouseout Atom = 0x4950a + Onmouseover Atom = 0x4a20b + Onmouseup Atom = 0x4ad09 + Onmousewheel Atom = 0x4b60c + Onoffline Atom = 0x4c209 + Ononline Atom = 0x4cb08 + Onpagehide Atom = 0x4d30a + Onpageshow Atom = 0x4fe0a + Onpause Atom = 0x50d07 + Onplay Atom = 0x51706 + Onplaying Atom = 0x51709 + Onpopstate Atom = 0x5200a + Onprogress Atom = 0x52a0a + Onratechange Atom = 0x53e0c + Onreset Atom = 0x54a07 + Onresize Atom = 0x55108 + Onscroll Atom = 0x55f08 + Onseeked Atom = 0x56708 + Onseeking Atom = 0x56f09 + Onselect Atom = 0x57808 + Onshow Atom = 0x58206 + Onsort Atom = 0x58b06 + Onstalled Atom = 0x59509 + Onstorage Atom = 0x59e09 + Onsubmit Atom = 0x5a708 + Onsuspend Atom = 0x5bb09 + Ontimeupdate Atom = 0xdb0c + Ontoggle Atom = 0x5c408 + Onunload Atom = 0x5cc08 + Onvolumechange Atom = 0x5d40e + Onwaiting Atom = 0x5e209 + Open Atom = 0x3cf04 + Optgroup Atom = 0xf608 + Optimum Atom = 0x5eb07 + Option Atom = 0x60006 + Output Atom = 0x49c06 + P Atom = 0xc01 + Param Atom = 0xc05 + Pattern Atom = 0x5107 + Ping Atom = 0x7704 + Placeholder Atom = 0xc30b + Plaintext Atom = 0xfd09 + Poster Atom = 0x15706 + Pre Atom = 0x25e03 + Preload Atom = 0x25e07 + Progress Atom = 0x52c08 + Prompt Atom = 0x5fa06 + Public Atom = 0x41e06 + Q Atom = 0x13101 + Radiogroup Atom = 0x30a + Readonly Atom = 0x2fb08 + Rel Atom = 0x25f03 + Required Atom = 0x1d008 + Reversed Atom = 0x5a08 + Rows Atom = 0x9204 + Rowspan Atom = 0x9207 + Rp Atom = 0x1c602 + Rt Atom = 0x13f02 + Ruby Atom = 0xaf04 + S Atom = 0x2c01 + Samp Atom = 0x4e04 + Sandbox Atom = 0xbb07 + Scope Atom = 0x2bd05 + Scoped Atom = 0x2bd06 + Script Atom = 0x3d406 + Seamless Atom = 0x31c08 + Section Atom = 0x4e207 + Select Atom = 0x57a06 + Selected Atom = 0x57a08 + Shape Atom = 0x4f905 + Size Atom = 0x55504 + Sizes Atom = 0x55505 + Small Atom = 0x18f05 + Sortable Atom = 0x58d08 + Sorted Atom = 0x19906 + Source Atom = 0x1aa06 + Spacer Atom = 0x2db06 + Span Atom = 0x9504 + Spellcheck Atom = 0x3230a + Src Atom = 0x3c303 + Srcdoc Atom = 0x3c306 + Srclang Atom = 0x41107 + Start Atom = 0x38605 + Step Atom = 0x5f704 + Strike Atom = 0x53306 + Strong Atom = 0x55906 + Style Atom = 0x61105 + Sub Atom = 0x5a903 + Summary Atom = 0x61607 + Sup Atom = 0x61d03 + Svg Atom = 0x62003 + System Atom = 0x62306 + Tabindex Atom = 0x46308 + Table Atom = 0x42d05 + Target Atom = 0x24b06 + Tbody Atom = 0x2e05 + Td Atom = 0x4702 + Template Atom = 0x62608 + Textarea Atom = 0x2f608 + Tfoot Atom = 0x8c05 + Th Atom = 0x22e02 + Thead Atom = 0x2d405 + Time Atom = 0xdd04 + Title Atom = 0xa105 + Tr Atom = 0x10502 + Track Atom = 0x10505 + Translate Atom = 0x14009 + Tt Atom = 0x5302 + Type Atom = 0x21404 + Typemustmatch Atom = 0x2140d + U Atom = 0xb01 + Ul Atom = 0x8a02 + Usemap Atom = 0x51106 + Value Atom = 0x4005 + Var Atom = 0x11503 + Video Atom = 0x28105 + Wbr Atom = 0x12103 + Width Atom = 0x50705 + Wrap Atom = 0x58704 + Xmp Atom = 0xc103 +) + +const hash0 = 0xc17da63e + +const maxAtomLen = 19 + +var table = [1 << 9]Atom{ + 0x1: 0x48a0b, // onmousemove + 0x2: 0x5e209, // onwaiting + 0x3: 0x1fa13, // onautocompleteerror + 0x4: 0x5fa06, // prompt + 0x7: 0x5eb07, // optimum + 0x8: 0x1604, // mark + 0xa: 0x5ad07, // itemref + 0xb: 0x4fe0a, // onpageshow + 0xc: 0x57a06, // select + 0xd: 0x17b09, // draggable + 0xe: 0x3e03, // nav + 0xf: 0x17507, // command + 0x11: 0xb01, // u + 0x14: 0x2d507, // headers + 0x15: 0x44a08, // datalist + 0x17: 0x4e04, // samp + 0x1a: 0x3fb09, // onkeydown + 0x1b: 0x55f08, // onscroll + 0x1c: 0x15003, // col + 0x20: 0x3c908, // itemprop + 0x21: 0x2780a, // http-equiv + 0x22: 0x61d03, // sup + 0x24: 0x1d008, // required + 0x2b: 0x25e07, // preload + 0x2c: 0x6040d, // onbeforeprint + 0x2d: 0x3600b, // ondragenter + 0x2e: 0x50902, // dt + 0x2f: 0x5a708, // onsubmit + 0x30: 0x27002, // hr + 0x31: 0x32f0d, // oncontextmenu + 0x33: 0x29c05, // image + 0x34: 0x50d07, // onpause + 0x35: 0x25906, // hgroup + 0x36: 0x7704, // ping + 0x37: 0x57808, // onselect + 0x3a: 0x11303, // div + 0x3b: 0x1fa0e, // onautocomplete + 0x40: 0x2eb02, // mi + 0x41: 0x31c08, // seamless + 0x42: 0x2807, // charset + 0x43: 0x8502, // id + 0x44: 0x5200a, // onpopstate + 0x45: 0x3ef03, // del + 0x46: 0x2cb07, // marquee + 0x47: 0x3309, // accesskey + 0x49: 0x8d06, // footer + 0x4a: 0x44e04, // list + 0x4b: 0x2b005, // ismap + 0x51: 0x33804, // menu + 0x52: 0x2f04, // body + 0x55: 0x9a08, // frameset + 0x56: 0x54a07, // onreset + 0x57: 0x12705, // blink + 0x58: 0xa105, // title + 0x59: 0x38807, // article + 0x5b: 0x22e02, // th + 0x5d: 0x13101, // q + 0x5e: 0x3cf04, // open + 0x5f: 0x2fa04, // area + 0x61: 0x44206, // onload + 0x62: 0xda04, // font + 0x63: 0xd604, // base + 0x64: 0x16207, // colspan + 0x65: 0x53707, // keytype + 0x66: 0x11e02, // dl + 0x68: 0x1b008, // fieldset + 0x6a: 0x2eb03, // min + 0x6b: 0x11503, // var + 0x6f: 0x2d506, // header + 0x70: 0x13f02, // rt + 0x71: 0x15008, // colgroup + 0x72: 0x23502, // mn + 0x74: 0x13a07, // onabort + 0x75: 0x3906, // keygen + 0x76: 0x4c209, // onoffline + 0x77: 0x21f09, // challenge + 0x78: 0x2b203, // map + 0x7a: 0x2e902, // h4 + 0x7b: 0x3b607, // onerror + 0x7c: 0x2e109, // maxlength + 0x7d: 0x2f505, // mtext + 0x7e: 0xbb07, // sandbox + 0x7f: 0x58b06, // onsort + 0x80: 0x100a, // malignmark + 0x81: 0x45d04, // meta + 0x82: 0x7b05, // async + 0x83: 0x2a702, // h3 + 0x84: 0x26702, // dd + 0x85: 0x27004, // href + 0x86: 0x6e0a, // mediagroup + 0x87: 0x19406, // coords + 0x88: 0x41107, // srclang + 0x89: 0x34d0a, // ondblclick + 0x8a: 0x4005, // value + 0x8c: 0xe908, // oncancel + 0x8e: 0x3230a, // spellcheck + 0x8f: 0x9a05, // frame + 0x91: 0x12403, // big + 0x94: 0x1f606, // action + 0x95: 0x6903, // dir + 0x97: 0x2fb08, // readonly + 0x99: 0x42d05, // table + 0x9a: 0x61607, // summary + 0x9b: 0x12103, // wbr + 0x9c: 0x30a, // radiogroup + 0x9d: 0x6c04, // name + 0x9f: 0x62306, // system + 0xa1: 0x15d05, // color + 0xa2: 0x7f06, // canvas + 0xa3: 0x25504, // html + 0xa5: 0x56f09, // onseeking + 0xac: 0x4f905, // shape + 0xad: 0x25f03, // rel + 0xae: 0x28510, // oncanplaythrough + 0xaf: 0x3760a, // ondragover + 0xb0: 0x62608, // template + 0xb1: 0x1d80d, // foreignObject + 0xb3: 0x9204, // rows + 0xb6: 0x44e07, // listing + 0xb7: 0x49c06, // output + 0xb9: 0x3310b, // contextmenu + 0xbb: 0x11f03, // low + 0xbc: 0x1c602, // rp + 0xbd: 0x5bb09, // onsuspend + 0xbe: 0x13606, // button + 0xbf: 0x4db04, // desc + 0xc1: 0x4e207, // section + 0xc2: 0x52a0a, // onprogress + 0xc3: 0x59e09, // onstorage + 0xc4: 0x2d204, // math + 0xc5: 0x4503, // alt + 0xc7: 0x8a02, // ul + 0xc8: 0x5107, // pattern + 0xc9: 0x4b60c, // onmousewheel + 0xca: 0x35709, // ondragend + 0xcb: 0xaf04, // ruby + 0xcc: 0xc01, // p + 0xcd: 0x31707, // onclose + 0xce: 0x24205, // meter + 0xcf: 0x11807, // bgsound + 0xd2: 0x25106, // height + 0xd4: 0x101, // b + 0xd5: 0x2c308, // itemtype + 0xd8: 0x1bb07, // caption + 0xd9: 0x10c08, // disabled + 0xdb: 0x33808, // menuitem + 0xdc: 0x62003, // svg + 0xdd: 0x18f05, // small + 0xde: 0x44a04, // data + 0xe0: 0x4cb08, // ononline + 0xe1: 0x2a206, // mglyph + 0xe3: 0x6505, // embed + 0xe4: 0x10502, // tr + 0xe5: 0x46b0b, // onloadstart + 0xe7: 0x3c306, // srcdoc + 0xeb: 0x5c408, // ontoggle + 0xed: 0xe703, // bdo + 0xee: 0x4702, // td + 0xef: 0x8305, // aside + 0xf0: 0x29402, // h2 + 0xf1: 0x52c08, // progress + 0xf2: 0x12c0a, // blockquote + 0xf4: 0xf005, // label + 0xf5: 0x601, // i + 0xf7: 0x9207, // rowspan + 0xfb: 0x51709, // onplaying + 0xfd: 0x2a103, // img + 0xfe: 0xf608, // optgroup + 0xff: 0x42307, // content + 0x101: 0x53e0c, // onratechange + 0x103: 0x3da0c, // onhashchange + 0x104: 0x4807, // details + 0x106: 0x40008, // download + 0x109: 0x14009, // translate + 0x10b: 0x4230f, // contenteditable + 0x10d: 0x36b0b, // ondragleave + 0x10e: 0x2106, // accept + 0x10f: 0x57a08, // selected + 0x112: 0x1f20a, // formaction + 0x113: 0x5b506, // center + 0x115: 0x45510, // onloadedmetadata + 0x116: 0x12804, // link + 0x117: 0xdd04, // time + 0x118: 0x19f0b, // crossorigin + 0x119: 0x3bd07, // onfocus + 0x11a: 0x58704, // wrap + 0x11b: 0x42204, // icon + 0x11d: 0x28105, // video + 0x11e: 0x4de05, // class + 0x121: 0x5d40e, // onvolumechange + 0x122: 0xaa06, // onblur + 0x123: 0x2b909, // itemscope + 0x124: 0x61105, // style + 0x127: 0x41e06, // public + 0x129: 0x2320e, // formnovalidate + 0x12a: 0x58206, // onshow + 0x12c: 0x51706, // onplay + 0x12d: 0x3c804, // cite + 0x12e: 0x2bc02, // ms + 0x12f: 0xdb0c, // ontimeupdate + 0x130: 0x10904, // kind + 0x131: 0x2470a, // formtarget + 0x135: 0x3af07, // onended + 0x136: 0x26506, // hidden + 0x137: 0x2c01, // s + 0x139: 0x2280a, // formmethod + 0x13a: 0x3e805, // input + 0x13c: 0x50b02, // h6 + 0x13d: 0xc902, // ol + 0x13e: 0x3420b, // oncuechange + 0x13f: 0x1e50d, // foreignobject + 0x143: 0x4e70e, // onbeforeunload + 0x144: 0x2bd05, // scope + 0x145: 0x39609, // onemptied + 0x146: 0x14b05, // defer + 0x147: 0xc103, // xmp + 0x148: 0x39f10, // ondurationchange + 0x149: 0x1903, // kbd + 0x14c: 0x47609, // onmessage + 0x14d: 0x60006, // option + 0x14e: 0x2eb09, // minlength + 0x14f: 0x32807, // checked + 0x150: 0xce08, // autoplay + 0x152: 0x202, // br + 0x153: 0x2360a, // novalidate + 0x156: 0x6307, // noembed + 0x159: 0x31007, // onclick + 0x15a: 0x47f0b, // onmousedown + 0x15b: 0x3a708, // onchange + 0x15e: 0x3f209, // oninvalid + 0x15f: 0x2bd06, // scoped + 0x160: 0x18808, // controls + 0x161: 0x30b05, // muted + 0x162: 0x58d08, // sortable + 0x163: 0x51106, // usemap + 0x164: 0x1b80a, // figcaption + 0x165: 0x35706, // ondrag + 0x166: 0x26b04, // high + 0x168: 0x3c303, // src + 0x169: 0x15706, // poster + 0x16b: 0x1670e, // annotation-xml + 0x16c: 0x5f704, // step + 0x16d: 0x4, // abbr + 0x16e: 0x1b06, // dialog + 0x170: 0x1202, // li + 0x172: 0x3ed02, // mo + 0x175: 0x1d803, // for + 0x176: 0x1a803, // ins + 0x178: 0x55504, // size + 0x179: 0x43210, // onlanguagechange + 0x17a: 0x8607, // default + 0x17b: 0x1a03, // bdi + 0x17c: 0x4d30a, // onpagehide + 0x17d: 0x6907, // dirname + 0x17e: 0x21404, // type + 0x17f: 0x1f204, // form + 0x181: 0x28509, // oncanplay + 0x182: 0x6103, // dfn + 0x183: 0x46308, // tabindex + 0x186: 0x6502, // em + 0x187: 0x27404, // lang + 0x189: 0x39108, // dropzone + 0x18a: 0x4080a, // onkeypress + 0x18b: 0x23c08, // datetime + 0x18c: 0x16204, // cols + 0x18d: 0x1, // a + 0x18e: 0x4420c, // onloadeddata + 0x190: 0xa605, // audio + 0x192: 0x2e05, // tbody + 0x193: 0x22c06, // method + 0x195: 0xf404, // loop + 0x196: 0x29606, // iframe + 0x198: 0x2d504, // head + 0x19e: 0x5f108, // manifest + 0x19f: 0xb309, // autofocus + 0x1a0: 0x14904, // code + 0x1a1: 0x55906, // strong + 0x1a2: 0x30308, // multiple + 0x1a3: 0xc05, // param + 0x1a6: 0x21107, // enctype + 0x1a7: 0x5b304, // face + 0x1a8: 0xfd09, // plaintext + 0x1a9: 0x26e02, // h1 + 0x1aa: 0x59509, // onstalled + 0x1ad: 0x3d406, // script + 0x1ae: 0x2db06, // spacer + 0x1af: 0x55108, // onresize + 0x1b0: 0x4a20b, // onmouseover + 0x1b1: 0x5cc08, // onunload + 0x1b2: 0x56708, // onseeked + 0x1b4: 0x2140d, // typemustmatch + 0x1b5: 0x1cc06, // figure + 0x1b6: 0x4950a, // onmouseout + 0x1b7: 0x25e03, // pre + 0x1b8: 0x50705, // width + 0x1b9: 0x19906, // sorted + 0x1bb: 0x5704, // nobr + 0x1be: 0x5302, // tt + 0x1bf: 0x1105, // align + 0x1c0: 0x3e607, // oninput + 0x1c3: 0x41807, // onkeyup + 0x1c6: 0x1c00c, // onafterprint + 0x1c7: 0x210e, // accept-charset + 0x1c8: 0x33c06, // itemid + 0x1c9: 0x3e809, // inputmode + 0x1cb: 0x53306, // strike + 0x1cc: 0x5a903, // sub + 0x1cd: 0x10505, // track + 0x1ce: 0x38605, // start + 0x1d0: 0xd608, // basefont + 0x1d6: 0x1aa06, // source + 0x1d7: 0x18206, // legend + 0x1d8: 0x2d405, // thead + 0x1da: 0x8c05, // tfoot + 0x1dd: 0x1ec06, // object + 0x1de: 0x6e05, // media + 0x1df: 0x1670a, // annotation + 0x1e0: 0x20d0b, // formenctype + 0x1e2: 0x3d208, // noscript + 0x1e4: 0x55505, // sizes + 0x1e5: 0x1fc0c, // autocomplete + 0x1e6: 0x9504, // span + 0x1e7: 0x9808, // noframes + 0x1e8: 0x24b06, // target + 0x1e9: 0x38f06, // ondrop + 0x1ea: 0x2b306, // applet + 0x1ec: 0x5a08, // reversed + 0x1f0: 0x2a907, // isindex + 0x1f3: 0x27008, // hreflang + 0x1f5: 0x2f302, // h5 + 0x1f6: 0x4f307, // address + 0x1fa: 0x2e103, // max + 0x1fb: 0xc30b, // placeholder + 0x1fc: 0x2f608, // textarea + 0x1fe: 0x4ad09, // onmouseup + 0x1ff: 0x3800b, // ondragstart +} + +const atomText = "abbradiogrouparamalignmarkbdialogaccept-charsetbodyaccesskey" + + "genavaluealtdetailsampatternobreversedfnoembedirnamediagroup" + + "ingasyncanvasidefaultfooterowspanoframesetitleaudionblurubya" + + "utofocusandboxmplaceholderautoplaybasefontimeupdatebdoncance" + + "labelooptgrouplaintextrackindisabledivarbgsoundlowbrbigblink" + + "blockquotebuttonabortranslatecodefercolgroupostercolorcolspa" + + "nnotation-xmlcommandraggablegendcontrolsmallcoordsortedcross" + + "originsourcefieldsetfigcaptionafterprintfigurequiredforeignO" + + "bjectforeignobjectformactionautocompleteerrorformenctypemust" + + "matchallengeformmethodformnovalidatetimeterformtargetheightm" + + "lhgroupreloadhiddenhigh1hreflanghttp-equivideoncanplaythroug" + + "h2iframeimageimglyph3isindexismappletitemscopeditemtypemarqu" + + "eematheaderspacermaxlength4minlength5mtextareadonlymultiplem" + + "utedonclickoncloseamlesspellcheckedoncontextmenuitemidoncuec" + + "hangeondblclickondragendondragenterondragleaveondragoverondr" + + "agstarticleondropzonemptiedondurationchangeonendedonerroronf" + + "ocusrcdocitempropenoscriptonhashchangeoninputmodeloninvalido" + + "nkeydownloadonkeypressrclangonkeyupublicontenteditableonlang" + + "uagechangeonloadeddatalistingonloadedmetadatabindexonloadsta" + + "rtonmessageonmousedownonmousemoveonmouseoutputonmouseoveronm" + + "ouseuponmousewheelonofflineononlineonpagehidesclassectionbef" + + "oreunloaddresshapeonpageshowidth6onpausemaponplayingonpopsta" + + "teonprogresstrikeytypeonratechangeonresetonresizestrongonscr" + + "ollonseekedonseekingonselectedonshowraponsortableonstalledon" + + "storageonsubmitemrefacenteronsuspendontoggleonunloadonvolume" + + "changeonwaitingoptimumanifestepromptoptionbeforeprintstylesu" + + "mmarysupsvgsystemplate" diff --git a/Godeps/_workspace/src/golang.org/x/net/html/atom/table_test.go b/Godeps/_workspace/src/golang.org/x/net/html/atom/table_test.go new file mode 100644 index 00000000000..0f2ecce4fd4 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/atom/table_test.go @@ -0,0 +1,351 @@ +// generated by go run gen.go -test; DO NOT EDIT + +package atom + +var testAtomList = []string{ + "a", + "abbr", + "abbr", + "accept", + "accept-charset", + "accesskey", + "action", + "address", + "align", + "alt", + "annotation", + "annotation-xml", + "applet", + "area", + "article", + "aside", + "async", + "audio", + "autocomplete", + "autofocus", + "autoplay", + "b", + "base", + "basefont", + "bdi", + "bdo", + "bgsound", + "big", + "blink", + "blockquote", + "body", + "br", + "button", + "canvas", + "caption", + "center", + "challenge", + "charset", + "checked", + "cite", + "cite", + "class", + "code", + "col", + "colgroup", + "color", + "cols", + "colspan", + "command", + "command", + "content", + "contenteditable", + "contextmenu", + "controls", + "coords", + "crossorigin", + "data", + "data", + "datalist", + "datetime", + "dd", + "default", + "defer", + "del", + "desc", + "details", + "dfn", + "dialog", + "dir", + "dirname", + "disabled", + "div", + "dl", + "download", + "draggable", + "dropzone", + "dt", + "em", + "embed", + "enctype", + "face", + "fieldset", + "figcaption", + "figure", + "font", + "footer", + "for", + "foreignObject", + "foreignobject", + "form", + "form", + "formaction", + "formenctype", + "formmethod", + "formnovalidate", + "formtarget", + "frame", + "frameset", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "headers", + "height", + "hgroup", + "hidden", + "high", + "hr", + "href", + "hreflang", + "html", + "http-equiv", + "i", + "icon", + "id", + "iframe", + "image", + "img", + "input", + "inputmode", + "ins", + "isindex", + "ismap", + "itemid", + "itemprop", + "itemref", + "itemscope", + "itemtype", + "kbd", + "keygen", + "keytype", + "kind", + "label", + "label", + "lang", + "legend", + "li", + "link", + "list", + "listing", + "loop", + "low", + "malignmark", + "manifest", + "map", + "mark", + "marquee", + "math", + "max", + "maxlength", + "media", + "mediagroup", + "menu", + "menuitem", + "meta", + "meter", + "method", + "mglyph", + "mi", + "min", + "minlength", + "mn", + "mo", + "ms", + "mtext", + "multiple", + "muted", + "name", + "nav", + "nobr", + "noembed", + "noframes", + "noscript", + "novalidate", + "object", + "ol", + "onabort", + "onafterprint", + "onautocomplete", + "onautocompleteerror", + "onbeforeprint", + "onbeforeunload", + "onblur", + "oncancel", + "oncanplay", + "oncanplaythrough", + "onchange", + "onclick", + "onclose", + "oncontextmenu", + "oncuechange", + "ondblclick", + "ondrag", + "ondragend", + "ondragenter", + "ondragleave", + "ondragover", + "ondragstart", + "ondrop", + "ondurationchange", + "onemptied", + "onended", + "onerror", + "onfocus", + "onhashchange", + "oninput", + "oninvalid", + "onkeydown", + "onkeypress", + "onkeyup", + "onlanguagechange", + "onload", + "onloadeddata", + "onloadedmetadata", + "onloadstart", + "onmessage", + "onmousedown", + "onmousemove", + "onmouseout", + "onmouseover", + "onmouseup", + "onmousewheel", + "onoffline", + "ononline", + "onpagehide", + "onpageshow", + "onpause", + "onplay", + "onplaying", + "onpopstate", + "onprogress", + "onratechange", + "onreset", + "onresize", + "onscroll", + "onseeked", + "onseeking", + "onselect", + "onshow", + "onsort", + "onstalled", + "onstorage", + "onsubmit", + "onsuspend", + "ontimeupdate", + "ontoggle", + "onunload", + "onvolumechange", + "onwaiting", + "open", + "optgroup", + "optimum", + "option", + "output", + "p", + "param", + "pattern", + "ping", + "placeholder", + "plaintext", + "poster", + "pre", + "preload", + "progress", + "prompt", + "public", + "q", + "radiogroup", + "readonly", + "rel", + "required", + "reversed", + "rows", + "rowspan", + "rp", + "rt", + "ruby", + "s", + "samp", + "sandbox", + "scope", + "scoped", + "script", + "seamless", + "section", + "select", + "selected", + "shape", + "size", + "sizes", + "small", + "sortable", + "sorted", + "source", + "spacer", + "span", + "span", + "spellcheck", + "src", + "srcdoc", + "srclang", + "start", + "step", + "strike", + "strong", + "style", + "style", + "sub", + "summary", + "sup", + "svg", + "system", + "tabindex", + "table", + "target", + "tbody", + "td", + "template", + "textarea", + "tfoot", + "th", + "thead", + "time", + "title", + "title", + "tr", + "track", + "translate", + "tt", + "type", + "typemustmatch", + "u", + "ul", + "usemap", + "value", + "var", + "video", + "wbr", + "width", + "wrap", + "xmp", +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/charset.go b/Godeps/_workspace/src/golang.org/x/net/html/charset/charset.go new file mode 100644 index 00000000000..81280e4ae05 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/charset.go @@ -0,0 +1,244 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package charset provides common text encodings for HTML documents. +// +// The mapping from encoding labels to encodings is defined at +// https://encoding.spec.whatwg.org/. +package charset + +import ( + "bytes" + "fmt" + "io" + "mime" + "strings" + "unicode/utf8" + + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/html" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/text/encoding" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/text/encoding/charmap" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/text/transform" +) + +// Lookup returns the encoding with the specified label, and its canonical +// name. It returns nil and the empty string if label is not one of the +// standard encodings for HTML. Matching is case-insensitive and ignores +// leading and trailing whitespace. +func Lookup(label string) (e encoding.Encoding, name string) { + label = strings.ToLower(strings.Trim(label, "\t\n\r\f ")) + enc := encodings[label] + return enc.e, enc.name +} + +// DetermineEncoding determines the encoding of an HTML document by examining +// up to the first 1024 bytes of content and the declared Content-Type. +// +// See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding +func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) { + if len(content) > 1024 { + content = content[:1024] + } + + for _, b := range boms { + if bytes.HasPrefix(content, b.bom) { + e, name = Lookup(b.enc) + return e, name, true + } + } + + if _, params, err := mime.ParseMediaType(contentType); err == nil { + if cs, ok := params["charset"]; ok { + if e, name = Lookup(cs); e != nil { + return e, name, true + } + } + } + + if len(content) > 0 { + e, name = prescan(content) + if e != nil { + return e, name, false + } + } + + // Try to detect UTF-8. + // First eliminate any partial rune at the end. + for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- { + b := content[i] + if b < 0x80 { + break + } + if utf8.RuneStart(b) { + content = content[:i] + break + } + } + hasHighBit := false + for _, c := range content { + if c >= 0x80 { + hasHighBit = true + break + } + } + if hasHighBit && utf8.Valid(content) { + return encoding.Nop, "utf-8", false + } + + // TODO: change default depending on user's locale? + return charmap.Windows1252, "windows-1252", false +} + +// NewReader returns an io.Reader that converts the content of r to UTF-8. +// It calls DetermineEncoding to find out what r's encoding is. +func NewReader(r io.Reader, contentType string) (io.Reader, error) { + preview := make([]byte, 1024) + n, err := io.ReadFull(r, preview) + switch { + case err == io.ErrUnexpectedEOF: + preview = preview[:n] + r = bytes.NewReader(preview) + case err != nil: + return nil, err + default: + r = io.MultiReader(bytes.NewReader(preview), r) + } + + if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop { + r = transform.NewReader(r, e.NewDecoder()) + } + return r, nil +} + +// NewReaderLabel returns a reader that converts from the specified charset to +// UTF-8. It uses Lookup to find the encoding that corresponds to label, and +// returns an error if Lookup returns nil. It is suitable for use as +// encoding/xml.Decoder's CharsetReader function. +func NewReaderLabel(label string, input io.Reader) (io.Reader, error) { + e, _ := Lookup(label) + if e == nil { + return nil, fmt.Errorf("unsupported charset: %q", label) + } + return transform.NewReader(input, e.NewDecoder()), nil +} + +func prescan(content []byte) (e encoding.Encoding, name string) { + z := html.NewTokenizer(bytes.NewReader(content)) + for { + switch z.Next() { + case html.ErrorToken: + return nil, "" + + case html.StartTagToken, html.SelfClosingTagToken: + tagName, hasAttr := z.TagName() + if !bytes.Equal(tagName, []byte("meta")) { + continue + } + attrList := make(map[string]bool) + gotPragma := false + + const ( + dontKnow = iota + doNeedPragma + doNotNeedPragma + ) + needPragma := dontKnow + + name = "" + e = nil + for hasAttr { + var key, val []byte + key, val, hasAttr = z.TagAttr() + ks := string(key) + if attrList[ks] { + continue + } + attrList[ks] = true + for i, c := range val { + if 'A' <= c && c <= 'Z' { + val[i] = c + 0x20 + } + } + + switch ks { + case "http-equiv": + if bytes.Equal(val, []byte("content-type")) { + gotPragma = true + } + + case "content": + if e == nil { + name = fromMetaElement(string(val)) + if name != "" { + e, name = Lookup(name) + if e != nil { + needPragma = doNeedPragma + } + } + } + + case "charset": + e, name = Lookup(string(val)) + needPragma = doNotNeedPragma + } + } + + if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma { + continue + } + + if strings.HasPrefix(name, "utf-16") { + name = "utf-8" + e = encoding.Nop + } + + if e != nil { + return e, name + } + } + } +} + +func fromMetaElement(s string) string { + for s != "" { + csLoc := strings.Index(s, "charset") + if csLoc == -1 { + return "" + } + s = s[csLoc+len("charset"):] + s = strings.TrimLeft(s, " \t\n\f\r") + if !strings.HasPrefix(s, "=") { + continue + } + s = s[1:] + s = strings.TrimLeft(s, " \t\n\f\r") + if s == "" { + return "" + } + if q := s[0]; q == '"' || q == '\'' { + s = s[1:] + closeQuote := strings.IndexRune(s, rune(q)) + if closeQuote == -1 { + return "" + } + return s[:closeQuote] + } + + end := strings.IndexAny(s, "; \t\n\f\r") + if end == -1 { + end = len(s) + } + return s[:end] + } + return "" +} + +var boms = []struct { + bom []byte + enc string +}{ + {[]byte{0xfe, 0xff}, "utf-16be"}, + {[]byte{0xff, 0xfe}, "utf-16le"}, + {[]byte{0xef, 0xbb, 0xbf}, "utf-8"}, +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/charset_test.go b/Godeps/_workspace/src/golang.org/x/net/html/charset/charset_test.go new file mode 100644 index 00000000000..584f05a3791 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/charset_test.go @@ -0,0 +1,236 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package charset + +import ( + "bytes" + "encoding/xml" + "io/ioutil" + "runtime" + "strings" + "testing" + + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/text/transform" +) + +func transformString(t transform.Transformer, s string) (string, error) { + r := transform.NewReader(strings.NewReader(s), t) + b, err := ioutil.ReadAll(r) + return string(b), err +} + +var testCases = []struct { + utf8, other, otherEncoding string +}{ + {"Résumé", "Résumé", "utf8"}, + {"Résumé", "R\xe9sum\xe9", "latin1"}, + {"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"}, + {"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"}, + {"Hello, world", "Hello, world", "ASCII"}, + {"Gdańsk", "Gda\xf1sk", "ISO-8859-2"}, + {"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"}, + {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"}, + {"latviešu", "latvie\xf0u", "ISO-8859-13"}, + {"Seònaid", "Se\xf2naid", "ISO-8859-14"}, + {"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"}, + {"românește", "rom\xe2ne\xbate", "ISO-8859-16"}, + {"nutraĵo", "nutra\xbco", "ISO-8859-3"}, + {"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"}, + {"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"}, + {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"}, + {"Kağan", "Ka\xf0an", "ISO-8859-9"}, + {"Résumé", "R\x8esum\x8e", "macintosh"}, + {"Gdańsk", "Gda\xf1sk", "windows-1250"}, + {"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"}, + {"Résumé", "R\xe9sum\xe9", "windows-1252"}, + {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"}, + {"Kağan", "Ka\xf0an", "windows-1254"}, + {"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"}, + {"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"}, + {"latviešu", "latvie\xf0u", "windows-1257"}, + {"Việt", "Vi\xea\xf2t", "windows-1258"}, + {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"}, + {"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"}, + {"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"}, + {"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"}, + {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"}, + {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"}, + {"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"}, + {"㧯", "\x82\x31\x89\x38", "gb18030"}, + {"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"}, + {"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"}, + {"ｲｳｴｵｶ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"}, + {"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"}, + {"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"}, + {"네이트 | 즐거움의 시작, 슈파스(Spaβ) NATE", "\xb3\xd7\xc0\xcc\xc6\xae | \xc1\xf1\xb0\xc5\xbf\xf2\xc0\xc7 \xbd\xc3\xc0\xdb, \xbd\xb4\xc6\xc4\xbd\xba(Spa\xa5\xe2) NATE", "EUC-KR"}, +} + +func TestDecode(t *testing.T) { + for _, tc := range testCases { + e, _ := Lookup(tc.otherEncoding) + if e == nil { + t.Errorf("%s: not found", tc.otherEncoding) + continue + } + s, err := transformString(e.NewDecoder(), tc.other) + if err != nil { + t.Errorf("%s: decode %q: %v", tc.otherEncoding, tc.other, err) + continue + } + if s != tc.utf8 { + t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.utf8) + } + } +} + +func TestEncode(t *testing.T) { + for _, tc := range testCases { + e, _ := Lookup(tc.otherEncoding) + if e == nil { + t.Errorf("%s: not found", tc.otherEncoding) + continue + } + s, err := transformString(e.NewEncoder(), tc.utf8) + if err != nil { + t.Errorf("%s: encode %q: %s", tc.otherEncoding, tc.utf8, err) + continue + } + if s != tc.other { + t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.other) + } + } +} + +// TestNames verifies that you can pass an encoding's name to Lookup and get +// the same encoding back (except for "replacement"). +func TestNames(t *testing.T) { + for _, e := range encodings { + if e.name == "replacement" { + continue + } + _, got := Lookup(e.name) + if got != e.name { + t.Errorf("got %q, want %q", got, e.name) + continue + } + } +} + +var sniffTestCases = []struct { + filename, declared, want string +}{ + {"HTTP-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"}, + {"UTF-16LE-BOM.html", "", "utf-16le"}, + {"UTF-16BE-BOM.html", "", "utf-16be"}, + {"meta-content-attribute.html", "text/html", "iso-8859-15"}, + {"meta-charset-attribute.html", "text/html", "iso-8859-15"}, + {"No-encoding-declaration.html", "text/html", "utf-8"}, + {"HTTP-vs-UTF-8-BOM.html", "text/html; charset=iso-8859-15", "utf-8"}, + {"HTTP-vs-meta-content.html", "text/html; charset=iso-8859-15", "iso-8859-15"}, + {"HTTP-vs-meta-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"}, + {"UTF-8-BOM-vs-meta-content.html", "text/html", "utf-8"}, + {"UTF-8-BOM-vs-meta-charset.html", "text/html", "utf-8"}, +} + +func TestSniff(t *testing.T) { + switch runtime.GOOS { + case "nacl": // platforms that don't permit direct file system access + t.Skipf("not supported on %q", runtime.GOOS) + } + + for _, tc := range sniffTestCases { + content, err := ioutil.ReadFile("testdata/" + tc.filename) + if err != nil { + t.Errorf("%s: error reading file: %v", tc.filename, err) + continue + } + + _, name, _ := DetermineEncoding(content, tc.declared) + if name != tc.want { + t.Errorf("%s: got %q, want %q", tc.filename, name, tc.want) + continue + } + } +} + +func TestReader(t *testing.T) { + switch runtime.GOOS { + case "nacl": // platforms that don't permit direct file system access + t.Skipf("not supported on %q", runtime.GOOS) + } + + for _, tc := range sniffTestCases { + content, err := ioutil.ReadFile("testdata/" + tc.filename) + if err != nil { + t.Errorf("%s: error reading file: %v", tc.filename, err) + continue + } + + r, err := NewReader(bytes.NewReader(content), tc.declared) + if err != nil { + t.Errorf("%s: error creating reader: %v", tc.filename, err) + continue + } + + got, err := ioutil.ReadAll(r) + if err != nil { + t.Errorf("%s: error reading from charset.NewReader: %v", tc.filename, err) + continue + } + + e, _ := Lookup(tc.want) + want, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader(content), e.NewDecoder())) + if err != nil { + t.Errorf("%s: error decoding with hard-coded charset name: %v", tc.filename, err) + continue + } + + if !bytes.Equal(got, want) { + t.Errorf("%s: got %q, want %q", tc.filename, got, want) + continue + } + } +} + +var metaTestCases = []struct { + meta, want string +}{ + {"", ""}, + {"text/html", ""}, + {"text/html; charset utf-8", ""}, + {"text/html; charset=latin-2", "latin-2"}, + {"text/html; charset; charset = utf-8", "utf-8"}, + {`charset="big5"`, "big5"}, + {"charset='shift_jis'", "shift_jis"}, +} + +func TestFromMeta(t *testing.T) { + for _, tc := range metaTestCases { + got := fromMetaElement(tc.meta) + if got != tc.want { + t.Errorf("%q: got %q, want %q", tc.meta, got, tc.want) + } + } +} + +func TestXML(t *testing.T) { + const s = "r\xe9sum\xe9" + + d := xml.NewDecoder(strings.NewReader(s)) + d.CharsetReader = NewReaderLabel + + var a struct { + Word string + } + err := d.Decode(&a) + if err != nil { + t.Fatalf("Decode: %v", err) + } + + want := "résumé" + if a.Word != want { + t.Errorf("got %q, want %q", a.Word, want) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/gen.go b/Godeps/_workspace/src/golang.org/x/net/html/charset/gen.go new file mode 100644 index 00000000000..828347f4aec --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/gen.go @@ -0,0 +1,111 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// Download https://encoding.spec.whatwg.org/encodings.json and use it to +// generate table.go. + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "strings" +) + +type enc struct { + Name string + Labels []string +} + +type group struct { + Encodings []enc + Heading string +} + +const specURL = "https://encoding.spec.whatwg.org/encodings.json" + +func main() { + resp, err := http.Get(specURL) + if err != nil { + log.Fatalf("error fetching %s: %s", specURL, err) + } + if resp.StatusCode != 200 { + log.Fatalf("error fetching %s: HTTP status %s", specURL, resp.Status) + } + defer resp.Body.Close() + + var groups []group + d := json.NewDecoder(resp.Body) + err = d.Decode(&groups) + if err != nil { + log.Fatalf("error reading encodings.json: %s", err) + } + + fmt.Println("// generated by go run gen.go; DO NOT EDIT") + fmt.Println() + fmt.Println("package charset") + fmt.Println() + + fmt.Println("import (") + fmt.Println(`"golang.org/x/text/encoding"`) + for _, pkg := range []string{"charmap", "japanese", "korean", "simplifiedchinese", "traditionalchinese", "unicode"} { + fmt.Printf("\"golang.org/x/text/encoding/%s\"\n", pkg) + } + fmt.Println(")") + fmt.Println() + + fmt.Println("var encodings = map[string]struct{e encoding.Encoding; name string} {") + for _, g := range groups { + for _, e := range g.Encodings { + goName, ok := miscNames[e.Name] + if !ok { + for k, v := range prefixes { + if strings.HasPrefix(e.Name, k) { + goName = v + e.Name[len(k):] + break + } + } + if goName == "" { + log.Fatalf("unrecognized encoding name: %s", e.Name) + } + } + + for _, label := range e.Labels { + fmt.Printf("%q: {%s, %q},\n", label, goName, e.Name) + } + } + } + fmt.Println("}") +} + +var prefixes = map[string]string{ + "iso-8859-": "charmap.ISO8859_", + "windows-": "charmap.Windows", +} + +var miscNames = map[string]string{ + "utf-8": "encoding.Nop", + "ibm866": "charmap.CodePage866", + "iso-8859-8-i": "charmap.ISO8859_8", + "koi8-r": "charmap.KOI8R", + "koi8-u": "charmap.KOI8U", + "macintosh": "charmap.Macintosh", + "x-mac-cyrillic": "charmap.MacintoshCyrillic", + "gbk": "simplifiedchinese.GBK", + "gb18030": "simplifiedchinese.GB18030", + "hz-gb-2312": "simplifiedchinese.HZGB2312", + "big5": "traditionalchinese.Big5", + "euc-jp": "japanese.EUCJP", + "iso-2022-jp": "japanese.ISO2022JP", + "shift_jis": "japanese.ShiftJIS", + "euc-kr": "korean.EUCKR", + "replacement": "encoding.Replacement", + "utf-16be": "unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)", + "utf-16le": "unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM)", + "x-user-defined": "charmap.XUserDefined", +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/table.go b/Godeps/_workspace/src/golang.org/x/net/html/charset/table.go new file mode 100644 index 00000000000..754dd298d8f --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/table.go @@ -0,0 +1,235 @@ +// generated by go run gen.go; DO NOT EDIT + +package charset + +import ( + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/text/encoding" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/text/encoding/charmap" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/text/encoding/japanese" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/text/encoding/korean" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/text/encoding/simplifiedchinese" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/text/encoding/traditionalchinese" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/text/encoding/unicode" +) + +var encodings = map[string]struct { + e encoding.Encoding + name string +}{ + "unicode-1-1-utf-8": {encoding.Nop, "utf-8"}, + "utf-8": {encoding.Nop, "utf-8"}, + "utf8": {encoding.Nop, "utf-8"}, + "866": {charmap.CodePage866, "ibm866"}, + "cp866": {charmap.CodePage866, "ibm866"}, + "csibm866": {charmap.CodePage866, "ibm866"}, + "ibm866": {charmap.CodePage866, "ibm866"}, + "csisolatin2": {charmap.ISO8859_2, "iso-8859-2"}, + "iso-8859-2": {charmap.ISO8859_2, "iso-8859-2"}, + "iso-ir-101": {charmap.ISO8859_2, "iso-8859-2"}, + "iso8859-2": {charmap.ISO8859_2, "iso-8859-2"}, + "iso88592": {charmap.ISO8859_2, "iso-8859-2"}, + "iso_8859-2": {charmap.ISO8859_2, "iso-8859-2"}, + "iso_8859-2:1987": {charmap.ISO8859_2, "iso-8859-2"}, + "l2": {charmap.ISO8859_2, "iso-8859-2"}, + "latin2": {charmap.ISO8859_2, "iso-8859-2"}, + "csisolatin3": {charmap.ISO8859_3, "iso-8859-3"}, + "iso-8859-3": {charmap.ISO8859_3, "iso-8859-3"}, + "iso-ir-109": {charmap.ISO8859_3, "iso-8859-3"}, + "iso8859-3": {charmap.ISO8859_3, "iso-8859-3"}, + "iso88593": {charmap.ISO8859_3, "iso-8859-3"}, + "iso_8859-3": {charmap.ISO8859_3, "iso-8859-3"}, + "iso_8859-3:1988": {charmap.ISO8859_3, "iso-8859-3"}, + "l3": {charmap.ISO8859_3, "iso-8859-3"}, + "latin3": {charmap.ISO8859_3, "iso-8859-3"}, + "csisolatin4": {charmap.ISO8859_4, "iso-8859-4"}, + "iso-8859-4": {charmap.ISO8859_4, "iso-8859-4"}, + "iso-ir-110": {charmap.ISO8859_4, "iso-8859-4"}, + "iso8859-4": {charmap.ISO8859_4, "iso-8859-4"}, + "iso88594": {charmap.ISO8859_4, "iso-8859-4"}, + "iso_8859-4": {charmap.ISO8859_4, "iso-8859-4"}, + "iso_8859-4:1988": {charmap.ISO8859_4, "iso-8859-4"}, + "l4": {charmap.ISO8859_4, "iso-8859-4"}, + "latin4": {charmap.ISO8859_4, "iso-8859-4"}, + "csisolatincyrillic": {charmap.ISO8859_5, "iso-8859-5"}, + "cyrillic": {charmap.ISO8859_5, "iso-8859-5"}, + "iso-8859-5": {charmap.ISO8859_5, "iso-8859-5"}, + "iso-ir-144": {charmap.ISO8859_5, "iso-8859-5"}, + "iso8859-5": {charmap.ISO8859_5, "iso-8859-5"}, + "iso88595": {charmap.ISO8859_5, "iso-8859-5"}, + "iso_8859-5": {charmap.ISO8859_5, "iso-8859-5"}, + "iso_8859-5:1988": {charmap.ISO8859_5, "iso-8859-5"}, + "arabic": {charmap.ISO8859_6, "iso-8859-6"}, + "asmo-708": {charmap.ISO8859_6, "iso-8859-6"}, + "csiso88596e": {charmap.ISO8859_6, "iso-8859-6"}, + "csiso88596i": {charmap.ISO8859_6, "iso-8859-6"}, + "csisolatinarabic": {charmap.ISO8859_6, "iso-8859-6"}, + "ecma-114": {charmap.ISO8859_6, "iso-8859-6"}, + "iso-8859-6": {charmap.ISO8859_6, "iso-8859-6"}, + "iso-8859-6-e": {charmap.ISO8859_6, "iso-8859-6"}, + "iso-8859-6-i": {charmap.ISO8859_6, "iso-8859-6"}, + "iso-ir-127": {charmap.ISO8859_6, "iso-8859-6"}, + "iso8859-6": {charmap.ISO8859_6, "iso-8859-6"}, + "iso88596": {charmap.ISO8859_6, "iso-8859-6"}, + "iso_8859-6": {charmap.ISO8859_6, "iso-8859-6"}, + "iso_8859-6:1987": {charmap.ISO8859_6, "iso-8859-6"}, + "csisolatingreek": {charmap.ISO8859_7, "iso-8859-7"}, + "ecma-118": {charmap.ISO8859_7, "iso-8859-7"}, + "elot_928": {charmap.ISO8859_7, "iso-8859-7"}, + "greek": {charmap.ISO8859_7, "iso-8859-7"}, + "greek8": {charmap.ISO8859_7, "iso-8859-7"}, + "iso-8859-7": {charmap.ISO8859_7, "iso-8859-7"}, + "iso-ir-126": {charmap.ISO8859_7, "iso-8859-7"}, + "iso8859-7": {charmap.ISO8859_7, "iso-8859-7"}, + "iso88597": {charmap.ISO8859_7, "iso-8859-7"}, + "iso_8859-7": {charmap.ISO8859_7, "iso-8859-7"}, + "iso_8859-7:1987": {charmap.ISO8859_7, "iso-8859-7"}, + "sun_eu_greek": {charmap.ISO8859_7, "iso-8859-7"}, + "csiso88598e": {charmap.ISO8859_8, "iso-8859-8"}, + "csisolatinhebrew": {charmap.ISO8859_8, "iso-8859-8"}, + "hebrew": {charmap.ISO8859_8, "iso-8859-8"}, + "iso-8859-8": {charmap.ISO8859_8, "iso-8859-8"}, + "iso-8859-8-e": {charmap.ISO8859_8, "iso-8859-8"}, + "iso-ir-138": {charmap.ISO8859_8, "iso-8859-8"}, + "iso8859-8": {charmap.ISO8859_8, "iso-8859-8"}, + "iso88598": {charmap.ISO8859_8, "iso-8859-8"}, + "iso_8859-8": {charmap.ISO8859_8, "iso-8859-8"}, + "iso_8859-8:1988": {charmap.ISO8859_8, "iso-8859-8"}, + "visual": {charmap.ISO8859_8, "iso-8859-8"}, + "csiso88598i": {charmap.ISO8859_8, "iso-8859-8-i"}, + "iso-8859-8-i": {charmap.ISO8859_8, "iso-8859-8-i"}, + "logical": {charmap.ISO8859_8, "iso-8859-8-i"}, + "csisolatin6": {charmap.ISO8859_10, "iso-8859-10"}, + "iso-8859-10": {charmap.ISO8859_10, "iso-8859-10"}, + "iso-ir-157": {charmap.ISO8859_10, "iso-8859-10"}, + "iso8859-10": {charmap.ISO8859_10, "iso-8859-10"}, + "iso885910": {charmap.ISO8859_10, "iso-8859-10"}, + "l6": {charmap.ISO8859_10, "iso-8859-10"}, + "latin6": {charmap.ISO8859_10, "iso-8859-10"}, + "iso-8859-13": {charmap.ISO8859_13, "iso-8859-13"}, + "iso8859-13": {charmap.ISO8859_13, "iso-8859-13"}, + "iso885913": {charmap.ISO8859_13, "iso-8859-13"}, + "iso-8859-14": {charmap.ISO8859_14, "iso-8859-14"}, + "iso8859-14": {charmap.ISO8859_14, "iso-8859-14"}, + "iso885914": {charmap.ISO8859_14, "iso-8859-14"}, + "csisolatin9": {charmap.ISO8859_15, "iso-8859-15"}, + "iso-8859-15": {charmap.ISO8859_15, "iso-8859-15"}, + "iso8859-15": {charmap.ISO8859_15, "iso-8859-15"}, + "iso885915": {charmap.ISO8859_15, "iso-8859-15"}, + "iso_8859-15": {charmap.ISO8859_15, "iso-8859-15"}, + "l9": {charmap.ISO8859_15, "iso-8859-15"}, + "iso-8859-16": {charmap.ISO8859_16, "iso-8859-16"}, + "cskoi8r": {charmap.KOI8R, "koi8-r"}, + "koi": {charmap.KOI8R, "koi8-r"}, + "koi8": {charmap.KOI8R, "koi8-r"}, + "koi8-r": {charmap.KOI8R, "koi8-r"}, + "koi8_r": {charmap.KOI8R, "koi8-r"}, + "koi8-u": {charmap.KOI8U, "koi8-u"}, + "csmacintosh": {charmap.Macintosh, "macintosh"}, + "mac": {charmap.Macintosh, "macintosh"}, + "macintosh": {charmap.Macintosh, "macintosh"}, + "x-mac-roman": {charmap.Macintosh, "macintosh"}, + "dos-874": {charmap.Windows874, "windows-874"}, + "iso-8859-11": {charmap.Windows874, "windows-874"}, + "iso8859-11": {charmap.Windows874, "windows-874"}, + "iso885911": {charmap.Windows874, "windows-874"}, + "tis-620": {charmap.Windows874, "windows-874"}, + "windows-874": {charmap.Windows874, "windows-874"}, + "cp1250": {charmap.Windows1250, "windows-1250"}, + "windows-1250": {charmap.Windows1250, "windows-1250"}, + "x-cp1250": {charmap.Windows1250, "windows-1250"}, + "cp1251": {charmap.Windows1251, "windows-1251"}, + "windows-1251": {charmap.Windows1251, "windows-1251"}, + "x-cp1251": {charmap.Windows1251, "windows-1251"}, + "ansi_x3.4-1968": {charmap.Windows1252, "windows-1252"}, + "ascii": {charmap.Windows1252, "windows-1252"}, + "cp1252": {charmap.Windows1252, "windows-1252"}, + "cp819": {charmap.Windows1252, "windows-1252"}, + "csisolatin1": {charmap.Windows1252, "windows-1252"}, + "ibm819": {charmap.Windows1252, "windows-1252"}, + "iso-8859-1": {charmap.Windows1252, "windows-1252"}, + "iso-ir-100": {charmap.Windows1252, "windows-1252"}, + "iso8859-1": {charmap.Windows1252, "windows-1252"}, + "iso88591": {charmap.Windows1252, "windows-1252"}, + "iso_8859-1": {charmap.Windows1252, "windows-1252"}, + "iso_8859-1:1987": {charmap.Windows1252, "windows-1252"}, + "l1": {charmap.Windows1252, "windows-1252"}, + "latin1": {charmap.Windows1252, "windows-1252"}, + "us-ascii": {charmap.Windows1252, "windows-1252"}, + "windows-1252": {charmap.Windows1252, "windows-1252"}, + "x-cp1252": {charmap.Windows1252, "windows-1252"}, + "cp1253": {charmap.Windows1253, "windows-1253"}, + "windows-1253": {charmap.Windows1253, "windows-1253"}, + "x-cp1253": {charmap.Windows1253, "windows-1253"}, + "cp1254": {charmap.Windows1254, "windows-1254"}, + "csisolatin5": {charmap.Windows1254, "windows-1254"}, + "iso-8859-9": {charmap.Windows1254, "windows-1254"}, + "iso-ir-148": {charmap.Windows1254, "windows-1254"}, + "iso8859-9": {charmap.Windows1254, "windows-1254"}, + "iso88599": {charmap.Windows1254, "windows-1254"}, + "iso_8859-9": {charmap.Windows1254, "windows-1254"}, + "iso_8859-9:1989": {charmap.Windows1254, "windows-1254"}, + "l5": {charmap.Windows1254, "windows-1254"}, + "latin5": {charmap.Windows1254, "windows-1254"}, + "windows-1254": {charmap.Windows1254, "windows-1254"}, + "x-cp1254": {charmap.Windows1254, "windows-1254"}, + "cp1255": {charmap.Windows1255, "windows-1255"}, + "windows-1255": {charmap.Windows1255, "windows-1255"}, + "x-cp1255": {charmap.Windows1255, "windows-1255"}, + "cp1256": {charmap.Windows1256, "windows-1256"}, + "windows-1256": {charmap.Windows1256, "windows-1256"}, + "x-cp1256": {charmap.Windows1256, "windows-1256"}, + "cp1257": {charmap.Windows1257, "windows-1257"}, + "windows-1257": {charmap.Windows1257, "windows-1257"}, + "x-cp1257": {charmap.Windows1257, "windows-1257"}, + "cp1258": {charmap.Windows1258, "windows-1258"}, + "windows-1258": {charmap.Windows1258, "windows-1258"}, + "x-cp1258": {charmap.Windows1258, "windows-1258"}, + "x-mac-cyrillic": {charmap.MacintoshCyrillic, "x-mac-cyrillic"}, + "x-mac-ukrainian": {charmap.MacintoshCyrillic, "x-mac-cyrillic"}, + "chinese": {simplifiedchinese.GBK, "gbk"}, + "csgb2312": {simplifiedchinese.GBK, "gbk"}, + "csiso58gb231280": {simplifiedchinese.GBK, "gbk"}, + "gb2312": {simplifiedchinese.GBK, "gbk"}, + "gb_2312": {simplifiedchinese.GBK, "gbk"}, + "gb_2312-80": {simplifiedchinese.GBK, "gbk"}, + "gbk": {simplifiedchinese.GBK, "gbk"}, + "iso-ir-58": {simplifiedchinese.GBK, "gbk"}, + "x-gbk": {simplifiedchinese.GBK, "gbk"}, + "gb18030": {simplifiedchinese.GB18030, "gb18030"}, + "hz-gb-2312": {simplifiedchinese.HZGB2312, "hz-gb-2312"}, + "big5": {traditionalchinese.Big5, "big5"}, + "big5-hkscs": {traditionalchinese.Big5, "big5"}, + "cn-big5": {traditionalchinese.Big5, "big5"}, + "csbig5": {traditionalchinese.Big5, "big5"}, + "x-x-big5": {traditionalchinese.Big5, "big5"}, + "cseucpkdfmtjapanese": {japanese.EUCJP, "euc-jp"}, + "euc-jp": {japanese.EUCJP, "euc-jp"}, + "x-euc-jp": {japanese.EUCJP, "euc-jp"}, + "csiso2022jp": {japanese.ISO2022JP, "iso-2022-jp"}, + "iso-2022-jp": {japanese.ISO2022JP, "iso-2022-jp"}, + "csshiftjis": {japanese.ShiftJIS, "shift_jis"}, + "ms_kanji": {japanese.ShiftJIS, "shift_jis"}, + "shift-jis": {japanese.ShiftJIS, "shift_jis"}, + "shift_jis": {japanese.ShiftJIS, "shift_jis"}, + "sjis": {japanese.ShiftJIS, "shift_jis"}, + "windows-31j": {japanese.ShiftJIS, "shift_jis"}, + "x-sjis": {japanese.ShiftJIS, "shift_jis"}, + "cseuckr": {korean.EUCKR, "euc-kr"}, + "csksc56011987": {korean.EUCKR, "euc-kr"}, + "euc-kr": {korean.EUCKR, "euc-kr"}, + "iso-ir-149": {korean.EUCKR, "euc-kr"}, + "korean": {korean.EUCKR, "euc-kr"}, + "ks_c_5601-1987": {korean.EUCKR, "euc-kr"}, + "ks_c_5601-1989": {korean.EUCKR, "euc-kr"}, + "ksc5601": {korean.EUCKR, "euc-kr"}, + "ksc_5601": {korean.EUCKR, "euc-kr"}, + "windows-949": {korean.EUCKR, "euc-kr"}, + "csiso2022kr": {encoding.Replacement, "replacement"}, + "iso-2022-kr": {encoding.Replacement, "replacement"}, + "iso-2022-cn": {encoding.Replacement, "replacement"}, + "iso-2022-cn-ext": {encoding.Replacement, "replacement"}, + "utf-16be": {unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM), "utf-16be"}, + "utf-16": {unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM), "utf-16le"}, + "utf-16le": {unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM), "utf-16le"}, + "x-user-defined": {charmap.XUserDefined, "x-user-defined"}, +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-charset.html b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-charset.html new file mode 100644 index 00000000000..9915fa0ee4f --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-charset.html @@ -0,0 +1,48 @@ + + + + HTTP charset + + + + + + + + + + + +

HTTP charset

+ + +
+ + +
 
+ + + + + +
+

The character encoding of a page can be set using the HTTP header charset declaration.

+

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.

The only character encoding declaration for this HTML file is in the HTTP header, which sets the encoding to ISO 8859-15.

+
+
+
Next test
HTML5
+

the-input-byte-stream-001
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-vs-UTF-8-BOM.html b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-vs-UTF-8-BOM.html new file mode 100644 index 00000000000..26e5d8b4ebf --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-vs-UTF-8-BOM.html @@ -0,0 +1,48 @@ + + + + HTTP vs UTF-8 BOM + + + + + + + + + + + +

HTTP vs UTF-8 BOM

+ + +
+ + +
 
+ + + + + +
+

A character encoding set in the HTTP header has lower precedence than the UTF-8 signature.

+

The HTTP header attempts to set the character encoding to ISO 8859-15. The page starts with a UTF-8 signature.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.

If the test is unsuccessful, the characters  should appear at the top of the page. These represent the bytes that make up the UTF-8 signature when encountered in the ISO 8859-15 encoding.

+
+
+
Next test
HTML5
+

the-input-byte-stream-034
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-vs-meta-charset.html b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-vs-meta-charset.html new file mode 100644 index 00000000000..2f07e95158e --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-vs-meta-charset.html @@ -0,0 +1,49 @@ + + + + HTTP vs meta charset + + + + + + + + + + + +

HTTP vs meta charset

+ + +
+ + +
 
+ + + + + +
+

The HTTP header has a higher precedence than an encoding declaration in a meta charset attribute.

+

The HTTP header attempts to set the character encoding to ISO 8859-15. The page contains an encoding declaration in a meta charset attribute that attempts to set the character encoding to ISO 8859-1.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-018
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-vs-meta-content.html b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-vs-meta-content.html new file mode 100644 index 00000000000..6853cddeccc --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/HTTP-vs-meta-content.html @@ -0,0 +1,49 @@ + + + + HTTP vs meta content + + + + + + + + + + + +

HTTP vs meta content

+ + +
+ + +
 
+ + + + + +
+

The HTTP header has a higher precedence than an encoding declaration in a meta content attribute.

+

The HTTP header attempts to set the character encoding to ISO 8859-15. The page contains an encoding declaration in a meta content attribute that attempts to set the character encoding to ISO 8859-1.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-016
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/No-encoding-declaration.html b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/No-encoding-declaration.html new file mode 100644 index 00000000000..612e26c6c52 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/No-encoding-declaration.html @@ -0,0 +1,47 @@ + + + + No encoding declaration + + + + + + + + + + + +

No encoding declaration

+ + +
+ + +
 
+ + + + + +
+

A page with no encoding information in HTTP, BOM, XML declaration or meta element will be treated as UTF-8.

+

The test on this page contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-015
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/README b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/README new file mode 100644 index 00000000000..38ef0f9f121 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/README @@ -0,0 +1,9 @@ +These test cases come from +http://www.w3.org/International/tests/repository/html5/the-input-byte-stream/results-basics + +Distributed under both the W3C Test Suite License +(http://www.w3.org/Consortium/Legal/2008/04-testsuite-license) +and the W3C 3-clause BSD License +(http://www.w3.org/Consortium/Legal/2008/03-bsd-license). +To contribute to a W3C Test Suite, see the policies and contribution +forms (http://www.w3.org/2004/10/27-testcases). diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-16BE-BOM.html b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-16BE-BOM.html new file mode 100644 index 00000000000..3abf7a9343c Binary files /dev/null and b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-16BE-BOM.html differ diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-16LE-BOM.html b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-16LE-BOM.html new file mode 100644 index 00000000000..76254c980c2 Binary files /dev/null and b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-16LE-BOM.html differ diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-8-BOM-vs-meta-charset.html b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-8-BOM-vs-meta-charset.html new file mode 100644 index 00000000000..83de43338ec --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-8-BOM-vs-meta-charset.html @@ -0,0 +1,49 @@ + + + + UTF-8 BOM vs meta charset + + + + + + + + + + + +

UTF-8 BOM vs meta charset

+ + +
+ + +
 
+ + + + + +
+

A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta charset attribute declares a different encoding.

+

The page contains an encoding declaration in a meta charset attribute that attempts to set the character encoding to ISO 8859-15, but the file starts with a UTF-8 signature.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-038
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-8-BOM-vs-meta-content.html b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-8-BOM-vs-meta-content.html new file mode 100644 index 00000000000..501aac2d6a5 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/UTF-8-BOM-vs-meta-content.html @@ -0,0 +1,48 @@ + + + + UTF-8 BOM vs meta content + + + + + + + + + + + +

UTF-8 BOM vs meta content

+ + +
+ + +
 
+ + + + + +
+

A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta content attribute declares a different encoding.

+

The page contains an encoding declaration in a meta content attribute that attempts to set the character encoding to ISO 8859-15, but the file starts with a UTF-8 signature.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-037
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/meta-charset-attribute.html b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/meta-charset-attribute.html new file mode 100644 index 00000000000..2d7d25aba14 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/meta-charset-attribute.html @@ -0,0 +1,48 @@ + + + + meta charset attribute + + + + + + + + + + + +

meta charset attribute

+ + +
+ + +
 
+ + + + + +
+

The character encoding of the page can be set by a meta element with charset attribute.

+

The only character encoding declaration for this HTML file is in the charset attribute of the meta element, which declares the encoding to be ISO 8859-15.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-009
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/meta-content-attribute.html b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/meta-content-attribute.html new file mode 100644 index 00000000000..1c3f228e7c9 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/charset/testdata/meta-content-attribute.html @@ -0,0 +1,48 @@ + + + + meta content attribute + + + + + + + + + + + +

meta content attribute

+ + +
+ + +
 
+ + + + + +
+

The character encoding of the page can be set by a meta element with http-equiv and content attributes.

+

The only character encoding declaration for this HTML file is in the content attribute of the meta element, which declares the encoding to be ISO 8859-15.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-007
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/Godeps/_workspace/src/golang.org/x/net/html/const.go b/Godeps/_workspace/src/golang.org/x/net/html/const.go new file mode 100644 index 00000000000..52f651ff6db --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/const.go @@ -0,0 +1,102 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +// Section 12.2.3.2 of the HTML5 specification says "The following elements +// have varying levels of special parsing rules". +// https://html.spec.whatwg.org/multipage/syntax.html#the-stack-of-open-elements +var isSpecialElementMap = map[string]bool{ + "address": true, + "applet": true, + "area": true, + "article": true, + "aside": true, + "base": true, + "basefont": true, + "bgsound": true, + "blockquote": true, + "body": true, + "br": true, + "button": true, + "caption": true, + "center": true, + "col": true, + "colgroup": true, + "dd": true, + "details": true, + "dir": true, + "div": true, + "dl": true, + "dt": true, + "embed": true, + "fieldset": true, + "figcaption": true, + "figure": true, + "footer": true, + "form": true, + "frame": true, + "frameset": true, + "h1": true, + "h2": true, + "h3": true, + "h4": true, + "h5": true, + "h6": true, + "head": true, + "header": true, + "hgroup": true, + "hr": true, + "html": true, + "iframe": true, + "img": true, + "input": true, + "isindex": true, + "li": true, + "link": true, + "listing": true, + "marquee": true, + "menu": true, + "meta": true, + "nav": true, + "noembed": true, + "noframes": true, + "noscript": true, + "object": true, + "ol": true, + "p": true, + "param": true, + "plaintext": true, + "pre": true, + "script": true, + "section": true, + "select": true, + "source": true, + "style": true, + "summary": true, + "table": true, + "tbody": true, + "td": true, + "template": true, + "textarea": true, + "tfoot": true, + "th": true, + "thead": true, + "title": true, + "tr": true, + "track": true, + "ul": true, + "wbr": true, + "xmp": true, +} + +func isSpecialElement(element *Node) bool { + switch element.Namespace { + case "", "html": + return isSpecialElementMap[element.Data] + case "svg": + return element.Data == "foreignObject" + } + return false +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/doc.go b/Godeps/_workspace/src/golang.org/x/net/html/doc.go new file mode 100644 index 00000000000..b453fe1e4c2 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/doc.go @@ -0,0 +1,106 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package html implements an HTML5-compliant tokenizer and parser. + +Tokenization is done by creating a Tokenizer for an io.Reader r. It is the +caller's responsibility to ensure that r provides UTF-8 encoded HTML. + + z := html.NewTokenizer(r) + +Given a Tokenizer z, the HTML is tokenized by repeatedly calling z.Next(), +which parses the next token and returns its type, or an error: + + for { + tt := z.Next() + if tt == html.ErrorToken { + // ... + return ... + } + // Process the current token. + } + +There are two APIs for retrieving the current token. The high-level API is to +call Token; the low-level API is to call Text or TagName / TagAttr. Both APIs +allow optionally calling Raw after Next but before Token, Text, TagName, or +TagAttr. In EBNF notation, the valid call sequence per token is: + + Next {Raw} [ Token | Text | TagName {TagAttr} ] + +Token returns an independent data structure that completely describes a token. +Entities (such as "<") are unescaped, tag names and attribute keys are +lower-cased, and attributes are collected into a []Attribute. For example: + + for { + if z.Next() == html.ErrorToken { + // Returning io.EOF indicates success. + return z.Err() + } + emitToken(z.Token()) + } + +The low-level API performs fewer allocations and copies, but the contents of +the []byte values returned by Text, TagName and TagAttr may change on the next +call to Next. For example, to extract an HTML page's anchor text: + + depth := 0 + for { + tt := z.Next() + switch tt { + case ErrorToken: + return z.Err() + case TextToken: + if depth > 0 { + // emitBytes should copy the []byte it receives, + // if it doesn't process it immediately. + emitBytes(z.Text()) + } + case StartTagToken, EndTagToken: + tn, _ := z.TagName() + if len(tn) == 1 && tn[0] == 'a' { + if tt == StartTagToken { + depth++ + } else { + depth-- + } + } + } + } + +Parsing is done by calling Parse with an io.Reader, which returns the root of +the parse tree (the document element) as a *Node. It is the caller's +responsibility to ensure that the Reader provides UTF-8 encoded HTML. For +example, to process each anchor node in depth-first order: + + doc, err := html.Parse(r) + if err != nil { + // ... + } + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + // Do something with n... + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + +The relevant specifications include: +https://html.spec.whatwg.org/multipage/syntax.html and +https://html.spec.whatwg.org/multipage/syntax.html#tokenization +*/ +package html + +// The tokenization algorithm implemented by this package is not a line-by-line +// transliteration of the relatively verbose state-machine in the WHATWG +// specification. A more direct approach is used instead, where the program +// counter implies the state, such as whether it is tokenizing a tag or a text +// node. Specification compliance is verified by checking expected and actual +// outputs over a test suite rather than aiming for algorithmic fidelity. + +// TODO(nigeltao): Does a DOM API belong in this package or a separate one? +// TODO(nigeltao): How does parsing interact with a JavaScript engine? diff --git a/Godeps/_workspace/src/golang.org/x/net/html/doctype.go b/Godeps/_workspace/src/golang.org/x/net/html/doctype.go new file mode 100644 index 00000000000..c484e5a94fb --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/doctype.go @@ -0,0 +1,156 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "strings" +) + +// parseDoctype parses the data from a DoctypeToken into a name, +// public identifier, and system identifier. It returns a Node whose Type +// is DoctypeNode, whose Data is the name, and which has attributes +// named "system" and "public" for the two identifiers if they were present. +// quirks is whether the document should be parsed in "quirks mode". +func parseDoctype(s string) (n *Node, quirks bool) { + n = &Node{Type: DoctypeNode} + + // Find the name. + space := strings.IndexAny(s, whitespace) + if space == -1 { + space = len(s) + } + n.Data = s[:space] + // The comparison to "html" is case-sensitive. + if n.Data != "html" { + quirks = true + } + n.Data = strings.ToLower(n.Data) + s = strings.TrimLeft(s[space:], whitespace) + + if len(s) < 6 { + // It can't start with "PUBLIC" or "SYSTEM". + // Ignore the rest of the string. + return n, quirks || s != "" + } + + key := strings.ToLower(s[:6]) + s = s[6:] + for key == "public" || key == "system" { + s = strings.TrimLeft(s, whitespace) + if s == "" { + break + } + quote := s[0] + if quote != '"' && quote != '\'' { + break + } + s = s[1:] + q := strings.IndexRune(s, rune(quote)) + var id string + if q == -1 { + id = s + s = "" + } else { + id = s[:q] + s = s[q+1:] + } + n.Attr = append(n.Attr, Attribute{Key: key, Val: id}) + if key == "public" { + key = "system" + } else { + key = "" + } + } + + if key != "" || s != "" { + quirks = true + } else if len(n.Attr) > 0 { + if n.Attr[0].Key == "public" { + public := strings.ToLower(n.Attr[0].Val) + switch public { + case "-//w3o//dtd w3 html strict 3.0//en//", "-/w3d/dtd html 4.0 transitional/en", "html": + quirks = true + default: + for _, q := range quirkyIDs { + if strings.HasPrefix(public, q) { + quirks = true + break + } + } + } + // The following two public IDs only cause quirks mode if there is no system ID. + if len(n.Attr) == 1 && (strings.HasPrefix(public, "-//w3c//dtd html 4.01 frameset//") || + strings.HasPrefix(public, "-//w3c//dtd html 4.01 transitional//")) { + quirks = true + } + } + if lastAttr := n.Attr[len(n.Attr)-1]; lastAttr.Key == "system" && + strings.ToLower(lastAttr.Val) == "http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd" { + quirks = true + } + } + + return n, quirks +} + +// quirkyIDs is a list of public doctype identifiers that cause a document +// to be interpreted in quirks mode. The identifiers should be in lower case. +var quirkyIDs = []string{ + "+//silmaril//dtd html pro v0r11 19970101//", + "-//advasoft ltd//dtd html 3.0 aswedit + extensions//", + "-//as//dtd html 3.0 aswedit + extensions//", + "-//ietf//dtd html 2.0 level 1//", + "-//ietf//dtd html 2.0 level 2//", + "-//ietf//dtd html 2.0 strict level 1//", + "-//ietf//dtd html 2.0 strict level 2//", + "-//ietf//dtd html 2.0 strict//", + "-//ietf//dtd html 2.0//", + "-//ietf//dtd html 2.1e//", + "-//ietf//dtd html 3.0//", + "-//ietf//dtd html 3.2 final//", + "-//ietf//dtd html 3.2//", + "-//ietf//dtd html 3//", + "-//ietf//dtd html level 0//", + "-//ietf//dtd html level 1//", + "-//ietf//dtd html level 2//", + "-//ietf//dtd html level 3//", + "-//ietf//dtd html strict level 0//", + "-//ietf//dtd html strict level 1//", + "-//ietf//dtd html strict level 2//", + "-//ietf//dtd html strict level 3//", + "-//ietf//dtd html strict//", + "-//ietf//dtd html//", + "-//metrius//dtd metrius presentational//", + "-//microsoft//dtd internet explorer 2.0 html strict//", + "-//microsoft//dtd internet explorer 2.0 html//", + "-//microsoft//dtd internet explorer 2.0 tables//", + "-//microsoft//dtd internet explorer 3.0 html strict//", + "-//microsoft//dtd internet explorer 3.0 html//", + "-//microsoft//dtd internet explorer 3.0 tables//", + "-//netscape comm. corp.//dtd html//", + "-//netscape comm. corp.//dtd strict html//", + "-//o'reilly and associates//dtd html 2.0//", + "-//o'reilly and associates//dtd html extended 1.0//", + "-//o'reilly and associates//dtd html extended relaxed 1.0//", + "-//softquad software//dtd hotmetal pro 6.0::19990601::extensions to html 4.0//", + "-//softquad//dtd hotmetal pro 4.0::19971010::extensions to html 4.0//", + "-//spyglass//dtd html 2.0 extended//", + "-//sq//dtd html 2.0 hotmetal + extensions//", + "-//sun microsystems corp.//dtd hotjava html//", + "-//sun microsystems corp.//dtd hotjava strict html//", + "-//w3c//dtd html 3 1995-03-24//", + "-//w3c//dtd html 3.2 draft//", + "-//w3c//dtd html 3.2 final//", + "-//w3c//dtd html 3.2//", + "-//w3c//dtd html 3.2s draft//", + "-//w3c//dtd html 4.0 frameset//", + "-//w3c//dtd html 4.0 transitional//", + "-//w3c//dtd html experimental 19960712//", + "-//w3c//dtd html experimental 970421//", + "-//w3c//dtd w3 html//", + "-//w3o//dtd w3 html 3.0//", + "-//webtechs//dtd mozilla html 2.0//", + "-//webtechs//dtd mozilla html//", +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/entity.go b/Godeps/_workspace/src/golang.org/x/net/html/entity.go new file mode 100644 index 00000000000..a50c04c60e9 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/entity.go @@ -0,0 +1,2253 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +// All entities that do not end with ';' are 6 or fewer bytes long. +const longestEntityWithoutSemicolon = 6 + +// entity is a map from HTML entity names to their values. The semicolon matters: +// https://html.spec.whatwg.org/multipage/syntax.html#named-character-references +// lists both "amp" and "amp;" as two separate entries. +// +// Note that the HTML5 list is larger than the HTML4 list at +// http://www.w3.org/TR/html4/sgml/entities.html +var entity = map[string]rune{ + "AElig;": '\U000000C6', + "AMP;": '\U00000026', + "Aacute;": '\U000000C1', + "Abreve;": '\U00000102', + "Acirc;": '\U000000C2', + "Acy;": '\U00000410', + "Afr;": '\U0001D504', + "Agrave;": '\U000000C0', + "Alpha;": '\U00000391', + "Amacr;": '\U00000100', + "And;": '\U00002A53', + "Aogon;": '\U00000104', + "Aopf;": '\U0001D538', + "ApplyFunction;": '\U00002061', + "Aring;": '\U000000C5', + "Ascr;": '\U0001D49C', + "Assign;": '\U00002254', + "Atilde;": '\U000000C3', + "Auml;": '\U000000C4', + "Backslash;": '\U00002216', + "Barv;": '\U00002AE7', + "Barwed;": '\U00002306', + "Bcy;": '\U00000411', + "Because;": '\U00002235', + "Bernoullis;": '\U0000212C', + "Beta;": '\U00000392', + "Bfr;": '\U0001D505', + "Bopf;": '\U0001D539', + "Breve;": '\U000002D8', + "Bscr;": '\U0000212C', + "Bumpeq;": '\U0000224E', + "CHcy;": '\U00000427', + "COPY;": '\U000000A9', + "Cacute;": '\U00000106', + "Cap;": '\U000022D2', + "CapitalDifferentialD;": '\U00002145', + "Cayleys;": '\U0000212D', + "Ccaron;": '\U0000010C', + "Ccedil;": '\U000000C7', + "Ccirc;": '\U00000108', + "Cconint;": '\U00002230', + "Cdot;": '\U0000010A', + "Cedilla;": '\U000000B8', + "CenterDot;": '\U000000B7', + "Cfr;": '\U0000212D', + "Chi;": '\U000003A7', + "CircleDot;": '\U00002299', + "CircleMinus;": '\U00002296', + "CirclePlus;": '\U00002295', + "CircleTimes;": '\U00002297', + "ClockwiseContourIntegral;": '\U00002232', + "CloseCurlyDoubleQuote;": '\U0000201D', + "CloseCurlyQuote;": '\U00002019', + "Colon;": '\U00002237', + "Colone;": '\U00002A74', + "Congruent;": '\U00002261', + "Conint;": '\U0000222F', + "ContourIntegral;": '\U0000222E', + "Copf;": '\U00002102', + "Coproduct;": '\U00002210', + "CounterClockwiseContourIntegral;": '\U00002233', + "Cross;": '\U00002A2F', + "Cscr;": '\U0001D49E', + "Cup;": '\U000022D3', + "CupCap;": '\U0000224D', + "DD;": '\U00002145', + "DDotrahd;": '\U00002911', + "DJcy;": '\U00000402', + "DScy;": '\U00000405', + "DZcy;": '\U0000040F', + "Dagger;": '\U00002021', + "Darr;": '\U000021A1', + "Dashv;": '\U00002AE4', + "Dcaron;": '\U0000010E', + "Dcy;": '\U00000414', + "Del;": '\U00002207', + "Delta;": '\U00000394', + "Dfr;": '\U0001D507', + "DiacriticalAcute;": '\U000000B4', + "DiacriticalDot;": '\U000002D9', + "DiacriticalDoubleAcute;": '\U000002DD', + "DiacriticalGrave;": '\U00000060', + "DiacriticalTilde;": '\U000002DC', + "Diamond;": '\U000022C4', + "DifferentialD;": '\U00002146', + "Dopf;": '\U0001D53B', + "Dot;": '\U000000A8', + "DotDot;": '\U000020DC', + "DotEqual;": '\U00002250', + "DoubleContourIntegral;": '\U0000222F', + "DoubleDot;": '\U000000A8', + "DoubleDownArrow;": '\U000021D3', + "DoubleLeftArrow;": '\U000021D0', + "DoubleLeftRightArrow;": '\U000021D4', + "DoubleLeftTee;": '\U00002AE4', + "DoubleLongLeftArrow;": '\U000027F8', + "DoubleLongLeftRightArrow;": '\U000027FA', + "DoubleLongRightArrow;": '\U000027F9', + "DoubleRightArrow;": '\U000021D2', + "DoubleRightTee;": '\U000022A8', + "DoubleUpArrow;": '\U000021D1', + "DoubleUpDownArrow;": '\U000021D5', + "DoubleVerticalBar;": '\U00002225', + "DownArrow;": '\U00002193', + "DownArrowBar;": '\U00002913', + "DownArrowUpArrow;": '\U000021F5', + "DownBreve;": '\U00000311', + "DownLeftRightVector;": '\U00002950', + "DownLeftTeeVector;": '\U0000295E', + "DownLeftVector;": '\U000021BD', + "DownLeftVectorBar;": '\U00002956', + "DownRightTeeVector;": '\U0000295F', + "DownRightVector;": '\U000021C1', + "DownRightVectorBar;": '\U00002957', + "DownTee;": '\U000022A4', + "DownTeeArrow;": '\U000021A7', + "Downarrow;": '\U000021D3', + "Dscr;": '\U0001D49F', + "Dstrok;": '\U00000110', + "ENG;": '\U0000014A', + "ETH;": '\U000000D0', + "Eacute;": '\U000000C9', + "Ecaron;": '\U0000011A', + "Ecirc;": '\U000000CA', + "Ecy;": '\U0000042D', + "Edot;": '\U00000116', + "Efr;": '\U0001D508', + "Egrave;": '\U000000C8', + "Element;": '\U00002208', + "Emacr;": '\U00000112', + "EmptySmallSquare;": '\U000025FB', + "EmptyVerySmallSquare;": '\U000025AB', + "Eogon;": '\U00000118', + "Eopf;": '\U0001D53C', + "Epsilon;": '\U00000395', + "Equal;": '\U00002A75', + "EqualTilde;": '\U00002242', + "Equilibrium;": '\U000021CC', + "Escr;": '\U00002130', + "Esim;": '\U00002A73', + "Eta;": '\U00000397', + "Euml;": '\U000000CB', + "Exists;": '\U00002203', + "ExponentialE;": '\U00002147', + "Fcy;": '\U00000424', + "Ffr;": '\U0001D509', + "FilledSmallSquare;": '\U000025FC', + "FilledVerySmallSquare;": '\U000025AA', + "Fopf;": '\U0001D53D', + "ForAll;": '\U00002200', + "Fouriertrf;": '\U00002131', + "Fscr;": '\U00002131', + "GJcy;": '\U00000403', + "GT;": '\U0000003E', + "Gamma;": '\U00000393', + "Gammad;": '\U000003DC', + "Gbreve;": '\U0000011E', + "Gcedil;": '\U00000122', + "Gcirc;": '\U0000011C', + "Gcy;": '\U00000413', + "Gdot;": '\U00000120', + "Gfr;": '\U0001D50A', + "Gg;": '\U000022D9', + "Gopf;": '\U0001D53E', + "GreaterEqual;": '\U00002265', + "GreaterEqualLess;": '\U000022DB', + "GreaterFullEqual;": '\U00002267', + "GreaterGreater;": '\U00002AA2', + "GreaterLess;": '\U00002277', + "GreaterSlantEqual;": '\U00002A7E', + "GreaterTilde;": '\U00002273', + "Gscr;": '\U0001D4A2', + "Gt;": '\U0000226B', + "HARDcy;": '\U0000042A', + "Hacek;": '\U000002C7', + "Hat;": '\U0000005E', + "Hcirc;": '\U00000124', + "Hfr;": '\U0000210C', + "HilbertSpace;": '\U0000210B', + "Hopf;": '\U0000210D', + "HorizontalLine;": '\U00002500', + "Hscr;": '\U0000210B', + "Hstrok;": '\U00000126', + "HumpDownHump;": '\U0000224E', + "HumpEqual;": '\U0000224F', + "IEcy;": '\U00000415', + "IJlig;": '\U00000132', + "IOcy;": '\U00000401', + "Iacute;": '\U000000CD', + "Icirc;": '\U000000CE', + "Icy;": '\U00000418', + "Idot;": '\U00000130', + "Ifr;": '\U00002111', + "Igrave;": '\U000000CC', + "Im;": '\U00002111', + "Imacr;": '\U0000012A', + "ImaginaryI;": '\U00002148', + "Implies;": '\U000021D2', + "Int;": '\U0000222C', + "Integral;": '\U0000222B', + "Intersection;": '\U000022C2', + "InvisibleComma;": '\U00002063', + "InvisibleTimes;": '\U00002062', + "Iogon;": '\U0000012E', + "Iopf;": '\U0001D540', + "Iota;": '\U00000399', + "Iscr;": '\U00002110', + "Itilde;": '\U00000128', + "Iukcy;": '\U00000406', + "Iuml;": '\U000000CF', + "Jcirc;": '\U00000134', + "Jcy;": '\U00000419', + "Jfr;": '\U0001D50D', + "Jopf;": '\U0001D541', + "Jscr;": '\U0001D4A5', + "Jsercy;": '\U00000408', + "Jukcy;": '\U00000404', + "KHcy;": '\U00000425', + "KJcy;": '\U0000040C', + "Kappa;": '\U0000039A', + "Kcedil;": '\U00000136', + "Kcy;": '\U0000041A', + "Kfr;": '\U0001D50E', + "Kopf;": '\U0001D542', + "Kscr;": '\U0001D4A6', + "LJcy;": '\U00000409', + "LT;": '\U0000003C', + "Lacute;": '\U00000139', + "Lambda;": '\U0000039B', + "Lang;": '\U000027EA', + "Laplacetrf;": '\U00002112', + "Larr;": '\U0000219E', + "Lcaron;": '\U0000013D', + "Lcedil;": '\U0000013B', + "Lcy;": '\U0000041B', + "LeftAngleBracket;": '\U000027E8', + "LeftArrow;": '\U00002190', + "LeftArrowBar;": '\U000021E4', + "LeftArrowRightArrow;": '\U000021C6', + "LeftCeiling;": '\U00002308', + "LeftDoubleBracket;": '\U000027E6', + "LeftDownTeeVector;": '\U00002961', + "LeftDownVector;": '\U000021C3', + "LeftDownVectorBar;": '\U00002959', + "LeftFloor;": '\U0000230A', + "LeftRightArrow;": '\U00002194', + "LeftRightVector;": '\U0000294E', + "LeftTee;": '\U000022A3', + "LeftTeeArrow;": '\U000021A4', + "LeftTeeVector;": '\U0000295A', + "LeftTriangle;": '\U000022B2', + "LeftTriangleBar;": '\U000029CF', + "LeftTriangleEqual;": '\U000022B4', + "LeftUpDownVector;": '\U00002951', + "LeftUpTeeVector;": '\U00002960', + "LeftUpVector;": '\U000021BF', + "LeftUpVectorBar;": '\U00002958', + "LeftVector;": '\U000021BC', + "LeftVectorBar;": '\U00002952', + "Leftarrow;": '\U000021D0', + "Leftrightarrow;": '\U000021D4', + "LessEqualGreater;": '\U000022DA', + "LessFullEqual;": '\U00002266', + "LessGreater;": '\U00002276', + "LessLess;": '\U00002AA1', + "LessSlantEqual;": '\U00002A7D', + "LessTilde;": '\U00002272', + "Lfr;": '\U0001D50F', + "Ll;": '\U000022D8', + "Lleftarrow;": '\U000021DA', + "Lmidot;": '\U0000013F', + "LongLeftArrow;": '\U000027F5', + "LongLeftRightArrow;": '\U000027F7', + "LongRightArrow;": '\U000027F6', + "Longleftarrow;": '\U000027F8', + "Longleftrightarrow;": '\U000027FA', + "Longrightarrow;": '\U000027F9', + "Lopf;": '\U0001D543', + "LowerLeftArrow;": '\U00002199', + "LowerRightArrow;": '\U00002198', + "Lscr;": '\U00002112', + "Lsh;": '\U000021B0', + "Lstrok;": '\U00000141', + "Lt;": '\U0000226A', + "Map;": '\U00002905', + "Mcy;": '\U0000041C', + "MediumSpace;": '\U0000205F', + "Mellintrf;": '\U00002133', + "Mfr;": '\U0001D510', + "MinusPlus;": '\U00002213', + "Mopf;": '\U0001D544', + "Mscr;": '\U00002133', + "Mu;": '\U0000039C', + "NJcy;": '\U0000040A', + "Nacute;": '\U00000143', + "Ncaron;": '\U00000147', + "Ncedil;": '\U00000145', + "Ncy;": '\U0000041D', + "NegativeMediumSpace;": '\U0000200B', + "NegativeThickSpace;": '\U0000200B', + "NegativeThinSpace;": '\U0000200B', + "NegativeVeryThinSpace;": '\U0000200B', + "NestedGreaterGreater;": '\U0000226B', + "NestedLessLess;": '\U0000226A', + "NewLine;": '\U0000000A', + "Nfr;": '\U0001D511', + "NoBreak;": '\U00002060', + "NonBreakingSpace;": '\U000000A0', + "Nopf;": '\U00002115', + "Not;": '\U00002AEC', + "NotCongruent;": '\U00002262', + "NotCupCap;": '\U0000226D', + "NotDoubleVerticalBar;": '\U00002226', + "NotElement;": '\U00002209', + "NotEqual;": '\U00002260', + "NotExists;": '\U00002204', + "NotGreater;": '\U0000226F', + "NotGreaterEqual;": '\U00002271', + "NotGreaterLess;": '\U00002279', + "NotGreaterTilde;": '\U00002275', + "NotLeftTriangle;": '\U000022EA', + "NotLeftTriangleEqual;": '\U000022EC', + "NotLess;": '\U0000226E', + "NotLessEqual;": '\U00002270', + "NotLessGreater;": '\U00002278', + "NotLessTilde;": '\U00002274', + "NotPrecedes;": '\U00002280', + "NotPrecedesSlantEqual;": '\U000022E0', + "NotReverseElement;": '\U0000220C', + "NotRightTriangle;": '\U000022EB', + "NotRightTriangleEqual;": '\U000022ED', + "NotSquareSubsetEqual;": '\U000022E2', + "NotSquareSupersetEqual;": '\U000022E3', + "NotSubsetEqual;": '\U00002288', + "NotSucceeds;": '\U00002281', + "NotSucceedsSlantEqual;": '\U000022E1', + "NotSupersetEqual;": '\U00002289', + "NotTilde;": '\U00002241', + "NotTildeEqual;": '\U00002244', + "NotTildeFullEqual;": '\U00002247', + "NotTildeTilde;": '\U00002249', + "NotVerticalBar;": '\U00002224', + "Nscr;": '\U0001D4A9', + "Ntilde;": '\U000000D1', + "Nu;": '\U0000039D', + "OElig;": '\U00000152', + "Oacute;": '\U000000D3', + "Ocirc;": '\U000000D4', + "Ocy;": '\U0000041E', + "Odblac;": '\U00000150', + "Ofr;": '\U0001D512', + "Ograve;": '\U000000D2', + "Omacr;": '\U0000014C', + "Omega;": '\U000003A9', + "Omicron;": '\U0000039F', + "Oopf;": '\U0001D546', + "OpenCurlyDoubleQuote;": '\U0000201C', + "OpenCurlyQuote;": '\U00002018', + "Or;": '\U00002A54', + "Oscr;": '\U0001D4AA', + "Oslash;": '\U000000D8', + "Otilde;": '\U000000D5', + "Otimes;": '\U00002A37', + "Ouml;": '\U000000D6', + "OverBar;": '\U0000203E', + "OverBrace;": '\U000023DE', + "OverBracket;": '\U000023B4', + "OverParenthesis;": '\U000023DC', + "PartialD;": '\U00002202', + "Pcy;": '\U0000041F', + "Pfr;": '\U0001D513', + "Phi;": '\U000003A6', + "Pi;": '\U000003A0', + "PlusMinus;": '\U000000B1', + "Poincareplane;": '\U0000210C', + "Popf;": '\U00002119', + "Pr;": '\U00002ABB', + "Precedes;": '\U0000227A', + "PrecedesEqual;": '\U00002AAF', + "PrecedesSlantEqual;": '\U0000227C', + "PrecedesTilde;": '\U0000227E', + "Prime;": '\U00002033', + "Product;": '\U0000220F', + "Proportion;": '\U00002237', + "Proportional;": '\U0000221D', + "Pscr;": '\U0001D4AB', + "Psi;": '\U000003A8', + "QUOT;": '\U00000022', + "Qfr;": '\U0001D514', + "Qopf;": '\U0000211A', + "Qscr;": '\U0001D4AC', + "RBarr;": '\U00002910', + "REG;": '\U000000AE', + "Racute;": '\U00000154', + "Rang;": '\U000027EB', + "Rarr;": '\U000021A0', + "Rarrtl;": '\U00002916', + "Rcaron;": '\U00000158', + "Rcedil;": '\U00000156', + "Rcy;": '\U00000420', + "Re;": '\U0000211C', + "ReverseElement;": '\U0000220B', + "ReverseEquilibrium;": '\U000021CB', + "ReverseUpEquilibrium;": '\U0000296F', + "Rfr;": '\U0000211C', + "Rho;": '\U000003A1', + "RightAngleBracket;": '\U000027E9', + "RightArrow;": '\U00002192', + "RightArrowBar;": '\U000021E5', + "RightArrowLeftArrow;": '\U000021C4', + "RightCeiling;": '\U00002309', + "RightDoubleBracket;": '\U000027E7', + "RightDownTeeVector;": '\U0000295D', + "RightDownVector;": '\U000021C2', + "RightDownVectorBar;": '\U00002955', + "RightFloor;": '\U0000230B', + "RightTee;": '\U000022A2', + "RightTeeArrow;": '\U000021A6', + "RightTeeVector;": '\U0000295B', + "RightTriangle;": '\U000022B3', + "RightTriangleBar;": '\U000029D0', + "RightTriangleEqual;": '\U000022B5', + "RightUpDownVector;": '\U0000294F', + "RightUpTeeVector;": '\U0000295C', + "RightUpVector;": '\U000021BE', + "RightUpVectorBar;": '\U00002954', + "RightVector;": '\U000021C0', + "RightVectorBar;": '\U00002953', + "Rightarrow;": '\U000021D2', + "Ropf;": '\U0000211D', + "RoundImplies;": '\U00002970', + "Rrightarrow;": '\U000021DB', + "Rscr;": '\U0000211B', + "Rsh;": '\U000021B1', + "RuleDelayed;": '\U000029F4', + "SHCHcy;": '\U00000429', + "SHcy;": '\U00000428', + "SOFTcy;": '\U0000042C', + "Sacute;": '\U0000015A', + "Sc;": '\U00002ABC', + "Scaron;": '\U00000160', + "Scedil;": '\U0000015E', + "Scirc;": '\U0000015C', + "Scy;": '\U00000421', + "Sfr;": '\U0001D516', + "ShortDownArrow;": '\U00002193', + "ShortLeftArrow;": '\U00002190', + "ShortRightArrow;": '\U00002192', + "ShortUpArrow;": '\U00002191', + "Sigma;": '\U000003A3', + "SmallCircle;": '\U00002218', + "Sopf;": '\U0001D54A', + "Sqrt;": '\U0000221A', + "Square;": '\U000025A1', + "SquareIntersection;": '\U00002293', + "SquareSubset;": '\U0000228F', + "SquareSubsetEqual;": '\U00002291', + "SquareSuperset;": '\U00002290', + "SquareSupersetEqual;": '\U00002292', + "SquareUnion;": '\U00002294', + "Sscr;": '\U0001D4AE', + "Star;": '\U000022C6', + "Sub;": '\U000022D0', + "Subset;": '\U000022D0', + "SubsetEqual;": '\U00002286', + "Succeeds;": '\U0000227B', + "SucceedsEqual;": '\U00002AB0', + "SucceedsSlantEqual;": '\U0000227D', + "SucceedsTilde;": '\U0000227F', + "SuchThat;": '\U0000220B', + "Sum;": '\U00002211', + "Sup;": '\U000022D1', + "Superset;": '\U00002283', + "SupersetEqual;": '\U00002287', + "Supset;": '\U000022D1', + "THORN;": '\U000000DE', + "TRADE;": '\U00002122', + "TSHcy;": '\U0000040B', + "TScy;": '\U00000426', + "Tab;": '\U00000009', + "Tau;": '\U000003A4', + "Tcaron;": '\U00000164', + "Tcedil;": '\U00000162', + "Tcy;": '\U00000422', + "Tfr;": '\U0001D517', + "Therefore;": '\U00002234', + "Theta;": '\U00000398', + "ThinSpace;": '\U00002009', + "Tilde;": '\U0000223C', + "TildeEqual;": '\U00002243', + "TildeFullEqual;": '\U00002245', + "TildeTilde;": '\U00002248', + "Topf;": '\U0001D54B', + "TripleDot;": '\U000020DB', + "Tscr;": '\U0001D4AF', + "Tstrok;": '\U00000166', + "Uacute;": '\U000000DA', + "Uarr;": '\U0000219F', + "Uarrocir;": '\U00002949', + "Ubrcy;": '\U0000040E', + "Ubreve;": '\U0000016C', + "Ucirc;": '\U000000DB', + "Ucy;": '\U00000423', + "Udblac;": '\U00000170', + "Ufr;": '\U0001D518', + "Ugrave;": '\U000000D9', + "Umacr;": '\U0000016A', + "UnderBar;": '\U0000005F', + "UnderBrace;": '\U000023DF', + "UnderBracket;": '\U000023B5', + "UnderParenthesis;": '\U000023DD', + "Union;": '\U000022C3', + "UnionPlus;": '\U0000228E', + "Uogon;": '\U00000172', + "Uopf;": '\U0001D54C', + "UpArrow;": '\U00002191', + "UpArrowBar;": '\U00002912', + "UpArrowDownArrow;": '\U000021C5', + "UpDownArrow;": '\U00002195', + "UpEquilibrium;": '\U0000296E', + "UpTee;": '\U000022A5', + "UpTeeArrow;": '\U000021A5', + "Uparrow;": '\U000021D1', + "Updownarrow;": '\U000021D5', + "UpperLeftArrow;": '\U00002196', + "UpperRightArrow;": '\U00002197', + "Upsi;": '\U000003D2', + "Upsilon;": '\U000003A5', + "Uring;": '\U0000016E', + "Uscr;": '\U0001D4B0', + "Utilde;": '\U00000168', + "Uuml;": '\U000000DC', + "VDash;": '\U000022AB', + "Vbar;": '\U00002AEB', + "Vcy;": '\U00000412', + "Vdash;": '\U000022A9', + "Vdashl;": '\U00002AE6', + "Vee;": '\U000022C1', + "Verbar;": '\U00002016', + "Vert;": '\U00002016', + "VerticalBar;": '\U00002223', + "VerticalLine;": '\U0000007C', + "VerticalSeparator;": '\U00002758', + "VerticalTilde;": '\U00002240', + "VeryThinSpace;": '\U0000200A', + "Vfr;": '\U0001D519', + "Vopf;": '\U0001D54D', + "Vscr;": '\U0001D4B1', + "Vvdash;": '\U000022AA', + "Wcirc;": '\U00000174', + "Wedge;": '\U000022C0', + "Wfr;": '\U0001D51A', + "Wopf;": '\U0001D54E', + "Wscr;": '\U0001D4B2', + "Xfr;": '\U0001D51B', + "Xi;": '\U0000039E', + "Xopf;": '\U0001D54F', + "Xscr;": '\U0001D4B3', + "YAcy;": '\U0000042F', + "YIcy;": '\U00000407', + "YUcy;": '\U0000042E', + "Yacute;": '\U000000DD', + "Ycirc;": '\U00000176', + "Ycy;": '\U0000042B', + "Yfr;": '\U0001D51C', + "Yopf;": '\U0001D550', + "Yscr;": '\U0001D4B4', + "Yuml;": '\U00000178', + "ZHcy;": '\U00000416', + "Zacute;": '\U00000179', + "Zcaron;": '\U0000017D', + "Zcy;": '\U00000417', + "Zdot;": '\U0000017B', + "ZeroWidthSpace;": '\U0000200B', + "Zeta;": '\U00000396', + "Zfr;": '\U00002128', + "Zopf;": '\U00002124', + "Zscr;": '\U0001D4B5', + "aacute;": '\U000000E1', + "abreve;": '\U00000103', + "ac;": '\U0000223E', + "acd;": '\U0000223F', + "acirc;": '\U000000E2', + "acute;": '\U000000B4', + "acy;": '\U00000430', + "aelig;": '\U000000E6', + "af;": '\U00002061', + "afr;": '\U0001D51E', + "agrave;": '\U000000E0', + "alefsym;": '\U00002135', + "aleph;": '\U00002135', + "alpha;": '\U000003B1', + "amacr;": '\U00000101', + "amalg;": '\U00002A3F', + "amp;": '\U00000026', + "and;": '\U00002227', + "andand;": '\U00002A55', + "andd;": '\U00002A5C', + "andslope;": '\U00002A58', + "andv;": '\U00002A5A', + "ang;": '\U00002220', + "ange;": '\U000029A4', + "angle;": '\U00002220', + "angmsd;": '\U00002221', + "angmsdaa;": '\U000029A8', + "angmsdab;": '\U000029A9', + "angmsdac;": '\U000029AA', + "angmsdad;": '\U000029AB', + "angmsdae;": '\U000029AC', + "angmsdaf;": '\U000029AD', + "angmsdag;": '\U000029AE', + "angmsdah;": '\U000029AF', + "angrt;": '\U0000221F', + "angrtvb;": '\U000022BE', + "angrtvbd;": '\U0000299D', + "angsph;": '\U00002222', + "angst;": '\U000000C5', + "angzarr;": '\U0000237C', + "aogon;": '\U00000105', + "aopf;": '\U0001D552', + "ap;": '\U00002248', + "apE;": '\U00002A70', + "apacir;": '\U00002A6F', + "ape;": '\U0000224A', + "apid;": '\U0000224B', + "apos;": '\U00000027', + "approx;": '\U00002248', + "approxeq;": '\U0000224A', + "aring;": '\U000000E5', + "ascr;": '\U0001D4B6', + "ast;": '\U0000002A', + "asymp;": '\U00002248', + "asympeq;": '\U0000224D', + "atilde;": '\U000000E3', + "auml;": '\U000000E4', + "awconint;": '\U00002233', + "awint;": '\U00002A11', + "bNot;": '\U00002AED', + "backcong;": '\U0000224C', + "backepsilon;": '\U000003F6', + "backprime;": '\U00002035', + "backsim;": '\U0000223D', + "backsimeq;": '\U000022CD', + "barvee;": '\U000022BD', + "barwed;": '\U00002305', + "barwedge;": '\U00002305', + "bbrk;": '\U000023B5', + "bbrktbrk;": '\U000023B6', + "bcong;": '\U0000224C', + "bcy;": '\U00000431', + "bdquo;": '\U0000201E', + "becaus;": '\U00002235', + "because;": '\U00002235', + "bemptyv;": '\U000029B0', + "bepsi;": '\U000003F6', + "bernou;": '\U0000212C', + "beta;": '\U000003B2', + "beth;": '\U00002136', + "between;": '\U0000226C', + "bfr;": '\U0001D51F', + "bigcap;": '\U000022C2', + "bigcirc;": '\U000025EF', + "bigcup;": '\U000022C3', + "bigodot;": '\U00002A00', + "bigoplus;": '\U00002A01', + "bigotimes;": '\U00002A02', + "bigsqcup;": '\U00002A06', + "bigstar;": '\U00002605', + "bigtriangledown;": '\U000025BD', + "bigtriangleup;": '\U000025B3', + "biguplus;": '\U00002A04', + "bigvee;": '\U000022C1', + "bigwedge;": '\U000022C0', + "bkarow;": '\U0000290D', + "blacklozenge;": '\U000029EB', + "blacksquare;": '\U000025AA', + "blacktriangle;": '\U000025B4', + "blacktriangledown;": '\U000025BE', + "blacktriangleleft;": '\U000025C2', + "blacktriangleright;": '\U000025B8', + "blank;": '\U00002423', + "blk12;": '\U00002592', + "blk14;": '\U00002591', + "blk34;": '\U00002593', + "block;": '\U00002588', + "bnot;": '\U00002310', + "bopf;": '\U0001D553', + "bot;": '\U000022A5', + "bottom;": '\U000022A5', + "bowtie;": '\U000022C8', + "boxDL;": '\U00002557', + "boxDR;": '\U00002554', + "boxDl;": '\U00002556', + "boxDr;": '\U00002553', + "boxH;": '\U00002550', + "boxHD;": '\U00002566', + "boxHU;": '\U00002569', + "boxHd;": '\U00002564', + "boxHu;": '\U00002567', + "boxUL;": '\U0000255D', + "boxUR;": '\U0000255A', + "boxUl;": '\U0000255C', + "boxUr;": '\U00002559', + "boxV;": '\U00002551', + "boxVH;": '\U0000256C', + "boxVL;": '\U00002563', + "boxVR;": '\U00002560', + "boxVh;": '\U0000256B', + "boxVl;": '\U00002562', + "boxVr;": '\U0000255F', + "boxbox;": '\U000029C9', + "boxdL;": '\U00002555', + "boxdR;": '\U00002552', + "boxdl;": '\U00002510', + "boxdr;": '\U0000250C', + "boxh;": '\U00002500', + "boxhD;": '\U00002565', + "boxhU;": '\U00002568', + "boxhd;": '\U0000252C', + "boxhu;": '\U00002534', + "boxminus;": '\U0000229F', + "boxplus;": '\U0000229E', + "boxtimes;": '\U000022A0', + "boxuL;": '\U0000255B', + "boxuR;": '\U00002558', + "boxul;": '\U00002518', + "boxur;": '\U00002514', + "boxv;": '\U00002502', + "boxvH;": '\U0000256A', + "boxvL;": '\U00002561', + "boxvR;": '\U0000255E', + "boxvh;": '\U0000253C', + "boxvl;": '\U00002524', + "boxvr;": '\U0000251C', + "bprime;": '\U00002035', + "breve;": '\U000002D8', + "brvbar;": '\U000000A6', + "bscr;": '\U0001D4B7', + "bsemi;": '\U0000204F', + "bsim;": '\U0000223D', + "bsime;": '\U000022CD', + "bsol;": '\U0000005C', + "bsolb;": '\U000029C5', + "bsolhsub;": '\U000027C8', + "bull;": '\U00002022', + "bullet;": '\U00002022', + "bump;": '\U0000224E', + "bumpE;": '\U00002AAE', + "bumpe;": '\U0000224F', + "bumpeq;": '\U0000224F', + "cacute;": '\U00000107', + "cap;": '\U00002229', + "capand;": '\U00002A44', + "capbrcup;": '\U00002A49', + "capcap;": '\U00002A4B', + "capcup;": '\U00002A47', + "capdot;": '\U00002A40', + "caret;": '\U00002041', + "caron;": '\U000002C7', + "ccaps;": '\U00002A4D', + "ccaron;": '\U0000010D', + "ccedil;": '\U000000E7', + "ccirc;": '\U00000109', + "ccups;": '\U00002A4C', + "ccupssm;": '\U00002A50', + "cdot;": '\U0000010B', + "cedil;": '\U000000B8', + "cemptyv;": '\U000029B2', + "cent;": '\U000000A2', + "centerdot;": '\U000000B7', + "cfr;": '\U0001D520', + "chcy;": '\U00000447', + "check;": '\U00002713', + "checkmark;": '\U00002713', + "chi;": '\U000003C7', + "cir;": '\U000025CB', + "cirE;": '\U000029C3', + "circ;": '\U000002C6', + "circeq;": '\U00002257', + "circlearrowleft;": '\U000021BA', + "circlearrowright;": '\U000021BB', + "circledR;": '\U000000AE', + "circledS;": '\U000024C8', + "circledast;": '\U0000229B', + "circledcirc;": '\U0000229A', + "circleddash;": '\U0000229D', + "cire;": '\U00002257', + "cirfnint;": '\U00002A10', + "cirmid;": '\U00002AEF', + "cirscir;": '\U000029C2', + "clubs;": '\U00002663', + "clubsuit;": '\U00002663', + "colon;": '\U0000003A', + "colone;": '\U00002254', + "coloneq;": '\U00002254', + "comma;": '\U0000002C', + "commat;": '\U00000040', + "comp;": '\U00002201', + "compfn;": '\U00002218', + "complement;": '\U00002201', + "complexes;": '\U00002102', + "cong;": '\U00002245', + "congdot;": '\U00002A6D', + "conint;": '\U0000222E', + "copf;": '\U0001D554', + "coprod;": '\U00002210', + "copy;": '\U000000A9', + "copysr;": '\U00002117', + "crarr;": '\U000021B5', + "cross;": '\U00002717', + "cscr;": '\U0001D4B8', + "csub;": '\U00002ACF', + "csube;": '\U00002AD1', + "csup;": '\U00002AD0', + "csupe;": '\U00002AD2', + "ctdot;": '\U000022EF', + "cudarrl;": '\U00002938', + "cudarrr;": '\U00002935', + "cuepr;": '\U000022DE', + "cuesc;": '\U000022DF', + "cularr;": '\U000021B6', + "cularrp;": '\U0000293D', + "cup;": '\U0000222A', + "cupbrcap;": '\U00002A48', + "cupcap;": '\U00002A46', + "cupcup;": '\U00002A4A', + "cupdot;": '\U0000228D', + "cupor;": '\U00002A45', + "curarr;": '\U000021B7', + "curarrm;": '\U0000293C', + "curlyeqprec;": '\U000022DE', + "curlyeqsucc;": '\U000022DF', + "curlyvee;": '\U000022CE', + "curlywedge;": '\U000022CF', + "curren;": '\U000000A4', + "curvearrowleft;": '\U000021B6', + "curvearrowright;": '\U000021B7', + "cuvee;": '\U000022CE', + "cuwed;": '\U000022CF', + "cwconint;": '\U00002232', + "cwint;": '\U00002231', + "cylcty;": '\U0000232D', + "dArr;": '\U000021D3', + "dHar;": '\U00002965', + "dagger;": '\U00002020', + "daleth;": '\U00002138', + "darr;": '\U00002193', + "dash;": '\U00002010', + "dashv;": '\U000022A3', + "dbkarow;": '\U0000290F', + "dblac;": '\U000002DD', + "dcaron;": '\U0000010F', + "dcy;": '\U00000434', + "dd;": '\U00002146', + "ddagger;": '\U00002021', + "ddarr;": '\U000021CA', + "ddotseq;": '\U00002A77', + "deg;": '\U000000B0', + "delta;": '\U000003B4', + "demptyv;": '\U000029B1', + "dfisht;": '\U0000297F', + "dfr;": '\U0001D521', + "dharl;": '\U000021C3', + "dharr;": '\U000021C2', + "diam;": '\U000022C4', + "diamond;": '\U000022C4', + "diamondsuit;": '\U00002666', + "diams;": '\U00002666', + "die;": '\U000000A8', + "digamma;": '\U000003DD', + "disin;": '\U000022F2', + "div;": '\U000000F7', + "divide;": '\U000000F7', + "divideontimes;": '\U000022C7', + "divonx;": '\U000022C7', + "djcy;": '\U00000452', + "dlcorn;": '\U0000231E', + "dlcrop;": '\U0000230D', + "dollar;": '\U00000024', + "dopf;": '\U0001D555', + "dot;": '\U000002D9', + "doteq;": '\U00002250', + "doteqdot;": '\U00002251', + "dotminus;": '\U00002238', + "dotplus;": '\U00002214', + "dotsquare;": '\U000022A1', + "doublebarwedge;": '\U00002306', + "downarrow;": '\U00002193', + "downdownarrows;": '\U000021CA', + "downharpoonleft;": '\U000021C3', + "downharpoonright;": '\U000021C2', + "drbkarow;": '\U00002910', + "drcorn;": '\U0000231F', + "drcrop;": '\U0000230C', + "dscr;": '\U0001D4B9', + "dscy;": '\U00000455', + "dsol;": '\U000029F6', + "dstrok;": '\U00000111', + "dtdot;": '\U000022F1', + "dtri;": '\U000025BF', + "dtrif;": '\U000025BE', + "duarr;": '\U000021F5', + "duhar;": '\U0000296F', + "dwangle;": '\U000029A6', + "dzcy;": '\U0000045F', + "dzigrarr;": '\U000027FF', + "eDDot;": '\U00002A77', + "eDot;": '\U00002251', + "eacute;": '\U000000E9', + "easter;": '\U00002A6E', + "ecaron;": '\U0000011B', + "ecir;": '\U00002256', + "ecirc;": '\U000000EA', + "ecolon;": '\U00002255', + "ecy;": '\U0000044D', + "edot;": '\U00000117', + "ee;": '\U00002147', + "efDot;": '\U00002252', + "efr;": '\U0001D522', + "eg;": '\U00002A9A', + "egrave;": '\U000000E8', + "egs;": '\U00002A96', + "egsdot;": '\U00002A98', + "el;": '\U00002A99', + "elinters;": '\U000023E7', + "ell;": '\U00002113', + "els;": '\U00002A95', + "elsdot;": '\U00002A97', + "emacr;": '\U00000113', + "empty;": '\U00002205', + "emptyset;": '\U00002205', + "emptyv;": '\U00002205', + "emsp;": '\U00002003', + "emsp13;": '\U00002004', + "emsp14;": '\U00002005', + "eng;": '\U0000014B', + "ensp;": '\U00002002', + "eogon;": '\U00000119', + "eopf;": '\U0001D556', + "epar;": '\U000022D5', + "eparsl;": '\U000029E3', + "eplus;": '\U00002A71', + "epsi;": '\U000003B5', + "epsilon;": '\U000003B5', + "epsiv;": '\U000003F5', + "eqcirc;": '\U00002256', + "eqcolon;": '\U00002255', + "eqsim;": '\U00002242', + "eqslantgtr;": '\U00002A96', + "eqslantless;": '\U00002A95', + "equals;": '\U0000003D', + "equest;": '\U0000225F', + "equiv;": '\U00002261', + "equivDD;": '\U00002A78', + "eqvparsl;": '\U000029E5', + "erDot;": '\U00002253', + "erarr;": '\U00002971', + "escr;": '\U0000212F', + "esdot;": '\U00002250', + "esim;": '\U00002242', + "eta;": '\U000003B7', + "eth;": '\U000000F0', + "euml;": '\U000000EB', + "euro;": '\U000020AC', + "excl;": '\U00000021', + "exist;": '\U00002203', + "expectation;": '\U00002130', + "exponentiale;": '\U00002147', + "fallingdotseq;": '\U00002252', + "fcy;": '\U00000444', + "female;": '\U00002640', + "ffilig;": '\U0000FB03', + "fflig;": '\U0000FB00', + "ffllig;": '\U0000FB04', + "ffr;": '\U0001D523', + "filig;": '\U0000FB01', + "flat;": '\U0000266D', + "fllig;": '\U0000FB02', + "fltns;": '\U000025B1', + "fnof;": '\U00000192', + "fopf;": '\U0001D557', + "forall;": '\U00002200', + "fork;": '\U000022D4', + "forkv;": '\U00002AD9', + "fpartint;": '\U00002A0D', + "frac12;": '\U000000BD', + "frac13;": '\U00002153', + "frac14;": '\U000000BC', + "frac15;": '\U00002155', + "frac16;": '\U00002159', + "frac18;": '\U0000215B', + "frac23;": '\U00002154', + "frac25;": '\U00002156', + "frac34;": '\U000000BE', + "frac35;": '\U00002157', + "frac38;": '\U0000215C', + "frac45;": '\U00002158', + "frac56;": '\U0000215A', + "frac58;": '\U0000215D', + "frac78;": '\U0000215E', + "frasl;": '\U00002044', + "frown;": '\U00002322', + "fscr;": '\U0001D4BB', + "gE;": '\U00002267', + "gEl;": '\U00002A8C', + "gacute;": '\U000001F5', + "gamma;": '\U000003B3', + "gammad;": '\U000003DD', + "gap;": '\U00002A86', + "gbreve;": '\U0000011F', + "gcirc;": '\U0000011D', + "gcy;": '\U00000433', + "gdot;": '\U00000121', + "ge;": '\U00002265', + "gel;": '\U000022DB', + "geq;": '\U00002265', + "geqq;": '\U00002267', + "geqslant;": '\U00002A7E', + "ges;": '\U00002A7E', + "gescc;": '\U00002AA9', + "gesdot;": '\U00002A80', + "gesdoto;": '\U00002A82', + "gesdotol;": '\U00002A84', + "gesles;": '\U00002A94', + "gfr;": '\U0001D524', + "gg;": '\U0000226B', + "ggg;": '\U000022D9', + "gimel;": '\U00002137', + "gjcy;": '\U00000453', + "gl;": '\U00002277', + "glE;": '\U00002A92', + "gla;": '\U00002AA5', + "glj;": '\U00002AA4', + "gnE;": '\U00002269', + "gnap;": '\U00002A8A', + "gnapprox;": '\U00002A8A', + "gne;": '\U00002A88', + "gneq;": '\U00002A88', + "gneqq;": '\U00002269', + "gnsim;": '\U000022E7', + "gopf;": '\U0001D558', + "grave;": '\U00000060', + "gscr;": '\U0000210A', + "gsim;": '\U00002273', + "gsime;": '\U00002A8E', + "gsiml;": '\U00002A90', + "gt;": '\U0000003E', + "gtcc;": '\U00002AA7', + "gtcir;": '\U00002A7A', + "gtdot;": '\U000022D7', + "gtlPar;": '\U00002995', + "gtquest;": '\U00002A7C', + "gtrapprox;": '\U00002A86', + "gtrarr;": '\U00002978', + "gtrdot;": '\U000022D7', + "gtreqless;": '\U000022DB', + "gtreqqless;": '\U00002A8C', + "gtrless;": '\U00002277', + "gtrsim;": '\U00002273', + "hArr;": '\U000021D4', + "hairsp;": '\U0000200A', + "half;": '\U000000BD', + "hamilt;": '\U0000210B', + "hardcy;": '\U0000044A', + "harr;": '\U00002194', + "harrcir;": '\U00002948', + "harrw;": '\U000021AD', + "hbar;": '\U0000210F', + "hcirc;": '\U00000125', + "hearts;": '\U00002665', + "heartsuit;": '\U00002665', + "hellip;": '\U00002026', + "hercon;": '\U000022B9', + "hfr;": '\U0001D525', + "hksearow;": '\U00002925', + "hkswarow;": '\U00002926', + "hoarr;": '\U000021FF', + "homtht;": '\U0000223B', + "hookleftarrow;": '\U000021A9', + "hookrightarrow;": '\U000021AA', + "hopf;": '\U0001D559', + "horbar;": '\U00002015', + "hscr;": '\U0001D4BD', + "hslash;": '\U0000210F', + "hstrok;": '\U00000127', + "hybull;": '\U00002043', + "hyphen;": '\U00002010', + "iacute;": '\U000000ED', + "ic;": '\U00002063', + "icirc;": '\U000000EE', + "icy;": '\U00000438', + "iecy;": '\U00000435', + "iexcl;": '\U000000A1', + "iff;": '\U000021D4', + "ifr;": '\U0001D526', + "igrave;": '\U000000EC', + "ii;": '\U00002148', + "iiiint;": '\U00002A0C', + "iiint;": '\U0000222D', + "iinfin;": '\U000029DC', + "iiota;": '\U00002129', + "ijlig;": '\U00000133', + "imacr;": '\U0000012B', + "image;": '\U00002111', + "imagline;": '\U00002110', + "imagpart;": '\U00002111', + "imath;": '\U00000131', + "imof;": '\U000022B7', + "imped;": '\U000001B5', + "in;": '\U00002208', + "incare;": '\U00002105', + "infin;": '\U0000221E', + "infintie;": '\U000029DD', + "inodot;": '\U00000131', + "int;": '\U0000222B', + "intcal;": '\U000022BA', + "integers;": '\U00002124', + "intercal;": '\U000022BA', + "intlarhk;": '\U00002A17', + "intprod;": '\U00002A3C', + "iocy;": '\U00000451', + "iogon;": '\U0000012F', + "iopf;": '\U0001D55A', + "iota;": '\U000003B9', + "iprod;": '\U00002A3C', + "iquest;": '\U000000BF', + "iscr;": '\U0001D4BE', + "isin;": '\U00002208', + "isinE;": '\U000022F9', + "isindot;": '\U000022F5', + "isins;": '\U000022F4', + "isinsv;": '\U000022F3', + "isinv;": '\U00002208', + "it;": '\U00002062', + "itilde;": '\U00000129', + "iukcy;": '\U00000456', + "iuml;": '\U000000EF', + "jcirc;": '\U00000135', + "jcy;": '\U00000439', + "jfr;": '\U0001D527', + "jmath;": '\U00000237', + "jopf;": '\U0001D55B', + "jscr;": '\U0001D4BF', + "jsercy;": '\U00000458', + "jukcy;": '\U00000454', + "kappa;": '\U000003BA', + "kappav;": '\U000003F0', + "kcedil;": '\U00000137', + "kcy;": '\U0000043A', + "kfr;": '\U0001D528', + "kgreen;": '\U00000138', + "khcy;": '\U00000445', + "kjcy;": '\U0000045C', + "kopf;": '\U0001D55C', + "kscr;": '\U0001D4C0', + "lAarr;": '\U000021DA', + "lArr;": '\U000021D0', + "lAtail;": '\U0000291B', + "lBarr;": '\U0000290E', + "lE;": '\U00002266', + "lEg;": '\U00002A8B', + "lHar;": '\U00002962', + "lacute;": '\U0000013A', + "laemptyv;": '\U000029B4', + "lagran;": '\U00002112', + "lambda;": '\U000003BB', + "lang;": '\U000027E8', + "langd;": '\U00002991', + "langle;": '\U000027E8', + "lap;": '\U00002A85', + "laquo;": '\U000000AB', + "larr;": '\U00002190', + "larrb;": '\U000021E4', + "larrbfs;": '\U0000291F', + "larrfs;": '\U0000291D', + "larrhk;": '\U000021A9', + "larrlp;": '\U000021AB', + "larrpl;": '\U00002939', + "larrsim;": '\U00002973', + "larrtl;": '\U000021A2', + "lat;": '\U00002AAB', + "latail;": '\U00002919', + "late;": '\U00002AAD', + "lbarr;": '\U0000290C', + "lbbrk;": '\U00002772', + "lbrace;": '\U0000007B', + "lbrack;": '\U0000005B', + "lbrke;": '\U0000298B', + "lbrksld;": '\U0000298F', + "lbrkslu;": '\U0000298D', + "lcaron;": '\U0000013E', + "lcedil;": '\U0000013C', + "lceil;": '\U00002308', + "lcub;": '\U0000007B', + "lcy;": '\U0000043B', + "ldca;": '\U00002936', + "ldquo;": '\U0000201C', + "ldquor;": '\U0000201E', + "ldrdhar;": '\U00002967', + "ldrushar;": '\U0000294B', + "ldsh;": '\U000021B2', + "le;": '\U00002264', + "leftarrow;": '\U00002190', + "leftarrowtail;": '\U000021A2', + "leftharpoondown;": '\U000021BD', + "leftharpoonup;": '\U000021BC', + "leftleftarrows;": '\U000021C7', + "leftrightarrow;": '\U00002194', + "leftrightarrows;": '\U000021C6', + "leftrightharpoons;": '\U000021CB', + "leftrightsquigarrow;": '\U000021AD', + "leftthreetimes;": '\U000022CB', + "leg;": '\U000022DA', + "leq;": '\U00002264', + "leqq;": '\U00002266', + "leqslant;": '\U00002A7D', + "les;": '\U00002A7D', + "lescc;": '\U00002AA8', + "lesdot;": '\U00002A7F', + "lesdoto;": '\U00002A81', + "lesdotor;": '\U00002A83', + "lesges;": '\U00002A93', + "lessapprox;": '\U00002A85', + "lessdot;": '\U000022D6', + "lesseqgtr;": '\U000022DA', + "lesseqqgtr;": '\U00002A8B', + "lessgtr;": '\U00002276', + "lesssim;": '\U00002272', + "lfisht;": '\U0000297C', + "lfloor;": '\U0000230A', + "lfr;": '\U0001D529', + "lg;": '\U00002276', + "lgE;": '\U00002A91', + "lhard;": '\U000021BD', + "lharu;": '\U000021BC', + "lharul;": '\U0000296A', + "lhblk;": '\U00002584', + "ljcy;": '\U00000459', + "ll;": '\U0000226A', + "llarr;": '\U000021C7', + "llcorner;": '\U0000231E', + "llhard;": '\U0000296B', + "lltri;": '\U000025FA', + "lmidot;": '\U00000140', + "lmoust;": '\U000023B0', + "lmoustache;": '\U000023B0', + "lnE;": '\U00002268', + "lnap;": '\U00002A89', + "lnapprox;": '\U00002A89', + "lne;": '\U00002A87', + "lneq;": '\U00002A87', + "lneqq;": '\U00002268', + "lnsim;": '\U000022E6', + "loang;": '\U000027EC', + "loarr;": '\U000021FD', + "lobrk;": '\U000027E6', + "longleftarrow;": '\U000027F5', + "longleftrightarrow;": '\U000027F7', + "longmapsto;": '\U000027FC', + "longrightarrow;": '\U000027F6', + "looparrowleft;": '\U000021AB', + "looparrowright;": '\U000021AC', + "lopar;": '\U00002985', + "lopf;": '\U0001D55D', + "loplus;": '\U00002A2D', + "lotimes;": '\U00002A34', + "lowast;": '\U00002217', + "lowbar;": '\U0000005F', + "loz;": '\U000025CA', + "lozenge;": '\U000025CA', + "lozf;": '\U000029EB', + "lpar;": '\U00000028', + "lparlt;": '\U00002993', + "lrarr;": '\U000021C6', + "lrcorner;": '\U0000231F', + "lrhar;": '\U000021CB', + "lrhard;": '\U0000296D', + "lrm;": '\U0000200E', + "lrtri;": '\U000022BF', + "lsaquo;": '\U00002039', + "lscr;": '\U0001D4C1', + "lsh;": '\U000021B0', + "lsim;": '\U00002272', + "lsime;": '\U00002A8D', + "lsimg;": '\U00002A8F', + "lsqb;": '\U0000005B', + "lsquo;": '\U00002018', + "lsquor;": '\U0000201A', + "lstrok;": '\U00000142', + "lt;": '\U0000003C', + "ltcc;": '\U00002AA6', + "ltcir;": '\U00002A79', + "ltdot;": '\U000022D6', + "lthree;": '\U000022CB', + "ltimes;": '\U000022C9', + "ltlarr;": '\U00002976', + "ltquest;": '\U00002A7B', + "ltrPar;": '\U00002996', + "ltri;": '\U000025C3', + "ltrie;": '\U000022B4', + "ltrif;": '\U000025C2', + "lurdshar;": '\U0000294A', + "luruhar;": '\U00002966', + "mDDot;": '\U0000223A', + "macr;": '\U000000AF', + "male;": '\U00002642', + "malt;": '\U00002720', + "maltese;": '\U00002720', + "map;": '\U000021A6', + "mapsto;": '\U000021A6', + "mapstodown;": '\U000021A7', + "mapstoleft;": '\U000021A4', + "mapstoup;": '\U000021A5', + "marker;": '\U000025AE', + "mcomma;": '\U00002A29', + "mcy;": '\U0000043C', + "mdash;": '\U00002014', + "measuredangle;": '\U00002221', + "mfr;": '\U0001D52A', + "mho;": '\U00002127', + "micro;": '\U000000B5', + "mid;": '\U00002223', + "midast;": '\U0000002A', + "midcir;": '\U00002AF0', + "middot;": '\U000000B7', + "minus;": '\U00002212', + "minusb;": '\U0000229F', + "minusd;": '\U00002238', + "minusdu;": '\U00002A2A', + "mlcp;": '\U00002ADB', + "mldr;": '\U00002026', + "mnplus;": '\U00002213', + "models;": '\U000022A7', + "mopf;": '\U0001D55E', + "mp;": '\U00002213', + "mscr;": '\U0001D4C2', + "mstpos;": '\U0000223E', + "mu;": '\U000003BC', + "multimap;": '\U000022B8', + "mumap;": '\U000022B8', + "nLeftarrow;": '\U000021CD', + "nLeftrightarrow;": '\U000021CE', + "nRightarrow;": '\U000021CF', + "nVDash;": '\U000022AF', + "nVdash;": '\U000022AE', + "nabla;": '\U00002207', + "nacute;": '\U00000144', + "nap;": '\U00002249', + "napos;": '\U00000149', + "napprox;": '\U00002249', + "natur;": '\U0000266E', + "natural;": '\U0000266E', + "naturals;": '\U00002115', + "nbsp;": '\U000000A0', + "ncap;": '\U00002A43', + "ncaron;": '\U00000148', + "ncedil;": '\U00000146', + "ncong;": '\U00002247', + "ncup;": '\U00002A42', + "ncy;": '\U0000043D', + "ndash;": '\U00002013', + "ne;": '\U00002260', + "neArr;": '\U000021D7', + "nearhk;": '\U00002924', + "nearr;": '\U00002197', + "nearrow;": '\U00002197', + "nequiv;": '\U00002262', + "nesear;": '\U00002928', + "nexist;": '\U00002204', + "nexists;": '\U00002204', + "nfr;": '\U0001D52B', + "nge;": '\U00002271', + "ngeq;": '\U00002271', + "ngsim;": '\U00002275', + "ngt;": '\U0000226F', + "ngtr;": '\U0000226F', + "nhArr;": '\U000021CE', + "nharr;": '\U000021AE', + "nhpar;": '\U00002AF2', + "ni;": '\U0000220B', + "nis;": '\U000022FC', + "nisd;": '\U000022FA', + "niv;": '\U0000220B', + "njcy;": '\U0000045A', + "nlArr;": '\U000021CD', + "nlarr;": '\U0000219A', + "nldr;": '\U00002025', + "nle;": '\U00002270', + "nleftarrow;": '\U0000219A', + "nleftrightarrow;": '\U000021AE', + "nleq;": '\U00002270', + "nless;": '\U0000226E', + "nlsim;": '\U00002274', + "nlt;": '\U0000226E', + "nltri;": '\U000022EA', + "nltrie;": '\U000022EC', + "nmid;": '\U00002224', + "nopf;": '\U0001D55F', + "not;": '\U000000AC', + "notin;": '\U00002209', + "notinva;": '\U00002209', + "notinvb;": '\U000022F7', + "notinvc;": '\U000022F6', + "notni;": '\U0000220C', + "notniva;": '\U0000220C', + "notnivb;": '\U000022FE', + "notnivc;": '\U000022FD', + "npar;": '\U00002226', + "nparallel;": '\U00002226', + "npolint;": '\U00002A14', + "npr;": '\U00002280', + "nprcue;": '\U000022E0', + "nprec;": '\U00002280', + "nrArr;": '\U000021CF', + "nrarr;": '\U0000219B', + "nrightarrow;": '\U0000219B', + "nrtri;": '\U000022EB', + "nrtrie;": '\U000022ED', + "nsc;": '\U00002281', + "nsccue;": '\U000022E1', + "nscr;": '\U0001D4C3', + "nshortmid;": '\U00002224', + "nshortparallel;": '\U00002226', + "nsim;": '\U00002241', + "nsime;": '\U00002244', + "nsimeq;": '\U00002244', + "nsmid;": '\U00002224', + "nspar;": '\U00002226', + "nsqsube;": '\U000022E2', + "nsqsupe;": '\U000022E3', + "nsub;": '\U00002284', + "nsube;": '\U00002288', + "nsubseteq;": '\U00002288', + "nsucc;": '\U00002281', + "nsup;": '\U00002285', + "nsupe;": '\U00002289', + "nsupseteq;": '\U00002289', + "ntgl;": '\U00002279', + "ntilde;": '\U000000F1', + "ntlg;": '\U00002278', + "ntriangleleft;": '\U000022EA', + "ntrianglelefteq;": '\U000022EC', + "ntriangleright;": '\U000022EB', + "ntrianglerighteq;": '\U000022ED', + "nu;": '\U000003BD', + "num;": '\U00000023', + "numero;": '\U00002116', + "numsp;": '\U00002007', + "nvDash;": '\U000022AD', + "nvHarr;": '\U00002904', + "nvdash;": '\U000022AC', + "nvinfin;": '\U000029DE', + "nvlArr;": '\U00002902', + "nvrArr;": '\U00002903', + "nwArr;": '\U000021D6', + "nwarhk;": '\U00002923', + "nwarr;": '\U00002196', + "nwarrow;": '\U00002196', + "nwnear;": '\U00002927', + "oS;": '\U000024C8', + "oacute;": '\U000000F3', + "oast;": '\U0000229B', + "ocir;": '\U0000229A', + "ocirc;": '\U000000F4', + "ocy;": '\U0000043E', + "odash;": '\U0000229D', + "odblac;": '\U00000151', + "odiv;": '\U00002A38', + "odot;": '\U00002299', + "odsold;": '\U000029BC', + "oelig;": '\U00000153', + "ofcir;": '\U000029BF', + "ofr;": '\U0001D52C', + "ogon;": '\U000002DB', + "ograve;": '\U000000F2', + "ogt;": '\U000029C1', + "ohbar;": '\U000029B5', + "ohm;": '\U000003A9', + "oint;": '\U0000222E', + "olarr;": '\U000021BA', + "olcir;": '\U000029BE', + "olcross;": '\U000029BB', + "oline;": '\U0000203E', + "olt;": '\U000029C0', + "omacr;": '\U0000014D', + "omega;": '\U000003C9', + "omicron;": '\U000003BF', + "omid;": '\U000029B6', + "ominus;": '\U00002296', + "oopf;": '\U0001D560', + "opar;": '\U000029B7', + "operp;": '\U000029B9', + "oplus;": '\U00002295', + "or;": '\U00002228', + "orarr;": '\U000021BB', + "ord;": '\U00002A5D', + "order;": '\U00002134', + "orderof;": '\U00002134', + "ordf;": '\U000000AA', + "ordm;": '\U000000BA', + "origof;": '\U000022B6', + "oror;": '\U00002A56', + "orslope;": '\U00002A57', + "orv;": '\U00002A5B', + "oscr;": '\U00002134', + "oslash;": '\U000000F8', + "osol;": '\U00002298', + "otilde;": '\U000000F5', + "otimes;": '\U00002297', + "otimesas;": '\U00002A36', + "ouml;": '\U000000F6', + "ovbar;": '\U0000233D', + "par;": '\U00002225', + "para;": '\U000000B6', + "parallel;": '\U00002225', + "parsim;": '\U00002AF3', + "parsl;": '\U00002AFD', + "part;": '\U00002202', + "pcy;": '\U0000043F', + "percnt;": '\U00000025', + "period;": '\U0000002E', + "permil;": '\U00002030', + "perp;": '\U000022A5', + "pertenk;": '\U00002031', + "pfr;": '\U0001D52D', + "phi;": '\U000003C6', + "phiv;": '\U000003D5', + "phmmat;": '\U00002133', + "phone;": '\U0000260E', + "pi;": '\U000003C0', + "pitchfork;": '\U000022D4', + "piv;": '\U000003D6', + "planck;": '\U0000210F', + "planckh;": '\U0000210E', + "plankv;": '\U0000210F', + "plus;": '\U0000002B', + "plusacir;": '\U00002A23', + "plusb;": '\U0000229E', + "pluscir;": '\U00002A22', + "plusdo;": '\U00002214', + "plusdu;": '\U00002A25', + "pluse;": '\U00002A72', + "plusmn;": '\U000000B1', + "plussim;": '\U00002A26', + "plustwo;": '\U00002A27', + "pm;": '\U000000B1', + "pointint;": '\U00002A15', + "popf;": '\U0001D561', + "pound;": '\U000000A3', + "pr;": '\U0000227A', + "prE;": '\U00002AB3', + "prap;": '\U00002AB7', + "prcue;": '\U0000227C', + "pre;": '\U00002AAF', + "prec;": '\U0000227A', + "precapprox;": '\U00002AB7', + "preccurlyeq;": '\U0000227C', + "preceq;": '\U00002AAF', + "precnapprox;": '\U00002AB9', + "precneqq;": '\U00002AB5', + "precnsim;": '\U000022E8', + "precsim;": '\U0000227E', + "prime;": '\U00002032', + "primes;": '\U00002119', + "prnE;": '\U00002AB5', + "prnap;": '\U00002AB9', + "prnsim;": '\U000022E8', + "prod;": '\U0000220F', + "profalar;": '\U0000232E', + "profline;": '\U00002312', + "profsurf;": '\U00002313', + "prop;": '\U0000221D', + "propto;": '\U0000221D', + "prsim;": '\U0000227E', + "prurel;": '\U000022B0', + "pscr;": '\U0001D4C5', + "psi;": '\U000003C8', + "puncsp;": '\U00002008', + "qfr;": '\U0001D52E', + "qint;": '\U00002A0C', + "qopf;": '\U0001D562', + "qprime;": '\U00002057', + "qscr;": '\U0001D4C6', + "quaternions;": '\U0000210D', + "quatint;": '\U00002A16', + "quest;": '\U0000003F', + "questeq;": '\U0000225F', + "quot;": '\U00000022', + "rAarr;": '\U000021DB', + "rArr;": '\U000021D2', + "rAtail;": '\U0000291C', + "rBarr;": '\U0000290F', + "rHar;": '\U00002964', + "racute;": '\U00000155', + "radic;": '\U0000221A', + "raemptyv;": '\U000029B3', + "rang;": '\U000027E9', + "rangd;": '\U00002992', + "range;": '\U000029A5', + "rangle;": '\U000027E9', + "raquo;": '\U000000BB', + "rarr;": '\U00002192', + "rarrap;": '\U00002975', + "rarrb;": '\U000021E5', + "rarrbfs;": '\U00002920', + "rarrc;": '\U00002933', + "rarrfs;": '\U0000291E', + "rarrhk;": '\U000021AA', + "rarrlp;": '\U000021AC', + "rarrpl;": '\U00002945', + "rarrsim;": '\U00002974', + "rarrtl;": '\U000021A3', + "rarrw;": '\U0000219D', + "ratail;": '\U0000291A', + "ratio;": '\U00002236', + "rationals;": '\U0000211A', + "rbarr;": '\U0000290D', + "rbbrk;": '\U00002773', + "rbrace;": '\U0000007D', + "rbrack;": '\U0000005D', + "rbrke;": '\U0000298C', + "rbrksld;": '\U0000298E', + "rbrkslu;": '\U00002990', + "rcaron;": '\U00000159', + "rcedil;": '\U00000157', + "rceil;": '\U00002309', + "rcub;": '\U0000007D', + "rcy;": '\U00000440', + "rdca;": '\U00002937', + "rdldhar;": '\U00002969', + "rdquo;": '\U0000201D', + "rdquor;": '\U0000201D', + "rdsh;": '\U000021B3', + "real;": '\U0000211C', + "realine;": '\U0000211B', + "realpart;": '\U0000211C', + "reals;": '\U0000211D', + "rect;": '\U000025AD', + "reg;": '\U000000AE', + "rfisht;": '\U0000297D', + "rfloor;": '\U0000230B', + "rfr;": '\U0001D52F', + "rhard;": '\U000021C1', + "rharu;": '\U000021C0', + "rharul;": '\U0000296C', + "rho;": '\U000003C1', + "rhov;": '\U000003F1', + "rightarrow;": '\U00002192', + "rightarrowtail;": '\U000021A3', + "rightharpoondown;": '\U000021C1', + "rightharpoonup;": '\U000021C0', + "rightleftarrows;": '\U000021C4', + "rightleftharpoons;": '\U000021CC', + "rightrightarrows;": '\U000021C9', + "rightsquigarrow;": '\U0000219D', + "rightthreetimes;": '\U000022CC', + "ring;": '\U000002DA', + "risingdotseq;": '\U00002253', + "rlarr;": '\U000021C4', + "rlhar;": '\U000021CC', + "rlm;": '\U0000200F', + "rmoust;": '\U000023B1', + "rmoustache;": '\U000023B1', + "rnmid;": '\U00002AEE', + "roang;": '\U000027ED', + "roarr;": '\U000021FE', + "robrk;": '\U000027E7', + "ropar;": '\U00002986', + "ropf;": '\U0001D563', + "roplus;": '\U00002A2E', + "rotimes;": '\U00002A35', + "rpar;": '\U00000029', + "rpargt;": '\U00002994', + "rppolint;": '\U00002A12', + "rrarr;": '\U000021C9', + "rsaquo;": '\U0000203A', + "rscr;": '\U0001D4C7', + "rsh;": '\U000021B1', + "rsqb;": '\U0000005D', + "rsquo;": '\U00002019', + "rsquor;": '\U00002019', + "rthree;": '\U000022CC', + "rtimes;": '\U000022CA', + "rtri;": '\U000025B9', + "rtrie;": '\U000022B5', + "rtrif;": '\U000025B8', + "rtriltri;": '\U000029CE', + "ruluhar;": '\U00002968', + "rx;": '\U0000211E', + "sacute;": '\U0000015B', + "sbquo;": '\U0000201A', + "sc;": '\U0000227B', + "scE;": '\U00002AB4', + "scap;": '\U00002AB8', + "scaron;": '\U00000161', + "sccue;": '\U0000227D', + "sce;": '\U00002AB0', + "scedil;": '\U0000015F', + "scirc;": '\U0000015D', + "scnE;": '\U00002AB6', + "scnap;": '\U00002ABA', + "scnsim;": '\U000022E9', + "scpolint;": '\U00002A13', + "scsim;": '\U0000227F', + "scy;": '\U00000441', + "sdot;": '\U000022C5', + "sdotb;": '\U000022A1', + "sdote;": '\U00002A66', + "seArr;": '\U000021D8', + "searhk;": '\U00002925', + "searr;": '\U00002198', + "searrow;": '\U00002198', + "sect;": '\U000000A7', + "semi;": '\U0000003B', + "seswar;": '\U00002929', + "setminus;": '\U00002216', + "setmn;": '\U00002216', + "sext;": '\U00002736', + "sfr;": '\U0001D530', + "sfrown;": '\U00002322', + "sharp;": '\U0000266F', + "shchcy;": '\U00000449', + "shcy;": '\U00000448', + "shortmid;": '\U00002223', + "shortparallel;": '\U00002225', + "shy;": '\U000000AD', + "sigma;": '\U000003C3', + "sigmaf;": '\U000003C2', + "sigmav;": '\U000003C2', + "sim;": '\U0000223C', + "simdot;": '\U00002A6A', + "sime;": '\U00002243', + "simeq;": '\U00002243', + "simg;": '\U00002A9E', + "simgE;": '\U00002AA0', + "siml;": '\U00002A9D', + "simlE;": '\U00002A9F', + "simne;": '\U00002246', + "simplus;": '\U00002A24', + "simrarr;": '\U00002972', + "slarr;": '\U00002190', + "smallsetminus;": '\U00002216', + "smashp;": '\U00002A33', + "smeparsl;": '\U000029E4', + "smid;": '\U00002223', + "smile;": '\U00002323', + "smt;": '\U00002AAA', + "smte;": '\U00002AAC', + "softcy;": '\U0000044C', + "sol;": '\U0000002F', + "solb;": '\U000029C4', + "solbar;": '\U0000233F', + "sopf;": '\U0001D564', + "spades;": '\U00002660', + "spadesuit;": '\U00002660', + "spar;": '\U00002225', + "sqcap;": '\U00002293', + "sqcup;": '\U00002294', + "sqsub;": '\U0000228F', + "sqsube;": '\U00002291', + "sqsubset;": '\U0000228F', + "sqsubseteq;": '\U00002291', + "sqsup;": '\U00002290', + "sqsupe;": '\U00002292', + "sqsupset;": '\U00002290', + "sqsupseteq;": '\U00002292', + "squ;": '\U000025A1', + "square;": '\U000025A1', + "squarf;": '\U000025AA', + "squf;": '\U000025AA', + "srarr;": '\U00002192', + "sscr;": '\U0001D4C8', + "ssetmn;": '\U00002216', + "ssmile;": '\U00002323', + "sstarf;": '\U000022C6', + "star;": '\U00002606', + "starf;": '\U00002605', + "straightepsilon;": '\U000003F5', + "straightphi;": '\U000003D5', + "strns;": '\U000000AF', + "sub;": '\U00002282', + "subE;": '\U00002AC5', + "subdot;": '\U00002ABD', + "sube;": '\U00002286', + "subedot;": '\U00002AC3', + "submult;": '\U00002AC1', + "subnE;": '\U00002ACB', + "subne;": '\U0000228A', + "subplus;": '\U00002ABF', + "subrarr;": '\U00002979', + "subset;": '\U00002282', + "subseteq;": '\U00002286', + "subseteqq;": '\U00002AC5', + "subsetneq;": '\U0000228A', + "subsetneqq;": '\U00002ACB', + "subsim;": '\U00002AC7', + "subsub;": '\U00002AD5', + "subsup;": '\U00002AD3', + "succ;": '\U0000227B', + "succapprox;": '\U00002AB8', + "succcurlyeq;": '\U0000227D', + "succeq;": '\U00002AB0', + "succnapprox;": '\U00002ABA', + "succneqq;": '\U00002AB6', + "succnsim;": '\U000022E9', + "succsim;": '\U0000227F', + "sum;": '\U00002211', + "sung;": '\U0000266A', + "sup;": '\U00002283', + "sup1;": '\U000000B9', + "sup2;": '\U000000B2', + "sup3;": '\U000000B3', + "supE;": '\U00002AC6', + "supdot;": '\U00002ABE', + "supdsub;": '\U00002AD8', + "supe;": '\U00002287', + "supedot;": '\U00002AC4', + "suphsol;": '\U000027C9', + "suphsub;": '\U00002AD7', + "suplarr;": '\U0000297B', + "supmult;": '\U00002AC2', + "supnE;": '\U00002ACC', + "supne;": '\U0000228B', + "supplus;": '\U00002AC0', + "supset;": '\U00002283', + "supseteq;": '\U00002287', + "supseteqq;": '\U00002AC6', + "supsetneq;": '\U0000228B', + "supsetneqq;": '\U00002ACC', + "supsim;": '\U00002AC8', + "supsub;": '\U00002AD4', + "supsup;": '\U00002AD6', + "swArr;": '\U000021D9', + "swarhk;": '\U00002926', + "swarr;": '\U00002199', + "swarrow;": '\U00002199', + "swnwar;": '\U0000292A', + "szlig;": '\U000000DF', + "target;": '\U00002316', + "tau;": '\U000003C4', + "tbrk;": '\U000023B4', + "tcaron;": '\U00000165', + "tcedil;": '\U00000163', + "tcy;": '\U00000442', + "tdot;": '\U000020DB', + "telrec;": '\U00002315', + "tfr;": '\U0001D531', + "there4;": '\U00002234', + "therefore;": '\U00002234', + "theta;": '\U000003B8', + "thetasym;": '\U000003D1', + "thetav;": '\U000003D1', + "thickapprox;": '\U00002248', + "thicksim;": '\U0000223C', + "thinsp;": '\U00002009', + "thkap;": '\U00002248', + "thksim;": '\U0000223C', + "thorn;": '\U000000FE', + "tilde;": '\U000002DC', + "times;": '\U000000D7', + "timesb;": '\U000022A0', + "timesbar;": '\U00002A31', + "timesd;": '\U00002A30', + "tint;": '\U0000222D', + "toea;": '\U00002928', + "top;": '\U000022A4', + "topbot;": '\U00002336', + "topcir;": '\U00002AF1', + "topf;": '\U0001D565', + "topfork;": '\U00002ADA', + "tosa;": '\U00002929', + "tprime;": '\U00002034', + "trade;": '\U00002122', + "triangle;": '\U000025B5', + "triangledown;": '\U000025BF', + "triangleleft;": '\U000025C3', + "trianglelefteq;": '\U000022B4', + "triangleq;": '\U0000225C', + "triangleright;": '\U000025B9', + "trianglerighteq;": '\U000022B5', + "tridot;": '\U000025EC', + "trie;": '\U0000225C', + "triminus;": '\U00002A3A', + "triplus;": '\U00002A39', + "trisb;": '\U000029CD', + "tritime;": '\U00002A3B', + "trpezium;": '\U000023E2', + "tscr;": '\U0001D4C9', + "tscy;": '\U00000446', + "tshcy;": '\U0000045B', + "tstrok;": '\U00000167', + "twixt;": '\U0000226C', + "twoheadleftarrow;": '\U0000219E', + "twoheadrightarrow;": '\U000021A0', + "uArr;": '\U000021D1', + "uHar;": '\U00002963', + "uacute;": '\U000000FA', + "uarr;": '\U00002191', + "ubrcy;": '\U0000045E', + "ubreve;": '\U0000016D', + "ucirc;": '\U000000FB', + "ucy;": '\U00000443', + "udarr;": '\U000021C5', + "udblac;": '\U00000171', + "udhar;": '\U0000296E', + "ufisht;": '\U0000297E', + "ufr;": '\U0001D532', + "ugrave;": '\U000000F9', + "uharl;": '\U000021BF', + "uharr;": '\U000021BE', + "uhblk;": '\U00002580', + "ulcorn;": '\U0000231C', + "ulcorner;": '\U0000231C', + "ulcrop;": '\U0000230F', + "ultri;": '\U000025F8', + "umacr;": '\U0000016B', + "uml;": '\U000000A8', + "uogon;": '\U00000173', + "uopf;": '\U0001D566', + "uparrow;": '\U00002191', + "updownarrow;": '\U00002195', + "upharpoonleft;": '\U000021BF', + "upharpoonright;": '\U000021BE', + "uplus;": '\U0000228E', + "upsi;": '\U000003C5', + "upsih;": '\U000003D2', + "upsilon;": '\U000003C5', + "upuparrows;": '\U000021C8', + "urcorn;": '\U0000231D', + "urcorner;": '\U0000231D', + "urcrop;": '\U0000230E', + "uring;": '\U0000016F', + "urtri;": '\U000025F9', + "uscr;": '\U0001D4CA', + "utdot;": '\U000022F0', + "utilde;": '\U00000169', + "utri;": '\U000025B5', + "utrif;": '\U000025B4', + "uuarr;": '\U000021C8', + "uuml;": '\U000000FC', + "uwangle;": '\U000029A7', + "vArr;": '\U000021D5', + "vBar;": '\U00002AE8', + "vBarv;": '\U00002AE9', + "vDash;": '\U000022A8', + "vangrt;": '\U0000299C', + "varepsilon;": '\U000003F5', + "varkappa;": '\U000003F0', + "varnothing;": '\U00002205', + "varphi;": '\U000003D5', + "varpi;": '\U000003D6', + "varpropto;": '\U0000221D', + "varr;": '\U00002195', + "varrho;": '\U000003F1', + "varsigma;": '\U000003C2', + "vartheta;": '\U000003D1', + "vartriangleleft;": '\U000022B2', + "vartriangleright;": '\U000022B3', + "vcy;": '\U00000432', + "vdash;": '\U000022A2', + "vee;": '\U00002228', + "veebar;": '\U000022BB', + "veeeq;": '\U0000225A', + "vellip;": '\U000022EE', + "verbar;": '\U0000007C', + "vert;": '\U0000007C', + "vfr;": '\U0001D533', + "vltri;": '\U000022B2', + "vopf;": '\U0001D567', + "vprop;": '\U0000221D', + "vrtri;": '\U000022B3', + "vscr;": '\U0001D4CB', + "vzigzag;": '\U0000299A', + "wcirc;": '\U00000175', + "wedbar;": '\U00002A5F', + "wedge;": '\U00002227', + "wedgeq;": '\U00002259', + "weierp;": '\U00002118', + "wfr;": '\U0001D534', + "wopf;": '\U0001D568', + "wp;": '\U00002118', + "wr;": '\U00002240', + "wreath;": '\U00002240', + "wscr;": '\U0001D4CC', + "xcap;": '\U000022C2', + "xcirc;": '\U000025EF', + "xcup;": '\U000022C3', + "xdtri;": '\U000025BD', + "xfr;": '\U0001D535', + "xhArr;": '\U000027FA', + "xharr;": '\U000027F7', + "xi;": '\U000003BE', + "xlArr;": '\U000027F8', + "xlarr;": '\U000027F5', + "xmap;": '\U000027FC', + "xnis;": '\U000022FB', + "xodot;": '\U00002A00', + "xopf;": '\U0001D569', + "xoplus;": '\U00002A01', + "xotime;": '\U00002A02', + "xrArr;": '\U000027F9', + "xrarr;": '\U000027F6', + "xscr;": '\U0001D4CD', + "xsqcup;": '\U00002A06', + "xuplus;": '\U00002A04', + "xutri;": '\U000025B3', + "xvee;": '\U000022C1', + "xwedge;": '\U000022C0', + "yacute;": '\U000000FD', + "yacy;": '\U0000044F', + "ycirc;": '\U00000177', + "ycy;": '\U0000044B', + "yen;": '\U000000A5', + "yfr;": '\U0001D536', + "yicy;": '\U00000457', + "yopf;": '\U0001D56A', + "yscr;": '\U0001D4CE', + "yucy;": '\U0000044E', + "yuml;": '\U000000FF', + "zacute;": '\U0000017A', + "zcaron;": '\U0000017E', + "zcy;": '\U00000437', + "zdot;": '\U0000017C', + "zeetrf;": '\U00002128', + "zeta;": '\U000003B6', + "zfr;": '\U0001D537', + "zhcy;": '\U00000436', + "zigrarr;": '\U000021DD', + "zopf;": '\U0001D56B', + "zscr;": '\U0001D4CF', + "zwj;": '\U0000200D', + "zwnj;": '\U0000200C', + "AElig": '\U000000C6', + "AMP": '\U00000026', + "Aacute": '\U000000C1', + "Acirc": '\U000000C2', + "Agrave": '\U000000C0', + "Aring": '\U000000C5', + "Atilde": '\U000000C3', + "Auml": '\U000000C4', + "COPY": '\U000000A9', + "Ccedil": '\U000000C7', + "ETH": '\U000000D0', + "Eacute": '\U000000C9', + "Ecirc": '\U000000CA', + "Egrave": '\U000000C8', + "Euml": '\U000000CB', + "GT": '\U0000003E', + "Iacute": '\U000000CD', + "Icirc": '\U000000CE', + "Igrave": '\U000000CC', + "Iuml": '\U000000CF', + "LT": '\U0000003C', + "Ntilde": '\U000000D1', + "Oacute": '\U000000D3', + "Ocirc": '\U000000D4', + "Ograve": '\U000000D2', + "Oslash": '\U000000D8', + "Otilde": '\U000000D5', + "Ouml": '\U000000D6', + "QUOT": '\U00000022', + "REG": '\U000000AE', + "THORN": '\U000000DE', + "Uacute": '\U000000DA', + "Ucirc": '\U000000DB', + "Ugrave": '\U000000D9', + "Uuml": '\U000000DC', + "Yacute": '\U000000DD', + "aacute": '\U000000E1', + "acirc": '\U000000E2', + "acute": '\U000000B4', + "aelig": '\U000000E6', + "agrave": '\U000000E0', + "amp": '\U00000026', + "aring": '\U000000E5', + "atilde": '\U000000E3', + "auml": '\U000000E4', + "brvbar": '\U000000A6', + "ccedil": '\U000000E7', + "cedil": '\U000000B8', + "cent": '\U000000A2', + "copy": '\U000000A9', + "curren": '\U000000A4', + "deg": '\U000000B0', + "divide": '\U000000F7', + "eacute": '\U000000E9', + "ecirc": '\U000000EA', + "egrave": '\U000000E8', + "eth": '\U000000F0', + "euml": '\U000000EB', + "frac12": '\U000000BD', + "frac14": '\U000000BC', + "frac34": '\U000000BE', + "gt": '\U0000003E', + "iacute": '\U000000ED', + "icirc": '\U000000EE', + "iexcl": '\U000000A1', + "igrave": '\U000000EC', + "iquest": '\U000000BF', + "iuml": '\U000000EF', + "laquo": '\U000000AB', + "lt": '\U0000003C', + "macr": '\U000000AF', + "micro": '\U000000B5', + "middot": '\U000000B7', + "nbsp": '\U000000A0', + "not": '\U000000AC', + "ntilde": '\U000000F1', + "oacute": '\U000000F3', + "ocirc": '\U000000F4', + "ograve": '\U000000F2', + "ordf": '\U000000AA', + "ordm": '\U000000BA', + "oslash": '\U000000F8', + "otilde": '\U000000F5', + "ouml": '\U000000F6', + "para": '\U000000B6', + "plusmn": '\U000000B1', + "pound": '\U000000A3', + "quot": '\U00000022', + "raquo": '\U000000BB', + "reg": '\U000000AE', + "sect": '\U000000A7', + "shy": '\U000000AD', + "sup1": '\U000000B9', + "sup2": '\U000000B2', + "sup3": '\U000000B3', + "szlig": '\U000000DF', + "thorn": '\U000000FE', + "times": '\U000000D7', + "uacute": '\U000000FA', + "ucirc": '\U000000FB', + "ugrave": '\U000000F9', + "uml": '\U000000A8', + "uuml": '\U000000FC', + "yacute": '\U000000FD', + "yen": '\U000000A5', + "yuml": '\U000000FF', +} + +// HTML entities that are two unicode codepoints. +var entity2 = map[string][2]rune{ + // TODO(nigeltao): Handle replacements that are wider than their names. + // "nLt;": {'\u226A', '\u20D2'}, + // "nGt;": {'\u226B', '\u20D2'}, + "NotEqualTilde;": {'\u2242', '\u0338'}, + "NotGreaterFullEqual;": {'\u2267', '\u0338'}, + "NotGreaterGreater;": {'\u226B', '\u0338'}, + "NotGreaterSlantEqual;": {'\u2A7E', '\u0338'}, + "NotHumpDownHump;": {'\u224E', '\u0338'}, + "NotHumpEqual;": {'\u224F', '\u0338'}, + "NotLeftTriangleBar;": {'\u29CF', '\u0338'}, + "NotLessLess;": {'\u226A', '\u0338'}, + "NotLessSlantEqual;": {'\u2A7D', '\u0338'}, + "NotNestedGreaterGreater;": {'\u2AA2', '\u0338'}, + "NotNestedLessLess;": {'\u2AA1', '\u0338'}, + "NotPrecedesEqual;": {'\u2AAF', '\u0338'}, + "NotRightTriangleBar;": {'\u29D0', '\u0338'}, + "NotSquareSubset;": {'\u228F', '\u0338'}, + "NotSquareSuperset;": {'\u2290', '\u0338'}, + "NotSubset;": {'\u2282', '\u20D2'}, + "NotSucceedsEqual;": {'\u2AB0', '\u0338'}, + "NotSucceedsTilde;": {'\u227F', '\u0338'}, + "NotSuperset;": {'\u2283', '\u20D2'}, + "ThickSpace;": {'\u205F', '\u200A'}, + "acE;": {'\u223E', '\u0333'}, + "bne;": {'\u003D', '\u20E5'}, + "bnequiv;": {'\u2261', '\u20E5'}, + "caps;": {'\u2229', '\uFE00'}, + "cups;": {'\u222A', '\uFE00'}, + "fjlig;": {'\u0066', '\u006A'}, + "gesl;": {'\u22DB', '\uFE00'}, + "gvertneqq;": {'\u2269', '\uFE00'}, + "gvnE;": {'\u2269', '\uFE00'}, + "lates;": {'\u2AAD', '\uFE00'}, + "lesg;": {'\u22DA', '\uFE00'}, + "lvertneqq;": {'\u2268', '\uFE00'}, + "lvnE;": {'\u2268', '\uFE00'}, + "nGg;": {'\u22D9', '\u0338'}, + "nGtv;": {'\u226B', '\u0338'}, + "nLl;": {'\u22D8', '\u0338'}, + "nLtv;": {'\u226A', '\u0338'}, + "nang;": {'\u2220', '\u20D2'}, + "napE;": {'\u2A70', '\u0338'}, + "napid;": {'\u224B', '\u0338'}, + "nbump;": {'\u224E', '\u0338'}, + "nbumpe;": {'\u224F', '\u0338'}, + "ncongdot;": {'\u2A6D', '\u0338'}, + "nedot;": {'\u2250', '\u0338'}, + "nesim;": {'\u2242', '\u0338'}, + "ngE;": {'\u2267', '\u0338'}, + "ngeqq;": {'\u2267', '\u0338'}, + "ngeqslant;": {'\u2A7E', '\u0338'}, + "nges;": {'\u2A7E', '\u0338'}, + "nlE;": {'\u2266', '\u0338'}, + "nleqq;": {'\u2266', '\u0338'}, + "nleqslant;": {'\u2A7D', '\u0338'}, + "nles;": {'\u2A7D', '\u0338'}, + "notinE;": {'\u22F9', '\u0338'}, + "notindot;": {'\u22F5', '\u0338'}, + "nparsl;": {'\u2AFD', '\u20E5'}, + "npart;": {'\u2202', '\u0338'}, + "npre;": {'\u2AAF', '\u0338'}, + "npreceq;": {'\u2AAF', '\u0338'}, + "nrarrc;": {'\u2933', '\u0338'}, + "nrarrw;": {'\u219D', '\u0338'}, + "nsce;": {'\u2AB0', '\u0338'}, + "nsubE;": {'\u2AC5', '\u0338'}, + "nsubset;": {'\u2282', '\u20D2'}, + "nsubseteqq;": {'\u2AC5', '\u0338'}, + "nsucceq;": {'\u2AB0', '\u0338'}, + "nsupE;": {'\u2AC6', '\u0338'}, + "nsupset;": {'\u2283', '\u20D2'}, + "nsupseteqq;": {'\u2AC6', '\u0338'}, + "nvap;": {'\u224D', '\u20D2'}, + "nvge;": {'\u2265', '\u20D2'}, + "nvgt;": {'\u003E', '\u20D2'}, + "nvle;": {'\u2264', '\u20D2'}, + "nvlt;": {'\u003C', '\u20D2'}, + "nvltrie;": {'\u22B4', '\u20D2'}, + "nvrtrie;": {'\u22B5', '\u20D2'}, + "nvsim;": {'\u223C', '\u20D2'}, + "race;": {'\u223D', '\u0331'}, + "smtes;": {'\u2AAC', '\uFE00'}, + "sqcaps;": {'\u2293', '\uFE00'}, + "sqcups;": {'\u2294', '\uFE00'}, + "varsubsetneq;": {'\u228A', '\uFE00'}, + "varsubsetneqq;": {'\u2ACB', '\uFE00'}, + "varsupsetneq;": {'\u228B', '\uFE00'}, + "varsupsetneqq;": {'\u2ACC', '\uFE00'}, + "vnsub;": {'\u2282', '\u20D2'}, + "vnsup;": {'\u2283', '\u20D2'}, + "vsubnE;": {'\u2ACB', '\uFE00'}, + "vsubne;": {'\u228A', '\uFE00'}, + "vsupnE;": {'\u2ACC', '\uFE00'}, + "vsupne;": {'\u228B', '\uFE00'}, +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/entity_test.go b/Godeps/_workspace/src/golang.org/x/net/html/entity_test.go new file mode 100644 index 00000000000..b53f866fa2d --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/entity_test.go @@ -0,0 +1,29 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "testing" + "unicode/utf8" +) + +func TestEntityLength(t *testing.T) { + // We verify that the length of UTF-8 encoding of each value is <= 1 + len(key). + // The +1 comes from the leading "&". This property implies that the length of + // unescaped text is <= the length of escaped text. + for k, v := range entity { + if 1+len(k) < utf8.RuneLen(v) { + t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v)) + } + if len(k) > longestEntityWithoutSemicolon && k[len(k)-1] != ';' { + t.Errorf("entity name %s is %d characters, but longestEntityWithoutSemicolon=%d", k, len(k), longestEntityWithoutSemicolon) + } + } + for k, v := range entity2 { + if 1+len(k) < utf8.RuneLen(v[0])+utf8.RuneLen(v[1]) { + t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v[0]) + string(v[1])) + } + } +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/escape.go b/Godeps/_workspace/src/golang.org/x/net/html/escape.go new file mode 100644 index 00000000000..d8561396200 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/escape.go @@ -0,0 +1,258 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "strings" + "unicode/utf8" +) + +// These replacements permit compatibility with old numeric entities that +// assumed Windows-1252 encoding. +// https://html.spec.whatwg.org/multipage/syntax.html#consume-a-character-reference +var replacementTable = [...]rune{ + '\u20AC', // First entry is what 0x80 should be replaced with. + '\u0081', + '\u201A', + '\u0192', + '\u201E', + '\u2026', + '\u2020', + '\u2021', + '\u02C6', + '\u2030', + '\u0160', + '\u2039', + '\u0152', + '\u008D', + '\u017D', + '\u008F', + '\u0090', + '\u2018', + '\u2019', + '\u201C', + '\u201D', + '\u2022', + '\u2013', + '\u2014', + '\u02DC', + '\u2122', + '\u0161', + '\u203A', + '\u0153', + '\u009D', + '\u017E', + '\u0178', // Last entry is 0x9F. + // 0x00->'\uFFFD' is handled programmatically. + // 0x0D->'\u000D' is a no-op. +} + +// unescapeEntity reads an entity like "<" from b[src:] and writes the +// corresponding "<" to b[dst:], returning the incremented dst and src cursors. +// Precondition: b[src] == '&' && dst <= src. +// attribute should be true if parsing an attribute value. +func unescapeEntity(b []byte, dst, src int, attribute bool) (dst1, src1 int) { + // https://html.spec.whatwg.org/multipage/syntax.html#consume-a-character-reference + + // i starts at 1 because we already know that s[0] == '&'. + i, s := 1, b[src:] + + if len(s) <= 1 { + b[dst] = b[src] + return dst + 1, src + 1 + } + + if s[i] == '#' { + if len(s) <= 3 { // We need to have at least "&#.". + b[dst] = b[src] + return dst + 1, src + 1 + } + i++ + c := s[i] + hex := false + if c == 'x' || c == 'X' { + hex = true + i++ + } + + x := '\x00' + for i < len(s) { + c = s[i] + i++ + if hex { + if '0' <= c && c <= '9' { + x = 16*x + rune(c) - '0' + continue + } else if 'a' <= c && c <= 'f' { + x = 16*x + rune(c) - 'a' + 10 + continue + } else if 'A' <= c && c <= 'F' { + x = 16*x + rune(c) - 'A' + 10 + continue + } + } else if '0' <= c && c <= '9' { + x = 10*x + rune(c) - '0' + continue + } + if c != ';' { + i-- + } + break + } + + if i <= 3 { // No characters matched. + b[dst] = b[src] + return dst + 1, src + 1 + } + + if 0x80 <= x && x <= 0x9F { + // Replace characters from Windows-1252 with UTF-8 equivalents. + x = replacementTable[x-0x80] + } else if x == 0 || (0xD800 <= x && x <= 0xDFFF) || x > 0x10FFFF { + // Replace invalid characters with the replacement character. + x = '\uFFFD' + } + + return dst + utf8.EncodeRune(b[dst:], x), src + i + } + + // Consume the maximum number of characters possible, with the + // consumed characters matching one of the named references. + + for i < len(s) { + c := s[i] + i++ + // Lower-cased characters are more common in entities, so we check for them first. + if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' { + continue + } + if c != ';' { + i-- + } + break + } + + entityName := string(s[1:i]) + if entityName == "" { + // No-op. + } else if attribute && entityName[len(entityName)-1] != ';' && len(s) > i && s[i] == '=' { + // No-op. + } else if x := entity[entityName]; x != 0 { + return dst + utf8.EncodeRune(b[dst:], x), src + i + } else if x := entity2[entityName]; x[0] != 0 { + dst1 := dst + utf8.EncodeRune(b[dst:], x[0]) + return dst1 + utf8.EncodeRune(b[dst1:], x[1]), src + i + } else if !attribute { + maxLen := len(entityName) - 1 + if maxLen > longestEntityWithoutSemicolon { + maxLen = longestEntityWithoutSemicolon + } + for j := maxLen; j > 1; j-- { + if x := entity[entityName[:j]]; x != 0 { + return dst + utf8.EncodeRune(b[dst:], x), src + j + 1 + } + } + } + + dst1, src1 = dst+i, src+i + copy(b[dst:dst1], b[src:src1]) + return dst1, src1 +} + +// unescape unescapes b's entities in-place, so that "a<b" becomes "a': + esc = ">" + case '"': + // """ is shorter than """. + esc = """ + case '\r': + esc = " " + default: + panic("unrecognized escape character") + } + s = s[i+1:] + if _, err := w.WriteString(esc); err != nil { + return err + } + i = strings.IndexAny(s, escapedChars) + } + _, err := w.WriteString(s) + return err +} + +// EscapeString escapes special characters like "<" to become "<". It +// escapes only five such characters: <, >, &, ' and ". +// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't +// always true. +func EscapeString(s string) string { + if strings.IndexAny(s, escapedChars) == -1 { + return s + } + var buf bytes.Buffer + escape(&buf, s) + return buf.String() +} + +// UnescapeString unescapes entities like "<" to become "<". It unescapes a +// larger range of entities than EscapeString escapes. For example, "á" +// unescapes to "á", as does "á" and "&xE1;". +// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't +// always true. +func UnescapeString(s string) string { + for _, c := range s { + if c == '&' { + return string(unescape([]byte(s), false)) + } + } + return s +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/escape_test.go b/Godeps/_workspace/src/golang.org/x/net/html/escape_test.go new file mode 100644 index 00000000000..b405d4b4a77 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/escape_test.go @@ -0,0 +1,97 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import "testing" + +type unescapeTest struct { + // A short description of the test case. + desc string + // The HTML text. + html string + // The unescaped text. + unescaped string +} + +var unescapeTests = []unescapeTest{ + // Handle no entities. + { + "copy", + "A\ttext\nstring", + "A\ttext\nstring", + }, + // Handle simple named entities. + { + "simple", + "& > <", + "& > <", + }, + // Handle hitting the end of the string. + { + "stringEnd", + "& &", + "& &", + }, + // Handle entities with two codepoints. + { + "multiCodepoint", + "text ⋛︀ blah", + "text \u22db\ufe00 blah", + }, + // Handle decimal numeric entities. + { + "decimalEntity", + "Delta = Δ ", + "Delta = Δ ", + }, + // Handle hexadecimal numeric entities. + { + "hexadecimalEntity", + "Lambda = λ = λ ", + "Lambda = λ = λ ", + }, + // Handle numeric early termination. + { + "numericEnds", + "&# &#x €43 © = ©f = ©", + "&# &#x €43 © = ©f = ©", + }, + // Handle numeric ISO-8859-1 entity replacements. + { + "numericReplacements", + "Footnote‡", + "Footnote‡", + }, +} + +func TestUnescape(t *testing.T) { + for _, tt := range unescapeTests { + unescaped := UnescapeString(tt.html) + if unescaped != tt.unescaped { + t.Errorf("TestUnescape %s: want %q, got %q", tt.desc, tt.unescaped, unescaped) + } + } +} + +func TestUnescapeEscape(t *testing.T) { + ss := []string{ + ``, + `abc def`, + `a & b`, + `a&b`, + `a & b`, + `"`, + `"`, + `"<&>"`, + `"<&>"`, + `3&5==1 && 0<1, "0<1", a+acute=á`, + `The special characters are: <, >, &, ' and "`, + } + for _, s := range ss { + if got := UnescapeString(EscapeString(s)); got != s { + t.Errorf("got %q want %q", got, s) + } + } +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/example_test.go b/Godeps/_workspace/src/golang.org/x/net/html/example_test.go new file mode 100644 index 00000000000..b5f92838658 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/example_test.go @@ -0,0 +1,40 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This example demonstrates parsing HTML data and walking the resulting tree. +package html_test + +import ( + "fmt" + "log" + "strings" + + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/html" +) + +func ExampleParse() { + s := `

Links:

` + doc, err := html.Parse(strings.NewReader(s)) + if err != nil { + log.Fatal(err) + } + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + for _, a := range n.Attr { + if a.Key == "href" { + fmt.Println(a.Val) + break + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + // Output: + // foo + // /bar/baz +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/foreign.go b/Godeps/_workspace/src/golang.org/x/net/html/foreign.go new file mode 100644 index 00000000000..d3b3844099b --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/foreign.go @@ -0,0 +1,226 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "strings" +) + +func adjustAttributeNames(aa []Attribute, nameMap map[string]string) { + for i := range aa { + if newName, ok := nameMap[aa[i].Key]; ok { + aa[i].Key = newName + } + } +} + +func adjustForeignAttributes(aa []Attribute) { + for i, a := range aa { + if a.Key == "" || a.Key[0] != 'x' { + continue + } + switch a.Key { + case "xlink:actuate", "xlink:arcrole", "xlink:href", "xlink:role", "xlink:show", + "xlink:title", "xlink:type", "xml:base", "xml:lang", "xml:space", "xmlns:xlink": + j := strings.Index(a.Key, ":") + aa[i].Namespace = a.Key[:j] + aa[i].Key = a.Key[j+1:] + } + } +} + +func htmlIntegrationPoint(n *Node) bool { + if n.Type != ElementNode { + return false + } + switch n.Namespace { + case "math": + if n.Data == "annotation-xml" { + for _, a := range n.Attr { + if a.Key == "encoding" { + val := strings.ToLower(a.Val) + if val == "text/html" || val == "application/xhtml+xml" { + return true + } + } + } + } + case "svg": + switch n.Data { + case "desc", "foreignObject", "title": + return true + } + } + return false +} + +func mathMLTextIntegrationPoint(n *Node) bool { + if n.Namespace != "math" { + return false + } + switch n.Data { + case "mi", "mo", "mn", "ms", "mtext": + return true + } + return false +} + +// Section 12.2.5.5. +var breakout = map[string]bool{ + "b": true, + "big": true, + "blockquote": true, + "body": true, + "br": true, + "center": true, + "code": true, + "dd": true, + "div": true, + "dl": true, + "dt": true, + "em": true, + "embed": true, + "h1": true, + "h2": true, + "h3": true, + "h4": true, + "h5": true, + "h6": true, + "head": true, + "hr": true, + "i": true, + "img": true, + "li": true, + "listing": true, + "menu": true, + "meta": true, + "nobr": true, + "ol": true, + "p": true, + "pre": true, + "ruby": true, + "s": true, + "small": true, + "span": true, + "strong": true, + "strike": true, + "sub": true, + "sup": true, + "table": true, + "tt": true, + "u": true, + "ul": true, + "var": true, +} + +// Section 12.2.5.5. +var svgTagNameAdjustments = map[string]string{ + "altglyph": "altGlyph", + "altglyphdef": "altGlyphDef", + "altglyphitem": "altGlyphItem", + "animatecolor": "animateColor", + "animatemotion": "animateMotion", + "animatetransform": "animateTransform", + "clippath": "clipPath", + "feblend": "feBlend", + "fecolormatrix": "feColorMatrix", + "fecomponenttransfer": "feComponentTransfer", + "fecomposite": "feComposite", + "feconvolvematrix": "feConvolveMatrix", + "fediffuselighting": "feDiffuseLighting", + "fedisplacementmap": "feDisplacementMap", + "fedistantlight": "feDistantLight", + "feflood": "feFlood", + "fefunca": "feFuncA", + "fefuncb": "feFuncB", + "fefuncg": "feFuncG", + "fefuncr": "feFuncR", + "fegaussianblur": "feGaussianBlur", + "feimage": "feImage", + "femerge": "feMerge", + "femergenode": "feMergeNode", + "femorphology": "feMorphology", + "feoffset": "feOffset", + "fepointlight": "fePointLight", + "fespecularlighting": "feSpecularLighting", + "fespotlight": "feSpotLight", + "fetile": "feTile", + "feturbulence": "feTurbulence", + "foreignobject": "foreignObject", + "glyphref": "glyphRef", + "lineargradient": "linearGradient", + "radialgradient": "radialGradient", + "textpath": "textPath", +} + +// Section 12.2.5.1 +var mathMLAttributeAdjustments = map[string]string{ + "definitionurl": "definitionURL", +} + +var svgAttributeAdjustments = map[string]string{ + "attributename": "attributeName", + "attributetype": "attributeType", + "basefrequency": "baseFrequency", + "baseprofile": "baseProfile", + "calcmode": "calcMode", + "clippathunits": "clipPathUnits", + "contentscripttype": "contentScriptType", + "contentstyletype": "contentStyleType", + "diffuseconstant": "diffuseConstant", + "edgemode": "edgeMode", + "externalresourcesrequired": "externalResourcesRequired", + "filterres": "filterRes", + "filterunits": "filterUnits", + "glyphref": "glyphRef", + "gradienttransform": "gradientTransform", + "gradientunits": "gradientUnits", + "kernelmatrix": "kernelMatrix", + "kernelunitlength": "kernelUnitLength", + "keypoints": "keyPoints", + "keysplines": "keySplines", + "keytimes": "keyTimes", + "lengthadjust": "lengthAdjust", + "limitingconeangle": "limitingConeAngle", + "markerheight": "markerHeight", + "markerunits": "markerUnits", + "markerwidth": "markerWidth", + "maskcontentunits": "maskContentUnits", + "maskunits": "maskUnits", + "numoctaves": "numOctaves", + "pathlength": "pathLength", + "patterncontentunits": "patternContentUnits", + "patterntransform": "patternTransform", + "patternunits": "patternUnits", + "pointsatx": "pointsAtX", + "pointsaty": "pointsAtY", + "pointsatz": "pointsAtZ", + "preservealpha": "preserveAlpha", + "preserveaspectratio": "preserveAspectRatio", + "primitiveunits": "primitiveUnits", + "refx": "refX", + "refy": "refY", + "repeatcount": "repeatCount", + "repeatdur": "repeatDur", + "requiredextensions": "requiredExtensions", + "requiredfeatures": "requiredFeatures", + "specularconstant": "specularConstant", + "specularexponent": "specularExponent", + "spreadmethod": "spreadMethod", + "startoffset": "startOffset", + "stddeviation": "stdDeviation", + "stitchtiles": "stitchTiles", + "surfacescale": "surfaceScale", + "systemlanguage": "systemLanguage", + "tablevalues": "tableValues", + "targetx": "targetX", + "targety": "targetY", + "textlength": "textLength", + "viewbox": "viewBox", + "viewtarget": "viewTarget", + "xchannelselector": "xChannelSelector", + "ychannelselector": "yChannelSelector", + "zoomandpan": "zoomAndPan", +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/node.go b/Godeps/_workspace/src/golang.org/x/net/html/node.go new file mode 100644 index 00000000000..619838f06b8 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/node.go @@ -0,0 +1,193 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/html/atom" +) + +// A NodeType is the type of a Node. +type NodeType uint32 + +const ( + ErrorNode NodeType = iota + TextNode + DocumentNode + ElementNode + CommentNode + DoctypeNode + scopeMarkerNode +) + +// Section 12.2.3.3 says "scope markers are inserted when entering applet +// elements, buttons, object elements, marquees, table cells, and table +// captions, and are used to prevent formatting from 'leaking'". +var scopeMarker = Node{Type: scopeMarkerNode} + +// A Node consists of a NodeType and some Data (tag name for element nodes, +// content for text) and are part of a tree of Nodes. Element nodes may also +// have a Namespace and contain a slice of Attributes. Data is unescaped, so +// that it looks like "a 0 { + return (*s)[i-1] + } + return nil +} + +// index returns the index of the top-most occurrence of n in the stack, or -1 +// if n is not present. +func (s *nodeStack) index(n *Node) int { + for i := len(*s) - 1; i >= 0; i-- { + if (*s)[i] == n { + return i + } + } + return -1 +} + +// insert inserts a node at the given index. +func (s *nodeStack) insert(i int, n *Node) { + (*s) = append(*s, nil) + copy((*s)[i+1:], (*s)[i:]) + (*s)[i] = n +} + +// remove removes a node from the stack. It is a no-op if n is not present. +func (s *nodeStack) remove(n *Node) { + i := s.index(n) + if i == -1 { + return + } + copy((*s)[i:], (*s)[i+1:]) + j := len(*s) - 1 + (*s)[j] = nil + *s = (*s)[:j] +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/node_test.go b/Godeps/_workspace/src/golang.org/x/net/html/node_test.go new file mode 100644 index 00000000000..471102f3a22 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/node_test.go @@ -0,0 +1,146 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "fmt" +) + +// checkTreeConsistency checks that a node and its descendants are all +// consistent in their parent/child/sibling relationships. +func checkTreeConsistency(n *Node) error { + return checkTreeConsistency1(n, 0) +} + +func checkTreeConsistency1(n *Node, depth int) error { + if depth == 1e4 { + return fmt.Errorf("html: tree looks like it contains a cycle") + } + if err := checkNodeConsistency(n); err != nil { + return err + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if err := checkTreeConsistency1(c, depth+1); err != nil { + return err + } + } + return nil +} + +// checkNodeConsistency checks that a node's parent/child/sibling relationships +// are consistent. +func checkNodeConsistency(n *Node) error { + if n == nil { + return nil + } + + nParent := 0 + for p := n.Parent; p != nil; p = p.Parent { + nParent++ + if nParent == 1e4 { + return fmt.Errorf("html: parent list looks like an infinite loop") + } + } + + nForward := 0 + for c := n.FirstChild; c != nil; c = c.NextSibling { + nForward++ + if nForward == 1e6 { + return fmt.Errorf("html: forward list of children looks like an infinite loop") + } + if c.Parent != n { + return fmt.Errorf("html: inconsistent child/parent relationship") + } + } + + nBackward := 0 + for c := n.LastChild; c != nil; c = c.PrevSibling { + nBackward++ + if nBackward == 1e6 { + return fmt.Errorf("html: backward list of children looks like an infinite loop") + } + if c.Parent != n { + return fmt.Errorf("html: inconsistent child/parent relationship") + } + } + + if n.Parent != nil { + if n.Parent == n { + return fmt.Errorf("html: inconsistent parent relationship") + } + if n.Parent == n.FirstChild { + return fmt.Errorf("html: inconsistent parent/first relationship") + } + if n.Parent == n.LastChild { + return fmt.Errorf("html: inconsistent parent/last relationship") + } + if n.Parent == n.PrevSibling { + return fmt.Errorf("html: inconsistent parent/prev relationship") + } + if n.Parent == n.NextSibling { + return fmt.Errorf("html: inconsistent parent/next relationship") + } + + parentHasNAsAChild := false + for c := n.Parent.FirstChild; c != nil; c = c.NextSibling { + if c == n { + parentHasNAsAChild = true + break + } + } + if !parentHasNAsAChild { + return fmt.Errorf("html: inconsistent parent/child relationship") + } + } + + if n.PrevSibling != nil && n.PrevSibling.NextSibling != n { + return fmt.Errorf("html: inconsistent prev/next relationship") + } + if n.NextSibling != nil && n.NextSibling.PrevSibling != n { + return fmt.Errorf("html: inconsistent next/prev relationship") + } + + if (n.FirstChild == nil) != (n.LastChild == nil) { + return fmt.Errorf("html: inconsistent first/last relationship") + } + if n.FirstChild != nil && n.FirstChild == n.LastChild { + // We have a sole child. + if n.FirstChild.PrevSibling != nil || n.FirstChild.NextSibling != nil { + return fmt.Errorf("html: inconsistent sole child's sibling relationship") + } + } + + seen := map[*Node]bool{} + + var last *Node + for c := n.FirstChild; c != nil; c = c.NextSibling { + if seen[c] { + return fmt.Errorf("html: inconsistent repeated child") + } + seen[c] = true + last = c + } + if last != n.LastChild { + return fmt.Errorf("html: inconsistent last relationship") + } + + var first *Node + for c := n.LastChild; c != nil; c = c.PrevSibling { + if !seen[c] { + return fmt.Errorf("html: inconsistent missing child") + } + delete(seen, c) + first = c + } + if first != n.FirstChild { + return fmt.Errorf("html: inconsistent first relationship") + } + + if len(seen) != 0 { + return fmt.Errorf("html: inconsistent forwards/backwards child list") + } + + return nil +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/parse.go b/Godeps/_workspace/src/golang.org/x/net/html/parse.go new file mode 100644 index 00000000000..4b9009b0be1 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/parse.go @@ -0,0 +1,2094 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "errors" + "fmt" + "io" + "strings" + + a "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/html/atom" +) + +// A parser implements the HTML5 parsing algorithm: +// https://html.spec.whatwg.org/multipage/syntax.html#tree-construction +type parser struct { + // tokenizer provides the tokens for the parser. + tokenizer *Tokenizer + // tok is the most recently read token. + tok Token + // Self-closing tags like
are treated as start tags, except that + // hasSelfClosingToken is set while they are being processed. + hasSelfClosingToken bool + // doc is the document root element. + doc *Node + // The stack of open elements (section 12.2.3.2) and active formatting + // elements (section 12.2.3.3). + oe, afe nodeStack + // Element pointers (section 12.2.3.4). + head, form *Node + // Other parsing state flags (section 12.2.3.5). + scripting, framesetOK bool + // im is the current insertion mode. + im insertionMode + // originalIM is the insertion mode to go back to after completing a text + // or inTableText insertion mode. + originalIM insertionMode + // fosterParenting is whether new elements should be inserted according to + // the foster parenting rules (section 12.2.5.3). + fosterParenting bool + // quirks is whether the parser is operating in "quirks mode." + quirks bool + // fragment is whether the parser is parsing an HTML fragment. + fragment bool + // context is the context element when parsing an HTML fragment + // (section 12.4). + context *Node +} + +func (p *parser) top() *Node { + if n := p.oe.top(); n != nil { + return n + } + return p.doc +} + +// Stop tags for use in popUntil. These come from section 12.2.3.2. +var ( + defaultScopeStopTags = map[string][]a.Atom{ + "": {a.Applet, a.Caption, a.Html, a.Table, a.Td, a.Th, a.Marquee, a.Object, a.Template}, + "math": {a.AnnotationXml, a.Mi, a.Mn, a.Mo, a.Ms, a.Mtext}, + "svg": {a.Desc, a.ForeignObject, a.Title}, + } +) + +type scope int + +const ( + defaultScope scope = iota + listItemScope + buttonScope + tableScope + tableRowScope + tableBodyScope + selectScope +) + +// popUntil pops the stack of open elements at the highest element whose tag +// is in matchTags, provided there is no higher element in the scope's stop +// tags (as defined in section 12.2.3.2). It returns whether or not there was +// such an element. If there was not, popUntil leaves the stack unchanged. +// +// For example, the set of stop tags for table scope is: "html", "table". If +// the stack was: +// ["html", "body", "font", "table", "b", "i", "u"] +// then popUntil(tableScope, "font") would return false, but +// popUntil(tableScope, "i") would return true and the stack would become: +// ["html", "body", "font", "table", "b"] +// +// If an element's tag is in both the stop tags and matchTags, then the stack +// will be popped and the function returns true (provided, of course, there was +// no higher element in the stack that was also in the stop tags). For example, +// popUntil(tableScope, "table") returns true and leaves: +// ["html", "body", "font"] +func (p *parser) popUntil(s scope, matchTags ...a.Atom) bool { + if i := p.indexOfElementInScope(s, matchTags...); i != -1 { + p.oe = p.oe[:i] + return true + } + return false +} + +// indexOfElementInScope returns the index in p.oe of the highest element whose +// tag is in matchTags that is in scope. If no matching element is in scope, it +// returns -1. +func (p *parser) indexOfElementInScope(s scope, matchTags ...a.Atom) int { + for i := len(p.oe) - 1; i >= 0; i-- { + tagAtom := p.oe[i].DataAtom + if p.oe[i].Namespace == "" { + for _, t := range matchTags { + if t == tagAtom { + return i + } + } + switch s { + case defaultScope: + // No-op. + case listItemScope: + if tagAtom == a.Ol || tagAtom == a.Ul { + return -1 + } + case buttonScope: + if tagAtom == a.Button { + return -1 + } + case tableScope: + if tagAtom == a.Html || tagAtom == a.Table { + return -1 + } + case selectScope: + if tagAtom != a.Optgroup && tagAtom != a.Option { + return -1 + } + default: + panic("unreachable") + } + } + switch s { + case defaultScope, listItemScope, buttonScope: + for _, t := range defaultScopeStopTags[p.oe[i].Namespace] { + if t == tagAtom { + return -1 + } + } + } + } + return -1 +} + +// elementInScope is like popUntil, except that it doesn't modify the stack of +// open elements. +func (p *parser) elementInScope(s scope, matchTags ...a.Atom) bool { + return p.indexOfElementInScope(s, matchTags...) != -1 +} + +// clearStackToContext pops elements off the stack of open elements until a +// scope-defined element is found. +func (p *parser) clearStackToContext(s scope) { + for i := len(p.oe) - 1; i >= 0; i-- { + tagAtom := p.oe[i].DataAtom + switch s { + case tableScope: + if tagAtom == a.Html || tagAtom == a.Table { + p.oe = p.oe[:i+1] + return + } + case tableRowScope: + if tagAtom == a.Html || tagAtom == a.Tr { + p.oe = p.oe[:i+1] + return + } + case tableBodyScope: + if tagAtom == a.Html || tagAtom == a.Tbody || tagAtom == a.Tfoot || tagAtom == a.Thead { + p.oe = p.oe[:i+1] + return + } + default: + panic("unreachable") + } + } +} + +// generateImpliedEndTags pops nodes off the stack of open elements as long as +// the top node has a tag name of dd, dt, li, option, optgroup, p, rp, or rt. +// If exceptions are specified, nodes with that name will not be popped off. +func (p *parser) generateImpliedEndTags(exceptions ...string) { + var i int +loop: + for i = len(p.oe) - 1; i >= 0; i-- { + n := p.oe[i] + if n.Type == ElementNode { + switch n.DataAtom { + case a.Dd, a.Dt, a.Li, a.Option, a.Optgroup, a.P, a.Rp, a.Rt: + for _, except := range exceptions { + if n.Data == except { + break loop + } + } + continue + } + } + break + } + + p.oe = p.oe[:i+1] +} + +// addChild adds a child node n to the top element, and pushes n onto the stack +// of open elements if it is an element node. +func (p *parser) addChild(n *Node) { + if p.shouldFosterParent() { + p.fosterParent(n) + } else { + p.top().AppendChild(n) + } + + if n.Type == ElementNode { + p.oe = append(p.oe, n) + } +} + +// shouldFosterParent returns whether the next node to be added should be +// foster parented. +func (p *parser) shouldFosterParent() bool { + if p.fosterParenting { + switch p.top().DataAtom { + case a.Table, a.Tbody, a.Tfoot, a.Thead, a.Tr: + return true + } + } + return false +} + +// fosterParent adds a child node according to the foster parenting rules. +// Section 12.2.5.3, "foster parenting". +func (p *parser) fosterParent(n *Node) { + var table, parent, prev *Node + var i int + for i = len(p.oe) - 1; i >= 0; i-- { + if p.oe[i].DataAtom == a.Table { + table = p.oe[i] + break + } + } + + if table == nil { + // The foster parent is the html element. + parent = p.oe[0] + } else { + parent = table.Parent + } + if parent == nil { + parent = p.oe[i-1] + } + + if table != nil { + prev = table.PrevSibling + } else { + prev = parent.LastChild + } + if prev != nil && prev.Type == TextNode && n.Type == TextNode { + prev.Data += n.Data + return + } + + parent.InsertBefore(n, table) +} + +// addText adds text to the preceding node if it is a text node, or else it +// calls addChild with a new text node. +func (p *parser) addText(text string) { + if text == "" { + return + } + + if p.shouldFosterParent() { + p.fosterParent(&Node{ + Type: TextNode, + Data: text, + }) + return + } + + t := p.top() + if n := t.LastChild; n != nil && n.Type == TextNode { + n.Data += text + return + } + p.addChild(&Node{ + Type: TextNode, + Data: text, + }) +} + +// addElement adds a child element based on the current token. +func (p *parser) addElement() { + p.addChild(&Node{ + Type: ElementNode, + DataAtom: p.tok.DataAtom, + Data: p.tok.Data, + Attr: p.tok.Attr, + }) +} + +// Section 12.2.3.3. +func (p *parser) addFormattingElement() { + tagAtom, attr := p.tok.DataAtom, p.tok.Attr + p.addElement() + + // Implement the Noah's Ark clause, but with three per family instead of two. + identicalElements := 0 +findIdenticalElements: + for i := len(p.afe) - 1; i >= 0; i-- { + n := p.afe[i] + if n.Type == scopeMarkerNode { + break + } + if n.Type != ElementNode { + continue + } + if n.Namespace != "" { + continue + } + if n.DataAtom != tagAtom { + continue + } + if len(n.Attr) != len(attr) { + continue + } + compareAttributes: + for _, t0 := range n.Attr { + for _, t1 := range attr { + if t0.Key == t1.Key && t0.Namespace == t1.Namespace && t0.Val == t1.Val { + // Found a match for this attribute, continue with the next attribute. + continue compareAttributes + } + } + // If we get here, there is no attribute that matches a. + // Therefore the element is not identical to the new one. + continue findIdenticalElements + } + + identicalElements++ + if identicalElements >= 3 { + p.afe.remove(n) + } + } + + p.afe = append(p.afe, p.top()) +} + +// Section 12.2.3.3. +func (p *parser) clearActiveFormattingElements() { + for { + n := p.afe.pop() + if len(p.afe) == 0 || n.Type == scopeMarkerNode { + return + } + } +} + +// Section 12.2.3.3. +func (p *parser) reconstructActiveFormattingElements() { + n := p.afe.top() + if n == nil { + return + } + if n.Type == scopeMarkerNode || p.oe.index(n) != -1 { + return + } + i := len(p.afe) - 1 + for n.Type != scopeMarkerNode && p.oe.index(n) == -1 { + if i == 0 { + i = -1 + break + } + i-- + n = p.afe[i] + } + for { + i++ + clone := p.afe[i].clone() + p.addChild(clone) + p.afe[i] = clone + if i == len(p.afe)-1 { + break + } + } +} + +// Section 12.2.4. +func (p *parser) acknowledgeSelfClosingTag() { + p.hasSelfClosingToken = false +} + +// An insertion mode (section 12.2.3.1) is the state transition function from +// a particular state in the HTML5 parser's state machine. It updates the +// parser's fields depending on parser.tok (where ErrorToken means EOF). +// It returns whether the token was consumed. +type insertionMode func(*parser) bool + +// setOriginalIM sets the insertion mode to return to after completing a text or +// inTableText insertion mode. +// Section 12.2.3.1, "using the rules for". +func (p *parser) setOriginalIM() { + if p.originalIM != nil { + panic("html: bad parser state: originalIM was set twice") + } + p.originalIM = p.im +} + +// Section 12.2.3.1, "reset the insertion mode". +func (p *parser) resetInsertionMode() { + for i := len(p.oe) - 1; i >= 0; i-- { + n := p.oe[i] + if i == 0 && p.context != nil { + n = p.context + } + + switch n.DataAtom { + case a.Select: + p.im = inSelectIM + case a.Td, a.Th: + p.im = inCellIM + case a.Tr: + p.im = inRowIM + case a.Tbody, a.Thead, a.Tfoot: + p.im = inTableBodyIM + case a.Caption: + p.im = inCaptionIM + case a.Colgroup: + p.im = inColumnGroupIM + case a.Table: + p.im = inTableIM + case a.Head: + p.im = inBodyIM + case a.Body: + p.im = inBodyIM + case a.Frameset: + p.im = inFramesetIM + case a.Html: + p.im = beforeHeadIM + default: + continue + } + return + } + p.im = inBodyIM +} + +const whitespace = " \t\r\n\f" + +// Section 12.2.5.4.1. +func initialIM(p *parser) bool { + switch p.tok.Type { + case TextToken: + p.tok.Data = strings.TrimLeft(p.tok.Data, whitespace) + if len(p.tok.Data) == 0 { + // It was all whitespace, so ignore it. + return true + } + case CommentToken: + p.doc.AppendChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return true + case DoctypeToken: + n, quirks := parseDoctype(p.tok.Data) + p.doc.AppendChild(n) + p.quirks = quirks + p.im = beforeHTMLIM + return true + } + p.quirks = true + p.im = beforeHTMLIM + return false +} + +// Section 12.2.5.4.2. +func beforeHTMLIM(p *parser) bool { + switch p.tok.Type { + case DoctypeToken: + // Ignore the token. + return true + case TextToken: + p.tok.Data = strings.TrimLeft(p.tok.Data, whitespace) + if len(p.tok.Data) == 0 { + // It was all whitespace, so ignore it. + return true + } + case StartTagToken: + if p.tok.DataAtom == a.Html { + p.addElement() + p.im = beforeHeadIM + return true + } + case EndTagToken: + switch p.tok.DataAtom { + case a.Head, a.Body, a.Html, a.Br: + p.parseImpliedToken(StartTagToken, a.Html, a.Html.String()) + return false + default: + // Ignore the token. + return true + } + case CommentToken: + p.doc.AppendChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return true + } + p.parseImpliedToken(StartTagToken, a.Html, a.Html.String()) + return false +} + +// Section 12.2.5.4.3. +func beforeHeadIM(p *parser) bool { + switch p.tok.Type { + case TextToken: + p.tok.Data = strings.TrimLeft(p.tok.Data, whitespace) + if len(p.tok.Data) == 0 { + // It was all whitespace, so ignore it. + return true + } + case StartTagToken: + switch p.tok.DataAtom { + case a.Head: + p.addElement() + p.head = p.top() + p.im = inHeadIM + return true + case a.Html: + return inBodyIM(p) + } + case EndTagToken: + switch p.tok.DataAtom { + case a.Head, a.Body, a.Html, a.Br: + p.parseImpliedToken(StartTagToken, a.Head, a.Head.String()) + return false + default: + // Ignore the token. + return true + } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return true + case DoctypeToken: + // Ignore the token. + return true + } + + p.parseImpliedToken(StartTagToken, a.Head, a.Head.String()) + return false +} + +// Section 12.2.5.4.4. +func inHeadIM(p *parser) bool { + switch p.tok.Type { + case TextToken: + s := strings.TrimLeft(p.tok.Data, whitespace) + if len(s) < len(p.tok.Data) { + // Add the initial whitespace to the current node. + p.addText(p.tok.Data[:len(p.tok.Data)-len(s)]) + if s == "" { + return true + } + p.tok.Data = s + } + case StartTagToken: + switch p.tok.DataAtom { + case a.Html: + return inBodyIM(p) + case a.Base, a.Basefont, a.Bgsound, a.Command, a.Link, a.Meta: + p.addElement() + p.oe.pop() + p.acknowledgeSelfClosingTag() + return true + case a.Script, a.Title, a.Noscript, a.Noframes, a.Style: + p.addElement() + p.setOriginalIM() + p.im = textIM + return true + case a.Head: + // Ignore the token. + return true + } + case EndTagToken: + switch p.tok.DataAtom { + case a.Head: + n := p.oe.pop() + if n.DataAtom != a.Head { + panic("html: bad parser state: element not found, in the in-head insertion mode") + } + p.im = afterHeadIM + return true + case a.Body, a.Html, a.Br: + p.parseImpliedToken(EndTagToken, a.Head, a.Head.String()) + return false + default: + // Ignore the token. + return true + } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return true + case DoctypeToken: + // Ignore the token. + return true + } + + p.parseImpliedToken(EndTagToken, a.Head, a.Head.String()) + return false +} + +// Section 12.2.5.4.6. +func afterHeadIM(p *parser) bool { + switch p.tok.Type { + case TextToken: + s := strings.TrimLeft(p.tok.Data, whitespace) + if len(s) < len(p.tok.Data) { + // Add the initial whitespace to the current node. + p.addText(p.tok.Data[:len(p.tok.Data)-len(s)]) + if s == "" { + return true + } + p.tok.Data = s + } + case StartTagToken: + switch p.tok.DataAtom { + case a.Html: + return inBodyIM(p) + case a.Body: + p.addElement() + p.framesetOK = false + p.im = inBodyIM + return true + case a.Frameset: + p.addElement() + p.im = inFramesetIM + return true + case a.Base, a.Basefont, a.Bgsound, a.Link, a.Meta, a.Noframes, a.Script, a.Style, a.Title: + p.oe = append(p.oe, p.head) + defer p.oe.remove(p.head) + return inHeadIM(p) + case a.Head: + // Ignore the token. + return true + } + case EndTagToken: + switch p.tok.DataAtom { + case a.Body, a.Html, a.Br: + // Drop down to creating an implied tag. + default: + // Ignore the token. + return true + } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return true + case DoctypeToken: + // Ignore the token. + return true + } + + p.parseImpliedToken(StartTagToken, a.Body, a.Body.String()) + p.framesetOK = true + return false +} + +// copyAttributes copies attributes of src not found on dst to dst. +func copyAttributes(dst *Node, src Token) { + if len(src.Attr) == 0 { + return + } + attr := map[string]string{} + for _, t := range dst.Attr { + attr[t.Key] = t.Val + } + for _, t := range src.Attr { + if _, ok := attr[t.Key]; !ok { + dst.Attr = append(dst.Attr, t) + attr[t.Key] = t.Val + } + } +} + +// Section 12.2.5.4.7. +func inBodyIM(p *parser) bool { + switch p.tok.Type { + case TextToken: + d := p.tok.Data + switch n := p.oe.top(); n.DataAtom { + case a.Pre, a.Listing: + if n.FirstChild == nil { + // Ignore a newline at the start of a 
 block.
+				if d != "" && d[0] == '\r' {
+					d = d[1:]
+				}
+				if d != "" && d[0] == '\n' {
+					d = d[1:]
+				}
+			}
+		}
+		d = strings.Replace(d, "\x00", "", -1)
+		if d == "" {
+			return true
+		}
+		p.reconstructActiveFormattingElements()
+		p.addText(d)
+		if p.framesetOK && strings.TrimLeft(d, whitespace) != "" {
+			// There were non-whitespace characters inserted.
+			p.framesetOK = false
+		}
+	case StartTagToken:
+		switch p.tok.DataAtom {
+		case a.Html:
+			copyAttributes(p.oe[0], p.tok)
+		case a.Base, a.Basefont, a.Bgsound, a.Command, a.Link, a.Meta, a.Noframes, a.Script, a.Style, a.Title:
+			return inHeadIM(p)
+		case a.Body:
+			if len(p.oe) >= 2 {
+				body := p.oe[1]
+				if body.Type == ElementNode && body.DataAtom == a.Body {
+					p.framesetOK = false
+					copyAttributes(body, p.tok)
+				}
+			}
+		case a.Frameset:
+			if !p.framesetOK || len(p.oe) < 2 || p.oe[1].DataAtom != a.Body {
+				// Ignore the token.
+				return true
+			}
+			body := p.oe[1]
+			if body.Parent != nil {
+				body.Parent.RemoveChild(body)
+			}
+			p.oe = p.oe[:1]
+			p.addElement()
+			p.im = inFramesetIM
+			return true
+		case a.Address, a.Article, a.Aside, a.Blockquote, a.Center, a.Details, a.Dir, a.Div, a.Dl, a.Fieldset, a.Figcaption, a.Figure, a.Footer, a.Header, a.Hgroup, a.Menu, a.Nav, a.Ol, a.P, a.Section, a.Summary, a.Ul:
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+		case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6:
+			p.popUntil(buttonScope, a.P)
+			switch n := p.top(); n.DataAtom {
+			case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6:
+				p.oe.pop()
+			}
+			p.addElement()
+		case a.Pre, a.Listing:
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+			// The newline, if any, will be dealt with by the TextToken case.
+			p.framesetOK = false
+		case a.Form:
+			if p.form == nil {
+				p.popUntil(buttonScope, a.P)
+				p.addElement()
+				p.form = p.top()
+			}
+		case a.Li:
+			p.framesetOK = false
+			for i := len(p.oe) - 1; i >= 0; i-- {
+				node := p.oe[i]
+				switch node.DataAtom {
+				case a.Li:
+					p.oe = p.oe[:i]
+				case a.Address, a.Div, a.P:
+					continue
+				default:
+					if !isSpecialElement(node) {
+						continue
+					}
+				}
+				break
+			}
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+		case a.Dd, a.Dt:
+			p.framesetOK = false
+			for i := len(p.oe) - 1; i >= 0; i-- {
+				node := p.oe[i]
+				switch node.DataAtom {
+				case a.Dd, a.Dt:
+					p.oe = p.oe[:i]
+				case a.Address, a.Div, a.P:
+					continue
+				default:
+					if !isSpecialElement(node) {
+						continue
+					}
+				}
+				break
+			}
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+		case a.Plaintext:
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+		case a.Button:
+			p.popUntil(defaultScope, a.Button)
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+			p.framesetOK = false
+		case a.A:
+			for i := len(p.afe) - 1; i >= 0 && p.afe[i].Type != scopeMarkerNode; i-- {
+				if n := p.afe[i]; n.Type == ElementNode && n.DataAtom == a.A {
+					p.inBodyEndTagFormatting(a.A)
+					p.oe.remove(n)
+					p.afe.remove(n)
+					break
+				}
+			}
+			p.reconstructActiveFormattingElements()
+			p.addFormattingElement()
+		case a.B, a.Big, a.Code, a.Em, a.Font, a.I, a.S, a.Small, a.Strike, a.Strong, a.Tt, a.U:
+			p.reconstructActiveFormattingElements()
+			p.addFormattingElement()
+		case a.Nobr:
+			p.reconstructActiveFormattingElements()
+			if p.elementInScope(defaultScope, a.Nobr) {
+				p.inBodyEndTagFormatting(a.Nobr)
+				p.reconstructActiveFormattingElements()
+			}
+			p.addFormattingElement()
+		case a.Applet, a.Marquee, a.Object:
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+			p.afe = append(p.afe, &scopeMarker)
+			p.framesetOK = false
+		case a.Table:
+			if !p.quirks {
+				p.popUntil(buttonScope, a.P)
+			}
+			p.addElement()
+			p.framesetOK = false
+			p.im = inTableIM
+			return true
+		case a.Area, a.Br, a.Embed, a.Img, a.Input, a.Keygen, a.Wbr:
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+			p.oe.pop()
+			p.acknowledgeSelfClosingTag()
+			if p.tok.DataAtom == a.Input {
+				for _, t := range p.tok.Attr {
+					if t.Key == "type" {
+						if strings.ToLower(t.Val) == "hidden" {
+							// Skip setting framesetOK = false
+							return true
+						}
+					}
+				}
+			}
+			p.framesetOK = false
+		case a.Param, a.Source, a.Track:
+			p.addElement()
+			p.oe.pop()
+			p.acknowledgeSelfClosingTag()
+		case a.Hr:
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+			p.oe.pop()
+			p.acknowledgeSelfClosingTag()
+			p.framesetOK = false
+		case a.Image:
+			p.tok.DataAtom = a.Img
+			p.tok.Data = a.Img.String()
+			return false
+		case a.Isindex:
+			if p.form != nil {
+				// Ignore the token.
+				return true
+			}
+			action := ""
+			prompt := "This is a searchable index. Enter search keywords: "
+			attr := []Attribute{{Key: "name", Val: "isindex"}}
+			for _, t := range p.tok.Attr {
+				switch t.Key {
+				case "action":
+					action = t.Val
+				case "name":
+					// Ignore the attribute.
+				case "prompt":
+					prompt = t.Val
+				default:
+					attr = append(attr, t)
+				}
+			}
+			p.acknowledgeSelfClosingTag()
+			p.popUntil(buttonScope, a.P)
+			p.parseImpliedToken(StartTagToken, a.Form, a.Form.String())
+			if action != "" {
+				p.form.Attr = []Attribute{{Key: "action", Val: action}}
+			}
+			p.parseImpliedToken(StartTagToken, a.Hr, a.Hr.String())
+			p.parseImpliedToken(StartTagToken, a.Label, a.Label.String())
+			p.addText(prompt)
+			p.addChild(&Node{
+				Type:     ElementNode,
+				DataAtom: a.Input,
+				Data:     a.Input.String(),
+				Attr:     attr,
+			})
+			p.oe.pop()
+			p.parseImpliedToken(EndTagToken, a.Label, a.Label.String())
+			p.parseImpliedToken(StartTagToken, a.Hr, a.Hr.String())
+			p.parseImpliedToken(EndTagToken, a.Form, a.Form.String())
+		case a.Textarea:
+			p.addElement()
+			p.setOriginalIM()
+			p.framesetOK = false
+			p.im = textIM
+		case a.Xmp:
+			p.popUntil(buttonScope, a.P)
+			p.reconstructActiveFormattingElements()
+			p.framesetOK = false
+			p.addElement()
+			p.setOriginalIM()
+			p.im = textIM
+		case a.Iframe:
+			p.framesetOK = false
+			p.addElement()
+			p.setOriginalIM()
+			p.im = textIM
+		case a.Noembed, a.Noscript:
+			p.addElement()
+			p.setOriginalIM()
+			p.im = textIM
+		case a.Select:
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+			p.framesetOK = false
+			p.im = inSelectIM
+			return true
+		case a.Optgroup, a.Option:
+			if p.top().DataAtom == a.Option {
+				p.oe.pop()
+			}
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+		case a.Rp, a.Rt:
+			if p.elementInScope(defaultScope, a.Ruby) {
+				p.generateImpliedEndTags()
+			}
+			p.addElement()
+		case a.Math, a.Svg:
+			p.reconstructActiveFormattingElements()
+			if p.tok.DataAtom == a.Math {
+				adjustAttributeNames(p.tok.Attr, mathMLAttributeAdjustments)
+			} else {
+				adjustAttributeNames(p.tok.Attr, svgAttributeAdjustments)
+			}
+			adjustForeignAttributes(p.tok.Attr)
+			p.addElement()
+			p.top().Namespace = p.tok.Data
+			if p.hasSelfClosingToken {
+				p.oe.pop()
+				p.acknowledgeSelfClosingTag()
+			}
+			return true
+		case a.Caption, a.Col, a.Colgroup, a.Frame, a.Head, a.Tbody, a.Td, a.Tfoot, a.Th, a.Thead, a.Tr:
+			// Ignore the token.
+		default:
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+		}
+	case EndTagToken:
+		switch p.tok.DataAtom {
+		case a.Body:
+			if p.elementInScope(defaultScope, a.Body) {
+				p.im = afterBodyIM
+			}
+		case a.Html:
+			if p.elementInScope(defaultScope, a.Body) {
+				p.parseImpliedToken(EndTagToken, a.Body, a.Body.String())
+				return false
+			}
+			return true
+		case a.Address, a.Article, a.Aside, a.Blockquote, a.Button, a.Center, a.Details, a.Dir, a.Div, a.Dl, a.Fieldset, a.Figcaption, a.Figure, a.Footer, a.Header, a.Hgroup, a.Listing, a.Menu, a.Nav, a.Ol, a.Pre, a.Section, a.Summary, a.Ul:
+			p.popUntil(defaultScope, p.tok.DataAtom)
+		case a.Form:
+			node := p.form
+			p.form = nil
+			i := p.indexOfElementInScope(defaultScope, a.Form)
+			if node == nil || i == -1 || p.oe[i] != node {
+				// Ignore the token.
+				return true
+			}
+			p.generateImpliedEndTags()
+			p.oe.remove(node)
+		case a.P:
+			if !p.elementInScope(buttonScope, a.P) {
+				p.parseImpliedToken(StartTagToken, a.P, a.P.String())
+			}
+			p.popUntil(buttonScope, a.P)
+		case a.Li:
+			p.popUntil(listItemScope, a.Li)
+		case a.Dd, a.Dt:
+			p.popUntil(defaultScope, p.tok.DataAtom)
+		case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6:
+			p.popUntil(defaultScope, a.H1, a.H2, a.H3, a.H4, a.H5, a.H6)
+		case a.A, a.B, a.Big, a.Code, a.Em, a.Font, a.I, a.Nobr, a.S, a.Small, a.Strike, a.Strong, a.Tt, a.U:
+			p.inBodyEndTagFormatting(p.tok.DataAtom)
+		case a.Applet, a.Marquee, a.Object:
+			if p.popUntil(defaultScope, p.tok.DataAtom) {
+				p.clearActiveFormattingElements()
+			}
+		case a.Br:
+			p.tok.Type = StartTagToken
+			return false
+		default:
+			p.inBodyEndTagOther(p.tok.DataAtom)
+		}
+	case CommentToken:
+		p.addChild(&Node{
+			Type: CommentNode,
+			Data: p.tok.Data,
+		})
+	}
+
+	return true
+}
+
+func (p *parser) inBodyEndTagFormatting(tagAtom a.Atom) {
+	// This is the "adoption agency" algorithm, described at
+	// https://html.spec.whatwg.org/multipage/syntax.html#adoptionAgency
+
+	// TODO: this is a fairly literal line-by-line translation of that algorithm.
+	// Once the code successfully parses the comprehensive test suite, we should
+	// refactor this code to be more idiomatic.
+
+	// Steps 1-4. The outer loop.
+	for i := 0; i < 8; i++ {
+		// Step 5. Find the formatting element.
+		var formattingElement *Node
+		for j := len(p.afe) - 1; j >= 0; j-- {
+			if p.afe[j].Type == scopeMarkerNode {
+				break
+			}
+			if p.afe[j].DataAtom == tagAtom {
+				formattingElement = p.afe[j]
+				break
+			}
+		}
+		if formattingElement == nil {
+			p.inBodyEndTagOther(tagAtom)
+			return
+		}
+		feIndex := p.oe.index(formattingElement)
+		if feIndex == -1 {
+			p.afe.remove(formattingElement)
+			return
+		}
+		if !p.elementInScope(defaultScope, tagAtom) {
+			// Ignore the tag.
+			return
+		}
+
+		// Steps 9-10. Find the furthest block.
+		var furthestBlock *Node
+		for _, e := range p.oe[feIndex:] {
+			if isSpecialElement(e) {
+				furthestBlock = e
+				break
+			}
+		}
+		if furthestBlock == nil {
+			e := p.oe.pop()
+			for e != formattingElement {
+				e = p.oe.pop()
+			}
+			p.afe.remove(e)
+			return
+		}
+
+		// Steps 11-12. Find the common ancestor and bookmark node.
+		commonAncestor := p.oe[feIndex-1]
+		bookmark := p.afe.index(formattingElement)
+
+		// Step 13. The inner loop. Find the lastNode to reparent.
+		lastNode := furthestBlock
+		node := furthestBlock
+		x := p.oe.index(node)
+		// Steps 13.1-13.2
+		for j := 0; j < 3; j++ {
+			// Step 13.3.
+			x--
+			node = p.oe[x]
+			// Step 13.4 - 13.5.
+			if p.afe.index(node) == -1 {
+				p.oe.remove(node)
+				continue
+			}
+			// Step 13.6.
+			if node == formattingElement {
+				break
+			}
+			// Step 13.7.
+			clone := node.clone()
+			p.afe[p.afe.index(node)] = clone
+			p.oe[p.oe.index(node)] = clone
+			node = clone
+			// Step 13.8.
+			if lastNode == furthestBlock {
+				bookmark = p.afe.index(node) + 1
+			}
+			// Step 13.9.
+			if lastNode.Parent != nil {
+				lastNode.Parent.RemoveChild(lastNode)
+			}
+			node.AppendChild(lastNode)
+			// Step 13.10.
+			lastNode = node
+		}
+
+		// Step 14. Reparent lastNode to the common ancestor,
+		// or for misnested table nodes, to the foster parent.
+		if lastNode.Parent != nil {
+			lastNode.Parent.RemoveChild(lastNode)
+		}
+		switch commonAncestor.DataAtom {
+		case a.Table, a.Tbody, a.Tfoot, a.Thead, a.Tr:
+			p.fosterParent(lastNode)
+		default:
+			commonAncestor.AppendChild(lastNode)
+		}
+
+		// Steps 15-17. Reparent nodes from the furthest block's children
+		// to a clone of the formatting element.
+		clone := formattingElement.clone()
+		reparentChildren(clone, furthestBlock)
+		furthestBlock.AppendChild(clone)
+
+		// Step 18. Fix up the list of active formatting elements.
+		if oldLoc := p.afe.index(formattingElement); oldLoc != -1 && oldLoc < bookmark {
+			// Move the bookmark with the rest of the list.
+			bookmark--
+		}
+		p.afe.remove(formattingElement)
+		p.afe.insert(bookmark, clone)
+
+		// Step 19. Fix up the stack of open elements.
+		p.oe.remove(formattingElement)
+		p.oe.insert(p.oe.index(furthestBlock)+1, clone)
+	}
+}
+
+// inBodyEndTagOther performs the "any other end tag" algorithm for inBodyIM.
+// "Any other end tag" handling from 12.2.5.5 The rules for parsing tokens in foreign content
+// https://html.spec.whatwg.org/multipage/syntax.html#parsing-main-inforeign
+func (p *parser) inBodyEndTagOther(tagAtom a.Atom) {
+	for i := len(p.oe) - 1; i >= 0; i-- {
+		if p.oe[i].DataAtom == tagAtom {
+			p.oe = p.oe[:i]
+			break
+		}
+		if isSpecialElement(p.oe[i]) {
+			break
+		}
+	}
+}
+
+// Section 12.2.5.4.8.
+func textIM(p *parser) bool {
+	switch p.tok.Type {
+	case ErrorToken:
+		p.oe.pop()
+	case TextToken:
+		d := p.tok.Data
+		if n := p.oe.top(); n.DataAtom == a.Textarea && n.FirstChild == nil {
+			// Ignore a newline at the start of a -->
+#errors
+#document
+| 
+|   
+|   
+|     -->
+#errors
+#document
+| 
+|   
+|   
+|     
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+#document
+| 
+|   
+|   
+|     
+#errors
+Line: 1 Col: 9 Unexpected end tag (strong). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end tag (strong) after the (implied) root element.
+Line: 1 Col: 13 Unexpected end tag (b) after the (implied) root element.
+Line: 1 Col: 18 Unexpected end tag (em) after the (implied) root element.
+Line: 1 Col: 22 Unexpected end tag (i) after the (implied) root element.
+Line: 1 Col: 26 Unexpected end tag (u) after the (implied) root element.
+Line: 1 Col: 35 Unexpected end tag (strike) after the (implied) root element.
+Line: 1 Col: 39 Unexpected end tag (s) after the (implied) root element.
+Line: 1 Col: 47 Unexpected end tag (blink) after the (implied) root element.
+Line: 1 Col: 52 Unexpected end tag (tt) after the (implied) root element.
+Line: 1 Col: 58 Unexpected end tag (pre) after the (implied) root element.
+Line: 1 Col: 64 Unexpected end tag (big) after the (implied) root element.
+Line: 1 Col: 72 Unexpected end tag (small) after the (implied) root element.
+Line: 1 Col: 79 Unexpected end tag (font) after the (implied) root element.
+Line: 1 Col: 88 Unexpected end tag (select) after the (implied) root element.
+Line: 1 Col: 93 Unexpected end tag (h1) after the (implied) root element.
+Line: 1 Col: 98 Unexpected end tag (h2) after the (implied) root element.
+Line: 1 Col: 103 Unexpected end tag (h3) after the (implied) root element.
+Line: 1 Col: 108 Unexpected end tag (h4) after the (implied) root element.
+Line: 1 Col: 113 Unexpected end tag (h5) after the (implied) root element.
+Line: 1 Col: 118 Unexpected end tag (h6) after the (implied) root element.
+Line: 1 Col: 125 Unexpected end tag (body) after the (implied) root element.
+Line: 1 Col: 130 Unexpected end tag (br). Treated as br element.
+Line: 1 Col: 134 End tag (a) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 140 This element (img) has no end tag.
+Line: 1 Col: 148 Unexpected end tag (title). Ignored.
+Line: 1 Col: 155 Unexpected end tag (span). Ignored.
+Line: 1 Col: 163 Unexpected end tag (style). Ignored.
+Line: 1 Col: 172 Unexpected end tag (script). Ignored.
+Line: 1 Col: 180 Unexpected end tag (table). Ignored.
+Line: 1 Col: 185 Unexpected end tag (th). Ignored.
+Line: 1 Col: 190 Unexpected end tag (td). Ignored.
+Line: 1 Col: 195 Unexpected end tag (tr). Ignored.
+Line: 1 Col: 203 This element (frame) has no end tag.
+Line: 1 Col: 210 This element (area) has no end tag.
+Line: 1 Col: 217 Unexpected end tag (link). Ignored.
+Line: 1 Col: 225 This element (param) has no end tag.
+Line: 1 Col: 230 This element (hr) has no end tag.
+Line: 1 Col: 238 This element (input) has no end tag.
+Line: 1 Col: 244 Unexpected end tag (col). Ignored.
+Line: 1 Col: 251 Unexpected end tag (base). Ignored.
+Line: 1 Col: 258 Unexpected end tag (meta). Ignored.
+Line: 1 Col: 269 This element (basefont) has no end tag.
+Line: 1 Col: 279 This element (bgsound) has no end tag.
+Line: 1 Col: 287 This element (embed) has no end tag.
+Line: 1 Col: 296 This element (spacer) has no end tag.
+Line: 1 Col: 300 Unexpected end tag (p). Ignored.
+Line: 1 Col: 305 End tag (dd) seen too early. Expected other end tag.
+Line: 1 Col: 310 End tag (dt) seen too early. Expected other end tag.
+Line: 1 Col: 320 Unexpected end tag (caption). Ignored.
+Line: 1 Col: 331 Unexpected end tag (colgroup). Ignored.
+Line: 1 Col: 339 Unexpected end tag (tbody). Ignored.
+Line: 1 Col: 347 Unexpected end tag (tfoot). Ignored.
+Line: 1 Col: 355 Unexpected end tag (thead). Ignored.
+Line: 1 Col: 365 End tag (address) seen too early. Expected other end tag.
+Line: 1 Col: 378 End tag (blockquote) seen too early. Expected other end tag.
+Line: 1 Col: 387 End tag (center) seen too early. Expected other end tag.
+Line: 1 Col: 393 Unexpected end tag (dir). Ignored.
+Line: 1 Col: 399 End tag (div) seen too early. Expected other end tag.
+Line: 1 Col: 404 End tag (dl) seen too early. Expected other end tag.
+Line: 1 Col: 415 End tag (fieldset) seen too early. Expected other end tag.
+Line: 1 Col: 425 End tag (listing) seen too early. Expected other end tag.
+Line: 1 Col: 432 End tag (menu) seen too early. Expected other end tag.
+Line: 1 Col: 437 End tag (ol) seen too early. Expected other end tag.
+Line: 1 Col: 442 End tag (ul) seen too early. Expected other end tag.
+Line: 1 Col: 447 End tag (li) seen too early. Expected other end tag.
+Line: 1 Col: 454 End tag (nobr) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 460 This element (wbr) has no end tag.
+Line: 1 Col: 476 End tag (button) seen too early. Expected other end tag.
+Line: 1 Col: 486 End tag (marquee) seen too early. Expected other end tag.
+Line: 1 Col: 495 End tag (object) seen too early. Expected other end tag.
+Line: 1 Col: 513 Unexpected end tag (html). Ignored.
+Line: 1 Col: 513 Unexpected end tag (frameset). Ignored.
+Line: 1 Col: 520 Unexpected end tag (head). Ignored.
+Line: 1 Col: 529 Unexpected end tag (iframe). Ignored.
+Line: 1 Col: 537 This element (image) has no end tag.
+Line: 1 Col: 547 This element (isindex) has no end tag.
+Line: 1 Col: 557 Unexpected end tag (noembed). Ignored.
+Line: 1 Col: 568 Unexpected end tag (noframes). Ignored.
+Line: 1 Col: 579 Unexpected end tag (noscript). Ignored.
+Line: 1 Col: 590 Unexpected end tag (optgroup). Ignored.
+Line: 1 Col: 599 Unexpected end tag (option). Ignored.
+Line: 1 Col: 611 Unexpected end tag (plaintext). Ignored.
+Line: 1 Col: 622 Unexpected end tag (textarea). Ignored.
+#document
+| 
+|   
+|   
+|     

+|     

+
+#data
+

+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end tag (strong) in table context caused voodoo mode.
+Line: 1 Col: 20 End tag (strong) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 24 Unexpected end tag (b) in table context caused voodoo mode.
+Line: 1 Col: 24 End tag (b) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 29 Unexpected end tag (em) in table context caused voodoo mode.
+Line: 1 Col: 29 End tag (em) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 33 Unexpected end tag (i) in table context caused voodoo mode.
+Line: 1 Col: 33 End tag (i) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 37 Unexpected end tag (u) in table context caused voodoo mode.
+Line: 1 Col: 37 End tag (u) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 46 Unexpected end tag (strike) in table context caused voodoo mode.
+Line: 1 Col: 46 End tag (strike) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 50 Unexpected end tag (s) in table context caused voodoo mode.
+Line: 1 Col: 50 End tag (s) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 58 Unexpected end tag (blink) in table context caused voodoo mode.
+Line: 1 Col: 58 Unexpected end tag (blink). Ignored.
+Line: 1 Col: 63 Unexpected end tag (tt) in table context caused voodoo mode.
+Line: 1 Col: 63 End tag (tt) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 69 Unexpected end tag (pre) in table context caused voodoo mode.
+Line: 1 Col: 69 End tag (pre) seen too early. Expected other end tag.
+Line: 1 Col: 75 Unexpected end tag (big) in table context caused voodoo mode.
+Line: 1 Col: 75 End tag (big) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 83 Unexpected end tag (small) in table context caused voodoo mode.
+Line: 1 Col: 83 End tag (small) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 90 Unexpected end tag (font) in table context caused voodoo mode.
+Line: 1 Col: 90 End tag (font) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 99 Unexpected end tag (select) in table context caused voodoo mode.
+Line: 1 Col: 99 Unexpected end tag (select). Ignored.
+Line: 1 Col: 104 Unexpected end tag (h1) in table context caused voodoo mode.
+Line: 1 Col: 104 End tag (h1) seen too early. Expected other end tag.
+Line: 1 Col: 109 Unexpected end tag (h2) in table context caused voodoo mode.
+Line: 1 Col: 109 End tag (h2) seen too early. Expected other end tag.
+Line: 1 Col: 114 Unexpected end tag (h3) in table context caused voodoo mode.
+Line: 1 Col: 114 End tag (h3) seen too early. Expected other end tag.
+Line: 1 Col: 119 Unexpected end tag (h4) in table context caused voodoo mode.
+Line: 1 Col: 119 End tag (h4) seen too early. Expected other end tag.
+Line: 1 Col: 124 Unexpected end tag (h5) in table context caused voodoo mode.
+Line: 1 Col: 124 End tag (h5) seen too early. Expected other end tag.
+Line: 1 Col: 129 Unexpected end tag (h6) in table context caused voodoo mode.
+Line: 1 Col: 129 End tag (h6) seen too early. Expected other end tag.
+Line: 1 Col: 136 Unexpected end tag (body) in the table row phase. Ignored.
+Line: 1 Col: 141 Unexpected end tag (br) in table context caused voodoo mode.
+Line: 1 Col: 141 Unexpected end tag (br). Treated as br element.
+Line: 1 Col: 145 Unexpected end tag (a) in table context caused voodoo mode.
+Line: 1 Col: 145 End tag (a) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 151 Unexpected end tag (img) in table context caused voodoo mode.
+Line: 1 Col: 151 This element (img) has no end tag.
+Line: 1 Col: 159 Unexpected end tag (title) in table context caused voodoo mode.
+Line: 1 Col: 159 Unexpected end tag (title). Ignored.
+Line: 1 Col: 166 Unexpected end tag (span) in table context caused voodoo mode.
+Line: 1 Col: 166 Unexpected end tag (span). Ignored.
+Line: 1 Col: 174 Unexpected end tag (style) in table context caused voodoo mode.
+Line: 1 Col: 174 Unexpected end tag (style). Ignored.
+Line: 1 Col: 183 Unexpected end tag (script) in table context caused voodoo mode.
+Line: 1 Col: 183 Unexpected end tag (script). Ignored.
+Line: 1 Col: 196 Unexpected end tag (th). Ignored.
+Line: 1 Col: 201 Unexpected end tag (td). Ignored.
+Line: 1 Col: 206 Unexpected end tag (tr). Ignored.
+Line: 1 Col: 214 This element (frame) has no end tag.
+Line: 1 Col: 221 This element (area) has no end tag.
+Line: 1 Col: 228 Unexpected end tag (link). Ignored.
+Line: 1 Col: 236 This element (param) has no end tag.
+Line: 1 Col: 241 This element (hr) has no end tag.
+Line: 1 Col: 249 This element (input) has no end tag.
+Line: 1 Col: 255 Unexpected end tag (col). Ignored.
+Line: 1 Col: 262 Unexpected end tag (base). Ignored.
+Line: 1 Col: 269 Unexpected end tag (meta). Ignored.
+Line: 1 Col: 280 This element (basefont) has no end tag.
+Line: 1 Col: 290 This element (bgsound) has no end tag.
+Line: 1 Col: 298 This element (embed) has no end tag.
+Line: 1 Col: 307 This element (spacer) has no end tag.
+Line: 1 Col: 311 Unexpected end tag (p). Ignored.
+Line: 1 Col: 316 End tag (dd) seen too early. Expected other end tag.
+Line: 1 Col: 321 End tag (dt) seen too early. Expected other end tag.
+Line: 1 Col: 331 Unexpected end tag (caption). Ignored.
+Line: 1 Col: 342 Unexpected end tag (colgroup). Ignored.
+Line: 1 Col: 350 Unexpected end tag (tbody). Ignored.
+Line: 1 Col: 358 Unexpected end tag (tfoot). Ignored.
+Line: 1 Col: 366 Unexpected end tag (thead). Ignored.
+Line: 1 Col: 376 End tag (address) seen too early. Expected other end tag.
+Line: 1 Col: 389 End tag (blockquote) seen too early. Expected other end tag.
+Line: 1 Col: 398 End tag (center) seen too early. Expected other end tag.
+Line: 1 Col: 404 Unexpected end tag (dir). Ignored.
+Line: 1 Col: 410 End tag (div) seen too early. Expected other end tag.
+Line: 1 Col: 415 End tag (dl) seen too early. Expected other end tag.
+Line: 1 Col: 426 End tag (fieldset) seen too early. Expected other end tag.
+Line: 1 Col: 436 End tag (listing) seen too early. Expected other end tag.
+Line: 1 Col: 443 End tag (menu) seen too early. Expected other end tag.
+Line: 1 Col: 448 End tag (ol) seen too early. Expected other end tag.
+Line: 1 Col: 453 End tag (ul) seen too early. Expected other end tag.
+Line: 1 Col: 458 End tag (li) seen too early. Expected other end tag.
+Line: 1 Col: 465 End tag (nobr) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 471 This element (wbr) has no end tag.
+Line: 1 Col: 487 End tag (button) seen too early. Expected other end tag.
+Line: 1 Col: 497 End tag (marquee) seen too early. Expected other end tag.
+Line: 1 Col: 506 End tag (object) seen too early. Expected other end tag.
+Line: 1 Col: 524 Unexpected end tag (html). Ignored.
+Line: 1 Col: 524 Unexpected end tag (frameset). Ignored.
+Line: 1 Col: 531 Unexpected end tag (head). Ignored.
+Line: 1 Col: 540 Unexpected end tag (iframe). Ignored.
+Line: 1 Col: 548 This element (image) has no end tag.
+Line: 1 Col: 558 This element (isindex) has no end tag.
+Line: 1 Col: 568 Unexpected end tag (noembed). Ignored.
+Line: 1 Col: 579 Unexpected end tag (noframes). Ignored.
+Line: 1 Col: 590 Unexpected end tag (noscript). Ignored.
+Line: 1 Col: 601 Unexpected end tag (optgroup). Ignored.
+Line: 1 Col: 610 Unexpected end tag (option). Ignored.
+Line: 1 Col: 622 Unexpected end tag (plaintext). Ignored.
+Line: 1 Col: 633 Unexpected end tag (textarea). Ignored.
+#document
+| 
+|   
+|   
+|     

+|     
+|       
+|         
+|     

+
+#data
+
+#errors
+Line: 1 Col: 10 Unexpected start tag (frameset). Expected DOCTYPE.
+Line: 1 Col: 10 Expected closing tag. Unexpected end of file.
+#document
+| 
+|   
+|   
diff --git a/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests10.dat b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests10.dat
new file mode 100644
index 00000000000..4f8df86f208
--- /dev/null
+++ b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests10.dat
@@ -0,0 +1,799 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+a
+#errors
+29: Bogus comment
+#document
+| 
+| 
+|   
+|   
+|     
+|     
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+35: Stray “svg” start tag.
+42: Stray end tag “svg”
+#document
+| 
+| 
+|   
+|   
+|     
+#errors
+43: Stray “svg” start tag.
+50: Stray end tag “svg”
+#document
+| 
+| 
+|   
+|   
+|     

+#errors
+34: Start tag “svg” seen in “table”.
+41: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|     
+
+#data
+
foo

+#errors
+34: Start tag “svg” seen in “table”.
+46: Stray end tag “g”.
+53: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|     
+
+#data
+
foobar

+#errors
+34: Start tag “svg” seen in “table”.
+46: Stray end tag “g”.
+58: Stray end tag “g”.
+65: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+
+#data
+
foobar

+#errors
+41: Start tag “svg” seen in “table”.
+53: Stray end tag “g”.
+65: Stray end tag “g”.
+72: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+|       
+
+#data
+
foobar

+#errors
+45: Start tag “svg” seen in “table”.
+57: Stray end tag “g”.
+69: Stray end tag “g”.
+76: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+|       
+|         
+
+#data
+
foobar

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
+|               
+|                 "foo"
+|               
+|                 "bar"
+
+#data
+
foobar
baz

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
+|               
+|                 "foo"
+|               
+|                 "bar"
+|             

+|               "baz"
+
+#data
+
foobar
baz

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|         

+|           "baz"
+
+#data
+
foobar
baz
quux
+#errors
+70: HTML start tag “p” in a foreign namespace context.
+81: “table” closed but “caption” was still open.
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|         

+|           "baz"
+|     

+|       "quux"
+
+#data
+
foobarbaz
quux
+#errors
+78: “table” closed but “caption” was still open.
+78: Unclosed elements on stack.
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|           "baz"
+|     

+|       "quux"
+
+#data
+foobar
baz
quux
+#errors
+44: Start tag “svg” seen in “table”.
+56: Stray end tag “g”.
+68: Stray end tag “g”.
+71: HTML start tag “p” in a foreign namespace context.
+71: Start tag “p” seen in “table”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+|     
+|       +|     

+|       "quux"
+
+#data
+


quux
+#errors
+50: Stray “svg” start tag.
+54: Stray “g” start tag.
+62: Stray end tag “g”
+66: Stray “g” start tag.
+74: Stray end tag “g”
+77: Stray “p” start tag.
+88: “table” end tag with “select” open.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
quux
+#errors
+36: Start tag “select” seen in “table”.
+42: Stray “svg” start tag.
+46: Stray “g” start tag.
+54: Stray end tag “g”
+58: Stray “g” start tag.
+66: Stray end tag “g”
+69: Stray “p” start tag.
+80: “table” end tag with “select” open.
+#document
+| 
+| 
+|   
+|   
+|     
+|     

+|       "quux"
+
+#data
+foobar
baz
+#errors
+41: Stray “svg” start tag.
+68: HTML start tag “p” in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+
+#data
+foobar
baz
+#errors
+34: Stray “svg” start tag.
+61: HTML start tag “p” in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+
+#data
+

+#errors
+31: Stray “svg” start tag.
+35: Stray “g” start tag.
+40: Stray end tag “g”
+44: Stray “g” start tag.
+49: Stray end tag “g”
+52: Stray “p” start tag.
+58: Stray “span” start tag.
+58: End of file seen and there were open elements.
+#document
+| 
+| 
+|   
+|   
+
+#data
+

+#errors
+42: Stray “svg” start tag.
+46: Stray “g” start tag.
+51: Stray end tag “g”
+55: Stray “g” start tag.
+60: Stray end tag “g”
+63: Stray “p” start tag.
+69: Stray “span” start tag.
+#document
+| 
+| 
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     
+|       xlink href="foo"
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+
+#data
+bar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+|       "bar"
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|     "a"
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|     "a"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|       
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|             
+|               "a"
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|             

+|               "a"
+
+#data
+
    a
+#errors
+40: HTML start tag “ul” in a foreign namespace context.
+41: End of file in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
    
+|           
+|           
      
+|             "a"
+
+#data
+
        a
+#errors
+35: HTML start tag “ul” in a foreign namespace context.
+36: End of file in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|         
          
+|           "a"
+
+#data
+
          
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
          
+|       
+|         
+|           
          
+
+#data
+
          
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
          
+|       
+|         
+|           
          
+
+#data
+
          
+#errors
+#document
+| 
+|   
+|   
+|     
          
+|       
+|         
+|           
+|             
          
+|             
          
+
+#data
+
          
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
          
+|           
+|             
          
+|               
+|       
+
+#data
+
          
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|           
+|             
          
+|               
          
+|       
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|       
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     
+|     

+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|       
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|           
+|             

+|               
+|                 
+|               
+|           
+|       
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|           
+|             
+|               
+|                 
+|               
+|             
+|           
+|       
diff --git a/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests11.dat b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests11.dat
new file mode 100644
index 00000000000..638cde479f7
--- /dev/null
+++ b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests11.dat
@@ -0,0 +1,482 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributename=""
+|       attributetype=""
+|       basefrequency=""
+|       baseprofile=""
+|       calcmode=""
+|       clippathunits=""
+|       contentscripttype=""
+|       contentstyletype=""
+|       diffuseconstant=""
+|       edgemode=""
+|       externalresourcesrequired=""
+|       filterres=""
+|       filterunits=""
+|       glyphref=""
+|       gradienttransform=""
+|       gradientunits=""
+|       kernelmatrix=""
+|       kernelunitlength=""
+|       keypoints=""
+|       keysplines=""
+|       keytimes=""
+|       lengthadjust=""
+|       limitingconeangle=""
+|       markerheight=""
+|       markerunits=""
+|       markerwidth=""
+|       maskcontentunits=""
+|       maskunits=""
+|       numoctaves=""
+|       pathlength=""
+|       patterncontentunits=""
+|       patterntransform=""
+|       patternunits=""
+|       pointsatx=""
+|       pointsaty=""
+|       pointsatz=""
+|       preservealpha=""
+|       preserveaspectratio=""
+|       primitiveunits=""
+|       refx=""
+|       refy=""
+|       repeatcount=""
+|       repeatdur=""
+|       requiredextensions=""
+|       requiredfeatures=""
+|       specularconstant=""
+|       specularexponent=""
+|       spreadmethod=""
+|       startoffset=""
+|       stddeviation=""
+|       stitchtiles=""
+|       surfacescale=""
+|       systemlanguage=""
+|       tablevalues=""
+|       targetx=""
+|       targety=""
+|       textlength=""
+|       viewbox=""
+|       viewtarget=""
+|       xchannelselector=""
+|       ychannelselector=""
+|       zoomandpan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
diff --git a/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests12.dat b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests12.dat
new file mode 100644
index 00000000000..63107d277b6
--- /dev/null
+++ b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests12.dat
@@ -0,0 +1,62 @@
+#data
+
foobazeggs
spam
quuxbar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     

+|       "foo"
+|       
+|         
+|           
+|             "baz"
+|         
+|           
+|             
+|               
+|                 "eggs"
+|             
+|               
+|                 

+|                   "spam"
+|                 
+|                   
+|                     
+|                       

+|                         
+|             
+|               "quux"
+|       "bar"
+
+#data
+foobazeggs
spam
quuxbar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "foo"
+|     
+|       
+|         
+|           "baz"
+|       
+|         
+|           
+|             
+|               "eggs"
+|           
+|             
+|               

+|                 "spam"
+|               
+|                 
+|                   
+|                     

+|                       
+|           
+|             "quux"
+|     "bar"
diff --git a/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests14.dat b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests14.dat
new file mode 100644
index 00000000000..b8713f88582
--- /dev/null
+++ b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests14.dat
@@ -0,0 +1,74 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|     
+
+#data
+
+#errors
+15: Unexpected start tag html
+#document
+| 
+| 
+|   abc:def="gh"
+|   
+|   
+|     
+
+#data
+
+#errors
+15: Unexpected start tag html
+#document
+| 
+| 
+|   xml:lang="bar"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   123="456"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   123="456"
+|   789="012"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     789="012"
diff --git a/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests15.dat b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests15.dat
new file mode 100644
index 00000000000..6ce1c0d1663
--- /dev/null
+++ b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests15.dat
@@ -0,0 +1,208 @@
+#data
+
 
X
+#errors
+Line: 1 Col: 31 Unexpected end tag (p). Ignored.
+Line: 1 Col: 36 Expected closing tag. Unexpected end of file.
+#document
+| 
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|     
+|       
+|         
+|           " "
+|           

+|             "X"
+
+#data
+

+
X
+#errors
+Line: 1 Col: 3 Unexpected start tag (p). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end tag (p). Ignored.
+Line: 2 Col: 4 Expected closing tag. Unexpected end of file.
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|     
+|       
+|         
+|           "
+"
+|           

+|             "X"
+
+#data
+ 
+#errors
+Line: 1 Col: 22 Unexpected end tag (html) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     " "
+
+#data
+
+#errors
+Line: 1 Col: 22 Unexpected end tag (body) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+Line: 1 Col: 6 Unexpected start tag (html). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end tag (html) after the (implied) root element.
+#document
+| 
+|   
+|   
+| 
+
+#data
+X
+#errors
+Line: 1 Col: 22 Unexpected end tag (body) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     
+|       "X"
+
+#data
+<!doctype html><table> X<meta></table>
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in table context caused voodoo mode.
+Line: 1 Col: 30 Unexpected start tag (meta) in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " X"
+|     <meta>
+|     <table>
+
+#data
+<!doctype html><table> x</table>
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x"
+|     <table>
+
+#data
+<!doctype html><table> x </table>
+#errors
+Line: 1 Col: 25 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x "
+|     <table>
+
+#data
+<!doctype html><table><tr> x</table>
+#errors
+Line: 1 Col: 28 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table>X<style> <tr>x </style> </table>
+#errors
+Line: 1 Col: 23 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "X"
+|     <table>
+|       <style>
+|         " <tr>x "
+|       " "
+
+#data
+<!doctype html><div><table><a>foo</a> <tr><td>bar</td> </tr></table></div>
+#errors
+Line: 1 Col: 30 Unexpected start tag (a) in table context caused voodoo mode.
+Line: 1 Col: 37 Unexpected end tag (a) in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+|       <a>
+|         "foo"
+|       <table>
+|         " "
+|         <tbody>
+|           <tr>
+|             <td>
+|               "bar"
+|             " "
+
+#data
+<frame></frame></frame><frameset><frame><frameset><frame></frameset><noframes></frameset><noframes>
+#errors
+6: Start tag seen without seeing a doctype first. Expected “<!DOCTYPE html>”.
+13: Stray start tag “frame”.
+21: Stray end tag “frame”.
+29: Stray end tag “frame”.
+39: “frameset” start tag after “body” already open.
+105: End of file seen inside an [R]CDATA element.
+105: End of file seen and there were open elements.
+XXX: These errors are wrong, please fix me!
+#document
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+|     <frameset>
+|       <frame>
+|     <noframes>
+|       "</frameset><noframes>"
+
+#data
+<!DOCTYPE html><object></html>
+#errors
+1: Expected closing tag. Unexpected end of file
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <object>
diff --git a/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests16.dat b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests16.dat
new file mode 100644
index 00000000000..c8ef66f0e6e
--- /dev/null
+++ b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests16.dat
@@ -0,0 +1,2299 @@
+#data
+<!doctype html><script>
+#errors
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script>a
+#errors
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "a"
+|   <body>
+
+#data
+<!doctype html><script><
+#errors
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<"
+|   <body>
+
+#data
+<!doctype html><script></
+#errors
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</"
+|   <body>
+
+#data
+<!doctype html><script></S
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</S"
+|   <body>
+
+#data
+<!doctype html><script></SC
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SC"
+|   <body>
+
+#data
+<!doctype html><script></SCR
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCR"
+|   <body>
+
+#data
+<!doctype html><script></SCRI
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRI"
+|   <body>
+
+#data
+<!doctype html><script></SCRIP
+#errors
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIP"
+|   <body>
+
+#data
+<!doctype html><script></SCRIPT
+#errors
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIPT"
+|   <body>
+
+#data
+<!doctype html><script></SCRIPT 
+#errors
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script></s
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</s"
+|   <body>
+
+#data
+<!doctype html><script></sc
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</sc"
+|   <body>
+
+#data
+<!doctype html><script></scr
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scr"
+|   <body>
+
+#data
+<!doctype html><script></scri
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scri"
+|   <body>
+
+#data
+<!doctype html><script></scrip
+#errors
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scrip"
+|   <body>
+
+#data
+<!doctype html><script></script
+#errors
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</script"
+|   <body>
+
+#data
+<!doctype html><script></script 
+#errors
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script><!
+#errors
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!"
+|   <body>
+
+#data
+<!doctype html><script><!a
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!a"
+|   <body>
+
+#data
+<!doctype html><script><!-
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!-"
+|   <body>
+
+#data
+<!doctype html><script><!-a
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!-a"
+|   <body>
+
+#data
+<!doctype html><script><!--
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<!doctype html><script><!--a
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--a"
+|   <body>
+
+#data
+<!doctype html><script><!--<
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<"
+|   <body>
+
+#data
+<!doctype html><script><!--<a
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<a"
+|   <body>
+
+#data
+<!doctype html><script><!--</
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--</"
+|   <body>
+
+#data
+<!doctype html><script><!--</script
+#errors
+Line: 1 Col: 35 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--</script"
+|   <body>
+
+#data
+<!doctype html><script><!--</script 
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<!doctype html><script><!--<s
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<s"
+|   <body>
+
+#data
+<!doctype html><script><!--<script
+#errors
+Line: 1 Col: 34 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script 
+#errors
+Line: 1 Col: 35 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script <
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <"
+|   <body>
+
+#data
+<!doctype html><script><!--<script <a
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </s
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </s"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script
+#errors
+Line: 1 Col: 43 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </scripta
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </scripta"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script 
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script>
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script>"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script/
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script/"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script <
+#errors
+Line: 1 Col: 45 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script <a
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script
+#errors
+Line: 1 Col: 52 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script 
+#errors
+Line: 1 Col: 53 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script/
+#errors
+Line: 1 Col: 53 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script -
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -a
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -<
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -<"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --a
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --<
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --<"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -->
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --><
+#errors
+Line: 1 Col: 39 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --><"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></
+#errors
+Line: 1 Col: 40 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script 
+#errors
+Line: 1 Col: 47 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script/
+#errors
+Line: 1 Col: 47 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script><\/script>--></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script><\/script>-->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></scr'+'ipt>--></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt>-->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>--><!--</script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>--><!--"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>-- ></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>-- >"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>- -></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- ->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>- - ></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- - >"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>-></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script>--!></script>X
+#errors
+Line: 1 Col: 49 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script>--!></script>X"
+|   <body>
+
+#data
+<!doctype html><script><!--<scr'+'ipt></script>--></script>
+#errors
+Line: 1 Col: 59 Unexpected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<scr'+'ipt>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><script><!--<script></scr'+'ipt></script>X
+#errors
+Line: 1 Col: 57 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt></script>X"
+|   <body>
+
+#data
+<!doctype html><style><!--<style></style>--></style>
+#errors
+Line: 1 Col: 52 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--<style>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><style><!--</style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><style><!--...</style>...--></style>
+#errors
+Line: 1 Col: 51 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|   <body>
+|     "...-->"
+
+#data
+<!doctype html><style><!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style></style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style>"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><style><!--...<style><!--...--!></style>--></style>
+#errors
+Line: 1 Col: 66 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--...<style><!--...--!>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><style><!--...</style><!-- --><style>@import ...</style>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|     <!--   -->
+|     <style>
+|       "@import ..."
+|   <body>
+
+#data
+<!doctype html><style>...<style><!--...</style><!-- --></style>
+#errors
+Line: 1 Col: 63 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "...<style><!--..."
+|     <!--   -->
+|   <body>
+
+#data
+<!doctype html><style>...<!--[if IE]><style>...</style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "...<!--[if IE]><style>..."
+|   <body>
+|     "X"
+
+#data
+<!doctype html><title><!--<title>-->
+#errors
+Line: 1 Col: 52 Unexpected end tag (title).
+#document
+| 
+| 
+|   
+|     
+|       "<!--<title>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><title></title>
+#errors
+#document
+| 
+| 
+|   
+|     
+|       ""
+|   
+
+#data
+foo/title><link></head><body>X
+#errors
+Line: 1 Col: 52 Unexpected end of file. Expected end tag (title).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <title>
+|       "foo/title><link></head><body>X"
+|   <body>
+
+#data
+<!doctype html><noscript><!--<noscript></noscript>--></noscript>
+#errors
+Line: 1 Col: 64 Unexpected end tag (noscript).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--<noscript>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><noscript><!--</noscript>X<noscript>--></noscript>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--"
+|   <body>
+|     "X"
+|     <noscript>
+|       "-->"
+
+#data
+<!doctype html><noscript><iframe></noscript>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<iframe>"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><noframes><!--<noframes></noframes>--></noframes>
+#errors
+Line: 1 Col: 64 Unexpected end tag (noframes).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noframes>
+|       "<!--<noframes>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><noframes><body><script><!--...</script></body></noframes></html>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noframes>
+|       "<body><script><!--...</script></body>"
+|   <body>
+
+#data
+<!doctype html><textarea><!--<textarea></textarea>--></textarea>
+#errors
+Line: 1 Col: 64 Unexpected end tag (textarea).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "<!--<textarea>"
+|     "-->"
+
+#data
+<!doctype html><textarea></textarea></textarea>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "</textarea>"
+
+#data
+<!doctype html><textarea><</textarea>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "<"
+
+#data
+<!doctype html><textarea>a<b</textarea>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "a<b"
+
+#data
+<!doctype html><iframe><!--<iframe></iframe>--></iframe>
+#errors
+Line: 1 Col: 56 Unexpected end tag (iframe).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "<!--<iframe>"
+|     "-->"
+
+#data
+<!doctype html><iframe>...<!--X->...<!--/X->...</iframe>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "...<!--X->...<!--/X->..."
+
+#data
+<!doctype html><xmp><!--<xmp></xmp>--></xmp>
+#errors
+Line: 1 Col: 44 Unexpected end tag (xmp).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <xmp>
+|       "<!--<xmp>"
+|     "-->"
+
+#data
+<!doctype html><noembed><!--<noembed></noembed>--></noembed>
+#errors
+Line: 1 Col: 60 Unexpected end tag (noembed).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <noembed>
+|       "<!--<noembed>"
+|     "-->"
+
+#data
+<script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 8 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script>a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "a"
+|   <body>
+
+#data
+<script><
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<"
+|   <body>
+
+#data
+<script></
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 10 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</"
+|   <body>
+
+#data
+<script></S
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</S"
+|   <body>
+
+#data
+<script></SC
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SC"
+|   <body>
+
+#data
+<script></SCR
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCR"
+|   <body>
+
+#data
+<script></SCRI
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRI"
+|   <body>
+
+#data
+<script></SCRIP
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 15 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIP"
+|   <body>
+
+#data
+<script></SCRIPT
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIPT"
+|   <body>
+
+#data
+<script></SCRIPT 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 17 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script></s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</s"
+|   <body>
+
+#data
+<script></sc
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</sc"
+|   <body>
+
+#data
+<script></scr
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scr"
+|   <body>
+
+#data
+<script></scri
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scri"
+|   <body>
+
+#data
+<script></scrip
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 15 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scrip"
+|   <body>
+
+#data
+<script></script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</script"
+|   <body>
+
+#data
+<script></script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 17 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script><!
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 10 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!"
+|   <body>
+
+#data
+<script><!a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!a"
+|   <body>
+
+#data
+<script><!-
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!-"
+|   <body>
+
+#data
+<script><!-a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!-a"
+|   <body>
+
+#data
+<script><!--
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<script><!--a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--a"
+|   <body>
+
+#data
+<script><!--<
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<"
+|   <body>
+
+#data
+<script><!--<a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<a"
+|   <body>
+
+#data
+<script><!--</
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--</"
+|   <body>
+
+#data
+<script><!--</script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--</script"
+|   <body>
+
+#data
+<script><!--</script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<script><!--<s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<s"
+|   <body>
+
+#data
+<script><!--<script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 19 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script"
+|   <body>
+
+#data
+<script><!--<script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script "
+|   <body>
+
+#data
+<script><!--<script <
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <"
+|   <body>
+
+#data
+<script><!--<script <a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <a"
+|   <body>
+
+#data
+<script><!--<script </
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </"
+|   <body>
+
+#data
+<script><!--<script </s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </s"
+|   <body>
+
+#data
+<script><!--<script </script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script"
+|   <body>
+
+#data
+<script><!--<script </scripta
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </scripta"
+|   <body>
+
+#data
+<script><!--<script </script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script>"
+|   <body>
+
+#data
+<script><!--<script </script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script/"
+|   <body>
+
+#data
+<script><!--<script </script <
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <"
+|   <body>
+
+#data
+<script><!--<script </script <a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <a"
+|   <body>
+
+#data
+<script><!--<script </script </
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </"
+|   <body>
+
+#data
+<script><!--<script </script </script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </script"
+|   <body>
+
+#data
+<script><!--<script </script </script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script </script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script </script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script -
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -"
+|   <body>
+
+#data
+<script><!--<script -a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -a"
+|   <body>
+
+#data
+<script><!--<script --
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --"
+|   <body>
+
+#data
+<script><!--<script --a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --a"
+|   <body>
+
+#data
+<script><!--<script -->
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --><
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --><"
+|   <body>
+
+#data
+<script><!--<script --></
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></"
+|   <body>
+
+#data
+<script><!--<script --></script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></script"
+|   <body>
+
+#data
+<script><!--<script --></script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --></script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script><\/script>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script><\/script>-->"
+|   <body>
+
+#data
+<script><!--<script></scr'+'ipt>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt>-->"
+|   <body>
+
+#data
+<script><!--<script></script><script></script></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>--><!--</script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>--><!--"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>-- ></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>-- >"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>- -></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- ->"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>- - ></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- - >"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>-></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>->"
+|   <body>
+
+#data
+<script><!--<script>--!></script>X
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 34 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script>--!></script>X"
+|   <body>
+
+#data
+<script><!--<scr'+'ipt></script>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 44 Unexpected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<scr'+'ipt>"
+|   <body>
+|     "-->"
+
+#data
+<script><!--<script></scr'+'ipt></script>X
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 42 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt></script>X"
+|   <body>
+
+#data
+<style><!--<style></style>--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--<style>"
+|   <body>
+|     "-->"
+
+#data
+<style><!--</style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--"
+|   <body>
+|     "X"
+
+#data
+<style><!--...</style>...--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 36 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|   <body>
+|     "...-->"
+
+#data
+<style><!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style></style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style>"
+|   <body>
+|     "X"
+
+#data
+<style><!--...<style><!--...--!></style>--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 51 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--...<style><!--...--!>"
+|   <body>
+|     "-->"
+
+#data
+<style><!--...</style><!-- --><style>@import ...</style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|     <!--   -->
+|     <style>
+|       "@import ..."
+|   <body>
+
+#data
+<style>...<style><!--...</style><!-- --></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 48 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "...<style><!--..."
+|     <!--   -->
+|   <body>
+
+#data
+<style>...<!--[if IE]><style>...</style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "...<!--[if IE]><style>..."
+|   <body>
+|     "X"
+
+#data
+<title><!--<title>-->
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end tag (title).
+#document
+| 
+|   
+|     
+|       "<!--<title>"
+|   <body>
+|     "-->"
+
+#data
+<title></title>
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+#document
+| 
+|   
+|     
+|       ""
+|   
+
+#data
+foo/title><link></head><body>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (title).
+#document
+| <html>
+|   <head>
+|     <title>
+|       "foo/title><link></head><body>X"
+|   <body>
+
+#data
+<noscript><!--<noscript></noscript>--></noscript>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (noscript).
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--<noscript>"
+|   <body>
+|     "-->"
+
+#data
+<noscript><!--</noscript>X<noscript>--></noscript>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--"
+|   <body>
+|     "X"
+|     <noscript>
+|       "-->"
+
+#data
+<noscript><iframe></noscript>X
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<iframe>"
+|   <body>
+|     "X"
+
+#data
+<noframes><!--<noframes></noframes>--></noframes>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noframes). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (noframes).
+#document
+| <html>
+|   <head>
+|     <noframes>
+|       "<!--<noframes>"
+|   <body>
+|     "-->"
+
+#data
+<noframes><body><script><!--...</script></body></noframes></html>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noframes). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noframes>
+|       "<body><script><!--...</script></body>"
+|   <body>
+
+#data
+<textarea><!--<textarea></textarea>--></textarea>
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (textarea).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "<!--<textarea>"
+|     "-->"
+
+#data
+<textarea></textarea></textarea>
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "</textarea>"
+
+#data
+<iframe><!--<iframe></iframe>--></iframe>
+#errors
+Line: 1 Col: 8 Unexpected start tag (iframe). Expected DOCTYPE.
+Line: 1 Col: 41 Unexpected end tag (iframe).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "<!--<iframe>"
+|     "-->"
+
+#data
+<iframe>...<!--X->...<!--/X->...</iframe>
+#errors
+Line: 1 Col: 8 Unexpected start tag (iframe). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "...<!--X->...<!--/X->..."
+
+#data
+<xmp><!--<xmp></xmp>--></xmp>
+#errors
+Line: 1 Col: 5 Unexpected start tag (xmp). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end tag (xmp).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <xmp>
+|       "<!--<xmp>"
+|     "-->"
+
+#data
+<noembed><!--<noembed></noembed>--></noembed>
+#errors
+Line: 1 Col: 9 Unexpected start tag (noembed). Expected DOCTYPE.
+Line: 1 Col: 45 Unexpected end tag (noembed).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <noembed>
+|       "<!--<noembed>"
+|     "-->"
+
+#data
+<!doctype html><table>
+
+#errors
+Line 2 Col 0 Unexpected end of file. Expected table content.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       "
+"
+
+#data
+<!doctype html><table><td><span><font></span><span>
+#errors
+Line 1 Col 26 Unexpected table cell start tag (td) in the table body phase.
+Line 1 Col 45 Unexpected end tag (span).
+Line 1 Col 51 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <span>
+|               <font>
+|             <font>
+|               <span>
+
+#data
+<!doctype html><form><table></form><form></table></form>
+#errors
+35: Stray end tag “form”.
+41: Start tag “form” seen in “table”.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <table>
+|         <form>
diff --git a/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests17.dat b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests17.dat
new file mode 100644
index 00000000000..7b555f888de
--- /dev/null
+++ b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests17.dat
@@ -0,0 +1,153 @@
+#data
+<!doctype html><table><tbody><select><tr>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><tr><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+
+#data
+<!doctype html><table><tr><td><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <select>
+|           <td>
+
+#data
+<!doctype html><table><tr><th><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <th>
+|             <select>
+|           <td>
+
+#data
+<!doctype html><table><caption><select><tr>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         <select>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><select><tr>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><th>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><tbody>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><thead>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><tfoot>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><caption>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><table><tr></table>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|     "a"
diff --git a/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests18.dat b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests18.dat
new file mode 100644
index 00000000000..680e1f068a6
--- /dev/null
+++ b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests18.dat
@@ -0,0 +1,269 @@
+#data
+<!doctype html><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+
+#data
+<!doctype html><table><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+
+#data
+<!doctype html><table><tbody><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+|       <tbody>
+
+#data
+<!doctype html><table><tbody><tr><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><tbody><tr><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><td><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <plaintext>
+|               "</plaintext>"
+
+#data
+<!doctype html><table><caption><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         <plaintext>
+|           "</plaintext>"
+
+#data
+<!doctype html><table><tr><style></script></style>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "abc"
+|     <table>
+|       <tbody>
+|         <tr>
+|           <style>
+|             "</script>"
+
+#data
+<!doctype html><table><tr><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "abc"
+|     <table>
+|       <tbody>
+|         <tr>
+|           <script>
+|             "</style>"
+
+#data
+<!doctype html><table><caption><style></script></style>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         <style>
+|           "</script>"
+|         "abc"
+
+#data
+<!doctype html><table><td><style></script></style>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <style>
+|               "</script>"
+|             "abc"
+
+#data
+<!doctype html><select><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <script>
+|         "</style>"
+|       "abc"
+
+#data
+<!doctype html><table><select><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <script>
+|         "</style>"
+|       "abc"
+|     <table>
+
+#data
+<!doctype html><table><tr><select><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <script>
+|         "</style>"
+|       "abc"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><frameset></frameset><noframes>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+
+#data
+<!doctype html><frameset></frameset><noframes>abc</noframes><!--abc-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+|   <!-- abc -->
+
+#data
+<!doctype html><frameset></frameset></html><noframes>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+
+#data
+<!doctype html><frameset></frameset></html><noframes>abc</noframes><!--abc-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+| <!-- abc -->
+
+#data
+<!doctype html><table><tr></tbody><tfoot>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|       <tfoot>
+
+#data
+<!doctype html><table><td><svg></svg>abc<td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <svg svg>
+|             "abc"
+|           <td>
diff --git a/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests19.dat b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests19.dat
new file mode 100644
index 00000000000..0d62f5a5b02
--- /dev/null
+++ b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests19.dat
@@ -0,0 +1,1237 @@
+#data
+<!doctype html><math><mn DefinitionUrl="foo">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <math math>
+|       <math mn>
+|         definitionURL="foo"
+
+#data
+<!doctype html><html></p><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <!-- foo -->
+|   <head>
+|   <body>
+
+#data
+<!doctype html><head></head></p><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <!-- foo -->
+|   <body>
+
+#data
+<!doctype html><body><p><pre>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <pre>
+
+#data
+<!doctype html><body><p><listing>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <listing>
+
+#data
+<!doctype html><p><plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <plaintext>
+
+#data
+<!doctype html><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <h1>
+
+#data
+<!doctype html><form><isindex>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+
+#data
+<!doctype html><isindex action="POST">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       action="POST"
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|       <hr>
+
+#data
+<!doctype html><isindex prompt="this is isindex">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "this is isindex"
+|         <input>
+|           name="isindex"
+|       <hr>
+
+#data
+<!doctype html><isindex type="hidden">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|           type="hidden"
+|       <hr>
+
+#data
+<!doctype html><isindex name="foo">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|       <hr>
+
+#data
+<!doctype html><ruby><p><rp>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <p>
+|       <rp>
+
+#data
+<!doctype html><ruby><div><span><rp>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <span>
+|           <rp>
+
+#data
+<!doctype html><ruby><div><p><rp>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <p>
+|         <rp>
+
+#data
+<!doctype html><ruby><p><rt>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <p>
+|       <rt>
+
+#data
+<!doctype html><ruby><div><span><rt>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <span>
+|           <rt>
+
+#data
+<!doctype html><ruby><div><p><rt>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <p>
+|         <rt>
+
+#data
+<!doctype html><math/><foo>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <math math>
+|     <foo>
+
+#data
+<!doctype html><svg/><foo>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <svg svg>
+|     <foo>
+
+#data
+<!doctype html><div></body><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+|   <!-- foo -->
+
+#data
+<!doctype html><h1><div><h3><span></h1>foo
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <h1>
+|       <div>
+|         <h3>
+|           <span>
+|         "foo"
+
+#data
+<!doctype html><p></h3>foo
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       "foo"
+
+#data
+<!doctype html><h3><li>abc</h2>foo
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <h3>
+|       <li>
+|         "abc"
+|     "foo"
+
+#data
+<!doctype html><table>abc<!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "abc"
+|     <table>
+|       <!-- foo -->
+
+#data
+<!doctype html><table>  <!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       "  "
+|       <!-- foo -->
+
+#data
+<!doctype html><table> b <!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " b "
+|     <table>
+|       <!-- foo -->
+
+#data
+<!doctype html><select><option><option>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+|       <option>
+
+#data
+<!doctype html><select><option></optgroup>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+
+#data
+<!doctype html><select><option></optgroup>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+
+#data
+<!doctype html><p><math><mi><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mi>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><mo><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mo>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><mn><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mn>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><ms><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math ms>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><mtext><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mtext>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><frameset></noframes>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html c=d><body></html><html a=b>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   a="b"
+|   c="d"
+|   <head>
+|   <body>
+
+#data
+<!doctype html><html c=d><frameset></frameset></html><html a=b>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   a="b"
+|   c="d"
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html><frameset></frameset></html><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+| <!-- foo -->
+
+#data
+<!doctype html><html><frameset></frameset></html>  
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   "  "
+
+#data
+<!doctype html><html><frameset></frameset></html>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html><frameset></frameset></html><p>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html><frameset></frameset></html></p>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<html><frameset></frameset></html><!doctype html>
+#errors
+#document
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><body><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!doctype html><p><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><p>a<frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       "a"
+
+#data
+<!doctype html><p> <frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><pre><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <pre>
+
+#data
+<!doctype html><listing><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <listing>
+
+#data
+<!doctype html><li><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <li>
+
+#data
+<!doctype html><dd><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <dd>
+
+#data
+<!doctype html><dt><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <dt>
+
+#data
+<!doctype html><button><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <button>
+
+#data
+<!doctype html><applet><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <applet>
+
+#data
+<!doctype html><marquee><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <marquee>
+
+#data
+<!doctype html><object><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <object>
+
+#data
+<!doctype html><table><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+
+#data
+<!doctype html><area><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <area>
+
+#data
+<!doctype html><basefont><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <basefont>
+|   <frameset>
+
+#data
+<!doctype html><bgsound><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <bgsound>
+|   <frameset>
+
+#data
+<!doctype html><br><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <br>
+
+#data
+<!doctype html><embed><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <embed>
+
+#data
+<!doctype html><img><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <img>
+
+#data
+<!doctype html><input><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <input>
+
+#data
+<!doctype html><keygen><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <keygen>
+
+#data
+<!doctype html><wbr><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <wbr>
+
+#data
+<!doctype html><hr><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <hr>
+
+#data
+<!doctype html><textarea></textarea><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+
+#data
+<!doctype html><xmp></xmp><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <xmp>
+
+#data
+<!doctype html><iframe></iframe><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+
+#data
+<!doctype html><select></select><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><svg></svg><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><math></math><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><svg><foreignObject><div> <frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><svg>a</svg><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <svg svg>
+|       "a"
+
+#data
+<!doctype html><svg> </svg><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<html>aaa<frameset></frameset>
+#errors
+#document
+| <html>
+|   <head>
+|   <body>
+|     "aaa"
+
+#data
+<html> a <frameset></frameset>
+#errors
+#document
+| <html>
+|   <head>
+|   <body>
+|     "a "
+
+#data
+<!doctype html><div><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><div><body><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+
+#data
+<!doctype html><p><math></p>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|     "a"
+
+#data
+<!doctype html><p><math><mn><span></p>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mn>
+|           <span>
+|             <p>
+|             "a"
+
+#data
+<!doctype html><math></html>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <math math>
+
+#data
+<!doctype html><meta charset="ascii">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <meta>
+|       charset="ascii"
+|   <body>
+
+#data
+<!doctype html><meta http-equiv="content-type" content="text/html;charset=ascii">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <meta>
+|       content="text/html;charset=ascii"
+|       http-equiv="content-type"
+|   <body>
+
+#data
+<!doctype html><head><!--aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa--><meta charset="utf8">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <!-- aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -->
+|     <meta>
+|       charset="utf8"
+|   <body>
+
+#data
+<!doctype html><html a=b><head></head><html c=d>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   a="b"
+|   c="d"
+|   <head>
+|   <body>
+
+#data
+<!doctype html><image/>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <img>
+
+#data
+<!doctype html>a<i>b<table>c<b>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "a"
+|     <i>
+|       "bc"
+|       <b>
+|         "de"
+|       "f"
+|       <table>
+
+#data
+<!doctype html><table><i>a<b>b<div>c<a>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|     <div>
+|       <b>
+|         <i>
+|           "c"
+|           <a>
+|             "d"
+|         <a>
+|           "e"
+|       <a>
+|         "f"
+|     <table>
+
+#data
+<!doctype html><i>a<b>b<div>c<a>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|     <div>
+|       <b>
+|         <i>
+|           "c"
+|           <a>
+|             "d"
+|         <a>
+|           "e"
+|       <a>
+|         "f"
+
+#data
+<!doctype html><table><i>a<b>b<div>c</i>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|       <div>
+|         <i>
+|           "c"
+|     <table>
+
+#data
+<!doctype html><table><i>a<b>b<div>c<a>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|     <div>
+|       <b>
+|         <i>
+|           "c"
+|           <a>
+|             "d"
+|         <a>
+|           "e"
+|       <a>
+|         "f"
+|     <table>
+
+#data
+<!doctype html><table><i>a<div>b<tr>c<b>d</i>e
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <div>
+|         "b"
+|     <i>
+|       "c"
+|       <b>
+|         "d"
+|     <b>
+|       "e"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><td><table><i>a<div>b<b>c</i>d
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <i>
+|               "a"
+|             <div>
+|               <i>
+|                 "b"
+|                 <b>
+|                   "c"
+|               <b>
+|                 "d"
+|             <table>
+
+#data
+<!doctype html><body><bgsound>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <bgsound>
+
+#data
+<!doctype html><body><basefont>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <basefont>
+
+#data
+<!doctype html><a><b></a><basefont>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <a>
+|       <b>
+|     <basefont>
+
+#data
+<!doctype html><a><b></a><bgsound>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <a>
+|       <b>
+|     <bgsound>
+
+#data
+<!doctype html><figcaption><article></figcaption>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <figcaption>
+|       <article>
+|     "a"
+
+#data
+<!doctype html><summary><article></summary>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <summary>
+|       <article>
+|     "a"
+
+#data
+<!doctype html><p><a><plaintext>b
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <a>
+|     <plaintext>
+|       <a>
+|         "b"
+
+#data
+<!DOCTYPE html><div>a<a></div>b<p>c</p>d
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+|       "a"
+|       <a>
+|     <a>
+|       "b"
+|       <p>
+|         "c"
+|       "d"
diff --git a/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests2.dat b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests2.dat
new file mode 100644
index 00000000000..60d85922162
--- /dev/null
+++ b/Godeps/_workspace/src/golang.org/x/net/html/testdata/webkit/tests2.dat
@@ -0,0 +1,763 @@
+#data
+<!DOCTYPE html>Test
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "Test"
+
+#data
+<textarea>test</div>test
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+Line: 1 Col: 24 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "test</div>test"
+
+#data
+<table><td>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected table cell start tag (td) in the table body phase.
+Line: 1 Col: 11 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+
+#data
+<table><td>test</tbody></table>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected table cell start tag (td) in the table body phase.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             "test"
+
+#data
+<frame>test
+#errors
+Line: 1 Col: 7 Unexpected start tag (frame). Expected DOCTYPE.
+Line: 1 Col: 7 Unexpected start tag frame. Ignored.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "test"
+
+#data
+<!DOCTYPE html><frameset>test
+#errors
+Line: 1 Col: 29 Unepxected characters in the frameset phase. Characters ignored.
+Line: 1 Col: 29 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!DOCTYPE html><frameset><!DOCTYPE html>
+#errors
+Line: 1 Col: 40 Unexpected DOCTYPE. Ignored.
+Line: 1 Col: 40 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!DOCTYPE html><font><p><b>test</font>
+#errors
+Line: 1 Col: 38 End tag (font) violates step 1, paragraph 3 of the adoption agency algorithm.
+Line: 1 Col: 38 End tag (font) violates step 1, paragraph 3 of the adoption agency algorithm.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <font>
+|     <p>
+|       <font>
+|         <b>
+|           "test"
+
+#data
+<!DOCTYPE html><dt><div><dd>
+#errors
+Line: 1 Col: 28 Missing end tag (div, dt).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <dt>
+|       <div>
+|     <dd>
+
+#data
+<script></x
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</x"
+|   <body>
+
+#data
+<table><plaintext><td>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 18 Unexpected start tag (plaintext) in table context caused voodoo mode.
+Line: 1 Col: 22 Unexpected end of file. Expected table content.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "<td>"
+|     <table>
+
+#data
+<plaintext></plaintext>
+#errors
+Line: 1 Col: 11 Unexpected start tag (plaintext). Expected DOCTYPE.
+Line: 1 Col: 23 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+
+#data
+<!DOCTYPE html><table><tr>TEST
+#errors
+Line: 1 Col: 30 Unexpected non-space characters in table context caused voodoo mode.
+Line: 1 Col: 30 Unexpected end of file. Expected table content.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "TEST"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!DOCTYPE html><body t1=1><body t2=2><body t3=3 t4=4>
+#errors
+Line: 1 Col: 37 Unexpected start tag (body).
+Line: 1 Col: 53 Unexpected start tag (body).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     t1="1"
+|     t2="2"
+|     t3="3"
+|     t4="4"
+
+#data
+</b test
+#errors
+Line: 1 Col: 8 Unexpected end of file in attribute name.
+Line: 1 Col: 8 End tag contains unexpected attributes.
+Line: 1 Col: 8 Unexpected end tag (b). Expected DOCTYPE.
+Line: 1 Col: 8 Unexpected end tag (b) after the (implied) root element.
+#document
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html></b test<b &=&>X
+#errors
+Line: 1 Col: 32 Named entity didn't end with ';'.
+Line: 1 Col: 33 End tag contains unexpected attributes.
+Line: 1 Col: 33 Unexpected end tag (b) after the (implied) root element.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "X"
+
+#data
+<!doctypehtml><scrIPt type=text/x-foobar;baz>X</SCRipt
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+Line: 1 Col: 54 Unexpected end of file in the tag name.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       type="text/x-foobar;baz"
+|       "X</SCRipt"
+|   <body>
+
+#data
+&
+#errors
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&"
+
+#data
+&#
+#errors
+Line: 1 Col: 1 Numeric entity expected. Got end of file instead.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#"
+
+#data
+&#X
+#errors
+Line: 1 Col: 3 Numeric entity expected but none found.
+Line: 1 Col: 3 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#X"
+
+#data
+&#x
+#errors
+Line: 1 Col: 3 Numeric entity expected but none found.
+Line: 1 Col: 3 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#x"
+
+#data
+-
+#errors
+Line: 1 Col: 4 Numeric entity didn't end with ';'.
+Line: 1 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "-"
+
+#data
+&x-test
+#errors
+Line: 1 Col: 1 Named entity expected. Got none.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&x-test"
+
+#data
+<!doctypehtml><p><li>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <li>
+
+#data
+<!doctypehtml><p><dt>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <dt>
+
+#data
+<!doctypehtml><p><dd>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <dd>
+
+#data
+<!doctypehtml><p><form>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+Line: 1 Col: 23 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <form>
+
+#data
+<!DOCTYPE html><p></P>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     "X"
+
+#data
+&
+#errors
+Line: 1 Col: 4 Named entity didn't end with ';'.
+Line: 1 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&"
+
+#data
+&AMp;
+#errors
+Line: 1 Col: 1 Named entity expected. Got none.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&AMp;"
+
+#data
+<!DOCTYPE html><html><head></head><body><thisISasillyTESTelementNameToMakeSureCrazyTagNamesArePARSEDcorrectLY>
+#errors
+Line: 1 Col: 110 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <thisisasillytestelementnametomakesurecrazytagnamesareparsedcorrectly>
+
+#data
+<!DOCTYPE html>X</body>X
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in the after body phase.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "XX"
+
+#data
+<!DOCTYPE html><!-- X
+#errors
+Line: 1 Col: 21 Unexpected end of file in comment.
+#document
+| <!DOCTYPE html>
+| <!--  X -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><table><caption>test TEST</caption><td>test
+#errors
+Line: 1 Col: 54 Unexpected table cell start tag (td) in the table body phase.
+Line: 1 Col: 58 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         "test TEST"
+|       <tbody>
+|         <tr>
+|           <td>
+|             "test"
+
+#data
+<!DOCTYPE html><select><option><optgroup>
+#errors
+Line: 1 Col: 41 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+|       <optgroup>
+
+#data
+<!DOCTYPE html><select><optgroup><option></optgroup><option><select><option>
+#errors
+Line: 1 Col: 68 Unexpected select start tag in the select phase treated as select end tag.
+Line: 1 Col: 76 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+|         <option>
+|       <option>
+|     <option>
+
+#data
+<!DOCTYPE html><select><optgroup><option><optgroup>
+#errors
+Line: 1 Col: 51 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+|         <option>
+|       <optgroup>
+
+#data
+<!DOCTYPE html><datalist><option>foo</datalist>bar
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <datalist>
+|       <option>
+|         "foo"
+|     "bar"
+
+#data
+<!DOCTYPE html><font><input><input></font>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <font>
+|       <input>
+|       <input>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX -->
+#errors
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX  -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX
+#errors
+Line: 1 Col: 29 Unexpected end of file in comment (-)
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX - XXX -->
+#errors
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX - XXX  -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<isindex test=x name=x>
+#errors
+Line: 1 Col: 23 Unexpected start tag (isindex). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected start tag isindex. Don't use it!
+#document
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|           test="x"
+|       <hr>
+
+#data
+test
+test
+#errors
+Line: 2 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "test
+test"
+
+#data
+<!DOCTYPE html><body><title>test</body>
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       "test</body>"
+
+#data
+<!DOCTYPE html><body><title>X
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       "X"
+|     <meta>
+|       name="z"
+|     <link>
+|       rel="foo"
+|     <style>
+|       "
+x { content:"</style" } "
+
+#data
+<!DOCTYPE html><select><optgroup></optgroup></select>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+
+#data
+ 
+ 
+#errors
+Line: 2 Col: 1 Unexpected End of file. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html>  <html>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><script>
+</script>  <title>x  
+#errors
+#document
+| 
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected start tag (script) that can be in head. Moved.
+#document
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+Line: 1 Col: 28 Unexpected start tag (style) that can be in head. Moved.
+#document
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+#document
+| 
+|   
+|   
+|   
+|     "x"
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (style).
+#document
+| 
+|   
+|      --> x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|      

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       ddd
+#errors
+#document
+| 
+|   
+|     
+#errors
+#document
+| 
+|   
+|   
+|     
+|     
  • 
+|       
+|         ",
+		"