From e8d7e5d1fa7d8477b91cb4dffeac57c7c20cb5c5 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Thu, 16 Feb 2017 17:59:53 -0500 Subject: [PATCH] mime: use sync.Map instead of RWMutex for type lookups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This provides a significant speedup for TypeByExtension and ExtensionsByType when using many CPU cores. updates #17973 updates #18177 name old time/op new time/op delta QEncodeWord 526ns ± 3% 525ns ± 3% ~ (p=0.990 n=15+28) QEncodeWord-6 945ns ± 7% 913ns ±20% ~ (p=0.220 n=14+28) QEncodeWord-48 1.02µs ± 2% 1.00µs ± 6% -2.22% (p=0.036 n=13+27) QDecodeWord 311ns ±18% 323ns ±20% ~ (p=0.107 n=16+28) QDecodeWord-6 595ns ±12% 612ns ±11% ~ (p=0.093 n=15+27) QDecodeWord-48 592ns ± 6% 606ns ± 8% +2.39% (p=0.045 n=16+26) QDecodeHeader 389ns ± 4% 394ns ± 8% ~ (p=0.161 n=12+26) QDecodeHeader-6 685ns ±12% 674ns ±20% ~ (p=0.773 n=14+27) QDecodeHeader-48 658ns ±13% 669ns ±14% ~ (p=0.457 n=16+28) TypeByExtension/.html 77.4ns ±15% 55.5ns ±13% -28.35% (p=0.000 n=8+8) TypeByExtension/.html-6 263ns ± 9% 10ns ±21% -96.29% (p=0.000 n=8+8) TypeByExtension/.html-48 175ns ± 5% 2ns ±16% -98.88% (p=0.000 n=8+8) TypeByExtension/.HTML 113ns ± 6% 97ns ± 6% -14.37% (p=0.000 n=8+8) TypeByExtension/.HTML-6 273ns ± 7% 17ns ± 4% -93.93% (p=0.000 n=7+8) TypeByExtension/.HTML-48 175ns ± 4% 4ns ± 4% -97.73% (p=0.000 n=8+8) TypeByExtension/.unused 116ns ± 4% 90ns ± 4% -22.89% (p=0.001 n=7+7) TypeByExtension/.unused-6 262ns ± 5% 15ns ± 4% -94.17% (p=0.000 n=8+8) TypeByExtension/.unused-48 176ns ± 4% 3ns ±10% -98.10% (p=0.000 n=8+8) ExtensionsByType/text/html 630ns ± 5% 522ns ± 5% -17.19% (p=0.000 n=8+7) ExtensionsByType/text/html-6 314ns ±20% 136ns ± 6% -56.80% (p=0.000 n=8+8) ExtensionsByType/text/html-48 298ns ± 4% 104ns ± 6% -65.06% (p=0.000 n=8+8) ExtensionsByType/text/html;_charset=utf-8 1.12µs ± 3% 1.05µs ± 7% -6.19% (p=0.004 n=8+7) ExtensionsByType/text/html;_charset=utf-8-6 402ns ±11% 307ns ± 4% -23.77% (p=0.000 n=8+8) ExtensionsByType/text/html;_charset=utf-8-48 422ns ± 3% 309ns ± 4% -26.86% (p=0.000 n=8+8) ExtensionsByType/application/octet-stream 810ns ± 2% 747ns ± 5% -7.74% (p=0.000 n=8+8) ExtensionsByType/application/octet-stream-6 289ns ± 9% 185ns ± 8% -36.15% (p=0.000 n=7+8) ExtensionsByType/application/octet-stream-48 267ns ± 6% 94ns ± 2% -64.91% (p=0.000 n=8+7) name old alloc/op new alloc/op delta QEncodeWord 48.0B ± 0% 48.0B ± 0% ~ (all equal) QEncodeWord-6 48.0B ± 0% 48.0B ± 0% ~ (all equal) QEncodeWord-48 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeWord 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeWord-6 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeWord-48 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeHeader 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeHeader-6 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeHeader-48 48.0B ± 0% 48.0B ± 0% ~ (all equal) TypeByExtension/.html 0.00B 0.00B ~ (all equal) TypeByExtension/.html-6 0.00B 0.00B ~ (all equal) TypeByExtension/.html-48 0.00B 0.00B ~ (all equal) TypeByExtension/.HTML 0.00B 0.00B ~ (all equal) TypeByExtension/.HTML-6 0.00B 0.00B ~ (all equal) TypeByExtension/.HTML-48 0.00B 0.00B ~ (all equal) TypeByExtension/.unused 0.00B 0.00B ~ (all equal) TypeByExtension/.unused-6 0.00B 0.00B ~ (all equal) TypeByExtension/.unused-48 0.00B 0.00B ~ (all equal) ExtensionsByType/text/html 192B ± 0% 176B ± 0% -8.33% (p=0.000 n=8+8) ExtensionsByType/text/html-6 192B ± 0% 176B ± 0% -8.33% (p=0.000 n=8+8) ExtensionsByType/text/html-48 192B ± 0% 176B ± 0% -8.33% (p=0.000 n=8+8) ExtensionsByType/text/html;_charset=utf-8 480B ± 0% 464B ± 0% -3.33% (p=0.000 n=8+8) ExtensionsByType/text/html;_charset=utf-8-6 480B ± 0% 464B ± 0% -3.33% (p=0.000 n=8+8) ExtensionsByType/text/html;_charset=utf-8-48 480B ± 0% 464B ± 0% -3.33% (p=0.000 n=8+8) ExtensionsByType/application/octet-stream 160B ± 0% 160B ± 0% ~ (all equal) ExtensionsByType/application/octet-stream-6 160B ± 0% 160B ± 0% ~ (all equal) ExtensionsByType/application/octet-stream-48 160B ± 0% 160B ± 0% ~ (all equal) name old allocs/op new allocs/op delta QEncodeWord 1.00 ± 0% 1.00 ± 0% ~ (all equal) QEncodeWord-6 1.00 ± 0% 1.00 ± 0% ~ (all equal) QEncodeWord-48 1.00 ± 0% 1.00 ± 0% ~ (all equal) QDecodeWord 2.00 ± 0% 2.00 ± 0% ~ (all equal) QDecodeWord-6 2.00 ± 0% 2.00 ± 0% ~ (all equal) QDecodeWord-48 2.00 ± 0% 2.00 ± 0% ~ (all equal) QDecodeHeader 2.00 ± 0% 2.00 ± 0% ~ (all equal) QDecodeHeader-6 2.00 ± 0% 2.00 ± 0% ~ (all equal) QDecodeHeader-48 2.00 ± 0% 2.00 ± 0% ~ (all equal) TypeByExtension/.html 0.00 0.00 ~ (all equal) TypeByExtension/.html-6 0.00 0.00 ~ (all equal) TypeByExtension/.html-48 0.00 0.00 ~ (all equal) TypeByExtension/.HTML 0.00 0.00 ~ (all equal) TypeByExtension/.HTML-6 0.00 0.00 ~ (all equal) TypeByExtension/.HTML-48 0.00 0.00 ~ (all equal) TypeByExtension/.unused 0.00 0.00 ~ (all equal) TypeByExtension/.unused-6 0.00 0.00 ~ (all equal) TypeByExtension/.unused-48 0.00 0.00 ~ (all equal) ExtensionsByType/text/html 3.00 ± 0% 3.00 ± 0% ~ (all equal) ExtensionsByType/text/html-6 3.00 ± 0% 3.00 ± 0% ~ (all equal) ExtensionsByType/text/html-48 3.00 ± 0% 3.00 ± 0% ~ (all equal) ExtensionsByType/text/html;_charset=utf-8 4.00 ± 0% 4.00 ± 0% ~ (all equal) ExtensionsByType/text/html;_charset=utf-8-6 4.00 ± 0% 4.00 ± 0% ~ (all equal) ExtensionsByType/text/html;_charset=utf-8-48 4.00 ± 0% 4.00 ± 0% ~ (all equal) ExtensionsByType/application/octet-stream 2.00 ± 0% 2.00 ± 0% ~ (all equal) ExtensionsByType/application/octet-stream-6 2.00 ± 0% 2.00 ± 0% ~ (all equal) ExtensionsByType/application/octet-stream-48 2.00 ± 0% 2.00 ± 0% ~ (all equal) https://perf.golang.org/search?q=upload:20170427.4 Change-Id: I35438be087ad6eb3d5da9119b395723ea5babaf6 Reviewed-on: https://go-review.googlesource.com/41990 Run-TryBot: Bryan Mills TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/mime/type.go | 108 ++++++++++++++++++++++++----------------------- 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/src/mime/type.go b/src/mime/type.go index d369259d8b1678..78fc6b6714e6ec 100644 --- a/src/mime/type.go +++ b/src/mime/type.go @@ -12,24 +12,48 @@ import ( ) var ( - mimeLock sync.RWMutex // guards following 3 maps - mimeTypes map[string]string // ".Z" => "application/x-compress" - mimeTypesLower map[string]string // ".z" => "application/x-compress" + mimeTypes sync.Map // map[string]string; ".Z" => "application/x-compress" + mimeTypesLower sync.Map // map[string]string; ".z" => "application/x-compress" // extensions maps from MIME type to list of lowercase file // extensions: "image/jpeg" => [".jpg", ".jpeg"] - extensions map[string][]string + extensionsMu sync.Mutex // Guards stores (but not loads) on extensions. + extensions sync.Map // map[string][]string; slice values are append-only. ) +func clearSyncMap(m *sync.Map) { + m.Range(func(k, _ interface{}) bool { + m.Delete(k) + return true + }) +} + // setMimeTypes is used by initMime's non-test path, and by tests. -// The two maps must not be the same, or nil. func setMimeTypes(lowerExt, mixExt map[string]string) { - if lowerExt == nil || mixExt == nil { - panic("nil map") + clearSyncMap(&mimeTypes) + clearSyncMap(&mimeTypesLower) + clearSyncMap(&extensions) + + for k, v := range lowerExt { + mimeTypesLower.Store(k, v) + } + for k, v := range mixExt { + mimeTypes.Store(k, v) + } + + extensionsMu.Lock() + defer extensionsMu.Unlock() + for k, v := range lowerExt { + justType, _, err := ParseMediaType(v) + if err != nil { + panic(err) + } + var exts []string + if ei, ok := extensions.Load(k); ok { + exts = ei.([]string) + } + extensions.Store(justType, append(exts, k)) } - mimeTypesLower = lowerExt - mimeTypes = mixExt - extensions = invert(lowerExt) } var builtinTypesLower = map[string]string{ @@ -45,29 +69,6 @@ var builtinTypesLower = map[string]string{ ".xml": "text/xml; charset=utf-8", } -func clone(m map[string]string) map[string]string { - m2 := make(map[string]string, len(m)) - for k, v := range m { - m2[k] = v - if strings.ToLower(k) != k { - panic("keys in builtinTypesLower must be lowercase") - } - } - return m2 -} - -func invert(m map[string]string) map[string][]string { - m2 := make(map[string][]string, len(m)) - for k, v := range m { - justType, _, err := ParseMediaType(v) - if err != nil { - panic(err) - } - m2[justType] = append(m2[justType], k) - } - return m2 -} - var once sync.Once // guards initMime var testInitMime, osInitMime func() @@ -76,7 +77,7 @@ func initMime() { if fn := testInitMime; fn != nil { fn() } else { - setMimeTypes(builtinTypesLower, clone(builtinTypesLower)) + setMimeTypes(builtinTypesLower, builtinTypesLower) osInitMime() } } @@ -100,12 +101,10 @@ func initMime() { // Text types have the charset parameter set to "utf-8" by default. func TypeByExtension(ext string) string { once.Do(initMime) - mimeLock.RLock() - defer mimeLock.RUnlock() // Case-sensitive lookup. - if v := mimeTypes[ext]; v != "" { - return v + if v, ok := mimeTypes.Load(ext); ok { + return v.(string) } // Case-insensitive lookup. @@ -118,7 +117,9 @@ func TypeByExtension(ext string) string { c := ext[i] if c >= utf8RuneSelf { // Slow path. - return mimeTypesLower[strings.ToLower(ext)] + si, _ := mimeTypesLower.Load(strings.ToLower(ext)) + s, _ := si.(string) + return s } if 'A' <= c && c <= 'Z' { lower = append(lower, c+('a'-'A')) @@ -126,9 +127,9 @@ func TypeByExtension(ext string) string { lower = append(lower, c) } } - // The conversion from []byte to string doesn't allocate in - // a map lookup. - return mimeTypesLower[string(lower)] + si, _ := mimeTypesLower.Load(string(lower)) + s, _ := si.(string) + return s } // ExtensionsByType returns the extensions known to be associated with the MIME @@ -142,13 +143,11 @@ func ExtensionsByType(typ string) ([]string, error) { } once.Do(initMime) - mimeLock.RLock() - defer mimeLock.RUnlock() - s, ok := extensions[justType] + s, ok := extensions.Load(justType) if !ok { return nil, nil } - return append([]string{}, s...), nil + return append([]string{}, s.([]string)...), nil } // AddExtensionType sets the MIME type associated with @@ -173,15 +172,20 @@ func setExtensionType(extension, mimeType string) error { } extLower := strings.ToLower(extension) - mimeLock.Lock() - defer mimeLock.Unlock() - mimeTypes[extension] = mimeType - mimeTypesLower[extLower] = mimeType - for _, v := range extensions[justType] { + mimeTypes.Store(extension, mimeType) + mimeTypesLower.Store(extLower, mimeType) + + extensionsMu.Lock() + defer extensionsMu.Unlock() + var exts []string + if ei, ok := extensions.Load(justType); ok { + exts = ei.([]string) + } + for _, v := range exts { if v == extLower { return nil } } - extensions[justType] = append(extensions[justType], extLower) + extensions.Store(justType, append(exts, extLower)) return nil }