From 40afb6d25011d6b015245ef9b9debfc4aee48d8a Mon Sep 17 00:00:00 2001 From: Patrick Gaskin Date: Fri, 11 Jun 2021 11:32:42 -0400 Subject: [PATCH] kepub: Switch to my forked+renamed html package instead of replacing it This will eliminate conflicts and other difficulties when building kepubify externally or as a library --- .appveyor.yml | 3 ++- .drone.star | 6 ++++-- .travis.yml | 3 ++- cmd/kepubify/kepubify.go | 2 +- go.mod | 5 ++--- go.sum | 12 ++++++------ internal/zip/zip.go | 9 +++++++++ internal/zip/ziptypes_go116.go | 28 ++++++++++++++++++++++++++++ internal/zip/ziptypes_go117.go | 28 ++++++++++++++++++++++++++++ kepub/convert.go | 2 +- kepub/convert_go116.go | 7 ++++--- kepub/convert_go117.go | 6 +++--- kepub/kobotest/main.go | 6 +++--- kepub/transform.go | 10 +++------- kepub/transform_test.go | 3 ++- 15 files changed, 98 insertions(+), 32 deletions(-) create mode 100644 internal/zip/zip.go create mode 100644 internal/zip/ziptypes_go116.go create mode 100644 internal/zip/ziptypes_go117.go diff --git a/.appveyor.yml b/.appveyor.yml index fe998f9..673437d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,8 +6,9 @@ build_script: - cmd: C:\go116\bin\go.exe run -mod=readonly ./cmd/kepubify --help test_script: -- cmd: C:\go116\bin\go.exe test -mod=readonly -run "^TestMod_" golang.org/x/net/html -v +- cmd: C:\go116\bin\go.exe test -mod=readonly -run "^TestMod_" github.com/pgaskin/kepubify/_/html/golang.org/x/net/html -v - cmd: C:\go116\bin\go.exe test -mod=readonly -v ./kepub - cmd: C:\go116\bin\go.exe test -mod=readonly -v ./cmd/kepubify +- cmd: C:\go116\bin\go.exe test -mod=readonly -tags zip117 -v ./cmd/kepubify deploy: off diff --git a/.drone.star b/.drone.star index 374ee25..ecae78f 100644 --- a/.drone.star +++ b/.drone.star @@ -17,11 +17,13 @@ pipeline = [{ "depends_on": pdep, } for (pname, psteps, pdep) in [ ("kepub", [ - ("test-html", "go test -mod=readonly -run \"^TestMod_\" golang.org/x/net/html -v"), + ("test-html", "go test -mod=readonly -run \"^TestMod_\" github.com/pgaskin/kepubify/_/html/golang.org/x/net/html -v"), ("test", "go test -mod=readonly ./kepub -v -cover"), + ("test-zip117", "go test -mod=readonly -tags zip117 ./kepub -v -cover"), ], []), ("kepubify", [ ("test", "go test -mod=readonly ./cmd/kepubify -v -cover"), + ("test-zip117", "go test -mod=readonly -tags zip117 ./cmd/kepubify -v -cover"), ("run", "go run -mod=readonly ./cmd/kepubify --help"), ], ["kepub"]), ("covergen", [ @@ -53,7 +55,7 @@ pipeline = [{ }, "command": [ "--platforms", platform, - "--build-cmd", "go env; CGO_ENABLED=%s go build -ldflags \"-s -w -X main.version=$(cat build/version)\" -o \"build/%s%s\" %s" % (cgo, app, suffix, "./cmd/" + app), + "--build-cmd", "go env; CGO_ENABLED=%s go build -ldflags \"-s -w -X main.version=$(cat build/version)\" -tags zip117 -o \"build/%s%s\" %s" % (cgo, app, suffix, "./cmd/" + app), ], } for (img, platform, suffix) in [ ("main", "linux/amd64", "-linux-64bit"), diff --git a/.travis.yml b/.travis.yml index 6e22463..bf2e0df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ go: script: - go run -mod=readonly ./cmd/kepubify --help -- go test -mod=readonly -run "^TestMod_" golang.org/x/net/html -v +- go test -mod=readonly -run "^TestMod_" github.com/pgaskin/kepubify/_/html/golang.org/x/net/html -v - go test -mod=readonly -v ./kepub +- go test -mod=readonly -v -tags zip117 ./kepub - go test -mod=readonly -v ./cmd/kepubify diff --git a/cmd/kepubify/kepubify.go b/cmd/kepubify/kepubify.go index 2fc6814..d100ab7 100644 --- a/cmd/kepubify/kepubify.go +++ b/cmd/kepubify/kepubify.go @@ -2,7 +2,6 @@ package main import ( - "archive/zip" "context" "fmt" "io" @@ -15,6 +14,7 @@ import ( "sync/atomic" "time" + "github.com/pgaskin/kepubify/v4/internal/zip" "github.com/pgaskin/kepubify/v4/kepub" "github.com/spf13/pflag" ) diff --git a/go.mod b/go.mod index 607df6c..b03a02c 100644 --- a/go.mod +++ b/go.mod @@ -13,11 +13,10 @@ require ( require ( github.com/beevik/etree v1.1.0 github.com/kr/smartypants v0.1.0 - golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 + github.com/pgaskin/kepubify/_/go116-zip.go117 v0.0.0-20210611152744-2d89b3182523 + github.com/pgaskin/kepubify/_/html v0.0.0-20210611145339-337924fbbaf0 golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520 ) // tests/utils require github.com/sergi/go-diff v1.1.0 - -replace golang.org/x/net => github.com/pgaskin/net v0.0.0-20200714194227-0135d2310fe0 // x/net/html options for kepubify diff --git a/go.sum b/go.sum index 4a04e8e..deebd32 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,12 @@ github.com/kr/smartypants v0.1.0/go.mod h1:EcTX9ge+SWNaGwbQvHwNICsMGavh98FLUqyOW github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/pgaskin/kepubify/_/go116-zip.go117 v0.0.0-20210611152744-2d89b3182523 h1:pYGj3rKTy+TDs5Z707kT+ztjoIDCy76lc2UPkZocAFM= +github.com/pgaskin/kepubify/_/go116-zip.go117 v0.0.0-20210611152744-2d89b3182523/go.mod h1:FNMbV/TSSnhqyzjq8jsS+VD0o/gwpuCH0dh8G1uQ/fw= +github.com/pgaskin/kepubify/_/html v0.0.0-20210611145339-337924fbbaf0 h1:qYHfG66zDgts5KsVYmDxHuIqEXOBZQ1lVdjS6tMAnHI= +github.com/pgaskin/kepubify/_/html v0.0.0-20210611145339-337924fbbaf0/go.mod h1:fxzoIpMFAReNKunZ+ttVbf3hNVrJGtrSZMI4olZizbs= github.com/pgaskin/koboutils/v2 v2.1.1 h1:Or5y+z8rXlip0Al8tiSj+Fb9NkuLhkcw1UPpzPPvKWY= github.com/pgaskin/koboutils/v2 v2.1.1/go.mod h1:wTzkDIlsxmUyfwfspGcm0Ap+HOxSUYV0S8kMYrf+0gM= -github.com/pgaskin/net v0.0.0-20200714194227-0135d2310fe0 h1:Xb0ptOMQ6+JL62R95PJm+ypDVTztCvWvEkHkgiuiKmM= -github.com/pgaskin/net v0.0.0-20200714194227-0135d2310fe0/go.mod h1:PqDaaxbNy6TMC9BuETpBkL6kfwza2vGdST0mVzKGmLc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= @@ -25,12 +27,10 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520 h1:Bx6FllMpG4NWDOfhMBz1VR2QYNp/SAOHPIAsaVmxfPo= golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/zip/zip.go b/internal/zip/zip.go new file mode 100644 index 0000000..f2ad136 --- /dev/null +++ b/internal/zip/zip.go @@ -0,0 +1,9 @@ +// Package zip is an ugly hack to allow kepubify to be build on Go 1.16 with a +// backported archive/zip from Go 1.17 for much better performance. To use it, +// build with `-tags zip117`. This tag does not have any effect on any version +// other than Go 1.16. Note that if other applications embed kepubify and use +// this option, they must also use the same backported module with +// (*kepub.Converter).Convert if they want the performance improvements. +package zip + +// TODO: remove this package and the build flags once kepubify switches to Go 1.17 diff --git a/internal/zip/ziptypes_go116.go b/internal/zip/ziptypes_go116.go new file mode 100644 index 0000000..648a5a6 --- /dev/null +++ b/internal/zip/ziptypes_go116.go @@ -0,0 +1,28 @@ +//go:build go1.16 && !go1.17 && !zip117 +// +build go1.16,!go1.17,!zip117 + +package zip + +import "archive/zip" + +const Deflate = zip.Deflate +const Store = zip.Store + +var FileInfoHeader = zip.FileInfoHeader +var NewReader = zip.NewReader +var NewWriter = zip.NewWriter +var OpenReader = zip.OpenReader +var RegisterCompressor = zip.RegisterCompressor +var RegisterDecompressor = zip.RegisterDecompressor + +type Compressor = zip.Compressor +type Decompressor = zip.Decompressor +type File = zip.File +type FileHeader = zip.FileHeader +type ReadCloser = zip.ReadCloser +type Reader = zip.Reader +type Writer = zip.Writer + +var ErrAlgorithm = zip.ErrAlgorithm +var ErrChecksum = zip.ErrChecksum +var ErrFormat = zip.ErrFormat diff --git a/internal/zip/ziptypes_go117.go b/internal/zip/ziptypes_go117.go new file mode 100644 index 0000000..60f5ddf --- /dev/null +++ b/internal/zip/ziptypes_go117.go @@ -0,0 +1,28 @@ +//go:build go1.17 || (go1.16 && zip117) +// +build go1.17 go1.16,zip117 + +package zip + +import "github.com/pgaskin/kepubify/_/go116-zip.go117/archive/zip" + +const Deflate = zip.Deflate +const Store = zip.Store + +var FileInfoHeader = zip.FileInfoHeader +var NewReader = zip.NewReader +var NewWriter = zip.NewWriter +var OpenReader = zip.OpenReader +var RegisterCompressor = zip.RegisterCompressor +var RegisterDecompressor = zip.RegisterDecompressor + +type Compressor = zip.Compressor +type Decompressor = zip.Decompressor +type File = zip.File +type FileHeader = zip.FileHeader +type ReadCloser = zip.ReadCloser +type Reader = zip.Reader +type Writer = zip.Writer + +var ErrAlgorithm = zip.ErrAlgorithm +var ErrChecksum = zip.ErrChecksum +var ErrFormat = zip.ErrFormat diff --git a/kepub/convert.go b/kepub/convert.go index 5c9227e..53a3337 100644 --- a/kepub/convert.go +++ b/kepub/convert.go @@ -1,7 +1,6 @@ package kepub import ( - "archive/zip" "bytes" "context" "encoding/xml" @@ -14,6 +13,7 @@ import ( "strings" "sync" + "github.com/pgaskin/kepubify/v4/internal/zip" "golang.org/x/sync/errgroup" ) diff --git a/kepub/convert_go116.go b/kepub/convert_go116.go index f9933dc..4531697 100644 --- a/kepub/convert_go116.go +++ b/kepub/convert_go116.go @@ -1,11 +1,12 @@ -//go:build go1.16 && !go1.17 -// +build go1.16,!go1.17 +//go:build go1.16 && !go1.17 && !zip117 +// +build go1.16,!go1.17,!zip117 package kepub import ( - "archive/zip" "io" + + "github.com/pgaskin/kepubify/v4/internal/zip" ) func zipCopyImpl(z *zip.Writer, f *zip.File) error { diff --git a/kepub/convert_go117.go b/kepub/convert_go117.go index e907c2f..37f232c 100644 --- a/kepub/convert_go117.go +++ b/kepub/convert_go117.go @@ -1,9 +1,9 @@ -//go:build go1.17 -// +build go1.17 +//go:build go1.17 || (go1.16 && zip117) +// +build go1.17 go1.16,zip117 package kepub -import "archive/zip" +import "github.com/pgaskin/kepubify/v4/internal/zip" func zipCopyImpl(z *zip.Writer, f *zip.File) error { return z.Copy(f) diff --git a/kepub/kobotest/main.go b/kepub/kobotest/main.go index 4fba51c..5bd5a20 100644 --- a/kepub/kobotest/main.go +++ b/kepub/kobotest/main.go @@ -11,11 +11,11 @@ import ( "os" "strings" - "golang.org/x/net/html" - "golang.org/x/net/html/atom" - "github.com/sergi/go-diff/diffmatchpatch" + "github.com/pgaskin/kepubify/_/html/golang.org/x/net/html" + "github.com/pgaskin/kepubify/_/html/golang.org/x/net/html/atom" + //go:linkname transformContentKoboSpans github.com/pgaskin/kepubify/v4/kepub.transformContentKoboSpans _ "unsafe" diff --git a/kepub/transform.go b/kepub/transform.go index 3ac40e7..d2b6949 100644 --- a/kepub/transform.go +++ b/kepub/transform.go @@ -10,11 +10,11 @@ import ( "strings" "unicode" - "golang.org/x/net/html" - "golang.org/x/net/html/atom" - "github.com/beevik/etree" "github.com/kr/smartypants" + + "github.com/pgaskin/kepubify/_/html/golang.org/x/net/html" + "github.com/pgaskin/kepubify/_/html/golang.org/x/net/html/atom" ) // TransformFileFilter returns true if a file should be filtered from the EPUB. @@ -433,10 +433,6 @@ func transformContentClean(doc *html.Node) { } } } - - // Note: The other cleanups for ensuring valid XHTML and other misc HTML - // fixes are now part of my forked golang.org/x/net/html package (see - // go test -run "^TestMod_" golang.org/x/net/html -v). } // withText adds text to a node and returns it. diff --git a/kepub/transform_test.go b/kepub/transform_test.go index f2495a7..958ce01 100644 --- a/kepub/transform_test.go +++ b/kepub/transform_test.go @@ -8,7 +8,8 @@ import ( "testing" "github.com/beevik/etree" - "golang.org/x/net/html" + + "github.com/pgaskin/kepubify/_/html/golang.org/x/net/html" ) func TestTransformContent(t *testing.T) {