From e066a5a2bc379dcc2dab3dc566b9ab9d91b0c0ea Mon Sep 17 00:00:00 2001 From: Wayback Archiver <66856220+waybackarchiver@users.noreply.github.com> Date: Sun, 1 May 2022 12:26:18 +0000 Subject: [PATCH] Several improvements from `telegra.ph` - Reduce the number of retries - Improve compatibility with non-headless mode - Use `http.Get` instead of `obelisk` to retrieve webpage - Allow the posting of a page without screenshot - Handles large pages with pagination --- go.mod | 20 ++++++++++---------- go.sum | 29 ++++++++++++++++++----------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index dceff909..041041ec 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,9 @@ module github.com/wabarc/wayback go 1.17 require ( + github.com/PuerkitoBio/goquery v1.8.0 github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030 - github.com/cenkalti/backoff/v4 v4.1.2 + github.com/cenkalti/backoff/v4 v4.1.3 github.com/cretz/bine v0.2.0 github.com/davecgh/go-spew v1.1.1 github.com/dghubble/go-twitter v0.0.0-20201011215211-4b180d0cc78d @@ -33,16 +34,17 @@ require ( github.com/wabarc/go-anonfile v0.1.0 github.com/wabarc/go-catbox v0.1.0 github.com/wabarc/helper v0.0.0-20220320101038-dbb2ea36249b + github.com/wabarc/imgbb v1.0.0 github.com/wabarc/ipfs-pinner v1.1.1-0.20220126131044-16299c0dd43d github.com/wabarc/logger v0.0.0-20210730133522-86bd3f31e792 github.com/wabarc/playback v0.0.0-20210718054702-cab6c6004933 github.com/wabarc/rivet v0.0.0-20220207154318-37fc56bcf4e1 github.com/wabarc/screenshot v1.5.1-0.20220318140348-632a135d50db - github.com/wabarc/telegra.ph v0.0.0-20220304132636-fce723bd6eae + github.com/wabarc/telegra.ph v0.0.0-20220501011455-b0b8c35c6d09 github.com/wabarc/warcraft v0.2.2-0.20211107142816-7beea5a75ab5 go.etcd.io/bbolt v1.3.6 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c gopkg.in/telebot.v3 v3.0.0-20220130115853-f0291132d3c3 maunium.net/go/mautrix v0.10.12 @@ -50,7 +52,6 @@ require ( require ( github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 // indirect - github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -59,15 +60,15 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cheggaaa/pb/v3 v3.0.8 // indirect - github.com/chromedp/cdproto v0.0.0-20220217222649-d8c14a5c6edf // indirect - github.com/chromedp/chromedp v0.7.8 // indirect + github.com/chromedp/cdproto v0.0.0-20220426083054-c58585551e6c // indirect + github.com/chromedp/chromedp v0.8.0 // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/dghubble/sling v1.3.0 // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect github.com/dop251/goja v0.0.0-20211211112501-fb27c91c26ed // indirect github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 // indirect - github.com/go-shiori/obelisk v0.0.0-20220220092206-60b3a6b76c5e // indirect + github.com/go-shiori/obelisk v0.0.0-20220314133127-347d73aad05b // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect @@ -122,13 +123,12 @@ require ( github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tdewolff/parse/v2 v2.5.27 // indirect + github.com/tdewolff/parse/v2 v2.5.29 // indirect github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect - github.com/wabarc/imgbb v1.0.0 // indirect github.com/wabarc/memento v0.0.0-20210703205719-adc2f8ab8bae // indirect github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect github.com/ybbus/httpretry v1.0.1 // indirect - golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect + golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect diff --git a/go.sum b/go.sum index bf146b22..f4ab277c 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,9 @@ github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030/go.mod h1:NJ github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -122,10 +123,13 @@ github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyV github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA= github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= -github.com/chromedp/cdproto v0.0.0-20220217222649-d8c14a5c6edf h1:1omDWNUsWxn2HpiMiMuyRmzjl9uG7RP3IE6GTlpgJWU= github.com/chromedp/cdproto v0.0.0-20220217222649-d8c14a5c6edf/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U= -github.com/chromedp/chromedp v0.7.8 h1:JFPIFb28LPjcx6l6mUUzLOTD/TgswcTtg7KrDn8S/2I= +github.com/chromedp/cdproto v0.0.0-20220321060548-7bc2623472b3/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0= +github.com/chromedp/cdproto v0.0.0-20220426083054-c58585551e6c h1:hIQEmCJsNOnN4SaWX80ebcucQ+7Nzm2URw9kpf63SBg= +github.com/chromedp/cdproto v0.0.0-20220426083054-c58585551e6c/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0= github.com/chromedp/chromedp v0.7.8/go.mod h1:HcIUFBa5vA+u2QI3+xljiU59llUQ8lgGoLzYSCBfmUA= +github.com/chromedp/chromedp v0.8.0 h1:+Cufl+QWWfbvyylGCtAUt34A2EI/kqxRM3wGHXMabU4= +github.com/chromedp/chromedp v0.8.0/go.mod h1:odCVV9o9i7HUKwHMFz9Y7T6s4Kbcz4GOyPlwKWopI9Q= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -226,8 +230,8 @@ github.com/go-shiori/go-readability v0.0.0-20210627123243-82cc33435520/go.mod h1 github.com/go-shiori/go-readability v0.0.0-20220215145315-dd6828d2f09b h1:yrGomo5CP7IvXwSwKbDeaJkhwa4BxfgOO/s1V7iOQm4= github.com/go-shiori/go-readability v0.0.0-20220215145315-dd6828d2f09b/go.mod h1:LTRGsNyO3/Y6u3ERbz17OiXy2qO1Y+/8QjXpg2ViyEY= github.com/go-shiori/obelisk v0.0.0-20201115143556-8de0d40b0a9b/go.mod h1:OlGUo0utWqPMcej0HirsoK2+65Je/ZPFNwasF/O3KYM= -github.com/go-shiori/obelisk v0.0.0-20220220092206-60b3a6b76c5e h1:HPKrY2DeewRa5hjuIuP6FwUeJf2C5CAOT2jKOdfyRMs= -github.com/go-shiori/obelisk v0.0.0-20220220092206-60b3a6b76c5e/go.mod h1:1qhWqSYJil8gAg6HZqLtAuvv1S5bh3xJwd4N/WCO0aM= +github.com/go-shiori/obelisk v0.0.0-20220314133127-347d73aad05b h1:k9qMjvbfeCA4glJTSP6TC++R/D5z7UNLemc3zX76sBI= +github.com/go-shiori/obelisk v0.0.0-20220314133127-347d73aad05b/go.mod h1:qKa73D7hc0YucHndvsCOgZ5Ap54XgSmZxaIytNAFUAQ= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -667,8 +671,9 @@ github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMT github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tdewolff/parse/v2 v2.5.5/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho= -github.com/tdewolff/parse/v2 v2.5.27 h1:PL3LzzXaOpmdrknnOlIeO2muIBHAwiKp6TxN1RbU5gI= github.com/tdewolff/parse/v2 v2.5.27/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho= +github.com/tdewolff/parse/v2 v2.5.29 h1:Uf0OtZL9YaUXTuHEOitdo9lD90P0XTwCjZi+KbGChuM= +github.com/tdewolff/parse/v2 v2.5.29/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho= github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4= github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/thoj/go-ircevent v0.0.0-20190807115034-8e7ce4b5a1eb h1:EavwSqheIJl3nb91HhkL73DwnT2Fk8W3yM7T7TuLZvA= @@ -716,11 +721,11 @@ github.com/wabarc/playback v0.0.0-20210718054702-cab6c6004933 h1:KZHhUK+ZRhe4OxQ github.com/wabarc/playback v0.0.0-20210718054702-cab6c6004933/go.mod h1:or8QaM2wUqIX45GfmLbky8AZKgOQm78VueJpqY2TwXY= github.com/wabarc/rivet v0.0.0-20220207154318-37fc56bcf4e1 h1:4TKACQ7hhNwQ4SUaIVXAibaqSs5r/4CrgaYL/+HtEak= github.com/wabarc/rivet v0.0.0-20220207154318-37fc56bcf4e1/go.mod h1:aWsIBu+Jr99P+G8KZ/NS/e6Yr5OEamt3i397U31QRu8= -github.com/wabarc/screenshot v1.4.1-0.20220304125241-bd18edec744e/go.mod h1:zacDw0LvNwstoCSqu2BYm8drXYOZ877TfDE6KhBEswY= +github.com/wabarc/screenshot v1.5.0/go.mod h1:zacDw0LvNwstoCSqu2BYm8drXYOZ877TfDE6KhBEswY= github.com/wabarc/screenshot v1.5.1-0.20220318140348-632a135d50db h1:CUnpFXuJer3n+UGPtJyA/lKLpfA/Wo0n+pbjGpzFA8Q= github.com/wabarc/screenshot v1.5.1-0.20220318140348-632a135d50db/go.mod h1:MJAoWa51BLqshJp+GQmoLJe4Q4Q6QCkcHMO1GYmucEE= -github.com/wabarc/telegra.ph v0.0.0-20220304132636-fce723bd6eae h1:qxDauBAL0F3P/Mc4bC0qCnM2UQq9LK9pP3BY/N7tXkI= -github.com/wabarc/telegra.ph v0.0.0-20220304132636-fce723bd6eae/go.mod h1:s4RHlPLzyJ7nX4+3UHp+WfVZb0hPefHUl319kVBPLtg= +github.com/wabarc/telegra.ph v0.0.0-20220501011455-b0b8c35c6d09 h1:suVFVJmwPmO9nll/IKv+ud5mwtCSN+IaH7dlvpsfPqk= +github.com/wabarc/telegra.ph v0.0.0-20220501011455-b0b8c35c6d09/go.mod h1:b+6k5eTiZAyEyPpTJZOi6t6DqeWp+i+lmCEmOYkQO6o= github.com/wabarc/warcraft v0.2.2-0.20211107142816-7beea5a75ab5 h1:jY/jqIy/ddCMWWWuTIeAazE5F4QW8HAIvlI69XMJ1ew= github.com/wabarc/warcraft v0.2.2-0.20211107142816-7beea5a75ab5/go.mod h1:/BbCwReBjlqHRaw8Yh+7sfAicOesiMYNhiFpuL1x8Rc= github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= @@ -878,8 +883,9 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -994,8 +1000,9 @@ golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=