Skip to content

Commit

Permalink
Several improvements from telegra.ph
Browse files Browse the repository at this point in the history 
- Reduce the number of retries
- Improve compatibility with non-headless mode
- Use `http.Get` instead of `obelisk` to retrieve webpage
- Allow the posting of a page without screenshot
- Handles large pages with pagination
  • Loading branch information
@waybackarchiver
waybackarchiver committed May 1, 2022
1 parent 1984c8a commit e066a5a
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 21 deletions.
20 changes: 10 additions & 10 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ module github.com/wabarc/wayback
go 1.17

require (
github.com/PuerkitoBio/goquery v1.8.0
github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030
github.com/cenkalti/backoff/v4 v4.1.2
github.com/cenkalti/backoff/v4 v4.1.3
github.com/cretz/bine v0.2.0
github.com/davecgh/go-spew v1.1.1
github.com/dghubble/go-twitter v0.0.0-20201011215211-4b180d0cc78d
Expand All @@ -33,24 +34,24 @@ require (
github.com/wabarc/go-anonfile v0.1.0
github.com/wabarc/go-catbox v0.1.0
github.com/wabarc/helper v0.0.0-20220320101038-dbb2ea36249b
github.com/wabarc/imgbb v1.0.0
github.com/wabarc/ipfs-pinner v1.1.1-0.20220126131044-16299c0dd43d
github.com/wabarc/logger v0.0.0-20210730133522-86bd3f31e792
github.com/wabarc/playback v0.0.0-20210718054702-cab6c6004933
github.com/wabarc/rivet v0.0.0-20220207154318-37fc56bcf4e1
github.com/wabarc/screenshot v1.5.1-0.20220318140348-632a135d50db
github.com/wabarc/telegra.ph v0.0.0-20220304132636-fce723bd6eae
github.com/wabarc/telegra.ph v0.0.0-20220501011455-b0b8c35c6d09
github.com/wabarc/warcraft v0.2.2-0.20211107142816-7beea5a75ab5
go.etcd.io/bbolt v1.3.6
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
gopkg.in/telebot.v3 v3.0.0-20220130115853-f0291132d3c3
maunium.net/go/mautrix v0.10.12
)

require (
github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 // indirect
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
Expand All @@ -59,15 +60,15 @@ require (
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheggaaa/pb/v3 v3.0.8 // indirect
github.com/chromedp/cdproto v0.0.0-20220217222649-d8c14a5c6edf // indirect
github.com/chromedp/chromedp v0.7.8 // indirect
github.com/chromedp/cdproto v0.0.0-20220426083054-c58585551e6c // indirect
github.com/chromedp/chromedp v0.8.0 // indirect
github.com/chromedp/sysutil v1.0.0 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/dghubble/sling v1.3.0 // indirect
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect
github.com/dop251/goja v0.0.0-20211211112501-fb27c91c26ed // indirect
github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 // indirect
github.com/go-shiori/obelisk v0.0.0-20220220092206-60b3a6b76c5e // indirect
github.com/go-shiori/obelisk v0.0.0-20220314133127-347d73aad05b // indirect
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
Expand Down Expand Up @@ -122,13 +123,12 @@ require (
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tdewolff/parse/v2 v2.5.27 // indirect
github.com/tdewolff/parse/v2 v2.5.29 // indirect
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
github.com/wabarc/imgbb v1.0.0 // indirect
github.com/wabarc/memento v0.0.0-20210703205719-adc2f8ab8bae // indirect
github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
github.com/ybbus/httpretry v1.0.1 // indirect
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/sourcemap.v1 v1.0.5 // indirect
Expand Down
29 changes: 18 additions & 11 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,9 @@ github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030/go.mod h1:NJ
github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo=
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4=
github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
Expand All @@ -122,10 +123,13 @@ github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyV
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA=
github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
github.com/chromedp/cdproto v0.0.0-20220217222649-d8c14a5c6edf h1:1omDWNUsWxn2HpiMiMuyRmzjl9uG7RP3IE6GTlpgJWU=
github.com/chromedp/cdproto v0.0.0-20220217222649-d8c14a5c6edf/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
github.com/chromedp/chromedp v0.7.8 h1:JFPIFb28LPjcx6l6mUUzLOTD/TgswcTtg7KrDn8S/2I=
github.com/chromedp/cdproto v0.0.0-20220321060548-7bc2623472b3/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0=
github.com/chromedp/cdproto v0.0.0-20220426083054-c58585551e6c h1:hIQEmCJsNOnN4SaWX80ebcucQ+7Nzm2URw9kpf63SBg=
github.com/chromedp/cdproto v0.0.0-20220426083054-c58585551e6c/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0=
github.com/chromedp/chromedp v0.7.8/go.mod h1:HcIUFBa5vA+u2QI3+xljiU59llUQ8lgGoLzYSCBfmUA=
github.com/chromedp/chromedp v0.8.0 h1:+Cufl+QWWfbvyylGCtAUt34A2EI/kqxRM3wGHXMabU4=
github.com/chromedp/chromedp v0.8.0/go.mod h1:odCVV9o9i7HUKwHMFz9Y7T6s4Kbcz4GOyPlwKWopI9Q=
github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
Expand Down Expand Up @@ -226,8 +230,8 @@ github.com/go-shiori/go-readability v0.0.0-20210627123243-82cc33435520/go.mod h1
github.com/go-shiori/go-readability v0.0.0-20220215145315-dd6828d2f09b h1:yrGomo5CP7IvXwSwKbDeaJkhwa4BxfgOO/s1V7iOQm4=
github.com/go-shiori/go-readability v0.0.0-20220215145315-dd6828d2f09b/go.mod h1:LTRGsNyO3/Y6u3ERbz17OiXy2qO1Y+/8QjXpg2ViyEY=
github.com/go-shiori/obelisk v0.0.0-20201115143556-8de0d40b0a9b/go.mod h1:OlGUo0utWqPMcej0HirsoK2+65Je/ZPFNwasF/O3KYM=
github.com/go-shiori/obelisk v0.0.0-20220220092206-60b3a6b76c5e h1:HPKrY2DeewRa5hjuIuP6FwUeJf2C5CAOT2jKOdfyRMs=
github.com/go-shiori/obelisk v0.0.0-20220220092206-60b3a6b76c5e/go.mod h1:1qhWqSYJil8gAg6HZqLtAuvv1S5bh3xJwd4N/WCO0aM=
github.com/go-shiori/obelisk v0.0.0-20220314133127-347d73aad05b h1:k9qMjvbfeCA4glJTSP6TC++R/D5z7UNLemc3zX76sBI=
github.com/go-shiori/obelisk v0.0.0-20220314133127-347d73aad05b/go.mod h1:qKa73D7hc0YucHndvsCOgZ5Ap54XgSmZxaIytNAFUAQ=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
Expand Down Expand Up @@ -667,8 +671,9 @@ github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMT
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tdewolff/parse/v2 v2.5.5/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho=
github.com/tdewolff/parse/v2 v2.5.27 h1:PL3LzzXaOpmdrknnOlIeO2muIBHAwiKp6TxN1RbU5gI=
github.com/tdewolff/parse/v2 v2.5.27/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho=
github.com/tdewolff/parse/v2 v2.5.29 h1:Uf0OtZL9YaUXTuHEOitdo9lD90P0XTwCjZi+KbGChuM=
github.com/tdewolff/parse/v2 v2.5.29/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho=
github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4=
github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
github.com/thoj/go-ircevent v0.0.0-20190807115034-8e7ce4b5a1eb h1:EavwSqheIJl3nb91HhkL73DwnT2Fk8W3yM7T7TuLZvA=
Expand Down Expand Up @@ -716,11 +721,11 @@ github.com/wabarc/playback v0.0.0-20210718054702-cab6c6004933 h1:KZHhUK+ZRhe4OxQ
github.com/wabarc/playback v0.0.0-20210718054702-cab6c6004933/go.mod h1:or8QaM2wUqIX45GfmLbky8AZKgOQm78VueJpqY2TwXY=
github.com/wabarc/rivet v0.0.0-20220207154318-37fc56bcf4e1 h1:4TKACQ7hhNwQ4SUaIVXAibaqSs5r/4CrgaYL/+HtEak=
github.com/wabarc/rivet v0.0.0-20220207154318-37fc56bcf4e1/go.mod h1:aWsIBu+Jr99P+G8KZ/NS/e6Yr5OEamt3i397U31QRu8=
github.com/wabarc/screenshot v1.4.1-0.20220304125241-bd18edec744e/go.mod h1:zacDw0LvNwstoCSqu2BYm8drXYOZ877TfDE6KhBEswY=
github.com/wabarc/screenshot v1.5.0/go.mod h1:zacDw0LvNwstoCSqu2BYm8drXYOZ877TfDE6KhBEswY=
github.com/wabarc/screenshot v1.5.1-0.20220318140348-632a135d50db h1:CUnpFXuJer3n+UGPtJyA/lKLpfA/Wo0n+pbjGpzFA8Q=
github.com/wabarc/screenshot v1.5.1-0.20220318140348-632a135d50db/go.mod h1:MJAoWa51BLqshJp+GQmoLJe4Q4Q6QCkcHMO1GYmucEE=
github.com/wabarc/telegra.ph v0.0.0-20220304132636-fce723bd6eae h1:qxDauBAL0F3P/Mc4bC0qCnM2UQq9LK9pP3BY/N7tXkI=
github.com/wabarc/telegra.ph v0.0.0-20220304132636-fce723bd6eae/go.mod h1:s4RHlPLzyJ7nX4+3UHp+WfVZb0hPefHUl319kVBPLtg=
github.com/wabarc/telegra.ph v0.0.0-20220501011455-b0b8c35c6d09 h1:suVFVJmwPmO9nll/IKv+ud5mwtCSN+IaH7dlvpsfPqk=
github.com/wabarc/telegra.ph v0.0.0-20220501011455-b0b8c35c6d09/go.mod h1:b+6k5eTiZAyEyPpTJZOi6t6DqeWp+i+lmCEmOYkQO6o=
github.com/wabarc/warcraft v0.2.2-0.20211107142816-7beea5a75ab5 h1:jY/jqIy/ddCMWWWuTIeAazE5F4QW8HAIvlI69XMJ1ew=
github.com/wabarc/warcraft v0.2.2-0.20211107142816-7beea5a75ab5/go.mod h1:/BbCwReBjlqHRaw8Yh+7sfAicOesiMYNhiFpuL1x8Rc=
github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs=
Expand Down Expand Up @@ -878,8 +883,9 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -994,8 +1000,9 @@ golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc=
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
Expand Down

0 comments on commit e066a5a

Please sign in to comment.