diff --git a/README.md b/README.md index ea6f9030..3239f056 100644 --- a/README.md +++ b/README.md @@ -162,8 +162,8 @@ There are several ways to fix this: - to: "https?://forbidden.com.*" on: 307 outcome: invalid - - from: "^http://.*" - to: "^https://.*" + - from: "http://.*" + to: "https://.*" outcome: follow ``` @@ -172,7 +172,7 @@ There are several ways to fix this: * The number of redirects allowed in a single redirect chain is limited and can be configured with the `maxRedirectFollows` parameter, also within `networking`. A number smaller than 0 disables the limit. -2. How does `xrefcheck` handle localhost links? +1. How does `xrefcheck` handle localhost links? * By default, `xrefcheck` will ignore links to localhost. * This behavior can be disabled by removing the corresponding entry from the `ignoreExternalRefsTo` list in the config file. diff --git a/src/Xrefcheck/Config.hs b/src/Xrefcheck/Config.hs index 74cfd951..263b4306 100644 --- a/src/Xrefcheck/Config.hs +++ b/src/Xrefcheck/Config.hs @@ -128,8 +128,8 @@ overrideConfig config defScanners = cScanners $ defConfig flavor defExclusions = cExclusions $ defConfig flavor - defNetworking = cNetworking $ defConfig flavor - defRedirectConfig = [] + defNetworking = cNetworking (defConfig flavor) + & ncExternalRefRedirectsL .~ [] overrideExclusions exclusionConfig = ExclusionConfig @@ -150,15 +150,12 @@ overrideConfig config , ncMaxRetries = overrideField ncMaxRetries , ncMaxTimeoutRetries = overrideField ncMaxTimeoutRetries , ncMaxRedirectFollows = overrideField ncMaxRedirectFollows - , ncExternalRefRedirects = externalRefRedirects + , ncExternalRefRedirects = overrideField ncExternalRefRedirects } where overrideField :: (forall f. NetworkingConfig' f -> Field f a) -> a overrideField field = fromMaybe (field defNetworking) $ field networkingConfig - externalRefRedirects :: RedirectConfig - externalRefRedirects = fromMaybe defRedirectConfig $ ncExternalRefRedirects networkingConfig - ----------------------------------------------------------- -- Yaml instances ----------------------------------------------------------- diff --git a/src/Xrefcheck/Config/Default.hs b/src/Xrefcheck/Config/Default.hs index ddaf2115..84b5de3e 100644 --- a/src/Xrefcheck/Config/Default.hs +++ b/src/Xrefcheck/Config/Default.hs @@ -9,6 +9,7 @@ import Universum import Text.Interpolation.Nyan +import Fmt (Builder) import Xrefcheck.Core import Xrefcheck.Util @@ -36,7 +37,7 @@ exclusions: # List of POSIX extended regular expressions. ignoreExternalRefsTo: # Ignore localhost links by default - - ^(https?|ftps?)://(localhost|127\\.0\\.0\\.1).* + - (https?|ftps?)://(localhost|127\\.0\\.0\\.1).* # Networking parameters. networking: @@ -101,18 +102,8 @@ networking: # # The first one that matches is applied, and the link is considered # as valid if none of them does match. - # - # If a value is provided for 'networking' but not for 'externalRefRedirects', - # then it will default to an empty list of rules and every redirect will pass. externalRefRedirects: - - from: .* - to: .* - on: permanent - outcome: invalid - - from: .* - to: .* - on: temporary - outcome: valid +#{interpolateIndentF 4 externalRefRedirects} # Parameters of scanners for various file types. scanners: @@ -128,7 +119,7 @@ scanners: |] where ignoreLocalRefsFrom :: NonEmpty Text - ignoreLocalRefsFrom = fromList $ case flavor of + ignoreLocalRefsFrom = fromList $ case flavor of GitHub -> [ ".github/pull_request_template.md" , ".github/issue_template.md" @@ -141,7 +132,7 @@ scanners: ] ignoreLocalRefsTo :: NonEmpty Text - ignoreLocalRefsTo = fromList $ case flavor of + ignoreLocalRefsTo = fromList $ case flavor of GitHub -> [ "../../../issues" , "../../../issues/*" @@ -154,3 +145,20 @@ scanners: , "../../merge_requests" , "../../merge_requests/*" ] + + externalRefRedirects :: Builder + externalRefRedirects = case flavor of + GitHub -> + [int|| + - on: permanent + outcome: invalid|] + GitLab -> + [int|| + - on: permanent + outcome: invalid + # GitLab redirects non-existing files to the repository's main page + # with a 302 code instead of answering with a 404 response. + - from: https?://gitlab.com/.*/-/blob/.* + to: https?://gitlab.com/.* + on: 302 + outcome: invalid|] diff --git a/tests/Test/Xrefcheck/RedirectConfigSpec.hs b/tests/Test/Xrefcheck/RedirectConfigSpec.hs index 25043f24..70fa2f31 100644 --- a/tests/Test/Xrefcheck/RedirectConfigSpec.hs +++ b/tests/Test/Xrefcheck/RedirectConfigSpec.hs @@ -49,7 +49,7 @@ test_redirectRequests = testGroup "Redirect config tests" [ testCase "Do match" $ do setRef <- newIORef mempty checkLinkAndProgressWithServer - (configMod [RedirectRule Nothing (regex "^.*/ok$") Nothing RROValid] []) + (configMod [RedirectRule Nothing (regex ".*/ok") Nothing RROValid] []) setRef mockRedirect (link "/permanent-redirect") @@ -58,7 +58,7 @@ test_redirectRequests = testGroup "Redirect config tests" , testCase "Do not match" $ do setRef <- newIORef mempty checkLinkAndProgressWithServer - (configMod [RedirectRule Nothing (regex "^.*/no-ok$") (Just RROPermanent) RROValid] []) + (configMod [RedirectRule Nothing (regex ".*/no-ok") (Just RROPermanent) RROValid] []) setRef mockRedirect (link "/permanent-redirect") @@ -69,7 +69,7 @@ test_redirectRequests = testGroup "Redirect config tests" [ testCase "Do match" $ do setRef <- newIORef mempty checkLinkAndProgressWithServer - (configMod [RedirectRule (regex "^.*/permanent-.*$") Nothing Nothing RROValid] []) + (configMod [RedirectRule (regex ".*/permanent-.*") Nothing Nothing RROValid] []) setRef mockRedirect (link "/permanent-redirect") @@ -78,7 +78,7 @@ test_redirectRequests = testGroup "Redirect config tests" , testCase "Do not match" $ do setRef <- newIORef mempty checkLinkAndProgressWithServer - (configMod [RedirectRule (regex "^.*/temporary-.*$") Nothing (Just RROPermanent) RROValid] []) + (configMod [RedirectRule (regex ".*/temporary-.*") Nothing (Just RROPermanent) RROValid] []) setRef mockRedirect (link "/permanent-redirect") @@ -89,7 +89,7 @@ test_redirectRequests = testGroup "Redirect config tests" [ testCase "Do match" $ do setRef <- newIORef mempty checkLinkAndProgressWithServer - (configMod [RedirectRule (regex "^.*/follow[0-9]$") (regex "^.*/ok$") (Just (RROCode 307)) RROInvalid] []) + (configMod [RedirectRule (regex ".*/follow[0-9]") (regex "^.*/ok$") (Just (RROCode 307)) RROInvalid] []) setRef mockRedirect (link "/follow3") @@ -98,7 +98,7 @@ test_redirectRequests = testGroup "Redirect config tests" , testCase "Do not match" $ do setRef <- newIORef mempty checkLinkAndProgressWithServer - (configMod [RedirectRule (regex "^.*/follow[0-9]$") (regex "^.*/ok$") (Just (RROCode 307)) RROInvalid] []) + (configMod [RedirectRule (regex ".*/follow[0-9]") (regex "^.*/ok$") (Just (RROCode 307)) RROInvalid] []) setRef mockRedirect (link "/follow2") @@ -137,7 +137,7 @@ test_redirectRequests = testGroup "Redirect config tests" , testCase "Mixed with ignore" $ do setRef <- newIORef mempty checkLinkAndProgressWithServer - (configMod [RedirectRule Nothing Nothing (Just (RROCode 307)) RROInvalid, RedirectRule Nothing Nothing Nothing RROFollow] (maybeToList (regex "^.*/follow3$"))) + (configMod [RedirectRule Nothing Nothing (Just (RROCode 307)) RROInvalid, RedirectRule Nothing Nothing Nothing RROFollow] (maybeToList (regex ".*/follow3"))) setRef mockRedirect (link "/follow1") diff --git a/tests/configs/github-config.yaml b/tests/configs/github-config.yaml index fd758858..adf55523 100644 --- a/tests/configs/github-config.yaml +++ b/tests/configs/github-config.yaml @@ -25,7 +25,7 @@ exclusions: # List of POSIX extended regular expressions. ignoreExternalRefsTo: # Ignore localhost links by default - - ^(https?|ftps?)://(localhost|127\.0\.0\.1).* + - (https?|ftps?)://(localhost|127\.0\.0\.1).* # Networking parameters. networking: @@ -90,18 +90,9 @@ networking: # # The first one that matches is applied, and the link is considered # as valid if none of them does match. - # - # If a value is provided for 'networking' but not for 'externalRefRedirects', - # then it will default to an empty list of rules and every redirect will pass. externalRefRedirects: - - from: .* - to: .* - on: permanent + - on: permanent outcome: invalid - - from: .* - to: .* - on: temporary - outcome: valid # Parameters of scanners for various file types. scanners: diff --git a/tests/golden/check-ignoreExternalRefsTo/config-check-disabled.yaml b/tests/golden/check-ignoreExternalRefsTo/config-check-disabled.yaml index df6ea981..351ac3d0 100644 --- a/tests/golden/check-ignoreExternalRefsTo/config-check-disabled.yaml +++ b/tests/golden/check-ignoreExternalRefsTo/config-check-disabled.yaml @@ -4,7 +4,7 @@ exclusions: ignoreExternalRefsTo: - - ^(https?|ftps?)://(localhost|127\.0\.0\.1).* + - (https?|ftps?)://(localhost|127\.0\.0\.1).* scanners: markdown: diff --git a/tests/golden/check-redirect-parse/bad-code.yaml b/tests/golden/check-redirect-parse/bad-code.yaml index b9233a8f..715e8dd9 100644 --- a/tests/golden/check-redirect-parse/bad-code.yaml +++ b/tests/golden/check-redirect-parse/bad-code.yaml @@ -9,5 +9,4 @@ scanners: networking: externalRefRedirects: - outcome: valid - to: ^.*$ on: 404 diff --git a/tests/golden/check-redirect-parse/bad-on.yaml b/tests/golden/check-redirect-parse/bad-on.yaml index 1c3d85bf..bd56376d 100644 --- a/tests/golden/check-redirect-parse/bad-on.yaml +++ b/tests/golden/check-redirect-parse/bad-on.yaml @@ -9,5 +9,4 @@ scanners: networking: externalRefRedirects: - outcome: valid - to: ^.*$ on: premanent diff --git a/tests/golden/check-redirect-parse/full-rule.yaml b/tests/golden/check-redirect-parse/full-rule.yaml index a0f7154d..69ff096a 100644 --- a/tests/golden/check-redirect-parse/full-rule.yaml +++ b/tests/golden/check-redirect-parse/full-rule.yaml @@ -9,5 +9,5 @@ scanners: networking: externalRefRedirects: - outcome: valid - to: ^https://.*$ + to: https://.* on: permanent diff --git a/tests/golden/check-redirect-parse/no-outcome.yaml b/tests/golden/check-redirect-parse/no-outcome.yaml index ff5eab86..9cf856c5 100644 --- a/tests/golden/check-redirect-parse/no-outcome.yaml +++ b/tests/golden/check-redirect-parse/no-outcome.yaml @@ -10,5 +10,5 @@ networking: externalRefRedirects: - outcome: valid on: temporary - - to: ^https://.*$ + - to: https://.* on: temporary diff --git a/tests/golden/check-redirect-parse/only-outcome-to.yaml b/tests/golden/check-redirect-parse/only-outcome-to.yaml index bcb1dc6a..d5ef9d75 100644 --- a/tests/golden/check-redirect-parse/only-outcome-to.yaml +++ b/tests/golden/check-redirect-parse/only-outcome-to.yaml @@ -9,4 +9,4 @@ scanners: networking: externalRefRedirects: - outcome: valid - to: ^https://.*$ + to: https://.*