Update scripts so url check is fixed

.github/workflows/pull_request.yml

@@ -93,12 +93,12 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: spell-check-results
-          path: resources/spell_check_results.tsv
+          path: check_reports/spell_check_results.tsv
 
       - name: Commit spell check errors
         run: |
           branch_name='preview-${{ github.event.pull_request.number }}'
-          git add --force resources/spell_check_results.tsv || echo "No changes to commit"
+          git add --force check_reports/spell_check_results.tsv || echo "No changes to commit"
           git commit -m 'Add spell check file' || echo "No changes to commit"
           git pull --set-upstream origin $branch_name --allow-unrelated-histories --strategy-option=ours
           git push --force origin $branch_name || echo "No changes to commit"
@@ -109,7 +109,7 @@ jobs:
           GH_PAT: ${{ secrets.GH_PAT }}
         run: |
           branch_name='preview-${{ github.event.pull_request.number }}'
-          sp_error_url=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/resources/spell_check_results.tsv
+          sp_error_url=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/check_reports/spell_check_results.tsv
           echo ::set-output name=time::$(date +'%Y-%m-%d')
           echo ::set-output name=commit_id::$GITHUB_SHA
           echo ::set-output name=sp_error_url::$sp_error_url
@@ -197,40 +197,17 @@ jobs:
       shell: bash
 
     - name: URLs checker
-      uses: urlstechie/urlchecker-action@master
-      with:
-        # Specify a branch
-        branch: preview-${{ github.event.pull_request.number }}
-
-        # Cleanup
-        cleanup: false
-
-        # A comma-separated list of file types to cover in the URL checks
-        file_types: .Rmd,.md
-
-        # Choose whether to include file with no URLs in the prints.
-        print_all: false
-
-        exclude_files: .github/PULL_REQUEST_TEMPLATE.md, docs/*
-
-        # A comma separated links to exclude during URL checks
-        exclude_urls: https://jhudatascience.org/{Course_Name}}
-
-        force_pass: true
-
-        # A path to a csv file to save results to
-        save: resources/url_checks.csv
-
-    - name: Count URL errors
       id: url_errors
       run: |
-        results=$(Rscript "scripts/url-check.R")
-        echo "::set-output name=url_results::$results"
+        chk_results=$(Rscript scripts/url-check.R)
+        echo $chk_results
+
+        echo "::set-output name=url_results::$chk_results"
 
     - name: Commit URL check
       run: |
         branch_name='preview-${{ github.event.pull_request.number }}'
-        git add --force resources/url_checks.csv || echo "No changes to commit"
+        git add --force check_reports/url_checks.tsv || echo "No changes to commit"
         git commit -m 'Add URL check file' || echo "No changes to commit"
         git fetch
         git merge -s recursive --strategy-option=theirs origin/${{ github.head_ref }} --allow-unrelated-histories
@@ -242,7 +219,7 @@ jobs:
         GH_PAT: ${{ secrets.GH_PAT }}
       run: |
         branch_name='preview-${{ github.event.pull_request.number }}'
-        url_errors=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/resources/url_checks.csv
+        url_errors=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/check_reports/url_checks.tsv
         echo ::set-output name=time::$(date +'%Y-%m-%d')
         echo ::set-output name=commit_id::$GITHUB_SHA
         echo ::set-output name=url_errors::$url_errors

.github/workflows/report-maker.yml

@@ -29,13 +29,13 @@ jobs:
         run: |
           if ${{inputs.check_type == 'spelling'}} ;then
             error_name='Spelling errors'
-            report_path='resources/spell_check_results.tsv'
+            report_path='check_reports/spell_check_results.tsv'
           elif ${{inputs.check_type == 'urls'}} ;then
             error_name='Broken URLs'
-            report_path='resources/url_checks.csv'
+            report_path='check_reports/url_checks.csv'
           elif ${{inputs.check_type ==  'quiz_format'}} ;then
             error_name='Quiz format errors'
-            report_path='question_error_report.tsv'
+            report_path='check_reports/question_error_report.tsv'
           fi
           echo $error_name
           echo $report_path
@@ -76,11 +76,7 @@ jobs:
       - name: Run quiz check
         if: ${{ inputs.check_type == 'quiz_format' }}
         run: |
-          Rscript -e "ottrpal::check_quizzes(quiz_dir = 'quizzes', write_report = TRUE, verbose = TRUE)"
-          chk_results=0
-          if [ -f question_error_report.tsv ]; then
-            chk_results=$(wc -l < question_error_report.tsv)
-          fi
+          chk_results=$(Rscript scripts/quiz-check.R)
           chk_results="$((chk_results-1))"
           echo $chk_results
 

scripts/quiz-check.R

@@ -0,0 +1,39 @@
+#!/usr/bin/env Rscript
+
+# Adapted for this jhudsl repository by Candace Savonen Mar 2022
+
+# Run spell check and save results
+
+library(magrittr)
+
+# Find .git root directory
+root_dir <- rprojroot::find_root(rprojroot::has_dir(".git"))
+
+output_file <- file.path(root_dir, 'check_reports', 'question_error_report.tsv')
+
+if (!dir.exists('check_reports')) {
+  dir.create('check_reports')
+}
+
+ottrpal::check_quizzes(quiz_dir = file.path(root_dir, 'quizzes'), write_report = TRUE, verbose = TRUE)
+
+if (file.exists("question_error_report.tsv")) {
+  quiz_errors <- readr::read_tsv("question_error_report.tsv")
+
+  file.copy('question_error_report.tsv', file.path(root_dir, 'check_reports'))
+  file.remove('question_error_report.tsv')
+
+  # Print out how many quiz check errors
+  write(nrow(quiz_errors), stdout())
+
+} else {
+  quiz_errors <- data.frame(errors = NA)
+
+  # Print out how many quiz check errors
+  write("1", stdout())
+}
+
+# Save question errors to file
+readr::write_tsv(quiz_errors, output_file)
+
+message(paste0("Saved to: ", output_file))

scripts/spell-check.R

@@ -13,10 +13,16 @@ library(magrittr)
 if (!("spelling" %in% installed.packages())){
   install.packages("spelling")
 }
-
 # Find .git root directory
 root_dir <- rprojroot::find_root(rprojroot::has_dir(".git"))
 
+# Set up output file directory
+output_file <- file.path(root_dir, 'check_reports', 'spell_check_results.tsv')
+
+if (!dir.exists('check_reports')) {
+  dir.create('check_reports')
+}
+
 # Read in dictionary
 dictionary <- readLines(file.path(root_dir, 'resources', 'dictionary.txt'))
 
@@ -40,16 +46,14 @@ if (nrow(sp_errors) > 0) {
     data.frame() %>%
     tidyr::unnest(cols = found) %>%
     tidyr::separate(found, into = c("file", "lines"), sep = ":")
+} else {
+  sp_errors <- data.frame(errors = NA)
 }
 
 # Print out how many spell check errors
 write(nrow(sp_errors), stdout())
 
-if (!dir.exists("resources")) {
-  dir.create("resources")
-}
-
-if (nrow(sp_errors) > 0) {
 # Save spell errors to file temporarily
-readr::write_tsv(sp_errors, file.path('resources', 'spell_check_results.tsv'))
-}
+readr::write_tsv(sp_errors, output_file)
+
+message(paste0("Saved to: ", output_file))

scripts/url-check.R

@@ -9,20 +9,24 @@ library(magrittr)
 # Find .git root directory
 root_dir <- rprojroot::find_root(rprojroot::has_dir(".git"))
 
-output_file <- file.path(root_dir, 'resources', 'url_checks.tsv')
+output_file <- file.path(root_dir, 'check_reports', 'url_checks.tsv')
+
+if (!dir.exists('check_reports')) {
+  dir.create('check_reports')
+}
 
 # Only declare `.Rmd` files but not the ones in the style-sets directory
 files <- list.files(path = root_dir, pattern = 'md$', full.names = TRUE)
 
 test_url <- function(url) {
   message(paste0("Testing: ", url))
-  url_status <- try(httr::response(url, as = "text"), silent = TRUE)
-  status <- ifelse(suppressWarnings(grepl("Could not resolve host", url_status)), "failed", "success")
+  url_status <- try(httr::GET(url), silent = TRUE)
+  status <- ifelse(suppressMessages(grepl("Could not resolve host", url_status)), "failed", "success")
   return(status)
 }
 
 get_urls <- function(file) {
-  # Read in a file and return the urls from it 
+  # Read in a file and return the urls from it
   content <- readLines(file)
   content <- grep("http|com$|www", content, value = TRUE)
   url_pattern <- "http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
@@ -42,16 +46,19 @@ all_urls <- lapply(files, get_urls)
 all_urls_df <- dplyr::bind_rows(all_urls)
 
 if (nrow(all_urls_df) > 0) {
-  all_urls_df <- all_urls_df %>% 
-    dplyr::filter(urls_status == "failed") %>% 
-    readr::write_tsv(output_file) 
+  all_urls_df <- all_urls_df %>%
+    dplyr::filter(urls_status == "failed")
+} else {
+  all_urls_df <- data.frame(errors = NA)
 }
 
 # Print out how many spell check errors
 write(nrow(all_urls_df), stdout())
 
-if (nrow(all_urls_df) > 0) {
-  # Save spell errors to file temporarily
-  readr::write_tsv(all_urls_df, output_file)
-}
+# Save spell errors to file temporarily
+readr::write_tsv(all_urls_df, output_file)
 
+message(paste0("Saved to: ", output_file))
+
+# Print out how many spell check errors
+write(nrow(all_urls_df), stdout())