From 41a339c540c4905a9a4d55a90f49c80bfd9ea7fa Mon Sep 17 00:00:00 2001 From: Candace Savonen Date: Thu, 31 Mar 2022 13:27:52 -0400 Subject: [PATCH 01/10] Update scripts --- .github/workflows/report-maker.yml | 12 +++------ scripts/quiz-check.R | 39 ++++++++++++++++++++++++++++++ scripts/spell-check.R | 20 +++++++++------ scripts/url-check.R | 27 ++++++++++++--------- 4 files changed, 71 insertions(+), 27 deletions(-) create mode 100644 scripts/quiz-check.R diff --git a/.github/workflows/report-maker.yml b/.github/workflows/report-maker.yml index d3cba9f8..20038873 100644 --- a/.github/workflows/report-maker.yml +++ b/.github/workflows/report-maker.yml @@ -29,13 +29,13 @@ jobs: run: | if ${{inputs.check_type == 'spelling'}} ;then error_name='Spelling errors' - report_path='resources/spell_check_results.tsv' + report_path='check_reports/spell_check_results.tsv' elif ${{inputs.check_type == 'urls'}} ;then error_name='Broken URLs' - report_path='resources/url_checks.csv' + report_path='check_reports/url_checks.csv' elif ${{inputs.check_type == 'quiz_format'}} ;then error_name='Quiz format errors' - report_path='question_error_report.tsv' + report_path='check_reports/question_error_report.tsv' fi echo $error_name echo $report_path @@ -76,11 +76,7 @@ jobs: - name: Run quiz check if: ${{ inputs.check_type == 'quiz_format' }} run: | - Rscript -e "ottrpal::check_quizzes(quiz_dir = 'quizzes', write_report = TRUE, verbose = TRUE)" - chk_results=0 - if [ -f question_error_report.tsv ]; then - chk_results=$(wc -l < question_error_report.tsv) - fi + chk_results=$(Rscript scripts/quiz-check.R) chk_results="$((chk_results-1))" echo $chk_results diff --git a/scripts/quiz-check.R b/scripts/quiz-check.R new file mode 100644 index 00000000..9c10ee92 --- /dev/null +++ b/scripts/quiz-check.R @@ -0,0 +1,39 @@ +#!/usr/bin/env Rscript + +# Adapted for this jhudsl repository by Candace Savonen Mar 2022 + +# Run spell check and save results + +library(magrittr) + +# Find .git root directory +root_dir <- rprojroot::find_root(rprojroot::has_dir(".git")) + +output_file <- file.path(root_dir, 'check_reports', 'question_error_report.tsv') + +if (!dir.exists('check_reports')) { + dir.create('check_reports') +} + +ottrpal::check_quizzes(quiz_dir = file.path(root_dir, 'quizzes'), write_report = TRUE, verbose = TRUE) + +if (file.exists("question_error_report.tsv")) { + quiz_errors <- readr::read_tsv("question_error_report.tsv") + + file.copy('question_error_report.tsv', file.path(root_dir, 'check_reports')) + file.remove('question_error_report.tsv') + + # Print out how many quiz check errors + write(nrow(quiz_errors), stdout()) + +} else { + quiz_errors <- data.frame(errors = NA) + + # Print out how many quiz check errors + write("1", stdout()) +} + +# Save question errors to file +readr::write_tsv(quiz_errors, output_file) + +message(paste0("Saved to: ", output_file)) diff --git a/scripts/spell-check.R b/scripts/spell-check.R index d3596a8d..a732fce3 100644 --- a/scripts/spell-check.R +++ b/scripts/spell-check.R @@ -13,10 +13,16 @@ library(magrittr) if (!("spelling" %in% installed.packages())){ install.packages("spelling") } - # Find .git root directory root_dir <- rprojroot::find_root(rprojroot::has_dir(".git")) +# Set up output file directory +output_file <- file.path(root_dir, 'check_reports', 'spell_check_results.tsv') + +if (!dir.exists('check_reports')) { + dir.create('check_reports') +} + # Read in dictionary dictionary <- readLines(file.path(root_dir, 'resources', 'dictionary.txt')) @@ -40,16 +46,14 @@ if (nrow(sp_errors) > 0) { data.frame() %>% tidyr::unnest(cols = found) %>% tidyr::separate(found, into = c("file", "lines"), sep = ":") +} else { + sp_errors <- data.frame(errors = NA) } # Print out how many spell check errors write(nrow(sp_errors), stdout()) -if (!dir.exists("resources")) { - dir.create("resources") -} - -if (nrow(sp_errors) > 0) { # Save spell errors to file temporarily -readr::write_tsv(sp_errors, file.path('resources', 'spell_check_results.tsv')) -} +readr::write_tsv(sp_errors, output_file) + +message(paste0("Saved to: ", output_file)) diff --git a/scripts/url-check.R b/scripts/url-check.R index f2447dc9..2a7f05c3 100644 --- a/scripts/url-check.R +++ b/scripts/url-check.R @@ -9,20 +9,24 @@ library(magrittr) # Find .git root directory root_dir <- rprojroot::find_root(rprojroot::has_dir(".git")) -output_file <- file.path(root_dir, 'resources', 'url_checks.tsv') +output_file <- file.path(root_dir, 'check_reports', 'url_checks.tsv') + +if (!dir.exists('check_reports')) { + dir.create('check_reports') +} # Only declare `.Rmd` files but not the ones in the style-sets directory files <- list.files(path = root_dir, pattern = 'md$', full.names = TRUE) test_url <- function(url) { message(paste0("Testing: ", url)) - url_status <- try(httr::response(url, as = "text"), silent = TRUE) - status <- ifelse(suppressWarnings(grepl("Could not resolve host", url_status)), "failed", "success") + url_status <- try(httr::GET(url), silent = TRUE) + status <- ifelse(suppressMessages(grepl("Could not resolve host", url_status)), "failed", "success") return(status) } get_urls <- function(file) { - # Read in a file and return the urls from it + # Read in a file and return the urls from it content <- readLines(file) content <- grep("http|com$|www", content, value = TRUE) url_pattern <- "http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+" @@ -42,16 +46,17 @@ all_urls <- lapply(files, get_urls) all_urls_df <- dplyr::bind_rows(all_urls) if (nrow(all_urls_df) > 0) { - all_urls_df <- all_urls_df %>% - dplyr::filter(urls_status == "failed") %>% - readr::write_tsv(output_file) + all_urls_df <- all_urls_df %>% + dplyr::filter(urls_status == "failed") %>% + readr::write_tsv(output_file) +} else { + all_urls_df <- data.frame(errors = NA) } # Print out how many spell check errors write(nrow(all_urls_df), stdout()) -if (nrow(all_urls_df) > 0) { - # Save spell errors to file temporarily - readr::write_tsv(all_urls_df, output_file) -} +# Save spell errors to file temporarily +readr::write_tsv(all_urls_df, output_file) +message(paste0("Saved to: ", output_file)) From 9906c99e5c21321bdc3634cd1c04c12f26d4b824 Mon Sep 17 00:00:00 2001 From: Candace Savonen Date: Thu, 31 Mar 2022 13:47:28 -0400 Subject: [PATCH 02/10] try to fix url checker --- .github/workflows/pull_request.yml | 31 +++++------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c9dab272..d19bed02 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -197,34 +197,13 @@ jobs: shell: bash - name: URLs checker - uses: urlstechie/urlchecker-action@master - with: - # Specify a branch - branch: preview-${{ github.event.pull_request.number }} - - # Cleanup - cleanup: false - - # A comma-separated list of file types to cover in the URL checks - file_types: .Rmd,.md - - # Choose whether to include file with no URLs in the prints. - print_all: false - - exclude_files: .github/PULL_REQUEST_TEMPLATE.md, docs/* - - # A comma separated links to exclude during URL checks - exclude_urls: https://jhudatascience.org/{Course_Name}} - - force_pass: true - - # A path to a csv file to save results to - save: resources/url_checks.csv - - - name: Count URL errors id: url_errors run: | - results=$(Rscript "scripts/url-check.R") + chk_results=$(Rscript scripts/url-check.R) + chk_results="$((chk_results-1))" + echo $chk_results + + echo "chk_results=$chk_results" >> $GITHUB_ENV echo "::set-output name=url_results::$results" - name: Commit URL check From 973453fa82a121032a15ff78a7ef9729829225d5 Mon Sep 17 00:00:00 2001 From: Candace Savonen Date: Thu, 31 Mar 2022 14:13:52 -0400 Subject: [PATCH 03/10] Add a broken url as a test --- 02-chapter_of_course.Rmd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/02-chapter_of_course.Rmd b/02-chapter_of_course.Rmd index dc240d2f..ae69326e 100644 --- a/02-chapter_of_course.Rmd +++ b/02-chapter_of_course.Rmd @@ -9,6 +9,8 @@ Every chapter needs to start out with this chunk of code: ottrpal::set_knitr_image_path() ``` +https://auighaiushguiahsufibhaisubfhv.com + ## Learning Objectives *Every chapter also needs Learning objectives that will look like this: From 4f28b63e43b58c0f54ea45ba1e02a240ff4c33e6 Mon Sep 17 00:00:00 2001 From: Candace Savonen Date: Thu, 31 Mar 2022 14:28:05 -0400 Subject: [PATCH 04/10] rearrange steps --- scripts/url-check.R | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/url-check.R b/scripts/url-check.R index 2a7f05c3..6de2247f 100644 --- a/scripts/url-check.R +++ b/scripts/url-check.R @@ -47,8 +47,7 @@ all_urls_df <- dplyr::bind_rows(all_urls) if (nrow(all_urls_df) > 0) { all_urls_df <- all_urls_df %>% - dplyr::filter(urls_status == "failed") %>% - readr::write_tsv(output_file) + dplyr::filter(urls_status == "failed") } else { all_urls_df <- data.frame(errors = NA) } @@ -57,6 +56,9 @@ if (nrow(all_urls_df) > 0) { write(nrow(all_urls_df), stdout()) # Save spell errors to file temporarily -readr::write_tsv(all_urls_df, output_file) +readr::write_tsv(data.frame(all_urls_df), output_file) message(paste0("Saved to: ", output_file)) + +# Print out how many spell check errors +write(nrow(all_urls_df), stdout()) From 25b84851c734ea295c63f2b849dae1a11f11499d Mon Sep 17 00:00:00 2001 From: Candace Savonen Date: Thu, 31 Mar 2022 14:30:14 -0400 Subject: [PATCH 05/10] get rid of -1 --- .github/workflows/pull_request.yml | 12 ++++++------ scripts/url-check.R | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d19bed02..3d4f4492 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -93,12 +93,12 @@ jobs: uses: actions/upload-artifact@v2 with: name: spell-check-results - path: resources/spell_check_results.tsv + path: report_checks/spell_check_results.tsv - name: Commit spell check errors run: | branch_name='preview-${{ github.event.pull_request.number }}' - git add --force resources/spell_check_results.tsv || echo "No changes to commit" + git add --force report_checks/spell_check_results.tsv || echo "No changes to commit" git commit -m 'Add spell check file' || echo "No changes to commit" git pull --set-upstream origin $branch_name --allow-unrelated-histories --strategy-option=ours git push --force origin $branch_name || echo "No changes to commit" @@ -109,7 +109,7 @@ jobs: GH_PAT: ${{ secrets.GH_PAT }} run: | branch_name='preview-${{ github.event.pull_request.number }}' - sp_error_url=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/resources/spell_check_results.tsv + sp_error_url=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/report_checks/spell_check_results.tsv echo ::set-output name=time::$(date +'%Y-%m-%d') echo ::set-output name=commit_id::$GITHUB_SHA echo ::set-output name=sp_error_url::$sp_error_url @@ -200,7 +200,7 @@ jobs: id: url_errors run: | chk_results=$(Rscript scripts/url-check.R) - chk_results="$((chk_results-1))" + chk_results="$((chk_results))" echo $chk_results echo "chk_results=$chk_results" >> $GITHUB_ENV @@ -209,7 +209,7 @@ jobs: - name: Commit URL check run: | branch_name='preview-${{ github.event.pull_request.number }}' - git add --force resources/url_checks.csv || echo "No changes to commit" + git add --force report_checks/url_checks.csv || echo "No changes to commit" git commit -m 'Add URL check file' || echo "No changes to commit" git fetch git merge -s recursive --strategy-option=theirs origin/${{ github.head_ref }} --allow-unrelated-histories @@ -221,7 +221,7 @@ jobs: GH_PAT: ${{ secrets.GH_PAT }} run: | branch_name='preview-${{ github.event.pull_request.number }}' - url_errors=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/resources/url_checks.csv + url_errors=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/report_checks/url_checks.csv echo ::set-output name=time::$(date +'%Y-%m-%d') echo ::set-output name=commit_id::$GITHUB_SHA echo ::set-output name=url_errors::$url_errors diff --git a/scripts/url-check.R b/scripts/url-check.R index 6de2247f..7fac989c 100644 --- a/scripts/url-check.R +++ b/scripts/url-check.R @@ -56,7 +56,7 @@ if (nrow(all_urls_df) > 0) { write(nrow(all_urls_df), stdout()) # Save spell errors to file temporarily -readr::write_tsv(data.frame(all_urls_df), output_file) +readr::write_tsv(all_urls_df, output_file) message(paste0("Saved to: ", output_file)) From 357f871464f627eddf1e66f4e4da5d73efdc0029 Mon Sep 17 00:00:00 2001 From: Candace Savonen Date: Thu, 31 Mar 2022 14:32:46 -0400 Subject: [PATCH 06/10] get rid of extra stuff --- .github/workflows/pull_request.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 3d4f4492..bff02bbd 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -200,7 +200,6 @@ jobs: id: url_errors run: | chk_results=$(Rscript scripts/url-check.R) - chk_results="$((chk_results))" echo $chk_results echo "chk_results=$chk_results" >> $GITHUB_ENV From 5e4ce40d62acada043cd05625bb97651e851bd74 Mon Sep 17 00:00:00 2001 From: Candace Savonen Date: Thu, 31 Mar 2022 14:42:26 -0400 Subject: [PATCH 07/10] Fix results Thing --- .github/workflows/pull_request.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index bff02bbd..3a4ea79f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -202,8 +202,7 @@ jobs: chk_results=$(Rscript scripts/url-check.R) echo $chk_results - echo "chk_results=$chk_results" >> $GITHUB_ENV - echo "::set-output name=url_results::$results" + echo "::set-output name=url_results::$chk_results" - name: Commit URL check run: | From 843ea693b1da75502a0ee9e6c13a59ce8fd93f98 Mon Sep 17 00:00:00 2001 From: Candace Savonen Date: Thu, 31 Mar 2022 14:47:35 -0400 Subject: [PATCH 08/10] Fix file path --- .github/workflows/pull_request.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 3a4ea79f..6212e933 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -93,12 +93,12 @@ jobs: uses: actions/upload-artifact@v2 with: name: spell-check-results - path: report_checks/spell_check_results.tsv + path: check_reports/spell_check_results.tsv - name: Commit spell check errors run: | branch_name='preview-${{ github.event.pull_request.number }}' - git add --force report_checks/spell_check_results.tsv || echo "No changes to commit" + git add --force check_reports/spell_check_results.tsv || echo "No changes to commit" git commit -m 'Add spell check file' || echo "No changes to commit" git pull --set-upstream origin $branch_name --allow-unrelated-histories --strategy-option=ours git push --force origin $branch_name || echo "No changes to commit" @@ -109,7 +109,7 @@ jobs: GH_PAT: ${{ secrets.GH_PAT }} run: | branch_name='preview-${{ github.event.pull_request.number }}' - sp_error_url=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/report_checks/spell_check_results.tsv + sp_error_url=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/check_reports/spell_check_results.tsv echo ::set-output name=time::$(date +'%Y-%m-%d') echo ::set-output name=commit_id::$GITHUB_SHA echo ::set-output name=sp_error_url::$sp_error_url @@ -207,7 +207,7 @@ jobs: - name: Commit URL check run: | branch_name='preview-${{ github.event.pull_request.number }}' - git add --force report_checks/url_checks.csv || echo "No changes to commit" + git add --force check_reports/url_checks.csv || echo "No changes to commit" git commit -m 'Add URL check file' || echo "No changes to commit" git fetch git merge -s recursive --strategy-option=theirs origin/${{ github.head_ref }} --allow-unrelated-histories @@ -219,7 +219,7 @@ jobs: GH_PAT: ${{ secrets.GH_PAT }} run: | branch_name='preview-${{ github.event.pull_request.number }}' - url_errors=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/report_checks/url_checks.csv + url_errors=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/check_reports/url_checks.csv echo ::set-output name=time::$(date +'%Y-%m-%d') echo ::set-output name=commit_id::$GITHUB_SHA echo ::set-output name=url_errors::$url_errors From 18a0cf877c515529f7cd3c218b86b14c77fe558f Mon Sep 17 00:00:00 2001 From: Candace Savonen Date: Thu, 31 Mar 2022 14:51:21 -0400 Subject: [PATCH 09/10] .csv -> .tsv --- .github/workflows/pull_request.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 6212e933..600025ae 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -207,7 +207,7 @@ jobs: - name: Commit URL check run: | branch_name='preview-${{ github.event.pull_request.number }}' - git add --force check_reports/url_checks.csv || echo "No changes to commit" + git add --force check_reports/url_checks.tsv || echo "No changes to commit" git commit -m 'Add URL check file' || echo "No changes to commit" git fetch git merge -s recursive --strategy-option=theirs origin/${{ github.head_ref }} --allow-unrelated-histories @@ -219,7 +219,7 @@ jobs: GH_PAT: ${{ secrets.GH_PAT }} run: | branch_name='preview-${{ github.event.pull_request.number }}' - url_errors=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/check_reports/url_checks.csv + url_errors=https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/$branch_name/check_reports/url_checks.tsv echo ::set-output name=time::$(date +'%Y-%m-%d') echo ::set-output name=commit_id::$GITHUB_SHA echo ::set-output name=url_errors::$url_errors From c42cdd7eb619d307a95301b524bc3703617919ae Mon Sep 17 00:00:00 2001 From: Candace Savonen Date: Thu, 31 Mar 2022 14:13:52 -0400 Subject: [PATCH 10/10] Revert "Add a broken url as a test" This reverts commit 973453fa82a121032a15ff78a7ef9729829225d5. --- 02-chapter_of_course.Rmd | 2 -- 1 file changed, 2 deletions(-) diff --git a/02-chapter_of_course.Rmd b/02-chapter_of_course.Rmd index ae69326e..dc240d2f 100644 --- a/02-chapter_of_course.Rmd +++ b/02-chapter_of_course.Rmd @@ -9,8 +9,6 @@ Every chapter needs to start out with this chunk of code: ottrpal::set_knitr_image_path() ``` -https://auighaiushguiahsufibhaisubfhv.com - ## Learning Objectives *Every chapter also needs Learning objectives that will look like this: