CRAN submission soon (#316)

* Removed duplicated output element (#291) * Removed duplicated output element * Updated package version and NEWS * Should fix #292 (#293) * Updated NEWS and increment version * Updated citation info in README * Should fix #299 (#300) * Should fix #299 * Added missing documentation objects and changed the name of one argument * Fixed another documentation issue * Removed deprecated SMC functions (#303) * Updated build version number * Increment version number to 1.3.2.9002 * Removed deprecated SMC functions (#301) * Updated NEWS (#301) * Logz issue 158 (#304) * Added exact partition function option * Updated tests * Updated SMC Mallows examples * Styling * More linting * Updated linter workflow Some linter were deprecated, ignored line on vignette needed update * Trying to please CodeFactor * Incremented version number and added entry to NEWS.md --------- Co-authored-by: Waldir Leoncio <w.l.netto@medisin.uio.no> * Added BugReport URL (#308) * Added URL for bug reports * Increment version number to 1.3.2.9004 * Small typo correction and documentation update * Website (#310) * Website * De-duplicating vignettes * Adding GH Action for pkgdown (#290) Literally the output of `usethis::use_github_action("pkgdown")` * Triggering `gh-pages` when pushing to PR #310 (tests) This is just for testing the online deployment and should be removed after merge (or just before). --------- Co-authored-by: Waldir Leoncio <w.l.netto@medisin.uio.no> * Small changes (#311) * Small changes * Fixed gh-pages bases for deployment (#290) The PR branch was left over. More importantly, the file listed "main" instead of "master". --------- Co-authored-by: Waldir Leoncio <w.l.netto@medisin.uio.no> * Corrected small typo in documentation * Cleaner unit test output (#314) * Increment version number to 1.3.2.9006 * Capturing output of test unit * Dirichlet concentration parameter is now actually being used (#315) * forwarded psi argument to run_mcmc * Incremented version and updated news.md * ready for cran submission * CRAN wanted a trailing slash in the URL to the website, since it becomes added automatically if one follows the URL --------- Co-authored-by: Waldir Leoncio <w.l.netto@medisin.uio.no>
ocbe-uio · Oct 4, 2023 · aea9a97 · aea9a97
1 parent 7ec6036
commit aea9a97
Show file tree

Hide file tree

Showing 84 changed files with 803 additions and 840 deletions.
diff --git a/.Rbuildignore b/.Rbuildignore
@@ -13,3 +13,6 @@ LICENSE
 ^Meta$
 ^\.github$
 ^CRAN-SUBMISSION$
+^_pkgdown\.yml$
+^docs$
+^pkgdown$
diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
@@ -57,20 +57,20 @@ jobs:
               "data-raw/",
               "tests/testthat.R",
               "R/RcppExports.R",
-              "vignettes/SMC-Mallows.Rmd" = 16
+              "vignettes/SMC-Mallows.Rmd" = 19
             )
             style_rules <- list(
               absolute_path_linter(), assignment_linter(), brace_linter(),
               commas_linter(), commented_code_linter(),
               equals_na_linter(), function_left_parentheses_linter(),
-              infix_spaces_linter(), no_tab_linter(),
+              infix_spaces_linter(), whitespace_linter(),
               nonportable_path_linter(),
-              pipe_continuation_linter(), seq_linter(), single_quotes_linter(),
+              pipe_continuation_linter(), seq_linter(), quotes_linter(),
               spaces_inside_linter(), spaces_left_parentheses_linter(),
               T_and_F_symbol_linter(), todo_comment_linter(),
               trailing_blank_lines_linter(), trailing_whitespace_linter(),
               undesirable_function_linter(),
-              unneeded_concatenation_linter()
+              unnecessary_concatenation_linter()
             )
             lint_package(linters = style_rules, exclusions = excluded_files)
           shell: Rscript {0}

diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml
@@ -0,0 +1,48 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+  release:
+    types: [published]
+  workflow_dispatch:
+
+name: pkgdown
+
+jobs:
+  pkgdown:
+    runs-on: ubuntu-latest
+    # Only restrict concurrency for non-PR jobs
+    concurrency:
+      group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }}
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: r-lib/actions/setup-pandoc@v2
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: any::pkgdown, local::.
+          needs: website
+
+      - name: Build site
+        run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE)
+        shell: Rscript {0}
+
+      - name: Deploy to GitHub pages 🚀
+        if: github.event_name != 'pull_request'
+        uses: JamesIves/github-pages-deploy-action@v4.4.1
+        with:
+          clean: false
+          branch: gh-pages
+          folder: docs
diff --git a/.gitignore b/.gitignore
@@ -9,3 +9,4 @@ doc
 Meta
 /doc/
 /Meta/
+docs
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,11 +1,14 @@
 Package: BayesMallows
 Type: Package
 Title: Bayesian Preference Learning with the Mallows Rank Model
-Version: 1.3.2
+Version: 1.4.0
 Authors@R: c(person("Oystein", "Sorensen",
                     email = "oystein.sorensen.1985@gmail.com",
                     role = c("aut", "cre"),
                     comment = c(ORCID = "0000-0003-0724-3542")),
+            person("Waldir", "Leoncio",
+                    email = "w.l.netto@medisin.uio.no",
+                    role = c("ctr")),
             person("Valeria", "Vitelli",
                     role = c("aut"),
                     email = "valeria.vitelli@medisin.uio.no",
@@ -22,14 +25,14 @@ Authors@R: c(person("Oystein", "Sorensen",
             person("Luca", "Tardella",
                     role = c("aut")),
             person("Anja", "Stein",
-                    role = c("aut")),
-            person("Waldir", "Leoncio",
-                    email = "w.l.netto@medisin.uio.no",
-                    role = c("ctr")))
+                    role = c("aut"))            
+                    )
 Maintainer: Oystein Sorensen <oystein.sorensen.1985@gmail.com>
 Description: An implementation of the Bayesian version of the Mallows rank model
     (Vitelli et al., Journal of Machine Learning Research, 2018 <https://jmlr.org/papers/v18/15-481.html>;
-    Crispino et al., Annals of Applied Statistics, 2019 <doi:10.1214/18-AOAS1203>). Both Metropolis-Hastings
+    Crispino et al., Annals of Applied Statistics, 2019 <doi:10.1214/18-AOAS1203>;
+    Sorensen et al., R Journal, 2020 <doi:10.32614/RJ-2020-026>;
+    Stein, PhD Thesis, 2023 <https://eprints.lancs.ac.uk/id/eprint/195759>). Both Metropolis-Hastings
     and sequential Monte Carlo algorithms for estimating the models are available. Cayley, footrule,
     Hamming, Kendall, Spearman, and Ulam distances are supported in the models. The rank data to be
     analyzed can be in the form of complete rankings, top-k rankings, partially missing rankings, as well
@@ -38,7 +41,8 @@ Description: An implementation of the Bayesian version of the Mallows rank model
     the partition function (normalizing constant) of the Mallows rank model, both with the importance
     sampling algorithm of Vitelli et al. and asymptotic approximation with the IPFP algorithm
     (Mukherjee, Annals of Statistics, 2016 <doi:10.1214/15-AOS1389>).
-URL: https://github.com/ocbe-uio/BayesMallows
+URL: https://github.com/ocbe-uio/BayesMallows, https://ocbe-uio.github.io/BayesMallows/
+BugReports: https://github.com/ocbe-uio/BayesMallows/issues
 License: GPL-3
 Encoding: UTF-8
 LazyData: true

diff --git a/NAMESPACE b/NAMESPACE
@@ -34,22 +34,16 @@ export(leap_and_shift_probs)
 export(lik_db_mix)
 export(metropolis_hastings_alpha)
 export(metropolis_hastings_aug_ranking)
-export(metropolis_hastings_aug_ranking_pseudo)
 export(metropolis_hastings_rho)
-export(plot_alpha_posterior)
 export(plot_elbow)
-export(plot_rho_posterior)
 export(plot_top_k)
 export(predict_top_k)
+export(prepare_partition_function)
 export(rank_distance)
 export(rank_freq_distr)
 export(sample_mallows)
 export(smc_mallows_new_item_rank)
-export(smc_mallows_new_item_rank_alpha_fixed)
 export(smc_mallows_new_users)
-export(smc_mallows_new_users_complete)
-export(smc_mallows_new_users_partial)
-export(smc_mallows_new_users_partial_alpha_fixed)
 importFrom(Rcpp,sourceCpp)
 importFrom(Rdpack,reprompt)
 importFrom(methods,is)

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,15 @@
+# BayesMallows 1.4.0
+
+* Bug fix: psi argument to compute_mallows() and compute_mallows_mixtures(), 
+  specifying the concentration parameter of the Dirichlet prior, is now 
+  forwarded to the underlying run_mcmc() function. Previously, this argument
+  has had no effect, and the default psi=10 has been used regardless of the 
+  input. Thanks to Lorenzo Zuccato for discovering this bug.
+* SMC functions now accept exact partition functions where these are available.
+* Removed SMC functions deprecated on version 1.2.0 (#301)
+* Website deployed at https://ocbe-uio.github.io/BayesMallows.
+* Reordering of authors, so Waldir Leoncio appears second in the list. 
+
 # BayesMallows 1.3.2
 
 * Fixed LTO compilation notes on CRAN.

diff --git a/R/BayesMallows.R b/R/BayesMallows.R
@@ -1,18 +1,18 @@
 #' BayesMallows: Bayesian Preference Learning with the Mallows Rank Model.
 #'
 #' @description The BayesMallows package provides functionality for fully
-#'   Bayesian analysis of preference or rank data. The package implements
-#'   the Bayesian Mallows model described in \insertCite{vitelli2018;textual}{BayesMallows},
-#'   which handles complete rankings, top-k rankings, ranks missing at random,
-#'   and consistent pairwise preference data, as well as mixtures of rank models.
-#'   Modeling of
-#'   pairwise preferences containing inconsistencies, as described in
-#'   \insertCite{crispino2019;textual}{BayesMallows}, is also supported. See also
-#'   \insertCite{sorensen2020;textual}{BayesMallows} for an overview of the methods
-#'   and a tutorial.
+#'   Bayesian analysis of preference or rank data. The package implements the
+#'   Bayesian Mallows model described in
+#'   \insertCite{vitelli2018;textual}{BayesMallows}, which handles complete
+#'   rankings, top-k rankings, ranks missing at random, and consistent pairwise
+#'   preference data, as well as mixtures of rank models. Modeling of pairwise
+#'   preferences containing inconsistencies, as described in
+#'   \insertCite{crispino2019;textual}{BayesMallows}, is also supported. See
+#'   also \insertCite{sorensen2020;textual}{BayesMallows} for an overview of the
+#'   methods and a tutorial.
 #'
-#' The documentation and examples for the following functions are likely most useful
-#' to get you started:
+#'   The documentation and examples for the following functions are likely most
+#'   useful to get you started:
 #' \itemize{
 #'  \item For analysis of rank or preference data, see \code{\link{compute_mallows}}.
 #'  \item For computation of multiple models with varying numbers of mixture components,
@@ -21,6 +21,9 @@
 #'  the importance sampling algorithm of \insertCite{vitelli2018;textual}{BayesMallows} or
 #'  the asymptotic algorithm of \insertCite{mukherjee2016;textual}{BayesMallows}, see
 #'  \code{\link{estimate_partition_function}}.
+#'  \item For sequential Monte Carlo algorithms developed in
+#'  \insertCite{steinSequentialInferenceMallows2023;textual}{BayesMallows}, see
+#'  \code{\link{smc_mallows_new_users}} and \code{\link{smc_mallows_new_item_rank}}.
 #' }
 #'
 #'

diff --git a/R/RcppExports.R b/R/RcppExports.R
@@ -361,7 +361,7 @@ leap_and_shift_probs <- function(rho, n_items, leap_size = 1L) {
     .Call(`_BayesMallows_leap_and_shift_probs`, rho, n_items, leap_size)
 }
 
-#' @title SMC-Mallows new users rank
+#' @title SMC-Mallows new item rank
 #' @description Function to perform resample-move SMC algorithm where we receive a new item ranks from an existing user
 #' at each time step. Each correction and augmentation is done by filling in the missing item ranks using pseudolikelihood augmentation.
 #' @param n_items Integer is the number of items in a ranking
@@ -375,7 +375,9 @@ leap_and_shift_probs <- function(rho, n_items, leap_size = 1L) {
 #' @param N Integer specifying the number of particles
 #' @param Time Integer specifying the number of time steps in the SMC algorithm
 #' @param logz_estimate Estimate of the partition function, computed with
-#' \code{\link{estimate_partition_function}} in the BayesMallow R package {estimate_partition_function}.
+#' \code{\link{estimate_partition_function}}.
+#' @param cardinalities Cardinalities for exact computation of partition function,
+#' returned from \code{\link{prepare_partition_function}}.
 #' @param mcmc_kernel_app Integer value for the number of applications we apply the MCMC move kernel
 #' @param alpha_prop_sd Numeric value of the standard deviation of the prior distribution for alpha
 #' @param lambda Strictly positive numeric value specifying the rate parameter
@@ -387,15 +389,22 @@ leap_and_shift_probs <- function(rho, n_items, leap_size = 1L) {
 #' SMC-Mallows algorithm. Defaults to \code{FALSE}.
 #' @param alpha_fixed Logical indicating whether to sample \code{alpha} or not.
 #' @param alpha numeric value of the scale parameter.
+#' @param aug_rankings_init Initial values for augmented rankings.
+#' @param rho_samples_init Initial values for rho samples.
+#' @param alpha_samples_init Initial values for alpha samples.
+#'
 #' @return a 3d matrix containing: the samples of: rho, alpha and the augmented rankings, and the effective sample size at each iteration of the SMC algorithm.
+#'
 #' @export
-smc_mallows_new_item_rank <- function(n_items, R_obs, N, Time, logz_estimate, mcmc_kernel_app, aug_rankings_init = NULL, rho_samples_init = NULL, alpha_samples_init = 0L, alpha = 0, alpha_prop_sd = 0.5, lambda = 0.1, alpha_max = 1e6, aug_method = "random", verbose = FALSE, alpha_fixed = FALSE, metric = "footrule", leap_size = 1L) {
-    .Call(`_BayesMallows_smc_mallows_new_item_rank`, n_items, R_obs, N, Time, logz_estimate, mcmc_kernel_app, aug_rankings_init, rho_samples_init, alpha_samples_init, alpha, alpha_prop_sd, lambda, alpha_max, aug_method, verbose, alpha_fixed, metric, leap_size)
+#'
+smc_mallows_new_item_rank <- function(n_items, R_obs, N, Time, logz_estimate, cardinalities, mcmc_kernel_app, aug_rankings_init = NULL, rho_samples_init = NULL, alpha_samples_init = 0L, alpha = 0, alpha_prop_sd = 0.5, lambda = 0.1, alpha_max = 1e6, aug_method = "random", verbose = FALSE, alpha_fixed = FALSE, metric = "footrule", leap_size = 1L) {
+    .Call(`_BayesMallows_smc_mallows_new_item_rank`, n_items, R_obs, N, Time, logz_estimate, cardinalities, mcmc_kernel_app, aug_rankings_init, rho_samples_init, alpha_samples_init, alpha, alpha_prop_sd, lambda, alpha_max, aug_method, verbose, alpha_fixed, metric, leap_size)
 }
 
 #' @title SMC-Mallows New Users
 #' @description Function to perform resample-move SMC algorithm where we
-#' receive new users with complete rankings at each time step
+#' receive new users with complete rankings at each time step. See Chapter 4
+#' of \insertCite{steinSequentialInferenceMallows2023}{BayesMallows}
 #'
 #' @param R_obs Matrix containing the full set of observed rankings of size
 #' n_assessors by n_items
@@ -411,8 +420,9 @@ smc_mallows_new_item_rank <- function(n_items, R_obs, N, Time, logz_estimate, mc
 #' @param N Integer specifying the number of particles
 #' @param Time Integer specifying the number of time steps in the SMC algorithm
 #' @param logz_estimate Estimate of the partition function, computed with
-#' \code{\link{estimate_partition_function}} in the BayesMallow R package
-#' {estimate_partition_function}.
+#' \code{\link{estimate_partition_function}}.
+#' @param cardinalities Cardinalities for exact evaluation of partition function,
+#' returned from \code{\link{prepare_partition_function}}.
 #' @param mcmc_kernel_app Integer value for the number of applications we
 #' apply the MCMC move kernel
 #' @param num_new_obs Integer value for the number of new observations
@@ -438,8 +448,8 @@ smc_mallows_new_item_rank <- function(n_items, R_obs, N, Time, logz_estimate, mc
 #'
 #' @example inst/examples/smc_mallows_new_users_complete_example.R
 #'
-smc_mallows_new_users <- function(R_obs, type, n_items, N, Time, mcmc_kernel_app, num_new_obs, alpha_prop_sd = 0.5, lambda = 0.1, alpha_max = 1e6, alpha = 0, aug_method = "random", logz_estimate = NULL, verbose = FALSE, metric = "footnote", leap_size = 1L) {
-    .Call(`_BayesMallows_smc_mallows_new_users`, R_obs, type, n_items, N, Time, mcmc_kernel_app, num_new_obs, alpha_prop_sd, lambda, alpha_max, alpha, aug_method, logz_estimate, verbose, metric, leap_size)
+smc_mallows_new_users <- function(R_obs, type, n_items, N, Time, mcmc_kernel_app, num_new_obs, alpha_prop_sd = 0.5, lambda = 0.1, alpha_max = 1e6, alpha = 0, aug_method = "random", logz_estimate = NULL, cardinalities = NULL, verbose = FALSE, metric = "footnote", leap_size = 1L) {
+    .Call(`_BayesMallows_smc_mallows_new_users`, R_obs, type, n_items, N, Time, mcmc_kernel_app, num_new_obs, alpha_prop_sd, lambda, alpha_max, alpha, aug_method, logz_estimate, cardinalities, verbose, metric, leap_size)
 }
 
 #' @title Metropolis-Hastings Alpha
@@ -457,8 +467,9 @@ smc_mallows_new_users <- function(R_obs, type, n_items, N, Time, mcmc_kernel_app
 #'   and \code{"ulam"}.
 #' @param rho Numeric vector specifying the current consensus ranking
 #' @param logz_estimate Estimate  grid of log of partition function,
-#'   computed with \code{\link{estimate_partition_function}} in
-#'   the BayesMallow R package {estimate_partition_function}.
+#'   computed with \code{\link{estimate_partition_function}}.
+#' @param cardinalities Cardinalities for exact computation of partition function,
+#' returned from \code{\link{prepare_partition_function}}.
 #' @param alpha_prop_sd Numeric value specifying the standard deviation of the
 #'   lognormal proposal distribution used for \eqn{\alpha} in the
 #'   Metropolis-Hastings algorithm. Defaults to \code{0.1}.
@@ -473,10 +484,9 @@ smc_mallows_new_users <- function(R_obs, type, n_items, N, Time, mcmc_kernel_app
 #' @importFrom stats dexp rlnorm runif
 #' @author Anja Stein
 #' @example /inst/examples/metropolis_hastings_alpha_example.R
-#'
 #' @export
-metropolis_hastings_alpha <- function(alpha, n_items, rankings, rho, logz_estimate, metric = "footrule", alpha_prop_sd = 0.5, alpha_max = 1e6, lambda = 0.1) {
-    .Call(`_BayesMallows_metropolis_hastings_alpha`, alpha, n_items, rankings, rho, logz_estimate, metric, alpha_prop_sd, alpha_max, lambda)
+metropolis_hastings_alpha <- function(alpha, n_items, rankings, rho, logz_estimate, cardinalities, metric = "footrule", alpha_prop_sd = 0.5, alpha_max = 1e6, lambda = 0.1) {
+    .Call(`_BayesMallows_metropolis_hastings_alpha`, alpha, n_items, rankings, rho, logz_estimate, cardinalities, metric, alpha_prop_sd, alpha_max, lambda)
 }
 
 #' @title Metropolis-Hastings Augmented Ranking
@@ -494,7 +504,6 @@ metropolis_hastings_alpha <- function(alpha, n_items, rankings, rho, logz_estima
 #' @param pseudo Boolean specifying whether to use pseudo proposal or not.s
 #' @return R_curr or R_obs A ranking sequence vector representing proposed augmented ranking for next iteration of MCMC chain
 #' @export
-#' @keywords internal
 metropolis_hastings_aug_ranking <- function(alpha, rho, n_items, partial_ranking, current_ranking, pseudo, metric = "footnote") {
     .Call(`_BayesMallows_metropolis_hastings_aug_ranking`, alpha, rho, n_items, partial_ranking, current_ranking, pseudo, metric)
 }

diff --git a/R/assess_convergence.R b/R/assess_convergence.R
@@ -147,9 +147,10 @@ trace_rtilde <- function(model_fit, items, assessors, ...) {
     items <- levels(model_fit$augmented_data$item)[items]
   }
 
-  df <- model_fit$augmented_data[model_fit$augmented_data$assessor %in% assessors &
-    model_fit$augmented_data$item %in% items, ,
-  drop = FALSE
+  df <- model_fit$augmented_data[
+    model_fit$augmented_data$assessor %in% assessors &
+      model_fit$augmented_data$item %in% items, ,
+    drop = FALSE
   ]
 
   df$assessor <- as.factor(df$assessor)

diff --git a/R/compute_consensus.R b/R/compute_consensus.R
@@ -227,7 +227,6 @@ find_cpc <- function(group_df, group_var = "cluster") {
 }
 
 .compute_map_consensus.consensus_BayesMallows <- function(model_fit, ...) {
-
   # Store the total number of iterations after burnin
   n_samples <- length(unique(model_fit$iteration))
 

diff --git a/R/compute_mallows.R b/R/compute_mallows.R
@@ -374,6 +374,7 @@ compute_mallows <- function(rankings = NULL,
     include_wcd = include_wcd,
     lambda = lambda,
     alpha_max = alpha_max,
+    psi = psi,
     leap_size = leap_size,
     alpha_prop_sd = alpha_prop_sd,
     alpha_init = alpha_init,

diff --git a/R/generate_initial_ranking.R b/R/generate_initial_ranking.R
@@ -90,12 +90,14 @@ generate_initial_ranking <- function(tc,
   if (is.null(cl)) {
     do.call(rbind, lapply(
       prefs, function(x, y, sr, r) create_ranks(as.matrix(x), y, sr, r),
-      n_items, shuffle_unranked, random))
+      n_items, shuffle_unranked, random
+    ))
   } else {
     do.call(rbind, parallel::parLapply(
       cl = cl, X = prefs,
       fun = function(x, y, sr, r) create_ranks(as.matrix(x), y, sr, r),
-      n_items, shuffle_unranked, random))
+      n_items, shuffle_unranked, random
+    ))
   }
 }
 

diff --git a/R/generate_transitive_closure.R b/R/generate_transitive_closure.R
@@ -81,7 +81,6 @@ generate_transitive_closure <- function(df, cl = NULL) {
 #'   upper ranked item.
 #' @keywords internal
 .generate_transitive_closure <- function(mat) {
-
   # This line was an answer to StackOverflow question 51794127
   my_set <- do.call(sets::set, apply(mat, 1, sets::as.tuple))
 

diff --git a/R/get_mallows_loglik.R b/R/get_mallows_loglik.R
@@ -31,7 +31,7 @@
 #' @example inst/examples/get_mallows_loglik_example.R
 #'
 get_mallows_loglik <- function(rho, alpha, weights, metric,
-                       rankings, obs_freq = NULL, log = TRUE) {
+                               rankings, obs_freq = NULL, log = TRUE) {
   if (!is.matrix(rankings)) {
     rankings <- matrix(rankings, nrow = 1)
   }
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,3 +9,4 @@ doc @@
     Meta
     /doc/
     /Meta/
+    docs