10 Defining some tests

When using litr to create packages that are not litr, one should be able to run tests along the way as we did above in testing the function add_text_to_file(). However, creating litr is a special case so we need to do something different for the tests that involve creating a .Rmd from template and then calling litr::render() on them (such as the tests in this section). In particular, we use eval=FALSE for these code blocks and then at the end of this document we will install the newly created version of litr and then call devtools::test(). Doing it this way is important for ensuring that the version of litr we are testing is the newest version, i.e. the version defined in this document.

To understand the reason we are doing it this way, imagine what would happen if instead we left eval=TRUE in the test in the next section. When we use rmarkdown::draft() to create a .Rmd file from template, the file it will give us will be an old version (namely the installed version of litr’s template) rather than the latest version.2 Furthermore, consider what happens when we call render() in the test below. This will start the knitting process on my-package.Rmd. However, inside my-package.Rmd, we have litr::setup() and litr::document(). When these are called in the knitting process, it will be the versions of the functions from the currently installed litr rather than the versions defined in this document.

Once we are done testing the new version of the package, we’d like to restore the state of litr to what it was previously. If we don’t do this, then this can lead to inadvertent circularity in which the next time we call litr::render("create-litr.Rmd"), we are using the version currently under development, which is bad because ultimately we need this version to be rendered by the previous version of litr. The following function implements this approach to testing litr:

#' Run tests for `litr` itself
#' 
#' Special function for testing `litr`.  The trick is to temporarily install
#' the new version of `litr`, run the test, and then put things back how it was
#' before.
#' 
#' Typical values for `install_old` could be
#' - `function() devtools::install("[location of old version]")`
#' - `function() remotes::install_github("jacobbien/litr-project@*release", subdir = "litr")`.
#' 
#' @param install_old A function that when run will install the old version
#' @param location_of_new Path to the new package directory
#' @keywords internal
test_litr <- function(install_old, location_of_new) {
  devtools::unload(params$package_name)
  devtools::install(location_of_new)
  out <- devtools::test(location_of_new)
  install_old()
  return(out)
}

Note: The call to devtools::unload() is to address an issue discussed here.

10.1 Testing check_unedited()

For our tests, we create a temporary directory (which we delete at the end). In this directory, we create a generating .Rmd file from one of the templates. We make repeated modifications to the package and each time verify that check_unedited() is FALSE with the modification and returns to TRUE when we put things back how they were. The modifications we try are the following:

  • Adding a file

  • Removing a file

  • Making a change to a file (in particular, adding a comment to an R file)

  • Changing something in the DESCRIPTION file (but not on the special litr line)

  • Changing the litr hash line itself

testthat::test_that("check_unedited works", {
  # Including this next line seems to be necessary for R CMD check on the cmd line:
  #Sys.setenv(RSTUDIO_PANDOC = "/Applications/RStudio.app/Contents/MacOS/pandoc")
  dir <- tempfile()
  fs::dir_create(dir)
  rmd_file <- file.path(dir, "my-package.Rmd")
  rmarkdown::draft(rmd_file,
                   template = "make-an-r-package",
                   package = "litr",
                   edit = FALSE)
  # create R package (named "rhello") from the Rmd template:
  render(rmd_file)
  package_path <- file.path(dir, "rhello")
  testthat::expect_true(check_unedited(package_path))

  # what if a file has been added?
  added_file <- file.path(package_path, "R", "say_hello2.R")
  writeLines("# Added something here.", added_file)
  testthat::expect_false(check_unedited(package_path))

  # what if we now remove it?
  fs::file_delete(added_file)
  testthat::expect_true(check_unedited(package_path))

  # what if a file is removed from package?
  rfile <- file.path(package_path, "R", "say_hello.R")
  fs::file_move(rfile, dir)
  testthat::expect_false(check_unedited(package_path))
  # now put it back
  fs::file_move(file.path(dir, "say_hello.R"), file.path(package_path, "R"))
  testthat::expect_true(check_unedited(package_path))

  # what if something is changed in a file?
  txt <- readLines(rfile)
  txt_mod <- txt
  txt_mod[3] <- paste0(txt[3], " # added a comment!!")
  writeLines(txt_mod, rfile)
  testthat::expect_false(check_unedited(package_path))
  # now put it back
  writeLines(txt, rfile)
  testthat::expect_true(check_unedited(package_path))

  # what if something is changed in the DESCRIPTION file?
  descfile <- file.path(package_path, "DESCRIPTION")
  txt <- readLines(descfile)
  txt_mod <- txt
  txt_mod[1] <- "Package: newname"
  writeLines(txt_mod, descfile)
  testthat::expect_false(check_unedited(package_path))
  # now put it back
  writeLines(txt, descfile)
  testthat::expect_true(check_unedited(package_path))

  # what if the special litr hash field is changed in the DESCRIPTION file?
  txt <- readLines(descfile)
  i_litr <- stringr::str_which(txt, description_litr_hash_field_name())
  txt_mod <- txt
  txt_mod[i_litr] <- paste0(txt_mod[i_litr], "a")
  writeLines(txt_mod, descfile)
  testthat::expect_false(check_unedited(package_path))
  # now put it back
  writeLines(txt, descfile)
  testthat::expect_true(check_unedited(package_path))

  fs::dir_delete(dir)
})

10.2 Testing get_params_used()

Let’s now test the get_params_used() function, making sure it behaves how we expect it to:

testthat::test_that("get_params_used works", {
  dir <- tempfile()
  if (fs::file_exists(dir)) fs::file_delete(dir)
  fs::dir_create(dir)
  rmd_file <- file.path(dir, "my-package.Rmd")
  rmarkdown::draft(rmd_file, template = "make-an-r-package", package = "litr",
                   edit = FALSE)
  default_params <- get_params_used(rmd_file, passed_params = list())
  testthat::expect_equal(
    default_params,
    rmarkdown::yaml_front_matter(rmd_file)$params
  )
  params1 <- default_params
  params1$package_parent_dir <- "dir"
  testthat::expect_equal(
    get_params_used(rmd_file, passed_params = list(package_parent_dir = "dir")),
    params1
  )
  params2 <- default_params
  params2$package_name <- "pkg"
  params2$package_parent_dir <- "dir"
  testthat::expect_equal(
    get_params_used(rmd_file,
                    passed_params = list(package_parent_dir = "dir",
                                         package_name = "pkg")),
    params2
  )
  fs::dir_delete(dir)
})

10.3 Testing chunk referencing

Here we test the handling of chunk references (as implemented in the document output hook set within setup()). In particular, we use a .Rmd that uses chunk references in several different ways. Within the .Rmd itself, we have tests that ensure that the code can still be run as expected.

fs::file_copy(
  path = file.path(
    "..", "source-files", "test-example-files", "create-rknuth.Rmd"
    ), 
  new_path = file.path("tests", "testthat"), 
  overwrite = TRUE
)
testthat::test_that('Knuth-style references work', {
  dir <- tempfile()
  if (fs::file_exists(dir)) fs::file_delete(dir)
  fs::dir_create(dir)
  rmd_file <- file.path(dir, 'create-rknuth.Rmd')
  fs::file_copy(path = testthat::test_path("create-rknuth.Rmd"), new_path = rmd_file)
  render(rmd_file)
  testthat::expect_true(fs::file_exists(file.path(dir, 'create-rknuth.html')))
  fs::dir_delete(dir)
})

10.4 Testing different ways of rendering

The mechanism by which rendering occurs depends on several factors:

  1. Whether litr::render() or rmarkdown::render() is being called.

  2. Whether there is a litr output format specified in the preamble of the .Rmd.

  3. Whether there is a litr output format being passed an argument to the render function.

In this section, we will test that one gets the same result regardless of how rendering was invoked.3 We will use variations on a base .Rmd file whose preamble is simply the following:

---
title: 'A Test'
params:
  package_name: 'pkg' # <-- change this to your package name
  package_parent_dir: '.' # <-- relative to this file location
---
fs::file_copy(
  path = file.path(
    "..", "source-files", "test-example-files", "create-pkg.Rmd"
    ), 
  new_path = file.path("tests", "testthat"), 
  overwrite = TRUE
)

There are 7 cases to consider (\(2^3-1\), since we exclude the case where rmarkdown::render() is called and no argument or preamble would indicate that this should be a litr-knit).

testthat::test_that('Rendering in all possible ways works', {
  
  # setup files for tests:
  dir <- tempfile()
  if (fs::file_exists(dir)) fs::file_delete(dir)
  fs::dir_create(dir)
  # .Rmd without output format in preamble
  rmd_file1 <- file.path(dir, 'create-pkg1.Rmd')
  fs::file_copy(testthat::test_path("create-pkg.Rmd"), rmd_file1)
  # .Rmd without output format in preamble
  rmd_file2 <- file.path(dir, 'create-pkg2.Rmd')
  fs::file_copy(rmd_file1, rmd_file2)
  litr:::add_text_to_file("output: litr::litr_html_document", rmd_file2, 3)
  # files names
  rmd_file <- file.path(dir, "create-pkg.Rmd")  
  html_file <- file.path(dir, "create-pkg.html")
  html_file_a <- file.path(dir, "a","create-pkg.html")
  pkg <- file.path(dir, "pkg")
  pkg_a <- file.path(dir, "a", "pkg")
  check_outputs_are_same <- function() {
    # html files should be the same:
    testthat::expect_equal(readLines(html_file_a), readLines(html_file))
    # packages should be the same (relying here on litr-hash in DESCRIPTION):
    testthat::expect_equal(readLines(file.path(pkg, "DESCRIPTION")),
                           readLines(file.path(pkg_a, "DESCRIPTION")))
  }

  ## Now test that all the cases give the same outputs:
  
  # Case 1: no preamble + litr::render()
  fs::file_copy(rmd_file1, rmd_file, overwrite = TRUE)
  render(rmd_file, output_file = html_file)
  if (fs::file_exists(file.path(dir, "a"))) fs::file_delete(file.path(dir, "a"))
  fs::dir_create(file.path(dir, "a"))
  fs::dir_copy(pkg, pkg_a)
  fs::dir_delete(pkg)
  fs::file_move(html_file, html_file_a)

  # Case 2: with preamble + litr::render()
  fs::file_copy(rmd_file2, rmd_file, overwrite = TRUE)
  render(rmd_file, output_file = html_file)
  check_outputs_are_same()
  
  # Case 3: no preamble + litr::render() with output format argument
  fs::file_copy(rmd_file1, rmd_file, overwrite = TRUE)
  render(rmd_file, output_format = litr::litr_html_document(),
         output_file = html_file)
  check_outputs_are_same()
  
  # Case 4: with preamble + litr::render() with output format argument
  fs::file_copy(rmd_file2, rmd_file, overwrite = TRUE)
  render(rmd_file, output_format = litr::litr_html_document(),
         output_file = html_file)
  check_outputs_are_same()

  # Case 5: with preamble + rmarkdown::render()
  fs::file_copy(rmd_file2, rmd_file, overwrite = TRUE)
  xfun::Rscript_call(rmarkdown::render,
                     list(input = rmd_file, output_file = html_file)
                     )
  check_outputs_are_same()

  # Case 6: no preamble + rmarkdown::render() with output format argument
  fs::file_copy(rmd_file1, rmd_file, overwrite = TRUE)
  xfun::Rscript_call(rmarkdown::render,
                     list(input = rmd_file,
                          output_format = litr::litr_html_document(),
                          output_file = html_file)
                     )
  check_outputs_are_same()

  # Case 7: with preamble + rmarkdown::render() with output format argument
  fs::file_copy(rmd_file2, rmd_file, overwrite = TRUE)
  xfun::Rscript_call(rmarkdown::render,
                     list(input = rmd_file,
                          output_format = litr::litr_html_document(),
                          output_file = html_file)
                     )
  check_outputs_are_same()
  
  fs::dir_delete(dir)
})

Let’s also make sure that we get the same R package output when using minimal_eval=TRUE as minimal_eval=TRUE.

testthat::test_that('Rendering with minimal_eval=TRUE works', {
  
  # setup files for tests:
  dir <- tempfile()
  if (fs::file_exists(dir)) fs::file_delete(dir)
  fs::dir_create(dir)
  rmd_file <- file.path(dir, 'create-pkg.Rmd')
  fs::file_copy(testthat::test_path("create-pkg.Rmd"), rmd_file)
  # .Rmd without output format in preamble
  html_file <- file.path(dir, "create-pkg.html")
  html_file_a <- file.path(dir, "a","create-pkg.html")
  pkg <- file.path(dir, "pkg")
  pkg_a <- file.path(dir, "a", "pkg")

  ## Now test that all the cases give the same outputs:
  
  # Case 1: minimal_eval = FALSE
  render(rmd_file, output_file = html_file, minimal_eval = FALSE)
  if (fs::file_exists(file.path(dir, "a"))) fs::file_delete(file.path(dir, "a"))
  fs::dir_create(file.path(dir, "a"))
  fs::dir_copy(pkg, pkg_a)
  fs::dir_delete(pkg)

  # Case 2: minimal_eval = TRUE passed to render
  render(rmd_file, output_file = html_file, minimal_eval = TRUE)
  testthat::expect_equal(readLines(file.path(pkg, "DESCRIPTION")),
                         readLines(file.path(pkg_a, "DESCRIPTION")))

  # Case 3: minimal_eval = TRUE passed to output format
  render(rmd_file,
         output_file = html_file,
         output_format = litr::litr_html_document(minimal_eval = TRUE)
         )
  testthat::expect_equal(readLines(file.path(pkg, "DESCRIPTION")),
                         readLines(file.path(pkg_a, "DESCRIPTION")))

  fs::dir_delete(dir)
})

10.5 Testing other templates

Let’s now make sure that each template can be knit without error.

testthat::test_that("templates can be knit", {
  dir <- tempfile()
  if (fs::file_exists(dir)) fs::file_delete(dir)
  fs::dir_create(dir)
  
  rmd_file <- file.path(dir, "create-rhello.Rmd")
  rmarkdown::draft(rmd_file,
                   template = "make-an-r-package",
                   package = "litr",
                   edit = FALSE)
  render(rmd_file)
  testthat::expect_true(fs::file_exists(file.path(dir, "create-rhello.html")))
  testthat::expect_true(fs::file_exists(file.path(dir, "rhello")))

  rmd_file <- file.path(dir, "create-rhasdata.Rmd")
  rmarkdown::draft(rmd_file,
                   template = "make-an-r-package-with-data",
                   package = "litr",
                   edit = FALSE)
  render(rmd_file)
  testthat::expect_true(fs::file_exists(file.path(dir, "create-rhasdata.html")))
  testthat::expect_true(fs::file_exists(file.path(dir, "rhasdata")))

  rmd_file <- file.path(dir, "create-withrcpp.Rmd")
  rmarkdown::draft(rmd_file,
                   template = "make-an-r-package-with-rcpp",
                   package = "litr",
                   edit = FALSE)
  render(rmd_file)
  testthat::expect_true(fs::file_exists(file.path(dir, "create-withrcpp.html")))
  testthat::expect_true(fs::file_exists(file.path(dir, "withrcpp")))

  rmd_file <- file.path(dir, "create-witharmadillo.Rmd")
  rmarkdown::draft(rmd_file,
                   template = "make-an-r-package-with-armadillo",
                   package = "litr",
                   edit = FALSE)
  render(rmd_file)
  testthat::expect_true(fs::file_exists(file.path(dir, "create-witharmadillo.Rmd")))
  testthat::expect_true(fs::file_exists(file.path(dir, "witharmadillo")))
    
  rmd_file <- file.path(dir, "create-withpkgdown.Rmd")
  rmarkdown::draft(rmd_file,
                   template = "make-an-r-package-with-extras",
                   package = "litr",
                   edit = FALSE)
  render(rmd_file)
  testthat::expect_true(fs::file_exists(file.path(dir, "create-withpkgdown.html")))
  testthat::expect_true(fs::file_exists(file.path(dir, "withpkgdown")))

  rmd_file <- file.path(dir, "create-frombookdown.Rmd")
  rmarkdown::draft(rmd_file,
                   template = "make-an-r-package-from-bookdown",
                   package = "litr",
                   edit = FALSE)
  prev_dir <- getwd()
  setwd(file.path(dir, "create-frombookdown"))
  fs::file_delete("create-frombookdown.Rmd")
  render("index.Rmd")
  setwd(prev_dir)
  testthat::expect_true(
    fs::file_exists(file.path(dir, "create-frombookdown", "_book", "index.html"))
    )
  testthat::expect_true(
    fs::file_exists(file.path(dir, "create-frombookdown", "frombookdown"))
    )

  fs::dir_delete(dir)
 })

Even though litr doesn’t directly use Rcpp, we’ll add it as a “Suggests” package since it would be required for running the above test.

usethis::use_package("Rcpp", type = "Suggests")
## ✔ Adding 'Rcpp' to Suggests field in DESCRIPTION
## • Use `requireNamespace("Rcpp", quietly = TRUE)` to test if package is installed
## • Then directly refer to functions with `Rcpp::fun()`

  1. If this were the only problem, we could get around this by using pkgload::package_file() to get the proper file; however, the next problem discussed was something that seemed quite hard to resolve.↩︎

  2. Note: When we call rmarkdown::render(), we call it in a fresh, non-interactive R session.↩︎