Merge branch 'main' into encore_repolarserr

.Rbuildignore

@@ -22,4 +22,9 @@
 ^src/Makevars$
 ^tools/libr_polars\.a$
 ^dev$
+
+^altdoc$
+
 ^DEVELOPMENT\.md$
+
+^\.venv_altdoc$

.github/workflows/docs.yaml

@@ -71,31 +71,35 @@ jobs:
           use-public-rspm: true
           Ncpus: 2
 
+        # altdoc uses quarto
+      - uses: quarto-dev/quarto-actions/setup@v2
+
       - name: Get requirements
         run: |
           make requirements-r
           make requirements-py
 
       - name: Build docs
         run: make docs
+        shell: bash
 
       - uses: webfactory/ssh-agent@v0.8.0
-        env: 
+        env:
           DEPLOY_DOCS: ${{ secrets.DEPLOY_DOCS  }}
-        if: ${{ (github.event_name != 'pull_request') && (github.repository_owner == 'pola-rs') }} 
+        if: ${{ (github.event_name != 'pull_request') && (github.repository_owner == 'pola-rs') }}
         with:
           ssh-private-key: ${{ secrets.DEPLOY_DOCS }}
 
         # https://www.mkdocs.org/user-guide/deploying-your-docs/
       - name: Build site and deploy to GitHub pages
-        env: 
+        env:
           DEPLOY_DOCS: ${{ secrets.DEPLOY_DOCS  }}
-        if: ${{ (github.event_name != 'pull_request') && (github.repository_owner == 'pola-rs') }} 
+        if: ${{ (github.event_name != 'pull_request') && (github.repository_owner == 'pola-rs') }}
         uses: JamesIves/github-pages-deploy-action@v4
         with:
           clean: true
           branch: main
-          folder: docs/site
+          folder: docs
           repository-name: rpolars/rpolars.github.io
           ssh-key: true
           clean-exclude: |

.gitignore

@@ -43,3 +43,8 @@ src/Makevars
 
 # Binary files
 tools/libr_polars.a
+
+# altdoc stuff
+altdoc/freeze.rds
+docs
+.venv_altdoc

DESCRIPTION

@@ -43,7 +43,8 @@ Suggests:
     vctrs,
     withr
 Config/Needs/website:
-    altdoc (>= 0.2.2),
+    etiennebacher/altdoc@5c69f3b,
+    future.apply,
     here,
     magrittr,
     pkgload,
@@ -54,7 +55,6 @@ Config/Needs/dev:
     RcppTOML,
     readr,
     rextendr,
-    robinlovelace/styler.equals,
     spelling,
     stringr,
     styler

DEVELOPMENT.md

@@ -97,3 +97,77 @@ usethis::use_version()
 ```r
 usethis::use_dev_version()
 ```
+
+## Check the performance via debug mode
+
+If you experience unexpected sluggish performance, when using polars in a given IDE, we'd like to hear about it. You can try to activate `pl$set_options(debug_polars = TRUE)` to profile what methods are being touched (not necessarily run) and how fast. Below is an example of good behavior.
+
+``` r
+library(polars)
+pl$set_options(debug_polars = TRUE)
+pl$DataFrame(iris)$select("Species")
+#> [TIME? ms]
+#> pl$DataFrame() -> [3.257ms]
+#> pl$lit() -> [2.721ms]
+#> pl$Series() -> [0.2244ms]
+#>    .pr$RPolarsSeries$new() -> [5.901ms]
+#> RPolarsExpr$alias() -> [20.62ms]
+#> pl$lit() -> [0.4537ms]
+#> pl$Series() -> [0.1681ms]
+#>    .pr$RPolarsSeries$new() -> [0.4008ms]
+#> RPolarsExpr$alias() -> [0.3057ms]
+#> pl$lit() -> [0.2573ms]
+#> pl$Series() -> [0.1891ms]
+#>    .pr$RPolarsSeries$new() -> [0.3707ms]
+#> RPolarsExpr$alias() -> [0.2408ms]
+#> pl$lit() -> [0.3285ms]
+#> pl$Series() -> [0.1342ms]
+#>    .pr$RPolarsSeries$new() -> [0.2878ms]
+#> RPolarsExpr$alias() -> [0.2875ms]
+#> pl$lit() -> [0.283ms]
+#> pl$Series() -> [0.1855ms]
+#>    .pr$RPolarsSeries$new() -> [9.417ms]
+#> RPolarsExpr$alias() -> [0.2825ms]
+#> pl$select() -> [0.1724ms]
+#>    .pr$RPolarsDataFrame$select() -> [45.21ms]
+#> RPolarsDataFrame$select() -> [0.2534ms]
+#>    .pr$RPolarsDataFrame$select() ->
+#> [6.062ms]
+#> RPolarsDataFrame$print() -> [0.2882ms]
+#>    .pr$RPolarsDataFrame$print() -> shape: (150, 1)
+#> ┌───────────┐
+#> │ Species   │
+#> │ ---       │
+#> │ cat       │
+#> ╞═══════════╡
+#> │ setosa    │
+#> │ setosa    │
+#> │ setosa    │
+#> │ setosa    │
+#> │ …         │
+#> │ virginica │
+#> │ virginica │
+#> │ virginica │
+#> │ virginica │
+#> └───────────┘
+```
+
+## Other tips
+
+<!-- TODO: Clean up -->
+
+To speed up the local rextendr::document() or R CMD check, run the following:
+
+```r
+source("inst/misc/develop_polars.R")
+
+#to rextendr:document() + not_cran + load packages + all_features
+load_polars()
+
+#to check package + reuses previous compilation in check, protects against deletion
+check_polars() #assumes rust target at `paste0(getwd(),"/src/rust")`
+```
+
+- The `RPOLARS_RUST_SOURCE` environment variable allows **polars** to recover the Cargo cache even if source files have been moved. Replace with your own absolute path to your local clone!
+- `filter_rcmdcheck.R` removes known warnings from final check report.
+- `unlink("check")` cleans up.

Makefile

@@ -1,7 +1,7 @@
 .DEFAULT_GOAL := help
 
 SHELL := /bin/bash
-VENV := .venv
+VENV := .venv_altdoc
 
 RUST_TOOLCHAIN_VERSION := $(shell Rscript -e 'read.dcf("DESCRIPTION", fields = "Config/polars/RustToolchainVersion", all = TRUE)[1, 1] |> cat()')
 
@@ -34,6 +34,7 @@ requirements-r:
 .PHONY: requirements-py
 requirements-py: .venv
 	$(VENV_BIN)/python -m pip install --upgrade pip
+	$(VENV_BIN)/pip install --upgrade mkdocs
 	$(VENV_BIN)/pip install --upgrade mkdocs-material
 
 .PHONY: requirements-rs
@@ -51,7 +52,7 @@ build: ## Compile polars R package with all features and generate Rd files
 	export NOT_CRAN=true \
 	&& export LIBR_POLARS_BUILD=true \
 	&& export RPOLARS_FULL_FEATURES=true \
-	&& Rscript -e 'if (!(require(arrow)&&require(nanoarrow))) warning("could not load arrow/nanoarrow, igonore changes to nanoarrow.Rd"); rextendr::document()'
+	&& Rscript -e 'if (!(require(arrow) && require(nanoarrow) && require(knitr))) warning("could not load arrow/nanoarrow/knitr, ignore changes to nanoarrow.Rd or knit_print.Rd"); rextendr::document()'
 
 .PHONY: install
 install: ## Install the R package
@@ -64,20 +65,18 @@ install: ## Install the R package
 all: fmt tools/lib-sums.tsv build test README.md LICENSE.note ## build -> test -> Update README.md, LICENSE.note
 
 .PHONY: docs
-docs: build install README.md docs/docs/reference_home.md ## Generate docs
-	cp docs/mkdocs.orig.yml docs/mkdocs.yml
-	Rscript -e 'altdoc::update_docs(custom_reference = "docs/make-docs.R")'
-	cd docs && ../$(VENV_BIN)/python3 -m mkdocs build
+docs: build install README.md altdoc/reference_home.md ## Generate docs
+	Rscript -e 'future::plan(future::multicore); source("altdoc/altdoc_preprocessing.R"); altdoc::render_docs(freeze = FALSE, parallel = TRUE)'
 
 .PHONY: docs-preview
 docs-preview: ## Preview docs on local server. Needs `make docs`
-	cd docs && ../$(VENV_BIN)/python3 -m mkdocs serve
+	Rscript -e 'altdoc::preview_docs()'
 
 README.md: README.Rmd build ## Update README.md
 	Rscript -e 'devtools::load_all(); rmarkdown::render("README.Rmd")'
 
-docs/docs/reference_home.md: docs/docs/reference_home.Rmd build ## Update the reference home page source
-	Rscript -e 'devtools::load_all(); rmarkdown::render("docs/docs/reference_home.Rmd")'
+altdoc/reference_home.md: altdoc/reference_home.Rmd build ## Update the reference home page source
+	Rscript -e 'devtools::load_all(); rmarkdown::render("altdoc/reference_home.Rmd")'
 
 LICENSE.note: src/rust/Cargo.lock ## Update LICENSE.note
 	Rscript -e 'rextendr::write_license_note(force = TRUE)'
@@ -93,12 +92,9 @@ test: build install ## Run fast unittests
 .PHONY: fmt
 fmt: fmt-rs fmt-r ## Format files
 
-GIT_DIF_TARGET ?=
-MODIFIED_R_FILES ?= $(shell R -s -e 'setdiff(system("git diff $(GIT_DIF_TARGET) --name-only | grep -e .*R$$ -e .*Rmd$$", intern = TRUE), "R/extendr-wrappers.R") |> cat()')
-
 .PHONY: fmt-r
-fmt-r: $(MODIFIED_R_FILES) ## Format R files
-	$(foreach file, $^, $(shell R -q -e 'styler::style_file("$(file)"); styler.equals::style_file("$(file)")' >/dev/null))
+fmt-r: ## Format R files, set envvar GIT_DIF_TARGET to git diff branch target. Default is "main".
+	Rscript -e 'source("./dev/styler_utils.R"); style_files()'
 
 .PHONY: fmt-rs
 fmt-rs: ## Format Rust files

NAMESPACE

@@ -4,22 +4,22 @@ S3method("!",RPolarsExpr)
 S3method("!=",RPolarsDataType)
 S3method("!=",RPolarsExpr)
 S3method("!=",RPolarsSeries)
-S3method("$",ExprBinNameSpace)
-S3method("$",ExprCatNameSpace)
-S3method("$",ExprDTNameSpace)
-S3method("$",ExprListNameSpace)
-S3method("$",ExprMetaNameSpace)
-S3method("$",ExprNameNameSpace)
-S3method("$",ExprStrNameSpace)
-S3method("$",ExprStructNameSpace)
-S3method("$",GroupBy)
 S3method("$",RPolarsChainedThen)
 S3method("$",RPolarsChainedWhen)
 S3method("$",RPolarsDataFrame)
 S3method("$",RPolarsDataType)
 S3method("$",RPolarsDataTypeVector)
 S3method("$",RPolarsErr)
 S3method("$",RPolarsExpr)
+S3method("$",RPolarsExprBinNameSpace)
+S3method("$",RPolarsExprCatNameSpace)
+S3method("$",RPolarsExprDTNameSpace)
+S3method("$",RPolarsExprListNameSpace)
+S3method("$",RPolarsExprMetaNameSpace)
+S3method("$",RPolarsExprNameNameSpace)
+S3method("$",RPolarsExprStrNameSpace)
+S3method("$",RPolarsExprStructNameSpace)
+S3method("$",RPolarsGroupBy)
 S3method("$",RPolarsLazyFrame)
 S3method("$",RPolarsLazyGroupBy)
 S3method("$",RPolarsProtoExprArray)
@@ -58,19 +58,19 @@ S3method(">",RPolarsExpr)
 S3method(">",RPolarsSeries)
 S3method(">=",RPolarsExpr)
 S3method(">=",RPolarsSeries)
-S3method("[",ExprListNameSpace)
 S3method("[",RPolarsDataFrame)
+S3method("[",RPolarsExprListNameSpace)
 S3method("[",RPolarsLazyFrame)
 S3method("[",RPolarsSeries)
 S3method("[",rpolars_raw_list)
-S3method("[[",GroupBy)
 S3method("[[",RPolarsChainedThen)
 S3method("[[",RPolarsChainedWhen)
 S3method("[[",RPolarsDataFrame)
 S3method("[[",RPolarsDataType)
 S3method("[[",RPolarsDataTypeVector)
 S3method("[[",RPolarsErr)
 S3method("[[",RPolarsExpr)
+S3method("[[",RPolarsGroupBy)
 S3method("[[",RPolarsLazyFrame)
 S3method("[[",RPolarsLazyGroupBy)
 S3method("[[",RPolarsProtoExprArray)
@@ -85,12 +85,12 @@ S3method("[[",RPolarsVecDataFrame)
 S3method("[[",RPolarsWhen)
 S3method("^",RPolarsExpr)
 S3method("|",RPolarsExpr)
-S3method(.DollarNames,GroupBy)
 S3method(.DollarNames,RPolarsChainedThen)
 S3method(.DollarNames,RPolarsChainedWhen)
 S3method(.DollarNames,RPolarsDataFrame)
 S3method(.DollarNames,RPolarsErr)
 S3method(.DollarNames,RPolarsExpr)
+S3method(.DollarNames,RPolarsGroupBy)
 S3method(.DollarNames,RPolarsLazyFrame)
 S3method(.DollarNames,RPolarsRField)
 S3method(.DollarNames,RPolarsRThreadHandle)
@@ -111,8 +111,8 @@ S3method(as.matrix,RPolarsDataFrame)
 S3method(as.matrix,RPolarsLazyFrame)
 S3method(as.vector,RPolarsSeries)
 S3method(as_polars_df,ArrowTabular)
-S3method(as_polars_df,GroupBy)
 S3method(as_polars_df,RPolarsDataFrame)
+S3method(as_polars_df,RPolarsGroupBy)
 S3method(as_polars_df,RPolarsLazyFrame)
 S3method(as_polars_df,RPolarsLazyGroupBy)
 S3method(as_polars_df,RPolarsSeries)
@@ -153,18 +153,19 @@ S3method(na.omit,RPolarsDataFrame)
 S3method(na.omit,RPolarsLazyFrame)
 S3method(names,RPolarsDataFrame)
 S3method(names,RPolarsLazyFrame)
-S3method(print,GroupBy)
 S3method(print,PTime)
 S3method(print,RPolarsChainedThen)
 S3method(print,RPolarsChainedWhen)
 S3method(print,RPolarsDataFrame)
 S3method(print,RPolarsDataType)
 S3method(print,RPolarsErr)
 S3method(print,RPolarsExpr)
+S3method(print,RPolarsGroupBy)
 S3method(print,RPolarsLazyFrame)
 S3method(print,RPolarsLazyGroupBy)
 S3method(print,RPolarsRField)
 S3method(print,RPolarsRThreadHandle)
+S3method(print,RPolarsSQLContext)
 S3method(print,RPolarsSeries)
 S3method(print,RPolarsThen)
 S3method(print,RPolarsWhen)
@@ -181,7 +182,6 @@ export(.pr)
 export(as_polars_df)
 export(as_polars_lf)
 export(as_polars_series)
-export(knit_print.RPolarsDataFrame)
 export(pl)
 importFrom(stats,median)
 importFrom(stats,na.omit)