CI improvements, Inference improvements for recursive functions, Downstream testing

.github/workflows/breakage.yml

@@ -0,0 +1,61 @@
+# Ref: https://securitylab.github.com/research/github-actions-preventing-pwn-requests
+# and: https://github.com/JuliaSmoothOptimizers/NLPModels.jl/blob/main/.github/workflows/breakage.yml
+name: Breakage
+
+# read-only repo token
+# no access to secrets
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  break:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        pkg: [
+          "JuliaDynamics/ConcurrentSim.jl",
+          "gerlero/Fronts.jl",
+          "QuantumSavory/QuantumSavory.jl",
+          "marcom/FoldRNA.jl",
+          "Tortar/IteratorSampling.jl",
+        ]
+        pkgversion: [latest]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # Install Julia
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: 1
+          arch: x64
+      - uses: julia-actions/cache@v1
+      - uses: julia-actions/julia-buildpkg@v1
+
+      # Breakage test
+      - name: 'Breakage of ${{ matrix.pkg }}, ${{ matrix.pkgversion }} version'
+        env:
+          URL: ${{ matrix.pkg }}
+          VERSION: ${{ matrix.pkgversion }}
+        run: |
+          set -v
+          mkdir -p ./pr
+          echo "${{ github.event.number }}" > ./pr/NR
+          git clone https://github.com/$URL
+          export PKG=$(echo $URL | cut -f2 -d/)
+          cd $PKG
+          TAG=$VERSION
+          export TAG
+          julia -e 'using Pkg;
+            PKG, TAG, VERSION = ENV["PKG"], ENV["TAG"], ENV["VERSION"]
+            joburl = joinpath(ENV["GITHUB_SERVER_URL"], ENV["GITHUB_REPOSITORY"], "actions/runs", ENV["GITHUB_RUN_ID"])
+            open("../pr/$PKG-$VERSION", "w") do io
+                TAG == "no_tag" && error("No tag for $VERSION")
+                pkg"activate .";
+                pkg"instantiate";
+                pkg"dev ../";
+                pkg"build";
+                pkg"test";
+            end'

.github/workflows/changelog-enforcer.yml

@@ -0,0 +1,15 @@
+name: "Changelog Enforcer"
+on:
+  pull_request:
+    # The specific activity types are listed here to include "labeled" and "unlabeled"
+    # (which are not included by default for the "pull_request" trigger).
+    # This is needed to allow skipping enforcement of the changelog in PRs with specific labels,
+    # as defined in the (optional) "skipLabels" property.
+    types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
+
+jobs:
+  # Enforces the update of a changelog file on every pull request 
+  changelog:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: dangoslen/changelog-enforcer@v3

.github/workflows/ci-julia-nightly.yml

@@ -28,16 +28,7 @@ jobs:
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - uses: actions/cache@v4
-        env:
-          cache-name: cache-artifacts
-        with:
-          path: ~/.julia/artifacts
-          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-test-${{ env.cache-name }}-
-            ${{ runner.os }}-test-
-            ${{ runner.os }}-
+      - uses: julia-actions/cache@v1
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
         env:

.github/workflows/ci.yml

@@ -13,6 +13,7 @@ jobs:
       matrix:
         version:
           - '1.6'
+          - '1.9'
           - '1'
         os:
           - ubuntu-latest
@@ -38,16 +39,7 @@ jobs:
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - uses: actions/cache@v4
-        env:
-          cache-name: cache-artifacts
-        with:
-          path: ~/.julia/artifacts
-          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-test-${{ env.cache-name }}-
-            ${{ runner.os }}-test-
-            ${{ runner.os }}-
+      - uses: julia-actions/cache@v1
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
         env:
@@ -65,6 +57,7 @@ jobs:
       - uses: julia-actions/setup-julia@v1
         with:
           version: '1'
+      - uses: julia-actions/cache@v1
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-docdeploy@v1
         env:

.github/workflows/downgrade.yml

@@ -0,0 +1,29 @@
+name: Downgrade
+on:
+  pull_request:
+    branches: [master, main]
+    paths-ignore:
+      - 'docs/**'
+  push:
+    branches: [master, main]
+    paths-ignore:
+      - 'docs/**'
+env:
+  PYTHON: ~
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: ['1.9']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+      - uses: cjdoris/julia-downgrade-compat-action@v1
+        with:
+          skip: Pkg,TOML,InteractiveUtils,Random,LinearAlgebra
+      - uses: julia-actions/cache@v1
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1

.github/workflows/invalidations.yml

@@ -20,6 +20,7 @@ jobs:
       with:
         version: '1'
     - uses: actions/checkout@v4
+    - uses: julia-actions/cache@v1
     - uses: julia-actions/julia-buildpkg@v1
     - uses: julia-actions/julia-invalidations@v1
       id: invs_pr

.github/workflows/spelling.yml

@@ -0,0 +1,15 @@
+name: Spell Check
+
+on: [pull_request]
+
+jobs:
+  typos-check:
+    name: Spell Check with Typos
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Actions Repository
+        uses: actions/checkout@v4
+      - name: Check spelling
+        uses: crate-ci/typos@master
+        with:
+            config: .typos.toml

.typos.toml

@@ -0,0 +1,9 @@
+[default.extend-words]
+ket = "ket"
+numer = "numer"
+
+[type.ipynb]
+# It detects false possitives in the base64 encoded images inside notebooks
+extend-glob = ["*.ipynb"]
+check-file = false
+

CHANGELOG.md

@@ -1,5 +1,9 @@
 # News
 
+## v0.6.8 - 2024-02-07
+
+- Redeploy the improved performance from v0.6.6 without the issues that caused them to be reverted in v0.6.7. The issue stemmed from lack of support for recursive resumable functions in the v0.6.6 improvements.
+- Significant additions to the CI and testing of the package to avoid such issues in the future.
 ## v0.6.7 - 2024-01-02
 
 - Fix stack overflow errors by reverting the changes introduced in v0.6.6.
@@ -10,7 +14,7 @@
 
 ## v0.6.5 - 2023-09-06
 
-- Fix to a performance regresion originally introduced by `@yieldfrom`.
+- Fix to a performance regression originally introduced by `@yieldfrom`.
 
 ## v0.6.4 - 2023-08-08
 
@@ -57,7 +61,7 @@
   * introduction of `let` block to allow variables not te be persisted between `@resumable function` calls (EXPERIMENTAL).
   * the `eltype` of a `@resumable function` based iterator is its return type if specified, otherwise `Any`.
 
-* 2018: v0.2 the iterator now behaves as a Python generator: only values that are explicitely yielded are generated; the return value is ignored and a warning is generated.
+* 2018: v0.2 the iterator now behaves as a Python generator: only values that are explicitly yielded are generated; the return value is ignored and a warning is generated.
 
 * 2017: v0.1 initial release that is Julia v0.6 compatible:
   * Introduction of the `@resumable` and the `@yield` macros.

Project.toml

@@ -5,11 +5,11 @@ license = "MIT"
 desc = "C# sharp style generators a.k.a. semi-coroutines for Julia."
 authors = ["Ben Lauwens <ben.lauwens@gmail.com>"]
 repo = "https://github.com/BenLauwens/ResumableFunctions.jl.git"
-version = "0.6.7"
+version = "0.6.8"
 
 [deps]
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 
 [compat]
-MacroTools = "0.5"
+MacroTools = "0.5.6"
 julia = "1.6"

docs/src/manual.md

@@ -272,7 +272,7 @@ DocTestSetup = nothing
 
 ## Iterator interface
 
-The interator interface is implemented for a `@resumable function`:
+The iterator interface is implemented for a `@resumable function`:
 
 ```@meta
 DocTestSetup = quote

src/macro.jl

@@ -39,15 +39,18 @@ macro resumable(expr::Expr)
   func_def[:body] = postwalk(transform_arg_yieldfrom, func_def[:body])
   func_def[:body] = postwalk(transform_yieldfrom, func_def[:body])
   func_def[:body] = postwalk(x->transform_for(x, ui8), func_def[:body])
-  slots = get_slots(copy(func_def), arg_dict, __module__)
+  inferfn, slots = get_slots(copy(func_def), arg_dict, __module__)
   type_name = gensym(Symbol(func_def[:name], :_FSMI))
   constr_def = copy(func_def)
-  if isempty(params)
-    struct_name = :($type_name <: ResumableFunctions.FiniteStateMachineIterator{$rtype})
+  slot_T = [gensym(s) for s in keys(slots)]
+  slot_T_sub = [:($k <: $v) for (k, v) in zip(slot_T, values(slots))]
+  struct_name = :($type_name{$(func_def[:whereparams]...), $(slot_T_sub...)} <: ResumableFunctions.FiniteStateMachineIterator{$rtype})
+  constr_def[:whereparams] = (func_def[:whereparams]..., slot_T_sub...)
+  # if there are no where or slot type parameters, we need to use the bare type
+  if isempty(params) && isempty(slot_T)
     constr_def[:name] = :($type_name)
   else
-    struct_name = :($type_name{$(func_def[:whereparams]...)} <: ResumableFunctions.FiniteStateMachineIterator{$rtype})
-    constr_def[:name] = :($type_name{$(params...)})
+    constr_def[:name] = :($type_name{$(params...), $(slot_T...)})
   end
   constr_def[:args] = tuple()
   constr_def[:kwargs] = tuple()
@@ -57,32 +60,44 @@ macro resumable(expr::Expr)
     fsmi._state = 0x00
     fsmi
   end
+  # the bare/fallback version of the constructor supplies default slot type parameters
+  # we only need to define this if there there are actually slot defaults to be filled
+  if !isempty(slot_T)
+    bareconstr_def = copy(constr_def)
+    if isempty(params)
+      bareconstr_def[:name] = :($type_name)
+    else
+      bareconstr_def[:name] = :($type_name{$(params...)})
+    end
+    bareconstr_def[:whereparams] = func_def[:whereparams]
+    bareconstr_def[:body] = :($(bareconstr_def[:name]){$(values(slots)...)}())
+    bareconst_expr = combinedef(bareconstr_def) |> flatten
+  else
+    bareconst_expr = nothing
+  end
   constr_expr = combinedef(constr_def) |> flatten
   type_expr = :(
     mutable struct $struct_name
       _state :: UInt8
-      $((:($slotname :: $slottype) for (slotname, slottype) in slots)...)
+      $((:($slotname :: $slottype) for (slotname, slottype) in zip(keys(slots), slot_T))...)
       $(constr_expr)
+      $(bareconst_expr)
     end
   )
   @debug type_expr|>MacroTools.striplines
   call_def = copy(func_def)
+  call_def[:rtype] = nothing
   if isempty(params)
-    call_def[:rtype] = nothing
-    call_def[:body] = quote
-      fsmi = $type_name()
-      $((arg !== Symbol("_") ? :(fsmi.$arg = $arg) : nothing for arg in args)...)
-      $((:(fsmi.$arg = $arg) for arg in kwargs)...)
-      fsmi
-    end
+    fsmi_name = type_name
   else
-    call_def[:rtype] = nothing
-    call_def[:body] = quote
-      fsmi = $type_name{$(params...)}()
-      $((arg !== Symbol("_") ? :(fsmi.$arg = $arg) : nothing for arg in args)...)
-      $((:(fsmi.$arg = $arg) for arg in kwargs)...)
-      fsmi
-    end
+    fsmi_name = :($type_name{$(params...)})
+  end
+  fwd_args, fwd_kwargs = forward_args(call_def)
+  call_def[:body] = quote
+    fsmi = ResumableFunctions.typed_fsmi($fsmi_name, $inferfn, $(fwd_args...), $(fwd_kwargs...))
+    $((arg !== Symbol("_") ? :(fsmi.$arg = $arg) : nothing for arg in args)...)
+    $((:(fsmi.$arg = $arg) for arg in kwargs)...)
+    fsmi
   end
   call_expr = combinedef(call_def) |> flatten
   @debug call_expr|>MacroTools.striplines
@@ -112,10 +127,15 @@ macro resumable(expr::Expr)
   func_def[:args] = [Expr(:kw, :(_arg::Any), nothing)]
   func_def[:kwargs] = []
   func_expr = combinedef(func_def) |> flatten
+  if inexpr(func_def[:body], call_def[:name])
+    @debug "recursion is present in a resumable function definition: falling back to no inference"
+    call_expr = postwalk(x->x==:(ResumableFunctions.typed_fsmi) ? :(ResumableFunctions.typed_fsmi_fallback) : x, call_expr)
+  end
   @debug func_expr|>MacroTools.striplines
   esc(quote
     $type_expr
     $func_expr
     Base.@__doc__($call_expr)
+    #$call_expr_recursive_escape
   end)
 end

src/transforms.jl

@@ -66,7 +66,7 @@ end
 
 
 """
-Function that replaces a `for` loop by a corresponding `while` loop saving explicitely the *iterator* and its *state*.
+Function that replaces a `for` loop by a corresponding `while` loop saving explicitly the *iterator* and its *state*.
 """
 function transform_for(expr, ui8::BoxedUInt8)
   @capture(expr, for element_ in iterator_ body_ end) || return expr
@@ -100,7 +100,7 @@ end
 """
 Function that replaces a variable `x` in an expression by `_fsmi.x` where `x` is a known slot.
 """
-function transform_slots(expr, symbols::Base.KeySet{Symbol, Dict{Symbol,Any}})
+function transform_slots(expr, symbols)
   expr isa Expr || return expr
   expr.head === :let && return transform_slots_let(expr, symbols)
   for i in 1:length(expr.args)
@@ -114,7 +114,7 @@ end
 """
 Function that handles `let` block
 """
-function transform_slots_let(expr::Expr, symbols::Base.KeySet{Symbol, Dict{Symbol,Any}})
+function transform_slots_let(expr::Expr, symbols)
   @capture(expr, let vars_; body_ end)
   locals = Set{Symbol}()
   (isa(vars, Expr) && vars.head==:(=))  || error("@resumable currently supports only single variable declarations in let blocks, i.e. only let blocks exactly of the form `let i=j; ...; end`. If you need multiple variables, please submit an issue on the issue tracker and consider contributing a patch.")
@@ -247,14 +247,16 @@ end
 """
 Function that replaces a `@yield ret` or `@yield` statement with
 ```julia
-  return ret
+  Base.inferencebarrier(ret)
 ```
+This version is used for inference only.
+It makes sure that `val = @yield ret` is inferred as `Any` rather than `typeof(ret)`.
 """
 function transform_yield(expr)
   _is_yield(expr) || return expr
   ret = length(expr.args) > 2 ? expr.args[3:end] : [nothing]
   quote
-    $(ret...)
+    Base.inferencebarrier($(ret...))
   end
 end