Update Clang.jl, regenerate bindings, add docs

.github/workflows/Documenter.yml

@@ -0,0 +1,19 @@
+name: Documenter
+on:
+  push:
+    branches: [main]
+    tags: [v*]
+  pull_request:
+
+jobs:
+  Documenter:
+    permissions:
+      contents: write
+    name: Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-docdeploy@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -48,7 +48,3 @@ jobs:
       - uses: julia-actions/julia-buildpkg@v1
         timeout-minutes: 3
       - uses: julia-actions/julia-runtest@v1
-      - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v1
-        with:
-          file: lcov.info

.github/workflows/generate_bindings.yml

@@ -6,14 +6,19 @@ on:
     paths:
       - .github/workflows/generate_bindings.yml
       - gen/**
+  pull_request:
+    branches: [main]
+    paths:
+      - .github/workflows/generate_bindings.yml
+      - gen/**
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
   cancel-in-progress: true
 jobs:
   generate-bindings:
     name: Generate bindings
     runs-on: ubuntu-22.04
-    timeout-minutes: 10
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v4
 
@@ -24,9 +29,21 @@ jobs:
       - name: Run the generator
         run: ./gen/generate.sh
 
-      - name: Create Pull Request
+      - name: Create Pull Request (on push)
+        if: ${{ github.event_name == 'push' }}
+        uses: peter-evans/create-pull-request@v6
+        with:
+          commit-message: "Regenerate bindings"
+          title: "Regenerate bindings"
+          reviewers: |
+            quinnj
+            Octogonapus
+
+      - name: Create Pull Request (on PR)
+        if: ${{ github.event_name == 'pull_request' }}
         uses: peter-evans/create-pull-request@v6
         with:
+          base: ${{ github.head_ref }}
           commit-message: "Regenerate bindings"
           title: "Regenerate bindings"
           reviewers: |

Project.toml

@@ -4,11 +4,13 @@ version = "1.0.0"
 
 [deps]
 CEnum = "fa961155-64e5-5f13-b03f-caf6b980ea82"
+LibAwsCommon = "c6e421ba-b5f8-4792-a1c4-42948de3ed9d"
 aws_c_io_jll = "13c41daa-f319-5298-b5eb-5754e0170d52"
 
 [compat]
 Aqua = "0.7"
 CEnum = "0.5"
+LibAwsCommon = "=1.0.0"
 aws_c_io_jll = "=0.14.6"
 julia = "1.6"
 

README.md

@@ -1,12 +1,7 @@
-# LibAwsCommon.jl
+[![](https://img.shields.io/badge/docs-stable-blue.svg)](https://JuliaServices.github.io/LibAwsIO.jl/stable)
+[![](https://img.shields.io/badge/docs-dev-blue.svg)](https://JuliaServices.github.io/LibAwsIO.jl/dev)
+[![CI](https://github.com/JuliaServices/LibAwsIO.jl/actions/workflows/ci.yml/badge.svg)](https://github.com/JuliaServices/LibAwsIO.jl/actions/workflows/ci.yml)
 
-Julia bindings for the [aws-c-common](https://github.com/awslabs/aws-c-common) library.
+# LibAwsIO.jl
 
-For documentation, please refer to the [lib.jl](https://github.com/JuliaServices/LibAwsCommon.jl/blob/main/src/lib.jl) source file where the original aws documentation is included.
-
-[![](https://img.shields.io/badge/docs-stable-blue.svg)](https://JuliaServices.github.io/LibAwsCommon.jl/stable)
-[![](https://img.shields.io/badge/docs-dev-blue.svg)](https://JuliaServices.github.io/LibAwsCommon.jl/dev)
-
-GitHub Actions : [![Build Status](https://github.com/JuliaServices/LibAwsCommon.jl/workflows/CI/badge.svg)](https://github.com/JuliaServices/LibAwsCommon.jl/actions?query=workflow%3ACI+branch%3Amaster)
-
-[![codecov.io](http://codecov.io/github/JuliaServices/LibAwsCommon.jl/coverage.svg?branch=master)](http://codecov.io/github/JuliaServices/LibAwsCommon.jl?branch=master)
+Julia bindings for the [aws-c-io](https://github.com/awslabs/aws-c-io) library.

docs/Project.toml

@@ -0,0 +1,4 @@
+[deps]
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+LibAwsCommon = "c6e421ba-b5f8-4792-a1c4-42948de3ed9d"
+LibAwsIO = "a5388770-19df-4151-b103-3d71de896ddf"

docs/make.jl

@@ -0,0 +1,20 @@
+using LibAwsIO
+using Documenter
+
+DocMeta.setdocmeta!(LibAwsIO, :DocTestSetup, :(using LibAwsIO); recursive=true)
+
+makedocs(;
+    modules=[LibAwsIO],
+    repo="https://github.com/JuliaServices/LibAwsIO.jl/blob/{commit}{path}#{line}",
+    sitename="LibAwsIO.jl",
+    format=Documenter.HTML(;
+        prettyurls=get(ENV, "CI", "false") == "true",
+        canonical="https://github.com/JuliaServices/LibAwsIO.jl",
+        assets=String[],
+        size_threshold=2_000_000, # 2 MB, we generate about 1 MB page
+        size_threshold_warn=2_000_000,
+    ),
+    pages=["Home" => "index.md"],
+)
+
+deploydocs(; repo="github.com/JuliaServices/LibAwsIO.jl", devbranch="main")

docs/src/index.md

@@ -0,0 +1,14 @@
+```@meta
+CurrentModule = LibAwsIO
+```
+
+# LibAwsIO
+
+Documentation for [LibAwsIO](https://github.com/JuliaServices/LibAwsIO.jl).
+
+```@index
+```
+
+```@autodocs
+Modules = [LibAwsIO]
+```

gen/Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.2"
 manifest_format = "2.0"
-project_hash = "997070a7b558e8e0faa0754e60e6833e12cbc2b5"
+project_hash = "91c208284b82a779e5b263a7d22a3c34a0736693"
 
 [[deps.ArgTools]]
 uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
@@ -21,9 +21,9 @@ version = "0.5.0"
 
 [[deps.Clang]]
 deps = ["CEnum", "Clang_jll", "Downloads", "Pkg", "TOML"]
-git-tree-sha1 = "846054622cb22aa63b5d51b5d84ec04b42d4d587"
+git-tree-sha1 = "2397d5da17ba4970f772a9888b208a0a1d77eb5d"
 uuid = "40e3b903-d033-50b4-a0cc-940c62c95e31"
-version = "0.17.8"
+version = "0.18.3"
 
 [[deps.Clang_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "TOML", "Zlib_jll", "libLLVM_jll"]
@@ -57,9 +57,9 @@ version = "1.3.1"
 
 [[deps.Git_jll]]
 deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"]
-git-tree-sha1 = "12945451c5d0e2d0dca0724c3a8d6448b46bbdf9"
+git-tree-sha1 = "d18fb8a1f3609361ebda9bf029b60fd0f120c809"
 uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb"
-version = "2.44.0+1"
+version = "2.44.0+2"
 
 [[deps.HistoricalStdlibVersions]]
 git-tree-sha1 = "c8b04a26eaa706b4da6968dfc27ae2d030547cba"
@@ -209,9 +209,9 @@ version = "1.2.13+1"
 
 [[deps.aws_c_cal_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "aws_c_common_jll"]
-git-tree-sha1 = "6b8ba9b8d5dfc8d9573a0c591bab74e40442a432"
+git-tree-sha1 = "7425bc9d167f7982c277d22609bc036a00ff1466"
 uuid = "70f11efc-bab2-57f1-b0f3-22aad4e67c4b"
-version = "0.6.10+0"
+version = "0.6.11+0"
 
 [[deps.aws_c_common_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl"]
@@ -221,9 +221,9 @@ version = "0.9.14+0"
 
 [[deps.aws_c_io_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "aws_c_cal_jll", "aws_c_common_jll", "s2n_tls_jll"]
-git-tree-sha1 = "9b62874ea33012d3ad904e4b5fb9c887441e2c9b"
+git-tree-sha1 = "8fe3ec4e9c43725e9df637fdd5a6846abc40f7bb"
 uuid = "13c41daa-f319-5298-b5eb-5754e0170d52"
-version = "0.14.6+0"
+version = "0.14.7+0"
 
 [[deps.libLLVM_jll]]
 deps = ["Artifacts", "Libdl"]
@@ -242,6 +242,6 @@ version = "17.4.0+2"
 
 [[deps.s2n_tls_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl"]
-git-tree-sha1 = "a9772e5e8c2c6e2c45b1d59b35f4847d22bb8516"
+git-tree-sha1 = "347948ecd0fa573c52131ff3eca523476b8f4894"
 uuid = "cddc5d3d-934d-5d3a-9747-62fc12ea3f48"
-version = "1.4.8+0"
+version = "1.4.12+0"

gen/Project.toml

@@ -5,6 +5,7 @@ aws_c_common_jll = "73048d1d-b8c4-5092-a58d-866c5e8d1e50"
 aws_c_io_jll = "13c41daa-f319-5298-b5eb-5754e0170d52"
 
 [compat]
-Clang = "0.17"
+Clang = "0.18.3"
 JLLPrefixes = "0.3"
-aws_c_io_jll = "=0.14.6"
+aws_c_common_jll = "=0.9.14"
+aws_c_io_jll = "=0.14.7"

gen/generate.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 dir=$(dirname "$0")
-julia +1.10 --project="$dir" -e 'using Pkg; Pkg.instantiate()'
-julia +1.10 --project="$dir" "$dir/generator.jl"
+julia --project="$dir" -e 'using Pkg; Pkg.instantiate()'
+julia --project="$dir" -t auto "$dir/generator.jl"

gen/generator.jl

@@ -5,47 +5,64 @@ import aws_c_common_jll, aws_c_io_jll
 
 cd(@__DIR__)
 
-function remove_itt_symbols!(dag::ExprDAG)
-    for i in eachindex(dag.nodes)
-        node = dag.nodes[i]
-        for expr in get_exprs(node)
-            node_name = if expr.head == :function
-                if expr.args[1].args[1] isa Expr # function is Module.name instead of just name
-                    expr.args[1].args[1].args[2]
-                else
-                    expr.args[1].args[1]
-                end
-            elseif expr.head == :struct
-                if expr.args[2] isa Expr # struct has type parameter
-                    expr.args[2].args[1]
-                else
-                    expr.args[2]
-                end
-            elseif expr.head == :const
-                expr.args[1].args[1]
-            end
-            # remove the node by renaming it to IGNORED, which we include in the generator's ignorelist
-            if contains(lowercase(string(node_name)), "itt")
-                dag.nodes[i] = ExprNode(:IGNORED, node.type, node.cursor, node.exprs, node.premature_exprs, node.adj)
-            end
+const refs_to_remove = []
+
+# This is called if the docs generated from the extract_c_comment_style method did not generate any lines.
+# We need to generate at least some docs so that cross-references work with Documenter.jl.
+function get_docs(node, docs)
+    # The macro node types (except for MacroDefault) seem to not generate code, but they will still emit docs and then
+    # you end up with docs stacked on top of each other, which is a Julia LoadError.
+    if node.type isa Generators.AbstractMacroNodeType && !(node.type isa Generators.MacroDefault)
+        return String[]
+    end
+
+    # don't generate empty docs because it makes Documenter.jl mad
+    if isempty(docs)
+        return ["Documentation not found."]
+    end
+
+    # remove references to things which don't exist because it causes Documenter.jl's cross_references check to fail
+    for ref in refs_to_remove
+        for doci in eachindex(docs)
+            docs[doci] = replace(docs[doci], "[`$ref`](@ref)" => "`$ref`")
         end
     end
-    return nothing
+
+    # # fix other random stuff
+    # for doci in eachindex(docs)
+    #     # fix some code that gets bogus references inserted
+    #     docs[doci] = replace(docs[doci], "for (struct [`aws_hash_iter`](@ref) iter = [`aws_hash_iter_begin`](@ref)(&map); ![`aws_hash_iter_done`](@ref)(&iter); [`aws_hash_iter_next`](@ref)(&iter)) { const key\\_type key = *(const key\\_type *)iter.element.key; value\\_type value = *(value\\_type *)iter.element.value; // etc. }" => "`for (struct aws_hash_iter iter = aws_hash_iter_begin(&map); !aws_hash_iter_done(&iter); aws_hash_iter_next(&iter)) { const key\\_type key = *(const key\\_type *)iter.element.key; value\\_type value = *(value\\_type *)iter.element.value; // etc. }`")
+    # end
+
+    return docs
+end
+
+function should_skip_target(target)
+    # aws_c_common_jll does not support i686 windows https://github.com/JuliaPackaging/Yggdrasil/blob/bbab3a916ae5543902b025a4a873cf9ee4a7de68/A/aws_c_common/build_tarballs.jl#L48-L49
+    return target == "i686-w64-mingw32"
+end
+
+# download toolchains in parallel
+Threads.@threads for target in JLLEnvs.JLL_ENV_TRIPLES
+    if should_skip_target(target)
+        continue
+    end
+    get_default_args(target) # downloads the toolchain
 end
 
 for target in JLLEnvs.JLL_ENV_TRIPLES
-    if target == "i686-w64-mingw32"
-        # aws_c_io_jll does not support i686 windows https://github.com/JuliaPackaging/Yggdrasil/blob/bbab3a916ae5543902b025a4a873cf9ee4a7de68/A/aws_c_common/build_tarballs.jl#L48-L49
+    if should_skip_target(target)
         continue
     end
     options = load_options(joinpath(@__DIR__, "generator.toml"))
     options["general"]["output_file_path"] = joinpath(@__DIR__, "..", "lib", "$target.jl")
+    options["general"]["callback_documentation"] = get_docs
 
-    header_dirs = []
     args = get_default_args(target)
     inc = JLLEnvs.get_pkg_include_dir(aws_c_common_jll, target)
-    push!(args, "-I$inc")
-    # push!(header_dirs, inc)
+    push!(args, "-isystem$inc")
+
+    header_dirs = []
     inc = JLLEnvs.get_pkg_include_dir(aws_c_io_jll, target)
     push!(args, "-I$inc")
     push!(header_dirs, inc)
@@ -63,14 +80,5 @@ for target in JLLEnvs.JLL_ENV_TRIPLES
     unique!(headers)
 
     ctx = create_context(headers, args, options)
-
-    # build without printing so we can do custom rewriting
-    build!(ctx, BUILDSTAGE_NO_PRINTING)
-
-    # the ITT symbols are just for aws-c-common's profiling stuff, we don't need to generate them and they cause
-    # problems with the generated code
-    # remove_itt_symbols!(ctx.dag)
-
-    # print
-    build!(ctx, BUILDSTAGE_PRINTING_ONLY)
+    build!(ctx)
 end

gen/generator.toml

@@ -1,6 +1,5 @@
 [general]
 library_name = "libaws_c_io"
-jll_pkg_extra = []
 smart_de_anonymize = true
 extract_c_comment_style = "doxygen"
 show_c_function_prototype = true
@@ -14,13 +13,12 @@ output_ignorelist = [
     "IGNORED",
     "STDCALL",
 ]
+generate_isystem_symbols = false
 
 auto_mutability = false
 auto_mutability_with_new = false
 auto_mutability_includelist = []
 
-export_symbol_prefixes = ["AWS_", "aws_"]
-
 # Julia's `@enum` do not allow duplicated values, so by default, C enums are translated to
 # CEnum.jl's `@cenum`.
 # if this entry is true, `@enum` is used and those duplicated enum constants are just commented.