stash

.github/actions/setup-java/action.yml

@@ -0,0 +1,12 @@
+name: Setup Java
+description: Installs needed jdk version
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-java@v4
+      with:
+        distribution: adopt
+        java-version: |
+          22
+          17
+        cache: gradle

.github/actions/setup-jextract/action.yml

@@ -0,0 +1,9 @@
+name: Setup JExtract
+description: Installs a jextract
+runs:
+  using: composite
+  steps:
+    - uses: oracle-actions/setup-java@v1
+      with:
+        website: jdk.java.net
+        release: jextract

.github/actions/setup-zig/action.yml

@@ -0,0 +1,9 @@
+name: Setup Zig
+description: Installs a zig compiler
+runs:
+  using: composite
+  steps:
+    - uses: goto-bus-stop/setup-zig@v2
+      with:
+        version: 0.13.0
+

.github/actions/update-ts-languages/action.yml

@@ -0,0 +1,10 @@
+name: Update TS Languages
+description: Runs scripts to update treesitter and its languages
+runs:
+  using: composite
+  steps:
+    - name: Update TS Languages
+      shell: bash
+      run: |
+        ./scripts/generate-ts.sh -jz
+        ./scripts/generate-languages.sh -jz

.github/workflows/ci-native.yml

@@ -5,30 +5,30 @@ on:
 
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - nickname: linux jdk17
-            java: 17
-            os: ubuntu-latest
-          - nickname: macos jdk17
-            java: 17
-            os: macos-latest
-          - nickname: windows jdk17
-            java: 17
-            os: windows-latest
-    name: CI Build ${{ matrix.nickname }}
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: adopt
-          java-version: |
-            22
-            17
-          cache: gradle
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-zig
+      - uses: ./.github/actions/setup-jextract
+      - uses: ./.github/actions/setup-java
+      - uses: ./.github/actions/update-ts-languages
       - name: Build
-        run: ./gradlew build
-
+        run: ./gradlew build -PshellIncludeTags=treesitter
+      - name: Publish Local Repo
+        run: ./gradlew publishMavenJavaPublicationToLocalRepository
+      - name: Upload Local Repo
+        uses: actions/upload-artifact@v4
+        with:
+          retention-days: 1
+          name: localrepo
+          path: |
+            build/publications/repos
+      - name: Upload Build Logs
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          retention-days: 1
+          name: buildlogs
+          path: |
+            */build/reports
+            */*/build/reports

.gitignore

@@ -17,6 +17,14 @@ spring-shell.log
 shell.log
 shell.log.*.gz
 shell.log.*.tmp
+shell-*.log
+shell-*.log.*.gz
+shell-*.log.*.tmp
+
+# Native Libs
+*.so
+*.dll
+*.jnilib
 
 # Visual Studio Code
 .vscode/*

.vscode/launch.json

@@ -109,6 +109,14 @@
             "console": "externalTerminal",
             "mainClass": "org.springframework.shell.samples.catalog.SpringShellApplication",
             "projectName": "spring-shell-sample-catalog"
+        },
+        {
+            "type": "java",
+            "name": "ffm interactive",
+            "request": "launch",
+            "mainClass": "org.springframework.shell.samples.ffm.SpringShellApplication",
+            "projectName": "spring-shell-sample-ffm",
+            "vmArgs": "--enable-native-access=ALL-UNNAMED"
         }
     ]
 }

.vscode/settings.json

@@ -6,6 +6,9 @@
         "org.springframework",
         "#"
     ],
+    "java.project.resourceFilters": [
+        "sample"
+    ],
     "java.checkstyle.configuration": "https://raw.githubusercontent.com/spring-cloud/spring-cloud-dataflow-build/main/spring-cloud-dataflow-build-tools/src/main/resources/checkstyle.xml",
     "java.checkstyle.properties": {
         "checkstyle.suppressions.file": "https://raw.githubusercontent.com/spring-cloud/spring-cloud-dataflow-build/main/spring-cloud-dataflow-build-tools/src/checkstyle/checkstyle-suppressions.xml",

_templates/init/tslanguage/gradle-build.txt

@@ -0,0 +1,16 @@
+---
+force: true
+to: spring-shell-treesitter-languages/spring-shell-treesitter-language-<%= language %>/spring-shell-treesitter-language-<%= language %>.gradle
+---
+plugins {
+	id 'org.springframework.shell.module'
+	id 'org.springframework.shell.toolchain'
+}
+
+description = 'Spring Shell Treesitter <%= h.changeCase.pascal(language) %>'
+
+dependencies {
+	management platform(project(":spring-shell-management"))
+	implementation project(':spring-shell-treesitter')
+	testImplementation 'org.springframework.boot:spring-boot-starter-test'
+}

_templates/init/tslanguage/main-language-class.txt

@@ -0,0 +1,56 @@
+---
+force: true
+to: spring-shell-treesitter-languages/spring-shell-treesitter-language-<%= language %>/src/main/java/org/springframework/shell/treesitter/language/<%= language %>/TreeSitterLanguage<%= h.changeCase.pascal(language) %>.java
+---
+/*
+ * Copyright 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.shell.treesitter.language.<%= language %>;
+
+import java.lang.foreign.MemorySegment;
+import java.util.List;
+
+import org.springframework.shell.treesitter.AbstractTreeSitterLanguage;
+import org.springframework.shell.treesitter.language.<%= language %>.ts.TreeSitter<%= h.changeCase.pascal(language) %>;
+
+public class TreeSitterLanguage<%= h.changeCase.pascal(language) %> extends AbstractTreeSitterLanguage {
+
+	@Override
+	public boolean supports(String languageName) {
+		return "<%= language %>".equals(languageName);
+	}
+
+	@Override
+	public List<String> supportedLanguages() {
+		return List.of("<%= language %>");
+	}
+
+	@Override
+	public TreeSitterLanguage<%= h.changeCase.pascal(language) %> language() {
+		return this;
+	}
+
+	@Override
+	public String highlightQuery() {
+		return resourceAsString(resourceLoader.getResource("classpath:org/springframework/shell/treesitter/queries/<%= language %>/highlights.scm"));
+	}
+
+	@Override
+	public MemorySegment initInternal() {
+		MemorySegment <%= language %> = TreeSitter<%= h.changeCase.pascal(language) %>.tree_sitter_<%= language %>();
+		return <%= language %>;
+	}
+
+}

_templates/init/tslanguage/test-language-class.txt

@@ -0,0 +1,42 @@
+---
+force: true
+to: spring-shell-treesitter-languages/spring-shell-treesitter-language-<%= language %>/src/test/java/org/springframework/shell/treesitter/language/<%= language %>/TreeSitterLanguage<%= h.changeCase.pascal(language) %>Tests.java
+---
+/*
+ * Copyright 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.shell.treesitter.language.<%= language %>;
+
+import java.lang.foreign.MemorySegment;
+
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.shell.treesitter.TreeSitterNativeLoader;
+import org.springframework.shell.treesitter.language.<%= language %>.ts.TreeSitter<%= h.changeCase.pascal(language) %>;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class TreeSitterLanguage<%= h.changeCase.pascal(language) %>Tests {
+
+	@Test
+	@Tag("treesitter")
+	void languageLoads() {
+		TreeSitterNativeLoader.initializeLanguage("<%= language %>");
+		MemorySegment segment = TreeSitter<%= h.changeCase.pascal(language) %>.tree_sitter_<%= language %>();
+		assertThat(segment).isNotNull();
+	}
+
+}

_templates/init/tslanguage/treesitter-factories.txt

@@ -0,0 +1,6 @@
+---
+force: true
+to: spring-shell-treesitter-languages/spring-shell-treesitter-language-<%= language %>/src/main/resources/META-INF/spring/treesitter.factories
+---
+org.springframework.shell.treesitter.TreeSitterLanguageProvider=\
+org.springframework.shell.treesitter.language.<%= language %>.TreeSitterLanguage<%= h.changeCase.pascal(language) %>

buildSrc/src/main/java/org/springframework/shell/gradle/JavaConventions.java

@@ -41,7 +41,7 @@ class JavaConventions {
 	private static final String SOURCE_AND_TARGET_COMPATIBILITY = "17";
 	private static final String INCLUDE_TAGS = "shellIncludeTags";
 	private static final String EXCLUDE_TAGS = "shellExcludeTags";
-	private static final String[] DEFAULT_EXCLUDE_TAGS = new String[] { };
+	private static final String[] DEFAULT_EXCLUDE_TAGS = new String[] { "treesitter" };
 
 	void apply(Project project) {
 		project.getPlugins().withType(JavaBasePlugin.class, java -> {

scripts/generate-languages.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+find_basedir() {
+  local basedir=$(cd -P -- "$(dirname -- "$0")" && cd .. && pwd -P)
+  echo "${basedir}"
+}
+
+dojextract='false'
+dozig='false'
+
+print_usage() {
+  echo "Usage: generate-languages.sh [-j] [-z]"
+}
+
+while getopts 'jz' flag; do
+  case "${flag}" in
+    j) dojextract='true' ;;
+    z) dozig='true' ;;
+    *) print_usage
+       exit 1 ;;
+  esac
+done
+
+TMPDIR=/tmp/spring-shell-tslanguages
+rm -fr $TMPDIR
+
+VALUES='
+tree-sitter,tree-sitter-java,v0.21.0,java
+tree-sitter,tree-sitter-json,v0.21.0,json
+tree-sitter-grammars,tree-sitter-yaml,v0.6.1,yaml
+'
+
+# tree-sitter,tree-sitter-python,v0.21.0,python
+# tree-sitter,tree-sitter-php,v0.22.8,php
+# tree-sitter,tree-sitter-cpp,v0.22.3,cpp
+# tree-sitter,tree-sitter-ruby,v0.21.0,ruby
+# tree-sitter,tree-sitter-c,v0.21.4,c
+# tree-sitter,tree-sitter-scala,v0.22.5,scala
+# tree-sitter,tree-sitter-go,v0.21.2,go
+# tree-sitter,tree-sitter-rust,v0.21.2,rust
+# tree-sitter,tree-sitter-javascript,v0.21.4,javascript
+# tree-sitter,tree-sitter-html,v0.20.4,html
+# tree-sitter,tree-sitter-css,v0.21.1,css
+# tree-sitter,tree-sitter-julia,v0.22.6,julia
+# tree-sitter,tree-sitter-typescript,v0.21.2,typescript
+# tree-sitter,tree-sitter-jsdoc,v0.21.0,jsdoc
+# tree-sitter,tree-sitter-regex,v1.0.0,regex
+# tree-sitter,tree-sitter-ocaml,v0.22.0,ocaml
+# tree-sitter,tree-sitter-c-scharp,v0.21.3,csharp
+# tree-sitter,tree-sitter-bash,v0.21.0,bash
+# tree-sitter-grammars,tree-sitter-markdown,v0.2.3,markdown
+# tree-sitter-grammars,tree-sitter-xml,v0.6.4,xml
+# tree-sitter-grammars,tree-sitter-lua,v0.1.0,lua
+
+export PROJECTBASEDIR=$(find_basedir)
+
+for VALUE in $VALUES;
+  do
+    GHOWNER=$(echo $VALUE | cut -f1 -d,)
+    GHREPO=$(echo $VALUE | cut -f2 -d,)
+    TAG=$(echo $VALUE | cut -f3 -d,)
+    LANGUAGEID=$(echo $VALUE | cut -f4 -d,)
+    LANGUAGENAME="${LANGUAGEID^}"
+    REPOPATH=$TMPDIR/$GHOWNER/$GHREPO
+    TARGETMODULEPATH=spring-shell-treesitter-languages/spring-shell-treesitter-language-$LANGUAGEID
+    git clone --depth 1 -b $TAG https://github.com/$GHOWNER/$GHREPO.git $REPOPATH
+
+    if [ "$dojextract" == "true" ]; then
+      npx hygen init tslanguage --language $LANGUAGEID
+      mkdir -p $TARGETMODULEPATH/src/ts
+      cp $REPOPATH/bindings/c/tree-sitter-$LANGUAGEID.h $TARGETMODULEPATH/src/ts/
+      mkdir -p $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/queries/$LANGUAGEID
+      cp $REPOPATH/queries/highlights.scm $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/queries/$LANGUAGEID/
+      jextract --header-class-name TreeSitter$LANGUAGENAME --output $TARGETMODULEPATH/src/main/java -t org.springframework.shell.treesitter.language.$LANGUAGEID.ts $TARGETMODULEPATH/src/ts/tree-sitter-$LANGUAGEID.h
+    fi
+
+    if [ "$dozig" == "true" ]; then
+      mkdir -p $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/libs/linux/x86_64
+      mkdir -p $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/libs/windows/x86_64
+      mkdir -p $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/libs/mac/x86_64
+      mkdir -p $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/libs/mac/arm64
+      ZIGFILES=$REPOPATH/src/parser.c
+      if [ -f $REPOPATH/src/scanner.c ]; then
+        ZIGFILES="$ZIGFILES $REPOPATH/src/scanner.c"
+      fi
+      zig cc -g0 -O2 -shared -target x86_64-linux-gnu -std=c11 -I $REPOPATH/src -o $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/libs/linux/x86_64/libtree-sitter-$LANGUAGEID.so $ZIGFILES
+      zig cc -g0 -O2 -shared -target x86_64-windows -std=c11 -I $REPOPATH/src -o $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/libs/windows/x86_64/tree-sitter-$LANGUAGEID.dll $ZIGFILES
+      rm $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/libs/windows/x86_64/*.pdb
+      rm $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/libs/windows/x86_64/*.lib
+      zig cc -g0 -O2 -shared -target x86_64-macos -std=c11 -I $REPOPATH/src -o $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/libs/mac/x86_64/libtree-sitter-$LANGUAGEID.jnilib $ZIGFILES
+      zig cc -g0 -O2 -shared -target aarch64-macos -std=c11 -I $REPOPATH/src -o $TARGETMODULEPATH/src/main/resources/org/springframework/shell/treesitter/libs/mac/arm64/libtree-sitter-$LANGUAGEID.jnilib $ZIGFILES
+    fi
+
+done;
+