Merge branch 'main' into main

.github/workflows/release.yml

@@ -18,105 +18,107 @@ jobs:
   release:
     runs-on: [ "self-hosted", "overseas" ]
     steps:
-    - name: Ensure workspace ownership
-      run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
+      - name: Ensure workspace ownership
+        run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
 
-    # https://github.com/actions/checkout/blob/v3/README.md
-    - name: Check out code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
+      # https://github.com/actions/checkout/blob/v3/README.md
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
+          fetch-depth: 0
+          fetch-tags: true
 
-    - name: Prepare release body
-      run: |
-        if [[ $GITHUB_EVENT_NAME == 'create' ]]; then
-          RELEASE_TAG=${GITHUB_REF#refs/tags/}
-          if [[ $RELEASE_TAG == 'nightly' ]]; then
-            PRERELEASE=true
+      - name: Prepare release body
+        run: |
+          if [[ $GITHUB_EVENT_NAME == 'create' ]]; then
+            RELEASE_TAG=${GITHUB_REF#refs/tags/}
+            if [[ $RELEASE_TAG == 'nightly' ]]; then
+              PRERELEASE=true
+            else
+              PRERELEASE=false
+            fi
+            echo "Workflow triggered by create tag: $RELEASE_TAG"
           else
-            PRERELEASE=false
+            RELEASE_TAG=nightly
+            PRERELEASE=true
+            echo "Workflow triggered by schedule"
           fi
-          echo "Workflow triggered by create tag: $RELEASE_TAG"
-        else
-          RELEASE_TAG=nightly
-          PRERELEASE=true
-          echo "Workflow triggered by schedule"
-        fi
-        echo "RELEASE_TAG=$RELEASE_TAG" >> $GITHUB_ENV
-        echo "PRERELEASE=$PRERELEASE" >> $GITHUB_ENV
-        RELEASE_DATETIME=$(date --rfc-3339=seconds)
-        echo Release $RELEASE_TAG created from $GITHUB_SHA at $RELEASE_DATETIME > release_body.md
+          echo "RELEASE_TAG=$RELEASE_TAG" >> $GITHUB_ENV
+          echo "PRERELEASE=$PRERELEASE" >> $GITHUB_ENV
+          RELEASE_DATETIME=$(date --rfc-3339=seconds)
+          echo Release $RELEASE_TAG created from $GITHUB_SHA at $RELEASE_DATETIME > release_body.md
 
-    - name: Move the existing mutable tag
-      # https://github.com/softprops/action-gh-release/issues/171
-      run: |
-        git fetch --tags
-        if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
-          # Determine if a given tag exists and matches a specific Git commit.
-          # actions/checkout@v4 fetch-tags doesn't work when triggered by schedule
-          if [ "$(git rev-parse -q --verify "refs/tags/$RELEASE_TAG")" = "$GITHUB_SHA" ]; then
-            echo "mutable tag $RELEASE_TAG exists and matches $GITHUB_SHA"
-          else
-            git tag -f $RELEASE_TAG $GITHUB_SHA
-            git push -f origin $RELEASE_TAG:refs/tags/$RELEASE_TAG
-            echo "created/moved mutable tag $RELEASE_TAG to $GITHUB_SHA"
+      - name: Move the existing mutable tag
+        # https://github.com/softprops/action-gh-release/issues/171
+        run: |
+          git fetch --tags
+          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+            # Determine if a given tag exists and matches a specific Git commit.
+            # actions/checkout@v4 fetch-tags doesn't work when triggered by schedule
+            if [ "$(git rev-parse -q --verify "refs/tags/$RELEASE_TAG")" = "$GITHUB_SHA" ]; then
+              echo "mutable tag $RELEASE_TAG exists and matches $GITHUB_SHA"
+            else
+              git tag -f $RELEASE_TAG $GITHUB_SHA
+              git push -f origin $RELEASE_TAG:refs/tags/$RELEASE_TAG
+              echo "created/moved mutable tag $RELEASE_TAG to $GITHUB_SHA"
+            fi
           fi
-        fi
 
-    - name: Create or overwrite a release
-      # https://github.com/actions/upload-release-asset has been replaced by https://github.com/softprops/action-gh-release
-      uses: softprops/action-gh-release@v2
-      with:
-        token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
-        prerelease: ${{ env.PRERELEASE }}
-        tag_name: ${{ env.RELEASE_TAG }}
-        # The body field does not support environment variable substitution directly.
-        body_path: release_body.md
+      - name: Create or overwrite a release
+        # https://github.com/actions/upload-release-asset has been replaced by https://github.com/softprops/action-gh-release
+        uses: softprops/action-gh-release@v2
+        with:
+          token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
+          prerelease: ${{ env.PRERELEASE }}
+          tag_name: ${{ env.RELEASE_TAG }}
+          # The body field does not support environment variable substitution directly.
+          body_path: release_body.md
 
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
 
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-    # https://github.com/marketplace/actions/docker-login
-    - name: Login to Docker Hub
-      uses: docker/login-action@v3
-      with:
-        username: infiniflow
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
+      # https://github.com/marketplace/actions/docker-login
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: infiniflow
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-    # https://github.com/marketplace/actions/build-and-push-docker-images
-    - name: Build and push full image
-      uses: docker/build-push-action@v6
-      with:
-        context: .
-        push: true
-        tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}
-        file: Dockerfile
-        platforms: linux/amd64
+      # https://github.com/marketplace/actions/build-and-push-docker-images
+      - name: Build and push full image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}
+          file: Dockerfile
+          platforms: linux/amd64
 
-    # https://github.com/marketplace/actions/build-and-push-docker-images
-    - name: Build and push slim image
-      uses: docker/build-push-action@v6
-      with:
-        context: .
-        push: true
-        tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}-slim
-        file: Dockerfile
-        build-args: LIGHTEN=1
-        platforms: linux/amd64
+      # https://github.com/marketplace/actions/build-and-push-docker-images
+      - name: Build and push slim image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}-slim
+          file: Dockerfile
+          build-args: LIGHTEN=1
+          platforms: linux/amd64
 
-    - name: Build ragflow-sdk
-      if: startsWith(github.ref, 'refs/tags/v')
-      run: |
-        cd sdk/python && \
-        poetry build
+      - name: Build ragflow-sdk
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          cd sdk/python && \
+          poetry build
 
-    - name: Publish package distributions to PyPI
-      if: startsWith(github.ref, 'refs/tags/v')
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        packages-dir: dist/
-        password: ${{ secrets.PYPI_API_TOKEN }}
-        verbose: true
+      - name: Publish package distributions to PyPI
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          verbose: true

.github/workflows/tests.yml

@@ -49,6 +49,12 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
 
+      # https://github.com/astral-sh/ruff-action
+      - name: Static check with Ruff
+        uses: astral-sh/ruff-action@v2
+        with:
+          version: ">=0.8.2"
+
       - name: Build ragflow:dev-slim
         run: |
           RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}

Dockerfile

@@ -120,7 +120,7 @@ RUN --mount=type=cache,id=ragflow_npm,target=/root/.npm,sharing=locked \
 
 COPY .git /ragflow/.git
 
-RUN version_info=$(git describe --tags --match=v* --always); \
+RUN version_info=$(git describe --tags --match=v* --first-parent --always); \
     if [ "$LIGHTEN" == "1" ]; then \
         version_info="$version_info slim"; \
     else \

agent/canvas.py

@@ -133,7 +133,8 @@ def __str__(self):
             "components": {}
         }
         for k in self.dsl.keys():
-            if k in ["components"]:continue
+            if k in ["components"]:
+                continue
             dsl[k] = deepcopy(self.dsl[k])
 
         for k, cpn in self.components.items():
@@ -158,7 +159,8 @@ def reset(self):
 
     def get_compnent_name(self, cid):
         for n in self.dsl["graph"]["nodes"]:
-            if cid == n["id"]: return n["data"]["name"]
+            if cid == n["id"]:
+                return n["data"]["name"]
         return ""
 
     def run(self, **kwargs):
@@ -173,7 +175,8 @@ def run(self, **kwargs):
             if kwargs.get("stream"):
                 for an in ans():
                     yield an
-            else: yield ans
+            else:
+                yield ans
             return
 
         if not self.path:
@@ -188,7 +191,8 @@ def run(self, **kwargs):
         def prepare2run(cpns):
             nonlocal ran, ans
             for c in cpns:
-                if self.path[-1] and c == self.path[-1][-1]: continue
+                if self.path[-1] and c == self.path[-1][-1]:
+                    continue
                 cpn = self.components[c]["obj"]
                 if cpn.component_name == "Answer":
                     self.answer.append(c)
@@ -197,7 +201,8 @@ def prepare2run(cpns):
                     if c not in without_dependent_checking:
                         cpids = cpn.get_dependent_components()
                         if any([cc not in self.path[-1] for cc in cpids]):
-                            if c not in waiting: waiting.append(c)
+                            if c not in waiting:
+                                waiting.append(c)
                             continue
                     yield "*'{}'* is running...🕞".format(self.get_compnent_name(c))
                     ans = cpn.run(self.history, **kwargs)
@@ -211,10 +216,12 @@ def prepare2run(cpns):
             logging.debug(f"Canvas.run: {ran} {self.path}")
             cpn_id = self.path[-1][ran]
             cpn = self.get_component(cpn_id)
-            if not cpn["downstream"]: break
+            if not cpn["downstream"]:
+                break
 
             loop = self._find_loop()
-            if loop: raise OverflowError(f"Too much loops: {loop}")
+            if loop:
+                raise OverflowError(f"Too much loops: {loop}")
 
             if cpn["obj"].component_name.lower() in ["switch", "categorize", "relevant"]:
                 switch_out = cpn["obj"].output()[1].iloc[0, 0]
@@ -283,27 +290,30 @@ def get_embedding_model(self):
 
     def _find_loop(self, max_loops=6):
         path = self.path[-1][::-1]
-        if len(path) < 2: return False
+        if len(path) < 2:
+            return False
 
         for i in range(len(path)):
             if path[i].lower().find("answer") >= 0:
                 path = path[:i]
                 break
 
-        if len(path) < 2: return False
+        if len(path) < 2:
+            return False
 
-        for l in range(2, len(path) // 2):
-            pat = ",".join(path[0:l])
+        for loc in range(2, len(path) // 2):
+            pat = ",".join(path[0:loc])
             path_str = ",".join(path)
-            if len(pat) >= len(path_str): return False
+            if len(pat) >= len(path_str):
+                return False
             loop = max_loops
             while path_str.find(pat) == 0 and loop >= 0:
                 loop -= 1
                 if len(pat)+1 >= len(path_str):
                     return False
                 path_str = path_str[len(pat)+1:]
             if loop < 0:
-                pat = " => ".join([p.split(":")[0] for p in path[0:l]])
+                pat = " => ".join([p.split(":")[0] for p in path[0:loc]])
                 return pat + " => " + pat
 
         return False

agent/component/__init__.py

@@ -39,3 +39,73 @@ def component_class(class_name):
     m = importlib.import_module("agent.component")
     c = getattr(m, class_name)
     return c
+
+__all__ = [
+    "Begin",
+    "BeginParam",
+    "Generate",
+    "GenerateParam",
+    "Retrieval",
+    "RetrievalParam",
+    "Answer",
+    "AnswerParam",
+    "Categorize",
+    "CategorizeParam",
+    "Switch",
+    "SwitchParam",
+    "Relevant",
+    "RelevantParam",
+    "Message",
+    "MessageParam",
+    "RewriteQuestion",
+    "RewriteQuestionParam",
+    "KeywordExtract",
+    "KeywordExtractParam",
+    "Concentrator",
+    "ConcentratorParam",
+    "Baidu",
+    "BaiduParam",
+    "DuckDuckGo",
+    "DuckDuckGoParam",
+    "Wikipedia",
+    "WikipediaParam",
+    "PubMed",
+    "PubMedParam",
+    "ArXiv",
+    "ArXivParam",
+    "Google",
+    "GoogleParam",
+    "Bing",
+    "BingParam",
+    "GoogleScholar",
+    "GoogleScholarParam",
+    "DeepL",
+    "DeepLParam",
+    "GitHub",
+    "GitHubParam",
+    "BaiduFanyi",
+    "BaiduFanyiParam",
+    "QWeather",
+    "QWeatherParam",
+    "ExeSQL",
+    "ExeSQLParam",
+    "YahooFinance",
+    "YahooFinanceParam",
+    "WenCai",
+    "WenCaiParam",
+    "Jin10",
+    "Jin10Param",
+    "TuShare",
+    "TuShareParam",
+    "AkShare",
+    "AkShareParam",
+    "Crawler",
+    "CrawlerParam",
+    "Invoke",
+    "InvokeParam",
+    "Template",
+    "TemplateParam",
+    "Email",
+    "EmailParam",
+    "component_class"
+]