multiple: pydantic 2 compatibility, v0.3 (#26443)

Signed-off-by: ChengZi <chen.zhang@zilliz.com>
Co-authored-by: Eugene Yurtsev <eyurtsev@gmail.com>
Co-authored-by: Bagatur <22008038+baskaryan@users.noreply.github.com>
Co-authored-by: Dan O'Donovan <dan.odonovan@gmail.com>
Co-authored-by: Tom Daniel Grande <tomdgrande@gmail.com>
Co-authored-by: Grande <Tom.Daniel.Grande@statsbygg.no>
Co-authored-by: Bagatur <baskaryan@gmail.com>
Co-authored-by: ccurme <chester.curme@gmail.com>
Co-authored-by: Harrison Chase <hw.chase.17@gmail.com>
Co-authored-by: Tomaz Bratanic <bratanic.tomaz@gmail.com>
Co-authored-by: ZhangShenao <15201440436@163.com>
Co-authored-by: Friso H. Kingma <fhkingma@gmail.com>
Co-authored-by: ChengZi <chen.zhang@zilliz.com>
Co-authored-by: Nuno Campos <nuno@langchain.dev>
Co-authored-by: Morgante Pell <morgantep@google.com>

.github/scripts/check_diff.py

@@ -2,10 +2,12 @@
 import json
 import os
 import sys
-import tomllib
 from collections import defaultdict
 from typing import Dict, List, Set
 from pathlib import Path
+import tomllib
+
+from get_min_versions import get_min_version_from_toml
 
 
 LANGCHAIN_DIRS = [
@@ -16,6 +18,12 @@
     "libs/experimental",
 ]
 
+# when set to True, we are ignoring core dependents
+# in order to be able to get CI to pass for each individual
+# package that depends on core
+# e.g. if you touch core, we don't then add textsplitters/etc to CI
+IGNORE_CORE_DEPENDENTS = False
+
 # ignored partners are removed from dependents
 # but still run if directly edited
 IGNORED_PARTNERS = [
@@ -99,44 +107,96 @@ def add_dependents(dirs_to_eval: Set[str], dependents: dict) -> List[str]:
 
 
 def _get_configs_for_single_dir(job: str, dir_: str) -> List[Dict[str, str]]:
-    if dir_ == "libs/core":
-        return [
-            {"working-directory": dir_, "python-version": f"3.{v}"}
-            for v in range(8, 13)
-        ]
-    min_python = "3.8"
-    max_python = "3.12"
+    if job == "test-pydantic":
+        return _get_pydantic_test_configs(dir_)
 
+    if dir_ == "libs/core":
+        py_versions = ["3.9", "3.10", "3.11", "3.12"]
     # custom logic for specific directories
-    if dir_ == "libs/partners/milvus":
+    elif dir_ == "libs/partners/milvus":
         # milvus poetry doesn't allow 3.12 because they
         # declare deps in funny way
-        max_python = "3.11"
+        py_versions = ["3.9", "3.11"]
 
-    if dir_ in ["libs/community", "libs/langchain"] and job == "extended-tests":
+    elif dir_ in ["libs/community", "libs/langchain"] and job == "extended-tests":
         # community extended test resolution in 3.12 is slow
         # even in uv
-        max_python = "3.11"
+        py_versions = ["3.9", "3.11"]
 
-    if dir_ == "libs/community" and job == "compile-integration-tests":
+    elif dir_ == "libs/community" and job == "compile-integration-tests":
         # community integration deps are slow in 3.12
-        max_python = "3.11"
+        py_versions = ["3.9", "3.11"]
+    else:
+        py_versions = ["3.9", "3.12"]
 
-    return [
-        {"working-directory": dir_, "python-version": min_python},
-        {"working-directory": dir_, "python-version": max_python},
+    return [{"working-directory": dir_, "python-version": py_v} for py_v in py_versions]
+
+
+def _get_pydantic_test_configs(
+    dir_: str, *, python_version: str = "3.11"
+) -> List[Dict[str, str]]:
+    with open("./libs/core/poetry.lock", "rb") as f:
+        core_poetry_lock_data = tomllib.load(f)
+    for package in core_poetry_lock_data["package"]:
+        if package["name"] == "pydantic":
+            core_max_pydantic_minor = package["version"].split(".")[1]
+            break
+
+    with open(f"./{dir_}/poetry.lock", "rb") as f:
+        dir_poetry_lock_data = tomllib.load(f)
+
+    for package in dir_poetry_lock_data["package"]:
+        if package["name"] == "pydantic":
+            dir_max_pydantic_minor = package["version"].split(".")[1]
+            break
+
+    core_min_pydantic_version = get_min_version_from_toml(
+        "./libs/core/pyproject.toml", "release", python_version, include=["pydantic"]
+    )["pydantic"]
+    core_min_pydantic_minor = core_min_pydantic_version.split(".")[1] if "." in core_min_pydantic_version else "0"
+    dir_min_pydantic_version = (
+        get_min_version_from_toml(
+            f"./{dir_}/pyproject.toml", "release", python_version, include=["pydantic"]
+        )
+        .get("pydantic", "0.0.0")
+    )
+    dir_min_pydantic_minor = dir_min_pydantic_version.split(".")[1] if "." in dir_min_pydantic_version else "0"
+
+    custom_mins = {
+        # depends on pydantic-settings 2.4 which requires pydantic 2.7
+        "libs/community": 7,
+    }
+
+    max_pydantic_minor = min(
+        int(dir_max_pydantic_minor),
+        int(core_max_pydantic_minor),
+    )
+    min_pydantic_minor = max(
+        int(dir_min_pydantic_minor),
+        int(core_min_pydantic_minor),
+        custom_mins.get(dir_, 0),
+    )
+
+    configs = [
+        {
+            "working-directory": dir_,
+            "pydantic-version": f"2.{v}.0",
+            "python-version": python_version,
+        }
+        for v in range(min_pydantic_minor, max_pydantic_minor + 1)
     ]
+    return configs
 
 
 def _get_configs_for_multi_dirs(
-    job: str, dirs_to_run: List[str], dependents: dict
+    job: str, dirs_to_run: Dict[str, Set[str]], dependents: dict
 ) -> List[Dict[str, str]]:
     if job == "lint":
         dirs = add_dependents(
             dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"],
             dependents,
         )
-    elif job in ["test", "compile-integration-tests", "dependencies"]:
+    elif job in ["test", "compile-integration-tests", "dependencies", "test-pydantic"]:
         dirs = add_dependents(
             dirs_to_run["test"] | dirs_to_run["extended-test"], dependents
         )
@@ -165,6 +225,7 @@ def _get_configs_for_multi_dirs(
         dirs_to_run["lint"] = all_package_dirs()
         dirs_to_run["test"] = all_package_dirs()
         dirs_to_run["extended-test"] = set(LANGCHAIN_DIRS)
+
     for file in files:
         if any(
             file.startswith(dir_)
@@ -182,8 +243,12 @@ def _get_configs_for_multi_dirs(
         if any(file.startswith(dir_) for dir_ in LANGCHAIN_DIRS):
             # add that dir and all dirs after in LANGCHAIN_DIRS
             # for extended testing
+
             found = False
             for dir_ in LANGCHAIN_DIRS:
+                if dir_ == "libs/core" and IGNORE_CORE_DEPENDENTS:
+                    dirs_to_run["extended-test"].add(dir_)
+                    continue
                 if file.startswith(dir_):
                     found = True
                 if found:
@@ -224,7 +289,6 @@ def _get_configs_for_multi_dirs(
 
     # we now have dirs_by_job
     # todo: clean this up
-
     map_job_to_configs = {
         job: _get_configs_for_multi_dirs(job, dirs_to_run, dependents)
         for job in [
@@ -233,6 +297,7 @@ def _get_configs_for_multi_dirs(
             "extended-tests",
             "compile-integration-tests",
             "dependencies",
+            "test-pydantic",
         ]
     }
     map_job_to_configs["test-doc-imports"] = (

.github/scripts/check_prerelease_dependencies.py

@@ -11,7 +11,7 @@
 
     # see if we're releasing an rc
     version = toml_data["tool"]["poetry"]["version"]
-    releasing_rc = "rc" in version
+    releasing_rc = "rc" in version or "dev" in version
 
     # if not, iterate through dependencies and make sure none allow prereleases
     if not releasing_rc:

.github/scripts/get_min_versions.py

@@ -1,4 +1,5 @@
 import sys
+from typing import Optional
 
 if sys.version_info >= (3, 11):
     import tomllib
@@ -7,6 +8,9 @@
     import tomli as tomllib
 
 from packaging.version import parse as parse_version
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
+
 import re
 
 MIN_VERSION_LIBS = [
@@ -15,6 +19,7 @@
     "langchain",
     "langchain-text-splitters",
     "SQLAlchemy",
+    "pydantic",
 ]
 
 SKIP_IF_PULL_REQUEST = ["langchain-core"]
@@ -45,7 +50,13 @@ def get_min_version(version: str) -> str:
     raise ValueError(f"Unrecognized version format: {version}")
 
 
-def get_min_version_from_toml(toml_path: str, versions_for: str):
+def get_min_version_from_toml(
+    toml_path: str,
+    versions_for: str,
+    python_version: str,
+    *,
+    include: Optional[list] = None,
+):
     # Parse the TOML file
     with open(toml_path, "rb") as file:
         toml_data = tomllib.load(file)
@@ -64,11 +75,20 @@ def get_min_version_from_toml(toml_path: str, versions_for: str):
             continue
         # Check if the lib is present in the dependencies
         if lib in dependencies:
+            if include and lib not in include:
+                continue
             # Get the version string
             version_string = dependencies[lib]
 
             if isinstance(version_string, dict):
                 version_string = version_string["version"]
+            if isinstance(version_string, list):
+                version_string = [
+                    vs
+                    for vs in version_string
+                    if check_python_version(python_version, vs["python"])
+                ][0]["version"]
+
 
             # Use parse_version to get the minimum supported version from version_string
             min_version = get_min_version(version_string)
@@ -79,13 +99,31 @@ def get_min_version_from_toml(toml_path: str, versions_for: str):
     return min_versions
 
 
+def check_python_version(version_string, constraint_string):
+    """
+    Check if the given Python version matches the given constraints.
+
+    :param version_string: A string representing the Python version (e.g. "3.8.5").
+    :param constraint_string: A string representing the package's Python version constraints (e.g. ">=3.6, <4.0").
+    :return: True if the version matches the constraints, False otherwise.
+    """
+    try:
+        version = Version(version_string)
+        constraints = SpecifierSet(constraint_string)
+        return version in constraints
+    except Exception as e:
+        print(f"Error: {e}")
+        return False
+
+
 if __name__ == "__main__":
     # Get the TOML file path from the command line argument
     toml_file = sys.argv[1]
     versions_for = sys.argv[2]
+    python_version = sys.argv[3]
     assert versions_for in ["release", "pull_request"]
 
     # Call the function to get the minimum versions
-    min_versions = get_min_version_from_toml(toml_file, versions_for)
+    min_versions = get_min_version_from_toml(toml_file, versions_for, python_version)
 
     print(" ".join([f"{lib}=={version}" for lib, version in min_versions.items()]))