Bring back pydantic 2 support (#11756) (#11757)

* Loosen pydantic maximum to <3 (allowing for pydantic 2)

* Add an internal pydantic shim for getting pydantic BaseSettings reguardless of pydantic v1 vs v2

* Add changie doc

(cherry picked from commit 70ad931)

Co-authored-by: Quigley Malcolm <[email protected]>

Author: github-actions[bot]

.changes/unreleased/Dependencies-20250620-123600.yaml

@@ -0,0 +1,6 @@
+kind: Dependencies
+body: Allow for either pydantic v1 and v2
+time: 2025-06-20T12:36:00.196384-05:00
+custom:
+  Author: QMalcolm
+  Issue: "11634"

core/dbt/_pydantic_shim.py

@@ -0,0 +1,26 @@
+# type: ignore
+
+"""Shim to allow support for both Pydantic 1 and Pydantic 2.
+
+dbt-core must support both major versions of Pydantic because dbt-core users might be using an environment with
+either version, and we can't restrict them to one or the other. Here, we essentially import all Pydantic objects
+from version 1 that we use. Throughout the repo, we import these objects from this file instead of from Pydantic
+directly, meaning that we essentially only use Pydantic 1 in dbt-core currently, but without forcing that restriction
+on dbt users. The development environment for this repo should be pinned to Pydantic 1 to ensure devs get appropriate
+type hints.
+"""
+
+from importlib.metadata import version
+
+pydantic_version = version("pydantic")
+# Pydantic uses semantic versioning, i.e. <major>.<minor>.<patch>, and we need to know the major
+pydantic_major = pydantic_version.split(".")[0]
+
+if pydantic_major == "1":
+    from pydantic import BaseSettings  # noqa: F401
+elif pydantic_major == "2":
+    from pydantic.v1 import BaseSettings  # noqa: F401
+else:
+    raise RuntimeError(
+        f"Currently only pydantic 1 and 2 are supported, found pydantic {pydantic_version}"
+    )

core/dbt/utils/artifact_upload.py

@@ -3,9 +3,9 @@
 import zipfile
 
 import requests
-from pydantic import BaseSettings
 
 import dbt.tracking
+from dbt._pydantic_shim import BaseSettings  # type: ignore
 from dbt.config.runtime import UnsetProfile, load_project
 from dbt.constants import MANIFEST_FILE_NAME, RUN_RESULTS_FILE_NAME
 from dbt.events.types import ArtifactUploadSkipped, ArtifactUploadSuccess

core/setup.py

@@ -75,14 +75,14 @@
         "dbt-common>=1.22.0,<2.0",
         "dbt-adapters>=1.15.2,<2.0",
         "dbt-protos>=1.0.315,<2.0",
+        "pydantic<3",
         # ----
         # Expect compatibility with all new versions of these packages, so lower bounds only.
         "packaging>20.9",
         "pytz>=2015.7",
         "pyyaml>=6.0",
         "daff>=1.3.46",
         "typing-extensions>=4.4",
-        "pydantic<2",
         # ----
     ],
     zip_safe=False,