[WIP] Thin dependencies significantly

* Remove lint dependencies from dev target, as linting cannot be done
  within the app.  Too many dependency conflicts (anthropic causes
  pydoclint to crash, mypy typing bloats image size ~200 MB, etc).

* Vendor the tiny bits of langchain_community we need to drop that.

Still to-do:

* Pull in lint deps for linting runs (these will fail in GHA right now)

* Finish PyPDF vendoring (two more classes need to be pulled in)

* Test everything fully and completely

Implements: AP-470

Author: awilfox

pyproject.toml

@@ -6,15 +6,13 @@ build-backend = "setuptools.build_meta"
 name = "willa"
 version = "0.0.1"
 dependencies = [
-    "anthropic == 0.67.0", # for pydoclint incompatibility
     "asyncpg",
     "awscli-local",
     "boto3",
     "chainlit ~= 2.7.1",
     "lancedb",
-    "langchain",
     "langchain_aws",
-    "langchain_community",
+    "langchain_core",
     "langchain_ollama",
     "langfuse",
     "langgraph",
@@ -51,7 +49,7 @@ debug = [
     "debugpy",
 ]
 dev = [
-    "willa[test,lint,debug]"
+    "willa[test,debug]"
 ]
 
 [project.scripts]

willa/config/__init__.py

@@ -9,15 +9,19 @@
 from dotenv import dotenv_values
 
 from langchain_aws import BedrockEmbeddings, ChatBedrockConverse
-from langchain_community.vectorstores import LanceDB
 from langchain_core.embeddings import Embeddings
 from langchain_core.language_models import BaseChatModel
 from langchain_core.prompts import ChatPromptTemplate, PromptTemplate
 from langchain_ollama import ChatOllama, OllamaEmbeddings
 from langfuse import Langfuse
 from langfuse.api.resources.commons.errors.not_found_error import NotFoundError
+<<<<<<< Updated upstream
 from langfuse.model import ChatPromptClient
+=======
+
+>>>>>>> Stashed changes
 from willa.errors.config import ImproperConfigurationError
+from willa.lcvendor.lancedb import LanceDB
 
 # Prompt if langfuse defined prompt can't be set
 FALLBACK_PROMPT = """You are a reference librarian who helps researchers answer

willa/etl/doc_proc.py

@@ -12,14 +12,14 @@
 from typing import Any, Optional, Union
 from unittest.mock import MagicMock
 
-from langchain_community.document_loaders import PyPDFLoader, PyPDFDirectoryLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_core.documents import Document
 from langchain_core.vectorstores.base import VectorStore
 from opentelemetry.util._decorator import _AgnosticContextManager
 from pymarc.record import Record
 
 from willa.config import CONFIG, get_langfuse_client
+from willa.lcvendor.pypdf import PyPDFLoader, PyPDFDirectoryLoader
 from willa.tind.format_validate_pymarc import pymarc_to_metadata
 
 

willa/lcvendor/__init__.py

@@ -0,0 +1,3 @@
+"""
+Vendored code from Langchain Community to reduce dependency bloat.
+"""