Merge pull request #5 from Azure-Samples/zavaproductsignite

Add Zava product data and search scripts for Ignite

Author: pamelafox

.env.sample

@@ -1,6 +1,6 @@
 AZURE_OPENAI_SERVICE="SERVICE-NAME"
 AZURE_OPENAI_DEPLOYMENT_NAME=DEPLOYMENT-NAME
-AZURE_OPENAI_ADA_DEPLOYMENT=ADA-DEPLOYMENT
+AZURE_OPENAI_EMBEDDING_DEPLOYMENT=EMBEDDING-DEPLOYMENT
 AZURE_SEARCH_SERVICE=SEARCH-SERVICE-NAME
 AZURE_VISION_ENDPOINT=https://YOUR-RESOURCE-NAME.cognitiveservices.azure.com
 AZURE_TENANT_ID=TENANT-ID

README.md

@@ -4,14 +4,14 @@ This repository contains many notebooks that explain how Azure AI Search works,
 
 ## Environment setup
 
-1. Run `azd up` on [azure-search-openai-demo](https://github.com/Azure-Samples/azure-search-openai-demo/) with GPT vision feature enabled. This will create the necessary resources for the Azure OpenAI, Azure AI Search services, and the Azure AI Vision service.
+1. Run `azd up` on [azure-search-openai-demo](https://github.com/Azure-Samples/azure-search-openai-demo/) with multimodal feature enabled. This will create the necessary resources for the Azure OpenAI, Azure AI Search services, and the Azure AI Vision service.
 
 2. Create a .env with these variables, with the values taken from `.azure/ENV-NAME/.env` in the azure-search-openai-demo repository.
 
     ```shell
     AZURE_OPENAI_SERVICE=YOUR-SERVICE-NAME
     AZURE_OPENAI_DEPLOYMENT_NAME=YOUR-OPENAI-DEPLOYMENT-NAME
-    AZURE_OPENAI_ADA_DEPLOYMENT=YOUR-EMBED-DEPLOYMENT-NAME
+    AZURE_OPENAI_EMBEDDING_DEPLOYMENT=YOUR-EMBED-DEPLOYMENT-NAME
     AZURE_SEARCH_SERVICE=YOUR-SEARCH-SERVICE-NAME
     AZURE_COMPUTERVISION_SERVICE=YOUR-COMPUTERVISION-SERVICE-NAME
     AZURE_TENANT_ID=YOUR-TENANT-ID
@@ -31,9 +31,9 @@ This repository contains many notebooks that explain how Azure AI Search works,
     pip install -r requirements.txt
     ```
 
-## Notebooks
+## Search on documents
 
-These are the available notebooks, in suggested order:
+These notebooks operate on the index from the [azure-search-openai-demo](https://github.com/Azure-Samples/azure-search-openai-demo/) repository, which contains chunked documents from a fictional company.
 
 * [Vector Embeddings Notebook](./vector_embeddings.ipynb)
 * [Azure AI Search Notebook](./azure_ai_search.ipynb)
@@ -43,3 +43,19 @@ These are the available notebooks, in suggested order:
 * [RAG Evaluation](./rag_eval.ipynb)
 
 You can find video recordings going through the notebooks [here](https://github.com/microsoft/aitour-rag-with-ai-search/tree/main/session-delivery-resources#video-recordings).
+
+## Search on product catalog
+
+You can also try out search techniques on a Zava product catalog.
+First, create the search index and upload the products by running:
+
+```shell
+python zava_product_upload.py
+```
+
+Then, explore the different search techniques with these Python scripts:
+
+* [Keyword Search](./zava_search_keyword.py)
+* [Vector Search](./zava_search_vector.py)
+* [Hybrid Search with RRF](./zava_search_rrf.py)
+* [Hybrid Search with RRF + Reranker](./zava_search_reranker.py)

azure_ai_search.ipynb

@@ -190,7 +190,7 @@
     "# Set up OpenAI client based on environment variables\n",
     "dotenv.load_dotenv()\n",
     "AZURE_OPENAI_SERVICE = os.getenv(\"AZURE_OPENAI_SERVICE\")\n",
-    "AZURE_OPENAI_ADA_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_ADA_DEPLOYMENT\")\n",
+    "AZURE_OPENAI_EMBEDDING_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYMENT\")\n",
     "\n",
     "token_provider = azure.identity.get_bearer_token_provider(azure_credential, \"https://cognitiveservices.azure.com/.default\")\n",
     "openai_client = openai.AzureOpenAI(\n",
@@ -199,7 +199,7 @@
     "    azure_ad_token_provider=token_provider)\n",
     "\n",
     "def get_embedding(text):\n",
-    "    get_embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_ADA_DEPLOYMENT, input=text)\n",
+    "    get_embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_EMBEDDING_DEPLOYMENT, input=text)\n",
     "    return get_embeddings_response.data[0].embedding"
    ]
   },

rag.ipynb

@@ -34,7 +34,7 @@
     "\n",
     "# Initialize Azure OpenAI client\n",
     "AZURE_OPENAI_SERVICE = os.getenv(\"AZURE_OPENAI_SERVICE\")\n",
-    "AZURE_OPENAI_ADA_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_ADA_DEPLOYMENT\")\n",
+    "AZURE_OPENAI_EMBEDDING_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYMENT\")\n",
     "\n",
     "token_provider = azure.identity.get_bearer_token_provider(azure_credential, \"https://cognitiveservices.azure.com/.default\")\n",
     "openai_client = openai.AzureOpenAI(\n",
@@ -43,7 +43,7 @@
     "    azure_ad_token_provider=token_provider)\n",
     "\n",
     "def get_embedding(text):\n",
-    "    get_embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_ADA_DEPLOYMENT, input=text)\n",
+    "    get_embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_EMBEDDING_DEPLOYMENT, input=text)\n",
     "    return get_embeddings_response.data[0].embedding\n",
     "\n",
     "# Initialize Azure search client\n",

rag_eval.ipynb

@@ -34,7 +34,7 @@
     "\n",
     "# Initialize Azure OpenAI client\n",
     "AZURE_OPENAI_SERVICE = os.getenv(\"AZURE_OPENAI_SERVICE\")\n",
-    "AZURE_OPENAI_ADA_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_ADA_DEPLOYMENT\")\n",
+    "AZURE_OPENAI_EMBEDDING_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYMENT\")\n",
     "\n",
     "token_provider = azure.identity.get_bearer_token_provider(azure_credential, \"https://cognitiveservices.azure.com/.default\")\n",
     "openai_client = openai.AzureOpenAI(\n",
@@ -43,7 +43,7 @@
     "    azure_ad_token_provider=token_provider)\n",
     "\n",
     "def get_embedding(text):\n",
-    "    get_embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_ADA_DEPLOYMENT, input=text)\n",
+    "    get_embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_EMBEDDING_DEPLOYMENT, input=text)\n",
     "    return get_embeddings_response.data[0].embedding\n",
     "\n",
     "# Initialize Azure search client\n",

render_table.py

@@ -0,0 +1,46 @@
+"""Utility for rendering search results as rich tables."""
+
+from typing import Any
+
+from rich.console import Console
+from rich.table import Table
+
+
+def render_product_results(results: list[dict[str, Any]], title: str, show_reranker: bool = False) -> None:
+    """Render product search results as a rich table.
+
+    Args:
+        results: List of search result documents
+        title: Title for the table
+        show_reranker: Whether to show reranker score column
+    """
+    console = Console()
+    table = Table(title=title, show_lines=True)
+
+    table.add_column("Score", justify="right", style="cyan", width=8)
+    if show_reranker:
+        table.add_column("Reranker", justify="right", style="red", width=8)
+    table.add_column("Name", style="magenta", width=25)
+    table.add_column("Description", style="white", width=50)
+    table.add_column("Categories", style="blue", width=20)
+    table.add_column("Price", justify="right", style="yellow", width=8)
+    table.add_column("SKU", style="green", width=12)
+
+    for doc in list(results):
+        row = [
+            f"{doc['@search.score']:.3f}",
+        ]
+        if show_reranker:
+            row.append(f"{doc['@search.reranker_score']:.3f}")
+        row.extend(
+            [
+                doc["name"],
+                doc["description"][:80] + "..." if len(doc["description"]) > 80 else doc["description"],
+                ", ".join(doc["categories"]),
+                f"${doc['price']:.2f}",
+                doc["sku"],
+            ]
+        )
+        table.add_row(*row)
+
+    console.print(table)

requirements.txt

@@ -1,9 +1,10 @@
-openai
+openai==1.109.1
 numpy
 pandas
 azure-identity
 azure-search-documents
 python-dotenv
 pillow
 azure-ai-evaluation
-ipykernel
+ipykernel
+rich

search_relevance.ipynb

@@ -16,7 +16,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -30,11 +30,11 @@
     "\n",
     "dotenv.load_dotenv()\n",
     "\n",
-    "azure_credential = azure.identity.AzureDeveloperCliCredential(tenant_id=os.getenv(\"AZURE_TENANT_ID\"))\n",
+    "azure_credential = azure.identity.DefaultAzureCredential()\n",
     "\n",
     "# Initialize Azure OpenAI client\n",
     "AZURE_OPENAI_SERVICE = os.getenv(\"AZURE_OPENAI_SERVICE\")\n",
-    "AZURE_OPENAI_ADA_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_ADA_DEPLOYMENT\")\n",
+    "AZURE_OPENAI_EMBEDDING_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYMENT\")\n",
     "\n",
     "token_provider = azure.identity.get_bearer_token_provider(azure_credential, \"https://cognitiveservices.azure.com/.default\")\n",
     "openai_client = openai.AzureOpenAI(\n",
@@ -43,7 +43,7 @@
     "    azure_ad_token_provider=token_provider)\n",
     "\n",
     "def get_embedding(text):\n",
-    "    get_embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_ADA_DEPLOYMENT, input=text)\n",
+    "    get_embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_EMBEDDING_DEPLOYMENT, input=text)\n",
     "    return get_embeddings_response.data[0].embedding\n",
     "\n",
     "# Initialize Azure search client\n",
@@ -72,9 +72,9 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Score: 0.81581\tMatching text: Not found\n",
-      "Score: 0.80998\tMatching text: Not found\n",
-      "Score: 0.80965\tMatching text: Not found\n"
+      "Score: 0.81510\tMatching text: Not found\n",
+      "Score: 0.80934\tMatching text: Not found\n",
+      "Score: 0.80891\tMatching text: Not found\n"
      ]
     }
    ],
@@ -239,7 +239,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": ".venv (3.11.9)",
    "language": "python",
    "name": "python3"
   },
@@ -253,7 +253,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.0"
+   "version": "3.11.9"
   },
   "orig_nbformat": 4
  },

vector_embeddings.ipynb

@@ -29,7 +29,7 @@
     "# Set up OpenAI client based on environment variables\n",
     "dotenv.load_dotenv()\n",
     "AZURE_OPENAI_SERVICE = os.getenv(\"AZURE_OPENAI_SERVICE\")\n",
-    "AZURE_OPENAI_ADA_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_ADA_DEPLOYMENT\")\n",
+    "AZURE_OPENAI_EMBEDDING_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYMENT\")\n",
     "\n",
     "azure_credential = azure.identity.AzureDeveloperCliCredential(tenant_id=os.getenv(\"AZURE_TENANT_ID\"))\n",
     "token_provider = azure.identity.get_bearer_token_provider(azure_credential,\n",
@@ -55,7 +55,7 @@
    "source": [
     "sentence = \"A dog just walked past my house and yipped yipped like a Martian\"\n",
     "\n",
-    "response = openai_client.embeddings.create(model=AZURE_OPENAI_ADA_DEPLOYMENT, input=sentence)\n",
+    "response = openai_client.embeddings.create(model=AZURE_OPENAI_EMBEDDING_DEPLOYMENT, input=sentence)\n",
     "\n",
     "vector = response.data[0].embedding"
    ]
@@ -1139,7 +1139,7 @@
     "              'djkshsjdkhfsjdfkhsd']\n",
     "\n",
     "def get_embeddings(sentences):\n",
-    "    embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_ADA_DEPLOYMENT, input=sentences)\n",
+    "    embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_EMBEDDING_DEPLOYMENT, input=sentences)\n",
     "    return [embedding_object.embedding for embedding_object in embeddings_response.data]\n",
     "\n",
     "embeddings1 = get_embeddings(sentences1)\n",
@@ -1277,7 +1277,7 @@
     "# Compute vector for query\n",
     "query = \"101 Dalmations\"\n",
     "\n",
-    "embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_ADA_DEPLOYMENT, input=[query])\n",
+    "embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_EMBEDDING_DEPLOYMENT, input=[query])\n",
     "vector = embeddings_response.data[0].embedding\n",
     "\n",
     "# Compute cosine similarity between query and each movie title\n",