[Chore] Datadog CI integration (#403)

## Problem

To continue driving performance improvements in the SDK and avoid major
regressions, we need to track performance over time. First step in that
is figuring out how to send data over to datadog with ddtrace so we can
begin to have a historical overview of how perf is changing over time.

Thanks to @ssmith-pc for pointing me in the right direction on how to
get started with this stuff.

## Solution

- Add ddtrace dev dependency
- Adjust CI configurations to set env variables expected by ddtrace

Probably later I will need to:
- Develop some benchmark tests specifically exercising the areas where
performance is most critical.
- Setup some dashboards on the datadog side where we can easily find and
interpret the data


Not sure if this link will be visible to others, but here's where I see
data flowing into datadog
https://app.datadoghq.com/ci/test-runs?saved-view-id=3068550

## Type of Change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update
- [x] Infrastructure change (CI configs, etc)
- [ ] Non-code change (docs, etc)
- [ ] None of the above: (explain here)

Author: jhamon

.github/actions/test-data-plane/action.yaml

@@ -29,6 +29,9 @@ inputs:
     description: 'The version of Python to use'
     required: false
     default: '3.9'
+  DATADOG_API_KEY:
+    description: 'The Datadog API key'
+    required: true
 
 outputs:
   index_name:
@@ -52,10 +55,15 @@ runs:
     - name: Run data plane tests
       id: data-plane-tests
       shell: bash
-      run: poetry run pytest tests/integration/data
+      run: poetry run pytest tests/integration/data --ddtrace
       env:
+        DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+        DD_API_KEY: ${{ inputs.DATADOG_API_KEY }}
+        DD_SITE: datadoghq.com
+        DD_ENV: ci
+        DD_SERVICE: pinecone-python-client
         PINECONE_API_KEY: ${{ inputs.PINECONE_API_KEY }}
         USE_GRPC: ${{ inputs.use_grpc }}
         METRIC: ${{ inputs.metric }}
         SPEC: ${{ inputs.spec }}
-        FRESHNESS_TIMEOUT_SECONDS: ${{ inputs.freshness_timeout_seconds }}
+        FRESHNESS_TIMEOUT_SECONDS: ${{ inputs.freshness_timeout_seconds }}

.github/actions/test-dependency-grpc/action.yaml

@@ -42,25 +42,25 @@ runs:
     - name: Install grpcio ${{ inputs.grpcio_version }}
       run: poetry add grpcio==${{ inputs.grpcio_version }}
       shell: bash
-    
+
     - name: Install lz4 ${{ inputs.lz4_version }}
       run: poetry add lz4==${{ inputs.lz4_version }}
       shell: bash
-    
+
     - name: Install protobuf ${{ inputs.protobuf_version }}
       run: poetry add protobuf==${{ inputs.protobuf_version }}
       shell: bash
-    
+
     - name: Install googleapis-common-protos ${{ inputs.googleapis_common_protos_version }}
       run: poetry add googleapis-common-protos==${{ inputs.googleapis_common_protos_version }}
       shell: bash
-    
+
     - uses: nick-fields/retry@v3
       with:
         timeout_minutes: 5
         max_attempts: 3
         retry_on: error
-        command: poetry run pytest tests/dependency/grpc -s -v
+        command: poetry run pytest tests/dependency/grpc -s -v --ddtrace
       env:
         PINECONE_API_KEY: ${{ inputs.PINECONE_API_KEY }}
-        INDEX_NAME: ${{ inputs.index_name }}  
+        INDEX_NAME: ${{ inputs.index_name }}

.github/actions/test-dependency-rest/action.yaml

@@ -39,7 +39,7 @@ runs:
         timeout_minutes: 5
         max_attempts: 3
         retry_on: error
-        command: poetry run pytest tests/dependency/rest -s -v
+        command: poetry run pytest tests/dependency/rest -s -v --ddtrace
       env:
         PINECONE_API_KEY: '${{ inputs.PINECONE_API_KEY }}'
-        INDEX_NAME: '${{ inputs.index_name }}'
+        INDEX_NAME: '${{ inputs.index_name }}'

.github/workflows/testing-dependency.yaml

@@ -3,6 +3,13 @@ name: Dependency Testing
 on:
   workflow_call: {}
 
+env:
+  DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+  DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+  DD_SITE: datadoghq.com
+  DD_ENV: ci
+  DD_SERVICE: pinecone-python-client
+
 jobs:
   dependency-matrix-setup:
     name: Deps setup
@@ -72,6 +79,7 @@ jobs:
           lz4_version: '${{ matrix.lz4_version }}'
           protobuf_version: '${{ matrix.protobuf_version }}'
           googleapis_common_protos_version: '${{ matrix.googleapis_common_protos_version }}'
+          DD_TAGS: 'test.configuration.python:${{ matrix.python_version}},test.configuration.grpcio:${{ matrix.grpcio_version }},test.configuration.lz4:${{ matrix.lz4_version }},test.configuration.protobuf:${{ matrix.protobuf_version }},test.configuration.googleapis_common_protos:${{ matrix.googleapis_common_protos_version }}'
 
   dependency-matrix-grpc-312:
     name: Deps (GRPC)
@@ -107,6 +115,7 @@ jobs:
           lz4_version: '${{ matrix.lz4_version }}'
           protobuf_version: '${{ matrix.protobuf_version }}'
           googleapis_common_protos_version: '${{ matrix.googleapis_common_protos_version }}'
+          DD_TAGS: 'test.configuration.python:${{ matrix.python_version}},test.configuration.grpcio:${{ matrix.grpcio_version }},test.configuration.lz4:${{ matrix.lz4_version }},test.configuration.protobuf:${{ matrix.protobuf_version }},test.configuration.googleapis_common_protos:${{ matrix.googleapis_common_protos_version }}'
 
   dependency-matrix-rest:
     name: Deps (REST)
@@ -132,7 +141,8 @@ jobs:
           index_name: '${{ needs.dependency-matrix-setup.outputs.index_name }}'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
           urllib3_version: '${{ matrix.urllib3_version }}'
-      
+          DD_TAGS: 'test.configuration.python:${{ matrix.python_version}},test.configuration.urllib3:${{ matrix.urllib3_version }}'
+
   dependency-matrix-rest-312:
     name: Deps (REST)
     runs-on: ubuntu-latest
@@ -156,6 +166,7 @@ jobs:
           index_name: '${{ needs.dependency-matrix-setup.outputs.index_name }}'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
           urllib3_version: '${{ matrix.urllib3_version }}'
+          DD_TAGS: 'test.configuration.python:${{ matrix.python_version}},test.configuration.urllib3:${{ matrix.urllib3_version }}'
 
   deps-cleanup:
     name: Deps cleanup
@@ -171,4 +182,4 @@ jobs:
       - uses: ./.github/actions/delete-index
         with:
           index_name: '${{ needs.dependency-matrix-setup.outputs.index_name }}'
-          PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
+          PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'

.github/workflows/testing-integration.yaml

@@ -2,6 +2,13 @@ name: "Integration Tests"
 'on':
   workflow_call: {}
 
+env:
+  DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+  DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+  DD_SITE: datadoghq.com
+  DD_ENV: ci
+  DD_SERVICE: pinecone-python-client
+
 jobs:
   plugin-inference:
     name: Test inference plugin
@@ -20,7 +27,7 @@ jobs:
         with:
           include_grpc: 'true'
       - name: 'Run integration tests'
-        run: poetry run pytest tests/integration/inference -s -vv
+        run: poetry run pytest tests/integration/inference -s -vv --ddtrace
         env:
           PINECONE_DEBUG_CURL: 'true'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
@@ -33,7 +40,7 @@ jobs:
       matrix:
         python_version: [3.8, 3.12]
         use_grpc: [true, false]
-        metric: 
+        metric:
           - cosine
           # - euclidean
           # - dotproduct
@@ -43,6 +50,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test-data-plane
         with:
+          DATADOG_API_KEY: '${{ secrets.DATADOG_API_KEY }}'
           python_version: '${{ matrix.python_version }}'
           use_grpc: '${{ matrix.use_grpc }}'
           metric: '${{ matrix.metric }}'
@@ -91,7 +99,7 @@ jobs:
         uses: ./.github/actions/setup-poetry
       - name: 'Run integration tests (REST, prod)'
         if: matrix.pineconeEnv == 'prod'
-        run: poetry run pytest tests/integration/control/pod -s -v
+        run: poetry run pytest tests/integration/control/pod -s -v --ddtrace
         env:
           PINECONE_DEBUG_CURL: 'true'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
@@ -101,7 +109,7 @@ jobs:
           METRIC: 'cosine'
       - name: 'Run integration tests (REST, staging)'
         if: matrix.pineconeEnv == 'staging'
-        run: poetry run pytest tests/integration/control/pod -s -v
+        run: poetry run pytest tests/integration/control/pod -s -v --ddtrace
         env:
           PINECONE_DEBUG_CURL: 'true'
           PINECONE_CONTROLLER_HOST: 'https://api-staging.pinecone.io'
@@ -136,7 +144,7 @@ jobs:
         uses: ./.github/actions/setup-poetry
       - name: 'Run integration tests (REST, prod)'
         if: matrix.pineconeEnv == 'prod'
-        run: poetry run pytest tests/integration/control/serverless -s -vv
+        run: poetry run pytest tests/integration/control/serverless -s -vv --ddtrace
         env:
           PINECONE_DEBUG_CURL: 'true'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
@@ -145,7 +153,7 @@ jobs:
           SERVERLESS_REGION: '${{ matrix.testConfig.serverless.region }}'
       - name: 'Run integration tests (REST, staging)'
         if: matrix.pineconeEnv == 'staging'
-        run: poetry run pytest tests/integration/control/serverless -s -vv
+        run: poetry run pytest tests/integration/control/serverless -s -vv --ddtrace
         env:
           PINECONE_DEBUG_CURL: 'true'
           PINECONE_CONTROLLER_HOST: 'https://api-staging.pinecone.io'

.github/workflows/testing-unit.yaml

@@ -2,6 +2,13 @@ name: "Unit Tests"
 'on':
   workflow_call: {}
 
+env:
+  DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+  DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+  DD_SITE: datadoghq.com
+  DD_ENV: ci
+  DD_SERVICE: pinecone-python-client
+
 jobs:
   unit-tests:
     name: Unit tests
@@ -30,10 +37,10 @@ jobs:
           include_grpc: '${{ matrix.use_grpc }}'
           include_types: true
       - name: Run unit tests (REST)
-        run: poetry run pytest --cov=pinecone --timeout=120 tests/unit
+        run: poetry run pytest --cov=pinecone --timeout=120 tests/unit --ddtrace
       - name: Run unit tests (GRPC)
         if: ${{ matrix.use_grpc == true }}
-        run: poetry run pytest --cov=pinecone/grpc --timeout=120 tests/unit_grpc
+        run: poetry run pytest --cov=pinecone/grpc --timeout=120 tests/unit_grpc --ddtrace
       - name: mypy check
         env:
           INCLUDE_GRPC: '${{ matrix.use_grpc }}'