WIP TASK: Use Responses API instead of deprecated Assistant API

.github/workflows/build.yml

@@ -36,5 +36,8 @@ jobs:
             - name: Set Neos Version
               run: composer require neos/neos ^${{ matrix.neos-versions }} --no-progress --no-interaction
 
+            - name: Run Linters
+              run: composer lint
+
             - name: Run Tests
               run: composer test

Classes/Command/KnowledgeCommandController.php

@@ -6,53 +6,132 @@
 
 use Neos\Flow\Annotations as Flow;
 use Neos\Flow\Cli\CommandController;
+use Neos\Media\Domain\Repository\DocumentRepository;
+use Sitegeist\Chatterbox\Domain\AssistantEntity;
+use Sitegeist\Chatterbox\Domain\AssistantEntityRepository;
+use Sitegeist\Chatterbox\Domain\Knowledge\DocumentCollection;
+use Sitegeist\Chatterbox\Domain\Knowledge\SourceOfKnowledgeContract;
+use Sitegeist\Chatterbox\Domain\Knowledge\SourceOfKnowledgeRepository;
+use Sitegeist\Chatterbox\Domain\Knowledge\VectorStoreReference;
+use Sitegeist\Chatterbox\Domain\Knowledge\VectorStoreReferenceRepository;
+use Sitegeist\Chatterbox\Domain\Knowledge\VectorStoreService;
 use Sitegeist\Chatterbox\Domain\OrganizationRepository;
+use Sitegeist\Flow\OpenAiClientFactory\AccountRepository;
+use Sitegeist\Flow\OpenAiClientFactory\OpenAiClientFactory;
 
 #[Flow\Scope('singleton')]
 class KnowledgeCommandController extends CommandController
 {
+    /**
+     * @var string
+     * @Flow\InjectConfiguration(path="context")
+     */
+    protected string $context;
+
     public function __construct(
-        private readonly OrganizationRepository $organizationRepository,
+        private readonly SourceOfKnowledgeRepository $sourceOfKnowledgeRepository,
+        private readonly AssistantEntityRepository $assistantEntityRepository,
+        private readonly VectorStoreService $vectorStoreService,
+        private readonly OpenAiClientFactory $clientFactory,
+        private readonly AccountRepository $accountRepository,
+        private readonly VectorStoreReferenceRepository $vectorStoreReferenceRepository,
     ) {
         parent::__construct();
     }
 
-    public function updatePoolCommand(?string $organizationId = null): void
+    public function listCommand(): void
     {
-        $this->outputLine('Updating knowledge pool');
-        $organizations = $organizationId
-            ? [$this->organizationRepository->findById($organizationId)]
-            : $this->organizationRepository->findAll();
-        $numberOfSources = 0;
-        foreach ($organizations as $organization) {
-            $numberOfSources += count($organization->library->findAllSourcesOfKnowledge());
+        $knowledges = $this->sourceOfKnowledgeRepository->findAll();
+        foreach ($knowledges as $knowledge) {
+            $this->outputLine($knowledge->getName()->value);
         }
-        $this->output->progressStart($numberOfSources);
+    }
 
-        foreach ($organizations as $organization) {
-            foreach ($organization->library->findAllSourcesOfKnowledge() as $sourceOfKnowledge) {
-                $organization->library->updateSourceOfKnowledge($sourceOfKnowledge);
-                $this->output->progressAdvance();
+    public function showCommand(string $name): void
+    {
+        $knowledge = $this->sourceOfKnowledgeRepository->findSourceByName($name);
+        if ($knowledge instanceof SourceOfKnowledgeContract) {
+            $documentCollection = $knowledge->getContent();
+            foreach ($documentCollection as $document) {
+                $this->outputLine('<info>%s.%s</info>', [$document->name,$document->type]);
+                $this->outputLine();
+                $this->output($document->content);
+                $this->outputLine();
+                $this->outputLine();
             }
+        } else {
+            $this->outputLine('Knowledge ´source %s was not found', [$name]);
+            $this->quit(1);
         }
-
-        $this->output->progressFinish();
-        $this->outputLine('');
-        $this->outputLine('Done');
     }
 
-    public function cleanPoolCommand(?string $organizationId = null): void
+    public function uploadCommand(?string $name = null, ?string $account = null): void
     {
-        $this->outputLine('Cleaning knowledge pool');
-        $organizations = $organizationId
-            ? [$this->organizationRepository->findById($organizationId)]
-            : $this->organizationRepository->findAll();
-        $this->output->progressStart(count($organizations));
-        foreach ($organizations as $organization) {
-            $organization->library->cleanKnowledgePool($organization->assistantDepartment);
-            $this->output->progressAdvance();
+        /**
+         * @var AssistantEntity[] $assistantEntities
+         */
+        $assistantEntities = $name
+            ? [$this->assistantEntityRepository->findOneByName($name)]
+            : $this->assistantEntityRepository->findAll();
+
+        /**
+         * @var array<string, string[]>
+         */
+        $accountsForKnowledgeSourceIds = [];
+
+        foreach ($assistantEntities as $assistantEntity) {
+            if ($account !== null && $assistantEntity->getAccount() !== $account) {
+                continue;
+            }
+            foreach ($assistantEntity->getKnowledgeSourceIdentifiers() as $knowledgeSourceIdentifier) {
+                $accountsForKnowledgeSourceIds[$knowledgeSourceIdentifier][] = $assistantEntity->getAccount();
+            }
+        }
+
+        $knowledgeSourceIdentifiersToUpdate = array_keys($accountsForKnowledgeSourceIds);
+
+        /**
+         * @var array<string, SourceOfKnowledgeContract> $knowledgeSources
+         */
+        $knowledgeSources = [];
+
+        foreach ($knowledgeSourceIdentifiersToUpdate as $knowledgeSourceIdentifier) {
+            $knowledgeSource = $this->sourceOfKnowledgeRepository->findSourceByName($knowledgeSourceIdentifier);
+            if ($knowledgeSource instanceof SourceOfKnowledgeContract) {
+                $knowledgeSources[$knowledgeSourceIdentifier] = $knowledgeSource;
+            }
+        }
+
+        foreach ($knowledgeSources as $knowledgeSourceIdentifier => $knowledgeSource) {
+            $uniqueAccountIdentifiers = array_unique($accountsForKnowledgeSourceIds[$knowledgeSourceIdentifier] ?? []);
+            foreach ($uniqueAccountIdentifiers as $accountIdentifier) {
+                $this->outputLine("- Uploading %s to %s", [$knowledgeSourceIdentifier, $accountIdentifier]);
+                $account = $this->accountRepository->findById($accountIdentifier);
+                $client = $this->clientFactory->createClientForAccountRecord($account);
+                $storeId = $this->vectorStoreService->upload($client, $knowledgeSource);
+
+                // persist reference
+                $existingVectorStoreReference = $this->vectorStoreReferenceRepository->findOneByAssistantAndKnowledgeSourceIdentifier(
+                    $accountIdentifier,
+                    $knowledgeSourceIdentifier
+                );
+
+                if ($existingVectorStoreReference instanceof VectorStoreReference) {
+                    $existingVectorStoreReference->updateVectorStoreId($storeId->value);
+                    $this->vectorStoreReferenceRepository->update($existingVectorStoreReference);
+                } else {
+                    $newVectorStoreReference  = new VectorStoreReference(
+                        $this->context,
+                        $accountIdentifier,
+                        $knowledgeSourceIdentifier,
+                        $storeId->value
+                    );
+                    $this->vectorStoreReferenceRepository->add($newVectorStoreReference);
+                }
+
+                $this->outputLine("- Cleanup old instances of %s on %s", [$knowledgeSourceIdentifier, $accountIdentifier]);
+                $this->vectorStoreService->cleanup($client, $knowledgeSource, $storeId);
+            }
         }
-        $this->outputLine('');
-        $this->outputLine('Done');
     }
 }

Classes/Command/OpenAiCommandController.php

@@ -0,0 +1,70 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Sitegeist\Chatterbox\Command;
+
+use Neos\Flow\Annotations as Flow;
+use Neos\Flow\Cli\CommandController;
+use Sitegeist\Chatterbox\Domain\Knowledge\SourceOfKnowledgeContract;
+use Sitegeist\Chatterbox\Domain\Knowledge\SourceOfKnowledgeRepository;
+use Sitegeist\Chatterbox\Domain\Knowledge\VectorStoreReferenceRepository;
+use Sitegeist\Chatterbox\Domain\OrganizationRepository;
+use Sitegeist\Flow\OpenAiClientFactory\AccountRepository;
+use Sitegeist\Flow\OpenAiClientFactory\OpenAiClientFactory;
+
+#[Flow\Scope('singleton')]
+class OpenAiCommandController extends CommandController
+{
+    public function __construct(
+        private readonly AccountRepository $accountRepository,
+        private readonly OpenAiClientFactory $clientFactory,
+        private readonly VectorStoreReferenceRepository $vectorStoreReferenceRepository,
+    ) {
+        parent::__construct();
+    }
+
+    public function deleteAllVectorStoresCommand(string $account): void
+    {
+        $account = $this->accountRepository->findById($account);
+
+        if (!$this->output->askConfirmation("Are you sure (Y/n)")) {
+            $this->quit(1);
+        }
+
+        $client = $this->clientFactory->createClientForAccountRecord($account);
+
+        $vectorStores = $client->vectorStores()->list();
+        $this->output->progressStart(count($vectorStores->data));
+        foreach ($vectorStores->data as $vectorStore) {
+            $this->output->progressAdvance();
+            $client->vectorStores()->delete($vectorStore->id);
+        }
+        $this->output->progressFinish();
+
+        // remove references aswell
+        $this->vectorStoreReferenceRepository->removeAll();
+
+        $this->outputLine();
+    }
+
+    public function deleteAllFilesCommand(string $account): void
+    {
+        $account = $this->accountRepository->findById($account);
+
+        if (!$this->output->askConfirmation("Are you sure (Y/n)")) {
+            $this->quit(1);
+        }
+
+        $client = $this->clientFactory->createClientForAccountRecord($account);
+
+        $files = $client->files()->list();
+        $this->output->progressStart(count($files->data));
+        foreach ($files->data as $fileResponse) {
+            $this->output->progressAdvance();
+            $client->files()->delete($fileResponse->id);
+        }
+        $this->output->progressFinish();
+        $this->outputLine();
+    }
+}

Classes/Command/ToolCommandController.php

@@ -7,30 +7,29 @@
 use Neos\Flow\Annotations as Flow;
 use Neos\Flow\Cli\CommandController;
 use Sitegeist\Chatterbox\Domain\OrganizationRepository;
+use Sitegeist\Chatterbox\Domain\Tools\ToolRepository;
 use Symfony\Component\Yaml\Yaml;
 
 #[Flow\Scope('singleton')]
 class ToolCommandController extends CommandController
 {
     public function __construct(
-        private readonly OrganizationRepository $organizationRepository,
+        private readonly ToolRepository $toolRepository,
     ) {
         parent::__construct();
     }
 
-    public function listCommand(string $organizationId): void
+    public function listCommand(): void
     {
-        $organization = $this->organizationRepository->findById($organizationId);
-        $tools = $organization->toolbox->findAll();
+        $tools = $this->toolRepository->findAll();
         foreach ($tools as $tool) {
             $this->outputLine($tool->getName());
         }
     }
 
-    public function showCommand(string $organizationId, string $toolName,): void
+    public function showCommand(string $toolName,): void
     {
-        $organization = $this->organizationRepository->findById($organizationId);
-        $tool = $organization->toolbox->findByName($toolName);
+        $tool = $this->toolRepository->findByName($toolName);
         if (!$tool) {
             $this->outputLine('no such tool');
             $this->quit();
@@ -44,10 +43,9 @@ public function showCommand(string $organizationId, string $toolName,): void
         }
     }
 
-    public function executeCommand(string $organizationId, string $toolName, string $parameters): void
+    public function executeCommand(string $toolName, string $parameters): void
     {
-        $organization = $this->organizationRepository->findById($organizationId);
-        $tool = $organization->toolbox->findByName($toolName);
+        $tool = $this->toolRepository->findByName($toolName);
         if (!$tool) {
             $this->outputLine('no such tool');
             $this->quit();