Skip to content

Adds asynchronous batch scanning #1131

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
mbusch3 wants to merge 1 commit into
base: dev
Choose a base branch
from
+42 −93
Draft

Adds asynchronous batch scanning #1131

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions src/Controller/SyncController.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,7 @@ public function requestContentSync(SessionService $sessionService, ContentItem $

// Add rescanned Issues to database
foreach ($report->getIssues() as $issue) {
if (isset($issue->isGeneric)) {
$lmsFetch->createGenericIssue($issue, $contentItem);
} else {
$lmsFetch->createIssue($issue, $contentItem);
}
$lmsFetch->createGenericIssue($issue, $contentItem);
}

$response->addMessage('Successfully scanned content', 'success', 5000);
Expand Down
88 changes: 14 additions & 74 deletions src/Services/LmsFetchService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,6 @@ public function refreshLmsContent(Course $course, User $user, $force = false)
3. ContentItems that are in the LMS but not in our database are added to our database.
*/
$contentItems = $lms->updateCourseContent($course, $user, $force);
$output->writeln("Found " . count($contentItems) . " updated content items in the LMS.");

$contentSections = $lms->getCourseSections($course, $user);

Expand All @@ -112,7 +111,6 @@ public function refreshLmsContent(Course $course, User $user, $force = false)
/* Step 4: Process the updated content with PhpAlly and link to report */
$this->scanContentItems($contentItems);

$output->writeln("Updating report now...");
/* Step 5: Update report from all active issues */
$this->updateReport($course, $user, count($contentItems));

Expand Down Expand Up @@ -231,94 +229,36 @@ private function scanContentItems(array $contentItems)
$scanner = $_ENV['ACCESSIBILITY_CHECKER'];
$equalAccessReports = null;

// $scanner = 'equalaccess_local';

// If we're using Equal Access Lambda, send all the requests to Lambda for the
// reports at once and save them all into an array (which should be in the same order as the ContentItems)
// if ($scanner == "equalaccess_lambda" && count($contentItems) > 0) {
// $equalAccessReports = $this->asyncReport->postMultipleArrayAsync($contentItems);
// }

// Scan each update content item for issues
/** @var \App\Entity\ContentItem $contentItem */

$index = 0;
$contentItemsById = array();
foreach ($contentItems as $contentItem) {
if($contentItem->getBody() == null) {
continue; // Skip content items that have no body
}
$contentItemsById[$contentItem->getId()] = $contentItem;
}

try {
// Scan the content item with the scanner set in the environment.
$report = $this->scanner->scanContentItem($contentItem, $equalAccessReports == null ? null : $equalAccessReports[$index++], $this->util);

if ($report) {
// TODO: Do something with report errors
if (count($report->getErrors())) {
foreach ($report->getErrors() as $error) {
$msg = $error . ', item = #' . $contentItem->getId();
$this->util->createMessage($msg, 'error', $contentItem->getCourse(), null, true);
}
}
try {
$reports = $this->scanner->scanContentItemArray($contentItemsById);

// Add Issues to report
foreach($reports as $id => $report) {
$contentItem = $contentItemsById[$id] ?? null;
if($contentItem !== null) {
foreach ($report->getIssues() as $issue) {
if(isset($issue->isGeneric)) {
$this->createGenericIssue($issue, $contentItem);
}
else {
$this->createIssue($issue, $contentItem);
}
$this->createGenericIssue($issue, $contentItem);
}
}
}
catch (\Exception $e) {
$this->util->createMessage($e->getMessage(), 'error', null, null, true);
throw $e; // Rethrow the exception to be caught by the controller
}
}
$this->doctrine->getManager()->flush();
}

public function createIssue(PhpAllyIssue $issue, ContentItem $contentItem)
{
$issueEntity = new Issue();
$meta = $contentItem->getCourse()->getInstitution()->getMetadata();
$issueType = self::ISSUE_TYPE_ERROR;

if (isset($meta['SUGGESTION_RULES'])) {
if (isset($meta['SUGGESTION_RULES'][$issue->getRuleId()])) {
$issueType = self::ISSUE_TYPE_SUGGESTION;
}
}
if (isset($_ENV['PHPALLY_SUGGESTION_RULES'])) {
if (strpos($_ENV['PHPALLY_SUGGESTION_RULES'], $issue->getRuleId()) !== false) {
$issueType = self::ISSUE_TYPE_SUGGESTION;
}
}

$scanner = $_ENV['ACCESSIBILITY_CHECKER'];
if ($scanner == 'equalaccess_lambda' || $scanner == 'equalaccess_local' || $scanner == 'equalaccess') {
$issueType = $this->equalAccess->getIssueType($issue->getMetadata());
if($issueType == 'pass') {
// If the issue is a pass, we don't create an issue for it
return null;
}
catch (\Exception $e) {
$this->util->createMessage($e->getMessage(), 'error', null, null, true);
throw $e; // Rethrow the exception to be caught by the controller
}

$issueEntity->setType($issueType);
$issueEntity->setStatus(Issue::$issueStatusActive);
$issueEntity->setContentItem($contentItem);
$issueEntity->setScanRuleId($issue->getRuleId());
$issueEntity->setHtml($issue->getHtml());
$issueEntity->setPreviewHtml($issue->getPreview());
$issueEntity->setMetadata($issue->getMetadata());

$contentItem->addIssue($issueEntity);

$this->doctrine->getManager()->persist($issueEntity);

return $issueEntity;

$this->doctrine->getManager()->flush();
}

public function createGenericIssue($issue, ContentItem $contentItem)
Expand Down
16 changes: 3 additions & 13 deletions src/Services/LocalApiAccessibilityService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,28 +33,24 @@ public function scanContentItem(ContentItem $contentItem) {
return $data;
}

public function scanMultipleContentItemsAsync(array $contentItems, int $concurrency = 5, bool $stopOnFailure = false): array
public function scanMultipleContentItemsAsync(array $contentItems, int $concurrency = 10, bool $stopOnFailure = false): array
{
// Initialize Guzzle client with base options
$client = new Client([
// TODO: the problem with this is that it does not matter if the scanner is the
// local or Lambda version, the URL is always the same location causing the
// local to be triggered.
'base_uri' => 'http://host.docker.internal:3000',
'timeout' => 30.0,
'http_errors' => false, // Don't throw exceptions for 4xx/5xx responses
]);

$output = new ConsoleOutput();
$output->writeln("Starting async scan of " . count($contentItems) . " content items");

// Initialize promises array
$promises = [];
$results = [];

// Create a promise for each content item
foreach ($contentItems as $contentItem) {
$id = $contentItem->getId();
foreach ($contentItems as $key => $contentItem) {
$id = $key;
//$html = HtmlService::clean($contentItem->getBody());
$html = $contentItem->getBody();

Expand Down Expand Up @@ -87,7 +83,6 @@ function ($response) use ($id, $output, &$results) {
$output->writeln("JSON decode error for item {$id}: " . json_last_error_msg());
$results[$id] = null;
} else {
$output->writeln("Successfully scanned content item {$id}");
$results[$id] = $result;
}
} catch (\Exception $e) {
Expand All @@ -112,10 +107,6 @@ function ($exception) use ($id, $output, &$results) {
$pool = new Promise\EachPromise($promises, [
// Execute N requests concurrently
'concurrency' => $concurrency,
// Invoked when a promise is fulfilled or rejected
'fulfilled' => function ($value, $idx, $aggregate) use ($output) {
$output->writeln("Completed request {$idx}");
},
'rejected' => function ($reason, $idx, $aggregate) use ($output, $stopOnFailure) {
$output->writeln("Failed request {$idx}: " . $reason->getMessage());

Expand All @@ -129,7 +120,6 @@ function ($exception) use ($id, $output, &$results) {
try {
// Wait for the pool to complete
$pool->promise()->wait();
$output->writeln("All content items scanned successfully");
} catch (\Exception $e) {
$output->writeln("Error during scanning process: " . $e->getMessage());
// Handle any uncaught exceptions from the promise pool
Expand Down
25 changes: 24 additions & 1 deletion src/Services/ScannerService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

use App\Response\ApiResponse;
use App\Services\LocalApiAccessibilityService;
use Symfony\Component\Console\Output\ConsoleOutput;

// Main scanner class, expects a phpAlly-styled JSON report from whichever scanner is run

Expand Down Expand Up @@ -56,7 +57,14 @@ public function scanContentItem(ContentItem $contentItem, $scannerReport = null,

$localService = new LocalApiAccessibilityService();
$json = $localService->scanContentItem($contentItem);
$report = $equalAccess->generateReport($json);
if(isset($json["error"])) {
$output = new ConsoleOutput();
$output->writeln("Failed to scan: " . json_encode($contentItem));
throw new \Exception("Error from accessibility checker: " . $json["error"]);
}
else if(isset($json["results"]) && is_array($json["results"])) {
$report = $equalAccess->generateReport($json);
}
}
else {
// Unknown scanner set in environment, should return error...
Expand All @@ -69,4 +77,19 @@ public function scanContentItem(ContentItem $contentItem, $scannerReport = null,

return $report;
}

public function scanContentItemArray(array $contentItems) {

$equalAccess = new EqualAccessService();
$localService = new LocalApiAccessibilityService();
$report = [];
$json = $localService->scanMultipleContentItemsAsync($contentItems);
foreach ($json as $id => $itemJson) {
if(isset($itemJson["results"]) && is_array($itemJson["results"])) {
$tempReport = $equalAccess->generateReport($itemJson);
$report[$id] = $tempReport;
}
}
return $report;
}
}
Loading