diff --git a/CHANGELOG.md b/CHANGELOG.md index 920ec82676..fc36b39ac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - **scoop-uninstall**: Allow access to `$bucket` in uninstall scripts ([#6380](https://github.com/ScoopInstaller/Scoop/issues/6380)) - **install:** Add separator at the end of notes, highlight suggestions ([#6418](https://github.com/ScoopInstaller/Scoop/issues/6418)) +- **virustotal:** Refactor into lib and integrate pre-download checks for install and update ([#6525](https://github.com/ScoopInstaller/Scoop/issues/6525)) + ### Bug Fixes diff --git a/bin/checkhashes.ps1 b/bin/checkhashes.ps1 index 6e6420fb2a..45ab735a2d 100644 --- a/bin/checkhashes.ps1 +++ b/bin/checkhashes.ps1 @@ -42,6 +42,7 @@ param( . "$PSScriptRoot\..\lib\core.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'get_hash' . "$PSScriptRoot\..\lib\buckets.ps1" . "$PSScriptRoot\..\lib\autoupdate.ps1" . "$PSScriptRoot\..\lib\json.ps1" diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index fbc3ebfdee..8b05d91ca3 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -1,5 +1,7 @@ # Must included with 'json.ps1' +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'get_hash' + function format_hash([String] $hash) { $hash = $hash.toLower() diff --git a/lib/download.ps1 b/lib/download.ps1 index 70641cca7f..1da0090e2d 100644 --- a/lib/download.ps1 +++ b/lib/download.ps1 @@ -1,8 +1,13 @@ # Description: Functions for downloading files +. "$PSScriptRoot\..\lib\core.ps1" +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'hash_for_url' +. "$PSScriptRoot\..\lib\helper\file-information.ps1" # 'Get-RemoteFileSize' +. "$PSScriptRoot\..\lib\virustotal.ps1" + ## Meta downloader -function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true) { +function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true, $check_virustotal = $false) { # we only want to show this warning once if (!$use_cache) { warn 'Cache is being ignored.' } @@ -14,8 +19,14 @@ function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture # download first if (Test-Aria2Enabled) { - Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash + Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash $check_virustotal } else { + $urls = if ($check_virustotal) { + Test-UrlsWithVirusTotal $app $urls $manifest $architecture + } else { + $urls + } + foreach ($url in $urls) { $fname = url_filename $url @@ -329,9 +340,14 @@ function get_filename_from_metalink($file) { return $filename } -function Invoke-CachedAria2Download ($app, $version, $manifest, $architecture, $dir, $cookies = $null, $use_cache = $true, $check_hash = $true) { +function Invoke-CachedAria2Download ($app, $version, $manifest, $architecture, $dir, $cookies = $null, $use_cache = $true, $check_hash = $true, $check_virustotal = $false) { $data = @{} $urls = @(script:url $manifest $architecture) + $urls = if ($check_virustotal) { + Test-UrlsWithVirusTotal $app $urls $manifest $architecture + } else { + $urls + } # aria2 input file $urlstxt = Join-Path $cachedir "$app.txt" @@ -457,9 +473,9 @@ function Invoke-CachedAria2Download ($app, $version, $manifest, $architecture, $ warn "Download failed! (Error $lastexitcode) $(aria_exit_code $lastexitcode)" warn $urlstxt_content warn $aria2 - warn $(new_issue_msg $app $bucket "download via aria2 failed") + warn $(new_issue_msg $app $bucket 'download via aria2 failed') - Write-Host "Fallback to default downloader ..." + Write-Host 'Fallback to default downloader ...' try { foreach ($url in $urls) { @@ -666,12 +682,6 @@ function get_magic_bytes_pretty($file, $glue = ' ') { return (get_magic_bytes $file | ForEach-Object { $_.ToString('x2') }) -join $glue } -Function Get-RemoteFileSize ($Uri) { - $response = Invoke-WebRequest -Uri $Uri -Method HEAD -UseBasicParsing - if (!$response.Headers.StatusCode) { - $response.Headers.'Content-Length' | ForEach-Object { [int]$_ } - } -} function ftp_file_size($url) { $request = [net.ftpwebrequest]::create($url) @@ -689,33 +699,18 @@ function url_remote_filename($url) { # this function extracts the original filename from the URL. $uri = (New-Object URI $url) $basename = Split-Path $uri.PathAndQuery -Leaf - If ($basename -match '.*[?=]+([\w._-]+)') { + if ($basename -match '.*[?=]+([\w._-]+)') { $basename = $matches[1] } - If (($basename -notlike '*.*') -or ($basename -match '^[v.\d]+$')) { + if (($basename -notlike '*.*') -or ($basename -match '^[v.\d]+$')) { $basename = Split-Path $uri.AbsolutePath -Leaf } - If (($basename -notlike '*.*') -and ($uri.Fragment -ne '')) { + if (($basename -notlike '*.*') -and ($uri.Fragment -ne '')) { $basename = $uri.Fragment.Trim('/', '#') } return $basename } -### Hash-related functions - -function hash_for_url($manifest, $url, $arch) { - $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null } - - if ($hashes.length -eq 0) { return $null } - - $urls = @(script:url $manifest $arch) - - $index = [array]::IndexOf($urls, $url) - if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." } - - @($hashes)[$index] -} - function check_hash($file, $hash, $app_name) { # returns (ok, err) if (!$hash) { @@ -751,19 +746,5 @@ function check_hash($file, $hash, $app_name) { return $true, $null } -function get_hash([String] $multihash) { - $type, $hash = $multihash -split ':' - if (!$hash) { - # no type specified, assume sha256 - $type, $hash = 'sha256', $multihash - } - - if (@('md5', 'sha1', 'sha256', 'sha512') -notcontains $type) { - return $null, "Hash type '$type' isn't supported." - } - - return $type, $hash.ToLower() -} - # Setup proxy globally setup_proxy diff --git a/lib/helper/file-information.ps1 b/lib/helper/file-information.ps1 new file mode 100644 index 0000000000..379d8204a4 --- /dev/null +++ b/lib/helper/file-information.ps1 @@ -0,0 +1,8 @@ +### Remote file information + +function Get-RemoteFileSize ($Uri) { + $response = Invoke-WebRequest -Uri $Uri -Method HEAD -UseBasicParsing + if (!$response.Headers.StatusCode) { + $response.Headers.'Content-Length' | ForEach-Object { [int]$_ } + } +} diff --git a/lib/helper/hash.ps1 b/lib/helper/hash.ps1 new file mode 100644 index 0000000000..4cb4826436 --- /dev/null +++ b/lib/helper/hash.ps1 @@ -0,0 +1,28 @@ +### Hash-related functions + +function hash_for_url($manifest, $url, $arch) { + $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null } + + if ($hashes.length -eq 0) { return $null } + + $urls = @(script:url $manifest $arch) + + $index = [array]::IndexOf($urls, $url) + if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." } + + @($hashes)[$index] +} + +function get_hash([String] $multihash) { + $type, $hash = $multihash -split ':' + if (!$hash) { + # no type specified, assume sha256 + $type, $hash = 'sha256', $multihash + } + + if (@('md5', 'sha1', 'sha256', 'sha512') -notcontains $type) { + return $null, "Hash type '$type' isn't supported." + } + + return $type, $hash.ToLower() +} diff --git a/lib/install.ps1 b/lib/install.ps1 index 6125697967..9b1b688e7d 100644 --- a/lib/install.ps1 +++ b/lib/install.ps1 @@ -1,3 +1,4 @@ + function nightly_version($quiet = $false) { if (!$quiet) { warn "This is a nightly version. Downloaded files won't be verified." @@ -5,7 +6,7 @@ function nightly_version($quiet = $false) { return "nightly-$(Get-Date -Format 'yyyyMMdd')" } -function install_app($app, $architecture, $global, $suggested, $use_cache = $true, $check_hash = $true) { +function install_app($app, $architecture, $global, $suggested, $use_cache = $true, $check_hash = $true, $check_virustotal = $false) { $app, $manifest, $bucket, $url = Get-Manifest $app if (!$manifest) { @@ -49,7 +50,7 @@ function install_app($app, $architecture, $global, $suggested, $use_cache = $tru $original_dir = $dir # keep reference to real (not linked) directory $persist_dir = persistdir $app $global - $fname = Invoke-ScoopDownload $app $version $manifest $bucket $architecture $dir $use_cache $check_hash + $fname = Invoke-ScoopDownload $app $version $manifest $bucket $architecture $dir $use_cache $check_hash $check_virustotal Invoke-Extraction -Path $dir -Name $fname -Manifest $manifest -ProcessorArchitecture $architecture Invoke-HookScript -HookType 'pre_install' -Manifest $manifest -ProcessorArchitecture $architecture diff --git a/lib/virustotal.ps1 b/lib/virustotal.ps1 new file mode 100644 index 0000000000..8cca736909 --- /dev/null +++ b/lib/virustotal.ps1 @@ -0,0 +1,350 @@ +. "$PSScriptRoot\json.ps1" # 'json_path' +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'hash_for_url' +. "$PSScriptRoot\..\lib\helper\file-information.ps1" # 'Get-RemoteFileSize' + +# Error codes +$_ERR_UNSAFE = 2 +$_ERR_EXCEPTION = 4 +$_ERR_NO_INFO = 8 +$_ERR_NO_API_KEY = 16 + +# Global state variables +$script:requests = 0 +$script:explained_rate_limit_sleeping = $False +$exit_code = 0 + +function ConvertTo-VirusTotalUrlId ($url) { + $url_id = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($url)) + $url_id = $url_id -replace '\+', '-' + $url_id = $url_id -replace '/', '_' + $url_id = $url_id -replace '=', '' + $url_id +} + +function Get-VirusTotalResultByHash ($hash, $url, $app, $api_key) { + $hash = $hash.ToLower() + $api_url = "https://www.virustotal.com/api/v3/files/$hash" + $headers = @{ + 'Accept' = 'application/json' + 'x-apikey' = $api_key + } + $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing + $result = $response.Content + $stats = json_path $result '$.data.attributes.last_analysis_stats' + [int]$malicious = json_path $stats '$.malicious' + [int]$suspicious = json_path $stats '$.suspicious' + [int]$timeout = json_path $stats '$.timeout' + [int]$undetected = json_path $stats '$.undetected' + [int]$unsafe = $malicious + $suspicious + [int]$total = $unsafe + $undetected + [int]$fileSize = json_path $result '$.data.attributes.size' + $report_hash = json_path $result '$.data.attributes.sha256' + $report_url = "https://www.virustotal.com/gui/file/$report_hash" + if ($total -eq 0) { + info "$app`: Analysis in progress." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $hash + 'App.HashType' = $null + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $report_url + 'FileReport.Hash' = $report_hash + 'UrlReport.Url' = $null + } + } else { + $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value + switch ($unsafe) { + 0 { success "$app`: $unsafe/$total, see $report_url" } + 1 { warn "$app`: $unsafe/$total, see $report_url" } + 2 { warn "$app`: $unsafe/$total, see $report_url" } + Default { warn "$([char]0x1b)[31m$app`: $unsafe/$total, see $report_url$([char]0x1b)[0m" } + } + $maliciousResults = $vendorResults | + Where-Object -Property category -EQ 'malicious' | + Select-Object -ExpandProperty engine_name + $suspiciousResults = $vendorResults | + Where-Object -Property category -EQ 'suspicious' | + Select-Object -ExpandProperty engine_name + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $hash + 'App.HashType' = $null + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $report_url + 'FileReport.Hash' = $report_hash + 'FileReport.MaliciousResults' = if ($maliciousResults) { $maliciousResults } else { @() } + 'FileReport.SuspiciousResults' = if ($suspiciousResults) { $suspiciousResults } else { @() } + 'FileReport.Malicious' = $malicious + 'FileReport.Suspicious' = $suspicious + 'FileReport.Timeout' = $timeout + 'FileReport.Undetected' = $undetected + 'FileReport.Unsafe' = $unsafe + 'FileReport.Total' = $total + 'UrlReport.Url' = $null + } + } + if ($unsafe -gt 0) { + $exit_code = $exit_code -bor $_ERR_UNSAFE + } +} + +function Get-VirusTotalResultByUrl ($url, $app, $api_key) { + $id = ConvertTo-VirusTotalUrlId $url + $api_url = "https://www.virustotal.com/api/v3/urls/$id" + $headers = @{ + 'Accept' = 'application/json' + 'x-apikey' = $api_key + } + $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing + $result = $response.Content + $id = json_path $result '$.data.id' + $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null + $last_analysis_date = json_path $result '$.data.attributes.last_analysis_date' 6>$null + $url_report_url = "https://www.virustotal.com/gui/url/$id" + info "$app`: Url report found." + if (!$hash) { + if (!$last_analysis_date) { + info "$app`: Analysis in progress." + } else { + info "$app`: Related file report not found." + warn "$app`: Manual file upload is required (instead of url submission)." + } + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $null + 'App.HashType' = $null + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url + 'UrlReport.Hash' = $null + } + } else { + info "$app`: Related file report found." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $null + 'App.HashType' = $null + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url + 'UrlReport.Hash' = $hash + } + } +} + +# Submit-ToVirusTotal +# - $url: where file to check can be downloaded +# - $app: Name of the application (used for reporting) +# - $do_scan: [boolean flag] whether to actually submit to VirusTotal +# This is a parameter instead of conditionnally calling +# the function to consolidate the warning message +# - $api_key: VirusTotal API key +# - $retrying: [boolean] Optional, for internal use to retry +# submitting the file after a delay if the rate limit is +# exceeded, without risking an infinite loop (as stack +# overflow) if the submission keeps failing. +function Submit-ToVirusTotal ($url, $app, $do_scan, $api_key, $retrying = $False) { + if (!$do_scan) { + warn "$app`: not found`: you can manually submit $url" + return + } + + try { + $script:requests += 1 + + $encoded_url = [System.Web.HttpUtility]::UrlEncode($url) + $api_url = 'https://www.virustotal.com/api/v3/urls' + $content_type = 'application/x-www-form-urlencoded' + $headers = @{ + 'Accept' = 'application/json' + 'x-apikey' = $api_key + 'Content-Type' = $content_type + } + $body = "url=$encoded_url" + $result = Invoke-WebRequest -Uri $api_url -Method POST -Headers $headers -ContentType $content_type -Body $body -UseBasicParsing + if ($result.StatusCode -eq 200) { + $id = ((json_path $result '$.data.id') -split '-')[1] + $url_report_url = "https://www.virustotal.com/gui/url/$id" + $fileSize = Get-RemoteFileSize $url + if ($fileSize -gt 80000000) { + info "$app`: Remote file size: $(filesize $fileSize). Large files might require manual file upload instead of url submission." + } + info "$app`: Analysis in progress." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $null + 'App.HashType' = $null + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url + 'UrlReport.Hash' = $null + } + return + } + + # EAFP: submission failed -> sleep, then retry + if (!$retrying) { + if (!$script:explained_rate_limit_sleeping) { + info 'VirusTotal API has rate limits. Waiting between requests...' + $script:explained_rate_limit_sleeping = $True + } + Start-Sleep -s (60 + $script:requests) + Submit-ToVirusTotal $url $app $do_scan $api_key $True + } else { + warn "$app`: VirusTotal submission of $url failed`:`n" + + "`tAPI returned $($result.StatusCode) after retrying" + } + } catch [Exception] { + warn "$app`: VirusTotal submission failed`: $($_.Exception.Message)" + return + } +} + +function Get-VirusTotalApiKey { + $api_key = get_config VIRUSTOTAL_API_KEY + if (!$api_key) { + abort ("VirusTotal API key is not configured`n" + + " You could get one from https://www.virustotal.com/gui/my-apikey and set with`n" + + " scoop config virustotal_api_key ") $_ERR_NO_API_KEY + } + return $api_key +} + +function Check-VirusTotalUrl($app, $url, $hash, $api_key, $scan) { + $isHashUnsupported = $false + $algo = $null + + if ($hash -match '(?[^:]+):(?.*)') { + $algo = $matches.algo + $hash = $matches.hash + if ($matches.algo -inotin 'md5', 'sha1', 'sha256') { + $hash = $null + $isHashUnsupported = $true + warn "$app`: Unsupported hash $($matches.algo). Will search by url instead." + } + } elseif ($hash) { + $algo = 'sha256' + } + + try { + if ($hash) { + $file_report = Get-VirusTotalResultByHash $hash $url $app $api_key + $file_report.'App.HashType' = $algo + return $file_report + } elseif (!$isHashUnsupported) { + warn "$app`: Hash not found. Will search by url instead." + } + } catch [Exception] { + $exit_code = $exit_code -bor $_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + $file_report_not_found = $true + warn "$app`: File report not found. Will search by url instead." + } else { + warn "$app`: VirusTotal file report query failed`: $($_.Exception.Message)" + if ($_.Exception.Response) { + warn "`tAPI returned $($_.Exception.Response.StatusCode)" + } + return + } + } + + try { + $url_report = Get-VirusTotalResultByUrl $url $app $api_key + $url_report.'App.Hash' = $hash + $url_report.'App.HashType' = $algo + if ($url_report.'UrlReport.Hash' -and ($file_report_not_found -eq $true) -and $hash) { + try { + $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app $api_key + if ($file_report.'FileReport.Hash' -ieq $matches['hash']) { + $file_report.'App.HashType' = $algo + $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' + return $file_report + } + } catch { + warn "$app`: Unable to get file report for $($url_report.'UrlReport.Hash')" + } + } + if (!$url_report.'UrlReport.Hash') { + Submit-ToVirusTotal $url $app $scan $api_key + return $url_report + } + } catch [Exception] { + $exit_code = $exit_code -bor $_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + Submit-ToVirusTotal $url $app $scan $api_key + return + } else { + warn "$app`: VirusTotal URL report query failed`: $($_.Exception.Message)" + if ($_.Exception.Response) { + warn "`tAPI returned $($_.Exception.Response.StatusCode)" + } + return + } + } + + try { + $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app $api_key + $file_report.'App.Hash' = $hash + $file_report.'App.HashType' = $algo + $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' + $file_report + warn "$app`: Unable to check hash match for $url" + } catch [Exception] { + $exit_code = $exit_code -bor $_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + Submit-ToVirusTotal $url $app $scan $api_key + $url_report + } else { + warn "$app`: VirusTotal file report query failed`: $($_.Exception.Message)" + if ($_.Exception.Response) { + warn "`tAPI returned $($_.Exception.Response.StatusCode)" + } + return + } + } +} + +function virustotal_check_app($app, $manifest, $architecture, $api_key, $scan) { + [int]$index = 0 + $urls = script:url $manifest $architecture + $urls | ForEach-Object { + $url = $_ + $index++ + if ($urls.GetType().IsArray) { + info "$app`: url $index" + } + $hash = hash_for_url $manifest $url $architecture + Check-VirusTotalUrl $app $url $hash $api_key $scan + } +} + +# return only the URLs that passed VirusTotal checks +function Test-UrlsWithVirusTotal($app, $urls, $manifest, $architecture) { + $safe_urls = @() + $api_key = Get-VirusTotalApiKey + + foreach ($url in $urls) { + $hash = hash_for_url $manifest $url $architecture + $reports = Check-VirusTotalUrl $app $url $hash $api_key $false + + $reports | ForEach-Object { + $file_report = $_ + $url = $file_report.'App.Url' + + if ($file_report.'FileReport.Unsafe' -eq 0) { + info "$app`: Safe URL: $url" + $safe_urls += $url + } else { + warn "$app`: Unsafe URL: $url" + } + } + } + + if ($safe_urls.Count -eq 0) { + abort "VirusTotal check for $app failed. Aborting before download." + } + + return $safe_urls +} diff --git a/libexec/scoop-config.ps1 b/libexec/scoop-config.ps1 index 6007bd6434..a420bf6a26 100644 --- a/libexec/scoop-config.ps1 +++ b/libexec/scoop-config.ps1 @@ -92,6 +92,9 @@ # API key used for uploading/scanning files using virustotal. # See: 'https://support.virustotal.com/hc/en-us/articles/115002088769-Please-give-me-an-API-key' # +# use_virustotal: $true|$false +# When set to $true, Scoop will always use VirusTotal to scan files before downloading. +# # cat_style: # When set to a non-empty string, Scoop will use 'bat' to display the manifest for # the `scoop cat` command and while doing manifest review. This requires 'bat' to be diff --git a/libexec/scoop-download.ps1 b/libexec/scoop-download.ps1 index 34a78ea952..7db9bb3c27 100644 --- a/libexec/scoop-download.ps1 +++ b/libexec/scoop-download.ps1 @@ -18,9 +18,11 @@ # -s, --skip-hash-check Skip hash verification (use with caution!) # -u, --no-update-scoop Don't update Scoop before downloading if it's outdated # -a, --arch <32bit|64bit|arm64> Use the specified architecture, if the app supports it +# -w, --virustotal-check Check the download against VirusTotal (may be slow) . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\json.ps1" # 'autoupdate.ps1' (indirectly) +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\autoupdate.ps1" # 'generate_user_manifest' (indirectly) . "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\manifest.ps1" # 'generate_user_manifest' 'Get-Manifest' @@ -30,12 +32,13 @@ if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\database.ps1" } -$opt, $apps, $err = getopt $args 'fsua:' 'force', 'skip-hash-check', 'no-update-scoop', 'arch=' +$opt, $apps, $err = getopt $args 'fsua:w' 'force', 'skip-hash-check', 'no-update-scoop', 'arch=', 'virustotal-check' if ($err) { error "scoop download: $err"; exit 1 } $check_hash = !($opt.s -or $opt.'skip-hash-check') $use_cache = !($opt.f -or $opt.force) $architecture = Get-DefaultArchitecture +$check_virustotal = $opt.w -or $opt.'virustotal-check' -or (get_config USE_VIRUSTOTAL $false) try { $architecture = Format-ArchitectureString ($opt.a + $opt.arch) } catch { @@ -101,9 +104,16 @@ foreach ($curr_app in $apps) { } if(Test-Aria2Enabled) { - Invoke-CachedAria2Download $app $version $manifest $architecture $cachedir $manifest.cookie $use_cache $curr_check_hash + Invoke-CachedAria2Download $app $version $manifest $architecture $cachedir $manifest.cookie $use_cache $curr_check_hash $check_virustotal } else { - foreach($url in script:url $manifest $architecture) { + $urls = @(script:url $manifest $architecture) + $urls = if ($check_virustotal) { + Test-UrlsWithVirusTotal $app $urls $manifest $architecture + } else { + $urls + } + + foreach($url in $urls) { try { Invoke-CachedDownload $app $version $url $null $manifest.cookie $use_cache } catch { diff --git a/libexec/scoop-info.ps1 b/libexec/scoop-info.ps1 index 1c147e17c0..f394dc7554 100644 --- a/libexec/scoop-info.ps1 +++ b/libexec/scoop-info.ps1 @@ -7,6 +7,7 @@ . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' . "$PSScriptRoot\..\lib\versions.ps1" # 'Get-InstalledVersion', 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\download.ps1" # 'Get-RemoteFileSize' +. "$PSScriptRoot\..\lib\helper\file-information.ps1" # 'Get-RemoteFileSize' $opt, $app, $err = getopt $args 'v' 'verbose' $original_app = $app diff --git a/libexec/scoop-install.ps1 b/libexec/scoop-install.ps1 index 6af199a8b8..dab7f0941a 100644 --- a/libexec/scoop-install.ps1 +++ b/libexec/scoop-install.ps1 @@ -24,6 +24,7 @@ # -i, --independent Don't install dependencies automatically # -k, --no-cache Don't use the download cache # -s, --skip-hash-check Skip hash validation (use with caution!) +# -w, --virustotal-check Check the download against VirusTotal (may be slow) # -u, --no-update-scoop Don't update Scoop before installing if it's outdated # -a, --arch <32bit|64bit|arm64> Use the specified architecture, if the app supports it @@ -43,11 +44,12 @@ if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\database.ps1" } -$opt, $apps, $err = getopt $args 'giksua:' 'global', 'independent', 'no-cache', 'skip-hash-check', 'no-update-scoop', 'arch=' +$opt, $apps, $err = getopt $args 'gikswua:' 'global', 'independent', 'no-cache', 'skip-hash-check', 'virustotal-check', 'no-update-scoop', 'arch=' if ($err) { error "scoop install: $err"; exit 1 } $global = $opt.g -or $opt.global $check_hash = !($opt.s -or $opt.'skip-hash-check') +$check_virustotal = $opt.w -or $opt.'virustotal-check' -or (get_config USE_VIRUSTOTAL $false) $independent = $opt.i -or $opt.independent $use_cache = !($opt.k -or $opt.'no-cache') $architecture = Get-DefaultArchitecture @@ -132,7 +134,7 @@ if ((Test-Aria2Enabled) -and (get_config 'aria2-warning-enabled' $true)) { warn "Should it cause issues, run 'scoop config aria2-enabled false' to disable it." warn "To disable this warning, run 'scoop config aria2-warning-enabled false'." } -$apps | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash } +$apps | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash $check_virustotal } show_suggestions $suggested diff --git a/libexec/scoop-update.ps1 b/libexec/scoop-update.ps1 index bc590a13f6..51f43218c6 100644 --- a/libexec/scoop-update.ps1 +++ b/libexec/scoop-update.ps1 @@ -11,6 +11,7 @@ # -i, --independent Don't install dependencies automatically # -k, --no-cache Don't use the download cache # -s, --skip-hash-check Skip hash validation (use with caution!) +# -w, --virustotal-check Check the download against VirusTotal (may be slow) # -q, --quiet Hide extraneous messages # -a, --all Update all apps (alternative to '*') @@ -21,6 +22,7 @@ . "$PSScriptRoot\..\lib\psmodules.ps1" . "$PSScriptRoot\..\lib\decompress.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\versions.ps1" . "$PSScriptRoot\..\lib\depends.ps1" . "$PSScriptRoot\..\lib\install.ps1" @@ -29,11 +31,12 @@ if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\database.ps1" } -$opt, $apps, $err = getopt $args 'gfiksqa' 'global', 'force', 'independent', 'no-cache', 'skip-hash-check', 'quiet', 'all' +$opt, $apps, $err = getopt $args 'gfikswqa' 'global', 'force', 'independent', 'no-cache', 'skip-hash-check', 'virustotal-check', 'quiet', 'all' if ($err) { error "scoop update: $err"; exit 1 } $global = $opt.g -or $opt.global $force = $opt.f -or $opt.force $check_hash = !($opt.s -or $opt.'skip-hash-check') +$check_virustotal = $opt.w -or $opt.'virustotal-check' -or (get_config USE_VIRUSTOTAL $false) $use_cache = !($opt.k -or $opt.'no-cache') $quiet = $opt.q -or $opt.quiet $independent = $opt.i -or $opt.independent @@ -258,7 +261,7 @@ function Sync-Bucket { } } -function update($app, $global, $quiet = $false, $independent, $suggested, $use_cache = $true, $check_hash = $true) { +function update($app, $global, $quiet = $false, $independent, $suggested, $use_cache = $true, $check_hash = $true, $check_virustotal = $false) { $old_version = Select-CurrentVersion -AppName $app -Global:$global $old_manifest = installed_manifest $app $old_version $global $install = install_info $app $old_version $global @@ -300,14 +303,22 @@ function update($app, $global, $quiet = $false, $independent, $suggested, $use_c } #endregion Workaround for #2952 + # can be multiple urls: if there are, then installer should go first to make 'installer.args' section work + $urls = @(script:url $manifest $architecture) + # region Workaround # Workaround for https://github.com/ScoopInstaller/Scoop/issues/2220 until install is refactored # Remove and replace whole region after proper fix Write-Host 'Downloading new version' if (Test-Aria2Enabled) { - Invoke-CachedAria2Download $app $version $manifest $architecture $cachedir $manifest.cookie $true $check_hash + Invoke-CachedAria2Download $app $version $manifest $architecture $cachedir $manifest.cookie $true $check_hash $check_virustotal } else { $urls = script:url $manifest $architecture + $urls = if ($check_virustotal) { + Test-UrlsWithVirusTotal $app $urls $manifest $architecture + } else { + $urls + } foreach ($url in $urls) { Invoke-CachedDownload $app $version $url $null $manifest.cookie $true @@ -376,12 +387,12 @@ function update($app, $global, $quiet = $false, $independent, $suggested, $use_c } if ($independent) { - install_app $app $architecture $global $suggested $use_cache $check_hash + install_app $app $architecture $global $suggested $use_cache $check_hash $check_virustotal } else { # Also add missing dependencies $apps = @(Get-Dependency $app $architecture) -ne $app ensure_none_failed $apps - $apps.Where({ !(installed $_) }) + $app | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash } + $apps.Where({ !(installed $_) }) + $app | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash $check_virustotal} } } @@ -462,7 +473,7 @@ if (-not ($apps -or $all)) { $suggested = @{} # $outdated is a list of ($app, $global) tuples - $outdated | ForEach-Object { update @_ $quiet $independent $suggested $use_cache $check_hash } + $outdated | ForEach-Object { update @_ $quiet $independent $suggested $use_cache $check_hash $check_virustotal } } exit 0 diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index d56eb6e8b4..02a427d58b 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -29,14 +29,13 @@ # -p, --passthru Return reports as objects . "$PSScriptRoot\..\lib\getopt.ps1" +. "$PSScriptRoot\..\lib\virustotal.ps1" . "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' -. "$PSScriptRoot\..\lib\json.ps1" # 'json_path' -. "$PSScriptRoot\..\lib\download.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\depends.ps1" # 'Get-Dependency' $opt, $apps, $err = getopt $args 'asnup' @('all', 'scan', 'no-depends', 'no-update-scoop', 'passthru') -if ($err) { error "scoop virustotal: $err"; exit 1 } +if ($err) { "scoop virustotal: $err"; exit 1 } $all = $apps -eq '*' -or $opt.a -or $opt.all if (!$apps -and !$all) { my_usage; exit 1 } $architecture = Get-DefaultArchitecture @@ -57,214 +56,8 @@ if (!$opt.n -and !$opt.'no-depends') { $apps = $apps | Get-Dependency -Architecture $architecture | Select-Object -Unique } -$_ERR_UNSAFE = 2 -$_ERR_EXCEPTION = 4 -$_ERR_NO_INFO = 8 -$_ERR_NO_API_KEY = 16 - -$exit_code = 0 - -# Global API key: -$api_key = get_config VIRUSTOTAL_API_KEY -if (!$api_key) { - abort ("VirusTotal API key is not configured`n" + - " You could get one from https://www.virustotal.com/gui/my-apikey and set with`n" + - " scoop config virustotal_api_key ") $_ERR_NO_API_KEY -} - -# Global flag to explain only once about sleep between requests -$explained_rate_limit_sleeping = $False - -# Requests counter to slow down requests submitted to VirusTotal as -# script execution progresses -$requests = 0 - -Function ConvertTo-VirusTotalUrlId ($url) { - $url_id = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($url)) - $url_id = $url_id -replace '\+', '-' - $url_id = $url_id -replace '/', '_' - $url_id = $url_id -replace '=', '' - $url_id -} - -Function Get-VirusTotalResultByHash ($hash, $url, $app) { - $hash = $hash.ToLower() - $api_url = "https://www.virustotal.com/api/v3/files/$hash" - $headers = @{} - $headers.Add('Accept', 'application/json') - $headers.Add('x-apikey', $api_key) - $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing - $result = $response.Content - $stats = json_path $result '$.data.attributes.last_analysis_stats' - [int]$malicious = json_path $stats '$.malicious' - [int]$suspicious = json_path $stats '$.suspicious' - [int]$timeout = json_path $stats '$.timeout' - [int]$undetected = json_path $stats '$.undetected' - [int]$unsafe = $malicious + $suspicious - [int]$total = $unsafe + $undetected - [int]$fileSize = json_path $result '$.data.attributes.size' - $report_hash = json_path $result '$.data.attributes.sha256' - $report_url = "https://www.virustotal.com/gui/file/$report_hash" - if ($total -eq 0) { - info "$app`: Analysis in progress." - [PSCustomObject] @{ - 'App.Name' = $app - 'App.Url' = $url - 'App.Hash' = $hash - 'App.HashType' = $null - 'App.Size' = filesize $fileSize - 'FileReport.Url' = $report_url - 'FileReport.Hash' = $report_hash - 'UrlReport.Url' = $null - } - } else { - $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value - switch ($unsafe) { - 0 { - success "$app`: $unsafe/$total, see $report_url" - } - 1 { - warn "$app`: $unsafe/$total, see $report_url" - } - 2 { - warn "$app`: $unsafe/$total, see $report_url" - } - Default { - warn "$([char]0x1b)[31m$app`: $unsafe/$total, see $report_url$([char]0x1b)[0m" - } - } - $maliciousResults = $vendorResults | - Where-Object -Property category -EQ 'malicious' | - Select-Object -ExpandProperty engine_name - $suspiciousResults = $vendorResults | - Where-Object -Property category -EQ 'suspicious' | - Select-Object -ExpandProperty engine_name - [PSCustomObject] @{ - 'App.Name' = $app - 'App.Url' = $url - 'App.Hash' = $hash - 'App.HashType' = $null - 'App.Size' = filesize $fileSize - 'FileReport.Url' = $report_url - 'FileReport.Hash' = $report_hash - 'FileReport.Malicious' = if ($maliciousResults) { $maliciousResults } else { 0 } - 'FileReport.Suspicious' = if ($suspiciousResults) { $suspiciousResults } else { 0 } - 'FileReport.Timeout' = $timeout - 'FileReport.Undetected' = $undetected - 'UrlReport.Url' = $null - } - } - if ($unsafe -gt 0) { - $Script:exit_code = $exit_code -bor $_ERR_UNSAFE - } -} - -Function Get-VirusTotalResultByUrl ($url, $app) { - $id = ConvertTo-VirusTotalUrlId $url - $api_url = "https://www.virustotal.com/api/v3/urls/$id" - $headers = @{} - $headers.Add('Accept', 'application/json') - $headers.Add('x-apikey', $api_key) - $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing - $result = $response.Content - $id = json_path $result '$.data.id' - $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null - $last_analysis_date = json_path $result '$.data.attributes.last_analysis_date' 6>$null - $url_report_url = "https://www.virustotal.com/gui/url/$id" - info "$app`: Url report found." - if (!$hash) { - if (!$last_analysis_date) { - info "$app`: Analysis in progress." - } else { - info "$app`: Related file report not found." - warn "$app`: Manual file upload is required (instead of url submission)." - } - [PSCustomObject] @{ - 'App.Name' = $app - 'App.Url' = $url - 'App.Hash' = $null - 'App.HashType' = $null - 'FileReport.Url' = $null - 'UrlReport.Url' = $url_report_url - 'UrlReport.Hash' = $null - } - } else { - info "$app`: Related file report found." - [PSCustomObject] @{ - 'App.Name' = $app - 'App.Url' = $url - 'App.Hash' = $null - 'App.HashType' = $null - 'FileReport.Url' = $null - 'UrlReport.Url' = $url_report_url - 'UrlReport.Hash' = $hash - } - } -} - -# Submit-ToVirusTotal -# - $url: where file to check can be downloaded -# - $app: Name of the application (used for reporting) -# - $do_scan: [boolean flag] whether to actually submit to VirusTotal -# This is a parameter instead of conditionnally calling -# the function to consolidate the warning message -# - $retrying: [boolean] Optional, for internal use to retry -# submitting the file after a delay if the rate limit is -# exceeded, without risking an infinite loop (as stack -# overflow) if the submission keeps failing. -Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { - if (!$do_scan) { - warn "$app`: not found`: you can manually submit $url" - return - } - - try { - $requests += 1 - - $encoded_url = [System.Web.HttpUtility]::UrlEncode($url) - $api_url = 'https://www.virustotal.com/api/v3/urls' - $content_type = 'application/x-www-form-urlencoded' - $headers = @{} - $headers.Add('Accept', 'application/json') - $headers.Add('x-apikey', $api_key) - $headers.Add('Content-Type', $content_type) - $body = "url=$encoded_url" - $result = Invoke-WebRequest -Uri $api_url -Method POST -Headers $headers -ContentType $content_type -Body $body -UseBasicParsing - if ($result.StatusCode -eq 200) { - $id = ((json_path $result '$.data.id') -split '-')[1] - $url_report_url = "https://www.virustotal.com/gui/url/$id" - $fileSize = Get-RemoteFileSize $url - if ($fileSize -gt 80000000) { - info "$app`: Remote file size: $(filesize $fileSize). Large files might require manual file upload instead of url submission." - } - info "$app`: Analysis in progress." - [PSCustomObject] @{ - 'App.Name' = $app - 'App.Url' = $url - 'App.Size' = filesize $fileSize - 'FileReport.Url' = $null - 'UrlReport.Url' = $url_report_url - } - return - } - - # EAFP: submission failed -> sleep, then retry - if (!$retrying) { - if (!$explained_rate_limit_sleeping) { - $explained_rate_limit_sleeping = $True - info "Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" - } - Start-Sleep -s (60 + $requests) - Submit-ToVirusTotal $url $app $do_scan $True - } else { - warn "$app`: VirusTotal submission of $url failed`:`n" + - "`tAPI returned $($result.StatusCode) after retrying" - } - } catch [Exception] { - warn "$app`: VirusTotal submission failed`: $($_.Exception.Message)" - return - } -} +$api_key = Get-VirusTotalApiKey +$scan = $opt.s -or $opt.scan $reports = $apps | ForEach-Object { $app = $_ @@ -275,107 +68,9 @@ $reports = $apps | ForEach-Object { return } - [int]$index = 0 - $urls = script:url $manifest $architecture - $urls | ForEach-Object { - $url = $_ - $index++ - if ($urls.GetType().IsArray) { - info "$app`: url $index" - } - $hash = hash_for_url $manifest $url $architecture - - try { - $isHashUnsupported = $false - if ($hash -match '(?[^:]+):(?.*)') { - $algo = $matches.algo - $hash = $matches.hash - if ($matches.algo -inotin 'md5', 'sha1', 'sha256') { - $hash = $null - $isHashUnsupported = $true - warn "$app`: Unsupported hash $($matches.algo). Will search by url instead." - } - } elseif ($hash) { - $algo = 'sha256' - } - if ($hash) { - $file_report = Get-VirusTotalResultByHash $hash $url $app - $file_report.'App.HashType' = $algo - $file_report - return - } elseif (!$isHashUnsupported) { - warn "$app`: Hash not found. Will search by url instead." - } - } catch [Exception] { - $exit_code = $exit_code -bor $_ERR_EXCEPTION - if ($_.Exception.Response.StatusCode -eq 404) { - $file_report_not_found = $true - warn "$app`: File report not found. Will search by url instead." - } else { - if ($_.Exception.Response.StatusCode -in 204, 429) { - abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code - } - warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" - return - } - } - - try { - $url_report = Get-VirusTotalResultByUrl $url $app - $url_report.'App.Hash' = $hash - $url_report.'App.HashType' = $algo - if ($url_report.'UrlReport.Hash' -and ($file_report_not_found -eq $true) -and $hash) { - if ($algo -eq 'sha256') { - if ($url_report.'UrlReport.Hash' -eq $hash) { - warn "$app`: Manual file upload is required (instead of url submission) for $url" - } else { - error "$app`: Hash not matched for $url" - } - } else { - error "$app`: Hash not matched or manual file upload is required (instead of url submission) for $url" - } - $url_report - return - } - if (!$url_report.'UrlReport.Hash') { - $url_report - return - } - } catch [Exception] { - $exit_code = $exit_code -bor $_ERR_EXCEPTION - if ($_.Exception.Response.StatusCode -eq 404) { - warn "$app`: Url report not found. Will submit $url" - Submit-ToVirusTotal $url $app ($opt.scan -or $opt.s) - return - } else { - if ($_.Exception.Response.StatusCode -in 204, 429) { - abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code - } - warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" - return - } - } - - try { - $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app - $file_report.'App.Hash' = $hash - $file_report.'App.HashType' = $algo - $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' - $file_report - warn "$app`: Unable to check hash match for $url" - } catch [Exception] { - $exit_code = $exit_code -bor $_ERR_EXCEPTION - if ($_.Exception.Response.StatusCode -eq 404) { - warn "$app`: File report not found for unknown reason. Manual file upload is required (instead of url submission)." - $url_report - } else { - if ($_.Exception.Response.StatusCode -in 204, 429) { - abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code - } - warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" - return - } - } + $report = virustotal_check_app $app $manifest $architecture $api_key $scan + if ($report) { + $report } } if ($opt.p -or $opt.'passthru') {