Skip to content

Commit bb3f3ce

Browse files
jvlflamejvlflame
authored
Merge pull request #403 from javinizer/maint/r18dev
2.6.0
2 parents b26a1d8 + e7660b9 commit bb3f3ce

7 files changed

Lines changed: 64 additions & 75 deletions

File tree

‎.github/CHANGELOG.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,29 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](http://keepachangelog.com/)
66
and this project adheres to [Semantic Versioning](http://semver.org/).
77

8+
## [2.6.0]
9+
10+
### Added
11+
12+
- Added setting `sleep` to add a delay between scraper threads
13+
14+
### Changed
15+
16+
- Reduced throttlelimit from 10 -> 3
17+
- Added custom UserAgent identifier to r18.dev scraper requests
18+
- Set JAVLibrary scrapers to disabled by default
19+
- Set throttlelimit to 1 by default
20+
21+
### Removed
22+
23+
- Removed all cloudflare validation from JAVLibrary scraper
24+
25+
## [2.5.18]
26+
27+
### Fixed
28+
29+
- Fixed age verification error on DmmJa
30+
831
## [2.5.17]
932

1033
### Fixed

‎src/Javinizer/Javinizer.psd1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313

1414
# Version number of this module.
1515

16-
ModuleVersion = '2.5.18'
16+
ModuleVersion = '2.6.0'
1717

1818
# Supported PSEditions
1919
# CompatiblePSEditions = @('Core')

‎src/Javinizer/Public/Get-R18DevData.ps1‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
$UserAgent = 'Javinizer (+https://github.com/javinizer/Javinizer)'
2+
13
function Get-R18DevData {
24
[CmdletBinding()]
35
param (
@@ -31,7 +33,7 @@ function Get-R18DevData {
3133

3234
try {
3335
Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$($MyInvocation.MyCommand.Name)] Performing [GET] on URL [$apiUrl]"
34-
$webRequest = (Invoke-WebRequest -Uri $apiUrl -Method Get -Verbose:$false).Content | ConvertFrom-Json
36+
$webRequest = (Invoke-WebRequest -Uri $apiUrl -UserAgent $UserAgent -Method Get -Verbose:$false).Content | ConvertFrom-Json
3537
} catch {
3638
Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$($MyInvocation.MyCommand.Name)] Error [GET] on URL [$Url]: $PSItem" -Action 'Continue'
3739
}

‎src/Javinizer/Public/Get-R18DevUrl.ps1‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
$UserAgent = 'Javinizer (+https://github.com/javinizer/Javinizer)'
2+
13
function Get-R18DevUrl {
24
[CmdletBinding()]
35
param (
@@ -34,7 +36,7 @@ function Get-R18DevUrl {
3436
# Try matching the video with Video ID
3537
try {
3638
Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Performing [GET] on URL [$searchUrl]"
37-
$webRequest = Invoke-WebRequest -Uri $searchUrl -Method Get -Verbose:$false | ConvertFrom-Json
39+
$webRequest = Invoke-WebRequest -Uri $searchUrl -UserAgent $UserAgent -Method Get -Verbose:$false | ConvertFrom-Json
3840
} catch {
3941
Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Error occured on [GET] on URL [$searchUrl]: $PSItem" -Action 'Continue'
4042
}
@@ -44,7 +46,7 @@ function Get-R18DevUrl {
4446

4547
try {
4648
Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Performing [GET] on Uri [$testUrl]"
47-
$webRequest = Invoke-WebRequest -Uri $testUrl -Method Get -Verbose:$false | ConvertFrom-Json
49+
$webRequest = Invoke-WebRequest -Uri $testUrl -UserAgent $UserAgent -Method Get -Verbose:$false | ConvertFrom-Json
4850
} catch {
4951
$webRequest = $null
5052
}

‎src/Javinizer/Public/Javinizer.ps1‎

Lines changed: 22 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -393,11 +393,6 @@ function Javinizer {
393393
[Parameter(ParameterSetName = 'Info')]
394394
[Switch]$AllResults,
395395

396-
[Parameter(ParameterSetName = 'Info')]
397-
[Parameter(ParameterSetName = 'Path')]
398-
[Parameter(ParameterSetName = 'Javlibrary')]
399-
[PSObject]$CfSession,
400-
401396
[Parameter(ParameterSetName = 'Emby')]
402397
[Switch]$SetEmbyThumbs,
403398

@@ -701,10 +696,6 @@ function Javinizer {
701696
}
702697

703698
'Info' {
704-
if (($Javlibrary -or $JavlibraryZh -or $JavlibraryJa) -or ($Find -like '*javlibrary*' -or $Find -like '*g46e*' -or $Find -like '*m45e*')) {
705-
$CfSession = Test-JavlibraryCf -Settings $Settings -CfSession $CfSession
706-
}
707-
708699
if ($Find -match 'https?:\/\/') {
709700
$urlObject = Get-JVUrlLocation -Url $Find -Settings $Settings
710701
$data = foreach ($item in $urlObject) {
@@ -756,7 +747,7 @@ function Javinizer {
756747
$data = Get-JVData -Id $Find -Javlibrary:$Javlibrary -JavlibraryJa:$JavlibraryJa -JavlibraryZh:$JavlibraryZh -Dmm:$Dmm `
757748
-DmmJa:$DmmJa -R18Dev:$R18Dev -Javbus:$Javbus -JavbusJa:$JavbusJa -JavbusZh:$JavbusZh -Jav321Ja:$Jav321Ja -JavlibraryBaseUrl $Settings.'javlibrary.baseurl' `
758749
-MgstageJa:$MgstageJa -Aventertainment:$Aventertainment -AventertainmentJa:$AventertainmentJa -Tokyohot:$Tokyohot -TokyohotJa:$TokyohotJa -TokyohotZh:$TokyohotZh -UncensorCsvPath $uncensorCsvPath -Strict:$Strict `
759-
-Javdb:$Javdb -JavdbZh:$JavdbZh -Session:$CfSession -JavdbSession:$Settings.'javdb.cookie.session' -AllResults:$AllResults
750+
-Javdb:$Javdb -JavdbZh:$JavdbZh -JavdbSession:$Settings.'javdb.cookie.session' -AllResults:$AllResults
760751
}
761752

762753
if ($Aggregated) {
@@ -885,14 +876,13 @@ function Javinizer {
885876
}
886877

887878
'Javlibrary' {
888-
$CfSession = Test-JavlibraryCf -Settings $Settings -CfSession $CfSession
889879
try {
890880
if (!($Path)) {
891881
$Path = $null -ne $Settings.'location.input' -and $Settings.'location.input' -ne '' ? $Settings.'location.input' : (Get-Location).Path
892882
}
893883

894884
$javlibraryBaseUrl = $Settings.'javlibrary.baseurl'
895-
$request = Invoke-WebRequest -Uri "$javlibraryBaseUrl/en/mv_owned.php" -WebSession $CfSession -UserAgent $CfSession.UserAgent -Verbose:$false -Headers @{
885+
$request = Invoke-WebRequest -Uri "$javlibraryBaseUrl/en/mv_owned.php" -Verbose:$false -Headers @{
896886
"method" = "GET"
897887
"authority" = "$javlibraryBaseUrl"
898888
"scheme" = "https"
@@ -940,7 +930,7 @@ function Javinizer {
940930
$index = 1
941931
foreach ($movieId in $unowned) {
942932
Write-Progress -Id 1 -Activity "Javinizer" -Status "Remaining Jobs: $($unowned.Count-$index)" -PercentComplete ($index / $unowned.Count * 100) -CurrentOperation "Setting owned: $movieId"
943-
Set-JavlibraryOwned -Id $movieId -UserId $Settings.'javlibrary.cookie.userid' -LoginSession $Settings.'javlibrary.cookie.session' -Session:$CfSession -BaseUrl $javlibraryBaseUrl
933+
Set-JavlibraryOwned -Id $movieId -UserId $Settings.'javlibrary.cookie.userid' -LoginSession $Settings.'javlibrary.cookie.session' -BaseUrl $javlibraryBaseUrl
944934
$index++
945935
}
946936
} else {
@@ -957,10 +947,6 @@ function Javinizer {
957947
}
958948

959949
'Path' {
960-
if (!$IsThread -and ($Settings.'scraper.movie.javlibrary' -or $Settings.'scraper.movie.javlibraryja' -or $Settings.'scraper.movie.javlibraryzh')) {
961-
$CfSession = Test-JavlibraryCf -Settings $Settings -CfSession $CfSession
962-
}
963-
964950
if (!($Path)) {
965951
try {
966952
# Default path to location.input in settings if not specified
@@ -1008,7 +994,7 @@ function Javinizer {
1008994
$mediaInfo = Get-JVMediaInfo -Path $movie.FullName
1009995
}
1010996

1011-
$javData = Get-JVData -Url $Url -Settings $Settings -UncensorCsvPath $uncensorCsvPath -Session:$CfSession -JavdbSession:$Settings.'javdb.cookie.session'
997+
$javData = Get-JVData -Url $Url -Settings $Settings -UncensorCsvPath $uncensorCsvPath -JavdbSession:$Settings.'javdb.cookie.session'
1012998
if ($null -ne $javData) {
1013999
$javAggregatedData = $javData | Get-JVAggregatedData -Settings $Settings -FileName $javMovies.BaseName -MediaInfo $mediaInfo | Test-JVData -RequiredFields $Settings.'sort.metadata.requiredfield'
10141000
if ($javAggregatedData.NullFields -eq '') {
@@ -1040,8 +1026,8 @@ function Javinizer {
10401026
}
10411027
}
10421028
} else {
1043-
if ($Settings.'throttlelimit' -lt 1 -or $Settings.'throttlelimit' -gt 10) {
1044-
Write-JVLog -Write $script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$($MyInvocation.MyCommand.Name)] Setting 'scraper.throttlelimit' must be within accepted values (1-10)"
1029+
if ($Settings.'throttlelimit' -lt 1 -or $Settings.'throttlelimit' -gt 3) {
1030+
Write-JVLog -Write $script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$($MyInvocation.MyCommand.Name)] Setting 'scraper.throttlelimit' must be within accepted values (1-3)"
10451031
}
10461032

10471033
if (!($PSboundParameters.ContainsKey('IsThread'))) {
@@ -1052,15 +1038,15 @@ function Javinizer {
10521038
Import-Module $using:jvModulePath
10531039
$jvMovie = $_
10541040
$Settings = $using:Settings
1055-
$jvData = Javinizer -IsThread -IsWeb -IsWebType $using:IsWebType -WebTempPath:$using:WebTempPath -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -CfSession:$using:CfSession -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
1041+
$jvData = Javinizer -IsThread -IsWeb -IsWebType $using:IsWebType -WebTempPath:$using:WebTempPath -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
10561042
Write-Output $jvData
10571043
}
10581044
} else {
10591045
$javMovies | Invoke-JVParallel -IsWeb -IsWebType 'sort' -MaxQueue $Settings.'throttlelimit' -Throttle $Settings.'throttlelimit' -Quiet:$true -ScriptBlock {
10601046
Import-Module $using:jvModulePath
10611047
$jvMovie = $_
10621048
$Settings = $using:Settings
1063-
Javinizer -IsThread -IsWeb -IsWebType $using:IsWebType -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -CfSession:$using:CfSession -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
1049+
Javinizer -IsThread -IsWeb -IsWebType $using:IsWebType -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
10641050
}
10651051
}
10661052

@@ -1069,20 +1055,31 @@ function Javinizer {
10691055
Import-Module $using:jvModulePath
10701056
$jvMovie = $_
10711057
$Settings = $using:Settings
1072-
$jvData = Javinizer -IsThread -Search -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -CfSession:$using:CfSession -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
1058+
$jvData = Javinizer -IsThread -Search -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
10731059
Write-Output $jvData
10741060
}
10751061
} else {
10761062
$javMovies | Invoke-JVParallel -MaxQueue $Settings.'throttlelimit' -Throttle $Settings.'throttlelimit' -Quiet:$HideProgress -ScriptBlock {
10771063
Import-Module $using:jvModulePath
10781064
$jvMovie = $_
10791065
$Settings = $using:Settings
1080-
Javinizer -IsThread -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -CfSession:$using:CfSession -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
1066+
Javinizer -IsThread -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
10811067
}
10821068
}
10831069
}
10841070

10851071
if ($PSboundParameters.ContainsKey('IsThread')) {
1072+
$sleepSeconds = $Settings.'sleep'
1073+
1074+
if (!($sleepSeconds)) {
1075+
# Default sleep to 5 seconds if not specified
1076+
$sleepSeconds = 5
1077+
}
1078+
1079+
if ($sleepSeconds -gt 0) {
1080+
Start-Sleep -Seconds $sleepSeconds
1081+
}
1082+
10861083
foreach ($movie in $javMovies) {
10871084
if (!$MoveToFolder -and ($DestinationPath -ne $movie.Directory)) {
10881085
$DestinationPath = $movie.Directory
@@ -1092,7 +1089,7 @@ function Javinizer {
10921089
$mediaInfo = Get-JVMediaInfo -Path $movie.FullName
10931090
}
10941091

1095-
$javData = Get-JVData -Id $movie.Id -Settings $Settings -UncensorCsvPath $uncensorCsvPath -Strict:$Strict -Session:$CfSession -JavdbSession:$Settings.'javdb.cookie.session'
1092+
$javData = Get-JVData -Id $movie.Id -Settings $Settings -UncensorCsvPath $uncensorCsvPath -Strict:$Strict -JavdbSession:$Settings.'javdb.cookie.session'
10961093
$javAggregatedData = $javData | Get-JVAggregatedData -Settings $Settings -FileName $movie.BaseName -MediaInfo $mediaInfo | Test-JVData -RequiredFields $Settings.'sort.metadata.requiredfield'
10971094

10981095
if ($PSBoundParameters.ContainsKey('IsWeb') -or $PSBoundParameters.ContainsKey('Search')) {

‎src/Javinizer/Universal/Repository/javinizergui.ps1‎

Lines changed: 1 addition & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
$cache:guiVersion = '2.5.17-1'
1+
$cache:guiVersion = '2.6.0-1'
22

33
# Define Javinizer module file paths
44
$cache:modulePath = (Get-InstalledModule -Name Javinizer).InstalledLocation
@@ -679,7 +679,6 @@ function Show-JVCfModal {
679679
$cf_chl_prog = (Get-UDElement -Id 'textbox-javlibrary-cfchlprog').value
680680
$cf_clearance = (Get-UDElement -Id 'textbox-javlibrary-cfclearance').value
681681
$UserAgent = (Get-UDElement -Id 'textbox-javlibrary-useragent').value
682-
$cache:cfSession = Get-CfSession -cf_chl_2 $cf_chl_2 -cf_chl_prog $cf_chl_prog -cf_clearance $cf_clearance -UserAgent $useragent -BaseUrl $cache:settings.'javlibrary.baseurl'
683682
} catch {
684683
Show-JVToast -Type Error -Message "$PSItem"
685684
Hide-UDModal
@@ -792,41 +791,6 @@ function Invoke-JavinizerWeb {
792791

793792
process {
794793
if (!($cache:inProgress)) {
795-
# Check if javlibrary cloudflare protection is enabled
796-
if ($cache:settings.'scraper.movie.javlibrary' -or $cache:settings.'scraper.movie.javlibraryja' -or $cache:settings.'scraper.movie.javlibraryzh') {
797-
if (-not ($cache:cfSession)) {
798-
try {
799-
Invoke-WebRequest -Uri $cache:settings.'javlibrary.baseurl' -Verbose:$false | Out-Null
800-
} catch {
801-
try {
802-
# Test with persisted settings
803-
if ($cache:settings.'javlibrary.cookie.cf_chl_2' -and $cache:settings.'javlibrary.cookie.cf_chl_prog' -and $cache:settings.'javlibrary.cookie.cf_clearance' -and $cache:settings.'javlibrary.browser.useragent') {
804-
$cache:cfSession = Get-CfSession -cf_chl_2:$cache:settings.'javlibrary.cookie.cf_chl_2' -cf_chl_prog:$cache:settings.'javlibrary.cookie.cf_chl_prog' -cf_clearance:$cache:settings.'javlibrary.cookie.cf_clearance' `
805-
-UserAgent:$cache:settings.'javlibrary.browser.useragent' -BaseUrl $cache:settings.'javlibrary.baseurl'
806-
807-
# Testing with the newly created session sometimes fails if there is no wait time
808-
Start-Sleep -Seconds 1
809-
810-
Invoke-WebRequest -Uri $cache:settings.'javlibrary.baseurl' -WebSession $cache:cfSession -UserAgent $cache:cfSession.UserAgent -Verbose:$false | Out-Null
811-
} else {
812-
Show-JVCfModal
813-
return
814-
}
815-
} catch {
816-
Show-JVCfModal
817-
return
818-
}
819-
}
820-
} else {
821-
try {
822-
Invoke-WebRequest -Uri $cache:settings.'javlibrary.baseurl' -WebSession $cache:cfSession -UserAgent $cache:cfSession.UserAgent -Verbose:$false | Out-Null
823-
} catch {
824-
Show-JVCfModal
825-
return
826-
}
827-
}
828-
}
829-
830794
Show-JVProgressModal -Sort
831795

832796
if ($interactive) {

0 commit comments

Comments
 (0)