diff --git a/go.mod b/go.mod
index 72df90e87..fcafa40d8 100644
--- a/go.mod
+++ b/go.mod
@@ -37,7 +37,7 @@ require (
github.com/lucasb-eyer/go-colorful v1.2.0
github.com/marcsauter/single v0.0.0-20201009143647-9f8d81240be2
github.com/markphelps/optional v0.11.0
- github.com/mattn/go-sqlite3 v1.14.19
+ github.com/mattn/go-sqlite3 v1.14.22
github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2
github.com/mholt/archiver v3.1.1+incompatible
github.com/mozillazg/go-slugify v0.2.0
@@ -54,7 +54,7 @@ require (
github.com/thoas/go-funk v0.9.3
github.com/tidwall/gjson v1.17.0
github.com/x-cray/logrus-prefixed-formatter v0.5.2
- github.com/xo/dburl v0.20.2
+ github.com/xo/dburl v0.21.1
golang.org/x/crypto v0.18.0
golang.org/x/net v0.20.0
golang.org/x/oauth2 v0.16.0
@@ -74,7 +74,7 @@ require (
github.com/antchfx/xpath v1.1.10 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.2.0 // indirect
- github.com/blevesearch/bleve_index_api v1.1.4
+ github.com/blevesearch/bleve_index_api v1.1.5
github.com/blevesearch/geo v0.1.18 // indirect
github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
github.com/blevesearch/gtreap v0.1.1 // indirect
@@ -149,6 +149,5 @@ require (
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/sourcemap.v1 v1.0.5 // indirect
- gopkg.in/yaml.v2 v2.4.0 // indirect
willnorris.com/go/gifresize v1.0.0 // indirect
)
diff --git a/go.sum b/go.sum
index ab86309d2..628a8d57f 100644
--- a/go.sum
+++ b/go.sum
@@ -42,10 +42,8 @@ github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjL
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg=
github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA=
-github.com/blevesearch/bleve_index_api v1.1.3 h1:aNyMEiWFviY/1zYm7JCr2lZRIiYX0TMtz3oymxxbApc=
-github.com/blevesearch/bleve_index_api v1.1.3/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
-github.com/blevesearch/bleve_index_api v1.1.4 h1:n9Ilxlb80g9DAhchR95IcVrzohamDSri0wPnkKnva50=
-github.com/blevesearch/bleve_index_api v1.1.4/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
+github.com/blevesearch/bleve_index_api v1.1.5 h1:0q05mzu6GT/kebzqKywCpou/eUea9wTKa7kfqX7QX+k=
+github.com/blevesearch/bleve_index_api v1.1.5/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw=
github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
@@ -105,8 +103,6 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m
github.com/emicklei/go-restful-openapi/v2 v2.9.1 h1:Of8B1rXdG81il5TTiSY+9Qrh7pYOr8aLdynHIpvo7fM=
github.com/emicklei/go-restful-openapi/v2 v2.9.1/go.mod h1:VKNgZyYviM1hnyrjD9RDzP2RuE94xTXxV+u6MGN4v4k=
github.com/emicklei/go-restful/v3 v3.7.3/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
-github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/emicklei/go-restful/v3 v3.11.2 h1:1onLa9DcsMYO9P+CXaL0dStDqQ2EHHXLiz+BtnqkLAU=
github.com/emicklei/go-restful/v3 v3.11.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -133,29 +129,19 @@ github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f/go.mod h1:D5ao98qkA
github.com/getlantern/systray v1.2.2 h1:dCEHtfmvkJG7HZ8lS/sLklTH4RKUcIsKrAD9sThoEBE=
github.com/getlantern/systray v1.2.2/go.mod h1:pXFOI1wwqwYXEhLPm9ZGjS2u/vVELeIgNMY5HvhHhcE=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q=
github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs=
github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
-github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
-github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU=
github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4=
github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
-github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8=
-github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
-github.com/go-openapi/spec v0.20.13 h1:XJDIN+dLH6vqXgafnl5SUIMnzaChQ6QTo0/UPMbkIaE=
-github.com/go-openapi/spec v0.20.13/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw=
github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do=
github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-openapi/swag v0.22.6 h1:dnqg1XfHXL9aBxSbktBqFR5CxVyVI+7fYWhAf1JOeTw=
github.com/go-openapi/swag v0.22.6/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0=
-github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo=
-github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
@@ -264,7 +250,6 @@ github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJ
github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
@@ -280,10 +265,10 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
-github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
-github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 h1:YocNLcTBdEdvY3iDK6jfWXvEaM5OCKkjxPKoJRdB3Gg=
@@ -339,14 +324,12 @@ github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/
github.com/putdotio/go-putio v1.7.1 h1:316PpOMO2a7H73foRxlpHmekeLso07et26Z00YlwQ2A=
github.com/putdotio/go-putio v1.7.1/go.mod h1:QhjpLhn3La/ea4FeJlp1qsiaFZDC0EIO8VUe8VEKMV0=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/robertkrimen/otto v0.2.1 h1:FVP0PJ0AHIjC+N4pKCG9yCDz6LHNPCwi/GKID5pGGF0=
-github.com/robertkrimen/otto v0.2.1/go.mod h1:UPwtJ1Xu7JrLcZjNWN8orJaM5n5YEtqL//farB5FlRY=
github.com/robertkrimen/otto v0.3.0 h1:5RI+8860NSxvXywDY9ddF5HcPw0puRsd8EgbXV0oqRE=
github.com/robertkrimen/otto v0.3.0/go.mod h1:uW9yN1CYflmUQYvAMS0m+ZiNo3dMzRUDQJX0jWbzgxw=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
-github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo=
github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc=
@@ -389,12 +372,8 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJ
github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
-github.com/xo/dburl v0.18.2 h1:9xqcVf+JEV7bcUa1OjCsoax06roohYFdye6xkvBKo50=
-github.com/xo/dburl v0.18.2/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco=
-github.com/xo/dburl v0.18.3 h1:z271VmL/pk00rAF+0JrwrsOLyQcEBqqjyH3qX7eeJIE=
-github.com/xo/dburl v0.18.3/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco=
-github.com/xo/dburl v0.20.2 h1:59zqIzahtfQ/X9E6fyp1ziHwjYEFy65opjpyQPmZC7w=
-github.com/xo/dburl v0.20.2/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco=
+github.com/xo/dburl v0.21.1 h1:n5mfH1fh51RQbvuaKKykGslodt8pZqyZJMNohVo2zK0=
+github.com/xo/dburl v0.21.1/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
@@ -408,12 +387,6 @@ golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
-golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
-golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
-golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -448,18 +421,10 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
-golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
-golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
-golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.2.0/go.mod h1:Cwn6afJ8jrQwYMxQDTpISoXmXW9I6qF6vDeuuoX3Ibs=
-golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0=
-golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM=
-golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
-golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -488,10 +453,6 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -500,10 +461,6 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
-golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/package.json b/package.json
index b4d9ac841..e81657043 100644
--- a/package.json
+++ b/package.json
@@ -15,9 +15,9 @@
"@fortawesome/fontawesome-free": "5.15.4",
"@fortawesome/fontawesome-svg-core": "1.2.36",
"@mdi/font": "7.4.47",
- "buefy": "0.9.27",
+ "buefy": "0.9.28",
"bulma-extensions": "6.2.7",
- "date-fns": "2.30.0",
+ "date-fns": "3.3.1",
"ky": "0.30.0",
"pretty-bytes": "6.1.1",
"videojs-hotkeys": "0.2.28",
@@ -35,8 +35,8 @@
"wampy": "6.4.2"
},
"devDependencies": {
- "@babel/core": "7.23.7",
- "@babel/eslint-parser": "7.23.3",
+ "@babel/core": "7.23.9",
+ "@babel/eslint-parser": "7.23.10",
"@vue/cli-plugin-babel": "5.0.8",
"@vue/cli-plugin-eslint": "5.0.8",
"@vue/cli-service": "5.0.8",
@@ -48,16 +48,16 @@
"eslint-plugin-node": "11.1.0",
"eslint-plugin-promise": "6.1.1",
"eslint-plugin-standard": "4.1.0",
- "eslint-plugin-vue": "9.20.1",
+ "eslint-plugin-vue": "9.21.1",
"less": "4.2.0",
- "less-loader": "11.1.4",
- "sass": "1.69.7",
- "sass-loader": "13.3.3",
+ "less-loader": "12.2.0",
+ "sass": "1.70.0",
+ "sass-loader": "14.0.0",
"simple-progress-webpack-plugin": "2.0.0",
"vue-cli-plugin-i18n": "2.3.2",
"vue-i18n-extract": "2.0.7",
"vue-template-compiler": "2.7.16",
- "webpack": "5.89.0"
+ "webpack": "5.90.1"
},
"eslintConfig": {
"root": true,
diff --git a/pkg/api/actors.go b/pkg/api/actors.go
index f698f0a1b..2a28b4d3c 100644
--- a/pkg/api/actors.go
+++ b/pkg/api/actors.go
@@ -710,11 +710,10 @@ func (i ActorResource) editActorExtRefs(req *restful.Request, resp *restful.Resp
var links []models.ExternalReferenceLink
- db, _ := models.GetDB()
- defer db.Close()
+ commonDb, _ := models.GetCommonDB()
// find any links that were removed
- db.Preload("ExternalReference").Where("internal_table = 'actors' and internal_db_id = ?", id).Find(&links)
+ commonDb.Preload("ExternalReference").Where("internal_table = 'actors' and internal_db_id = ?", id).Find(&links)
for _, link := range links {
found := false
for _, url := range urls {
@@ -724,7 +723,7 @@ func (i ActorResource) editActorExtRefs(req *restful.Request, resp *restful.Resp
}
}
if !found {
- db.Delete(&link)
+ commonDb.Delete(&link)
models.AddActionActor(actor.ID, "edit_actor", "delete", "external_reference_link", link.ExternalReference.ExternalURL)
}
}
diff --git a/pkg/api/external_references.go b/pkg/api/external_references.go
index 0666ec855..71ad0bd14 100644
--- a/pkg/api/external_references.go
+++ b/pkg/api/external_references.go
@@ -1,11 +1,26 @@
package api
import (
+ "net/http"
+ "time"
+
restfulspec "github.com/emicklei/go-restful-openapi/v2"
"github.com/emicklei/go-restful/v3"
+ "github.com/xbapps/xbvr/pkg/models"
)
-//var RequestBody []byte
+// var RequestBody []byte
+type RequestEditExtRefLink struct {
+ ID uint `json:"id"`
+ ExternalReferenceID uint `json:"external_reference_id"`
+ ExternalSource string `json:"external_source"`
+ ExternalId string `json:"external_id"`
+ MatchType int `json:"match_type"`
+ InternalTable string `json:"internal_table"`
+ InternalDbId uint `json:"internal_db_id"`
+ InternalNameId string `json:"internal_name_id"`
+ DeleteDate time.Time `json:"delete_date"`
+}
type ExternalReference struct{}
@@ -39,5 +54,116 @@ func (i ExternalReference) WebService() *restful.WebService {
Metadata(restfulspec.KeyOpenAPITags, tags))
ws.Route(ws.GET("/generic/scrape_by_site/{site-id}").To(i.genericActorScraperBySite).
Metadata(restfulspec.KeyOpenAPITags, tags))
+ ws.Route(ws.POST("/edit_link").To(i.editExtRefLink).
+ Metadata(restfulspec.KeyOpenAPITags, tags).
+ Writes())
+ ws.Route(ws.DELETE("/delete_extref").To(i.deleteExtRefLink).
+ Metadata(restfulspec.KeyOpenAPITags, tags).
+ Writes())
+ ws.Route(ws.DELETE("/delete_extref_source").To(i.deleteExtRefSource).
+ Metadata(restfulspec.KeyOpenAPITags, tags).
+ Writes())
+ ws.Route(ws.DELETE("/delete_extref_source_links/all").To(i.deleteExtRefSourceLinks).
+ Metadata(restfulspec.KeyOpenAPITags, tags).
+ Writes())
+ ws.Route(ws.DELETE("/delete_extref_source_links/keep_manual").To(i.deleteExtRefSourceLinksKeepManualMatches).
+ Metadata(restfulspec.KeyOpenAPITags, tags).
+ Writes())
return ws
}
+
+func (i ExternalReference) editExtRefLink(req *restful.Request, resp *restful.Response) {
+ var r RequestEditExtRefLink
+ err := req.ReadEntity(&r)
+ if err != nil {
+ log.Error(err)
+ return
+ }
+
+ var extreflink models.ExternalReferenceLink
+ if r.ID > 0 {
+ extreflink.ExternalReference.GetIfExist(r.ID)
+ } else {
+ extreflink.FindByExternaID(r.ExternalSource, r.ExternalId)
+ }
+ extreflink.InternalTable = r.InternalTable
+ extreflink.InternalDbId = r.InternalDbId
+ extreflink.InternalNameId = r.InternalNameId
+ extreflink.MatchType = r.MatchType
+ extreflink.Save()
+ resp.WriteHeaderAndEntity(http.StatusOK, extreflink)
+}
+func (i ExternalReference) deleteExtRefLink(req *restful.Request, resp *restful.Response) {
+ // delete a single external_reference_link
+ var r RequestEditExtRefLink
+ err := req.ReadEntity(&r)
+ if err != nil {
+ log.Error(err)
+ return
+ }
+
+ var extreflink models.ExternalReferenceLink
+ if r.ID > 0 {
+ extreflink.ExternalReference.GetIfExist(r.ID)
+ } else {
+ extreflink.FindByExternaID(r.ExternalSource, r.ExternalId)
+ }
+ extreflink.ExternalReference.Delete()
+ extreflink.Delete()
+ resp.WriteHeaderAndEntity(http.StatusOK, nil)
+}
+func (i ExternalReference) deleteExtRefSource(req *restful.Request, resp *restful.Response) {
+ // deletes all external_reference_links and external_references for a source
+ var r RequestEditExtRefLink
+ err := req.ReadEntity(&r)
+ if err != nil {
+ log.Error(err)
+ return
+ }
+ commonDb, _ := models.GetCommonDB()
+
+ commonDb.Where("external_source = ?", r.ExternalSource).Delete(models.ExternalReferenceLink{})
+ commonDb.Where("external_source = ?", r.ExternalSource).Delete(models.ExternalReference{})
+
+ resp.WriteHeaderAndEntity(http.StatusOK, nil)
+}
+func (i ExternalReference) deleteExtRefSourceLinks(req *restful.Request, resp *restful.Response) {
+ // deletes external_reference_links for a source
+ var r RequestEditExtRefLink
+ err := req.ReadEntity(&r)
+ if err != nil {
+ log.Error(err)
+ return
+ }
+ commonDb, _ := models.GetCommonDB()
+ commonDb.Where("external_source like ?", r.ExternalSource).Delete(models.ExternalReferenceLink{})
+
+ resp.WriteHeaderAndEntity(http.StatusOK, nil)
+}
+func (i ExternalReference) deleteExtRefSourceLinksKeepManualMatches(req *restful.Request, resp *restful.Response) {
+ // deletes external_reference_links for a source, but keeps links the user has manually set, ie match_type = 99999
+ var r RequestEditExtRefLink
+ err := req.ReadEntity(&r)
+ if err != nil {
+ log.Error(err)
+ return
+ }
+ db, _ := models.GetDB()
+ defer db.Close()
+
+ if r.DeleteDate.IsZero() {
+ db.Where("external_source like ? and match_type not in (99999, -1)", r.ExternalSource).Delete(models.ExternalReferenceLink{})
+ } else {
+ // Fetch records to delete
+ var recordsToDelete []models.ExternalReferenceLink
+ db.Debug().Joins("JOIN external_references ON external_reference_links.external_reference_id = external_references.id").
+ Where("external_reference_links.external_source LIKE ? AND match_type NOT IN (99999, -1) AND external_references.external_date >= ?", r.ExternalSource, r.DeleteDate).
+ Find(&recordsToDelete)
+ for _, record := range recordsToDelete {
+ db.Debug().Delete(&record)
+ }
+
+ }
+
+ resp.WriteHeaderAndEntity(http.StatusOK, nil)
+}
diff --git a/pkg/api/options.go b/pkg/api/options.go
index 6262806b5..0e918ed3d 100644
--- a/pkg/api/options.go
+++ b/pkg/api/options.go
@@ -62,12 +62,16 @@ type RequestSaveOptionsWeb struct {
}
type RequestSaveOptionsAdvanced struct {
- ShowInternalSceneId bool `json:"showInternalSceneId"`
- ShowHSPApiLink bool `json:"showHSPApiLink"`
- ShowSceneSearchField bool `json:"showSceneSearchField"`
- StashApiKey string `json:"stashApiKey"`
- ScrapeActorAfterScene bool `json:"scrapeActorAfterScene"`
- UseImperialEntry bool `json:"useImperialEntry"`
+ ShowInternalSceneId bool `json:"showInternalSceneId"`
+ ShowHSPApiLink bool `json:"showHSPApiLink"`
+ ShowSceneSearchField bool `json:"showSceneSearchField"`
+ StashApiKey string `json:"stashApiKey"`
+ ScrapeActorAfterScene bool `json:"scrapeActorAfterScene"`
+ UseImperialEntry bool `json:"useImperialEntry"`
+ LinkScenesAfterSceneScraping bool `json:"linkScenesAfterSceneScraping"`
+ UseAltSrcInFileMatching bool `json:"useAltSrcInFileMatching"`
+ UseAltSrcInScriptFilters bool `json:"useAltSrcInScriptFilters"`
+ IgnoreReleasedBefore time.Time `json:"ignoreReleasedBefore"`
}
type RequestSaveOptionsFunscripts struct {
@@ -165,6 +169,18 @@ type RequestSaveOptionsTaskSchedule struct {
StashdbRescrapeHourStart int `json:"stashdbRescrapeHourStart"`
StashdbRescrapeHourEnd int `json:"stashdbRescrapeHourEnd"`
StashdbRescrapeStartDelay int `json:"stashdbRescrapeStartDelay"`
+
+ LinkScenesEnabled bool `json:"linkScenesEnabled"`
+ LinkScenesHourInterval int `json:"linkScenesHourInterval"`
+ LinkScenesUseRange bool `json:"linkScenesUseRange"`
+ LinkScenesMinuteStart int `json:"linkScenesMinuteStart"`
+ LinkScenesHourStart int `json:"linkScenesHourStart"`
+ LinkScenesHourEnd int `json:"linkScenesHourEnd"`
+ LinkScenesStartDelay int `json:"linkScenesStartDelay"`
+}
+type RequestSaveSiteMatchParams struct {
+ SiteId string `json:"site"`
+ MatchParams models.AltSrcMatchParams `json:"match_params"`
}
type RequestCuepointsResponse struct {
@@ -172,10 +188,19 @@ type RequestCuepointsResponse struct {
Actions []string `json:"actions"`
}
type RequestSCustomSiteCreate struct {
- Url string `json:"scraperUrl"`
- Name string `json:"scraperName"`
- Avatar string `json:"scraperAvatar"`
- Company string `json:"scraperCompany"`
+ Url string `json:"scraperUrl"`
+ Name string `json:"scraperName"`
+ Avatar string `json:"scraperAvatar"`
+ Company string `json:"scraperCompany"`
+ MasterSiteId string `json:"masterSiteId"`
+}
+
+type GetStorageResponse struct {
+ Volumes []models.Volume `json:"volumes"`
+ MatchOhash bool `json:"match_ohash"`
+}
+type RequestSaveOptionsStorage struct {
+ MatchOhash bool `json:"match_ohash"`
}
type ConfigResource struct{}
@@ -208,12 +233,20 @@ func (i ConfigResource) WebService() *restful.WebService {
ws.Route(ws.PUT("/sites/subscribed/{site}").To(i.toggleSubscribed).
Metadata(restfulspec.KeyOpenAPITags, tags))
+ ws.Route(ws.PUT("/sites/limit_scraping/{site}").To(i.toggleLimitScraping).
+ Metadata(restfulspec.KeyOpenAPITags, tags))
+
ws.Route(ws.POST("/scraper/force-site-update").To(i.forceSiteUpdate).
Metadata(restfulspec.KeyOpenAPITags, tags))
ws.Route(ws.POST("/scraper/delete-scenes").To(i.deleteScenes).
Metadata(restfulspec.KeyOpenAPITags, tags))
+ ws.Route(ws.GET("/site/match_params/{site}").To(i.siteMatchParams).
+ Metadata(restfulspec.KeyOpenAPITags, tags))
+ ws.Route(ws.POST("/site/save_match_params").To(i.saveSiteMatchParams).
+ Metadata(restfulspec.KeyOpenAPITags, tags))
+
// "Storage" section endpoints
ws.Route(ws.GET("/storage").To(i.listStorage).
Metadata(restfulspec.KeyOpenAPITags, tags))
@@ -225,6 +258,9 @@ func (i ConfigResource) WebService() *restful.WebService {
Param(ws.PathParameter("storage-id", "Storage ID").DataType("int")).
Metadata(restfulspec.KeyOpenAPITags, tags))
+ ws.Route(ws.PUT("/storage").To(i.saveOptionsStorage).
+ Metadata(restfulspec.KeyOpenAPITags, tags))
+
// "DLNA" section endpoints
ws.Route(ws.PUT("/interface/dlna").To(i.saveOptionsDLNA).
Metadata(restfulspec.KeyOpenAPITags, tags))
@@ -313,6 +349,10 @@ func (i ConfigResource) toggleSubscribed(req *restful.Request, resp *restful.Res
i.toggleSiteField(req, resp, "Subscribed")
}
+func (i ConfigResource) toggleLimitScraping(req *restful.Request, resp *restful.Response) {
+ i.toggleSiteField(req, resp, "LimitScraping")
+}
+
func (i ConfigResource) toggleSiteField(req *restful.Request, resp *restful.Response, field string) {
db, _ := models.GetDB()
defer db.Close()
@@ -335,6 +375,9 @@ func (i ConfigResource) toggleSiteField(req *restful.Request, resp *restful.Resp
site.Subscribed = !site.Subscribed
log.Infof("Toggling %s %v", id, site.Subscribed)
db.Model(&models.Scene{}).Where("scraper_id = ?", site.ID).Update("is_subscribed", site.Subscribed)
+ case "LimitScraping":
+ site.LimitScraping = !site.LimitScraping
+ db.Model(&models.Scene{}).Where("scraper_id = ?", site.ID).Update("limit_scraping", site.LimitScraping)
}
site.Save()
@@ -361,6 +404,49 @@ func (i ConfigResource) listSitesWithDB(req *restful.Request, resp *restful.Resp
resp.WriteHeaderAndEntity(http.StatusOK, sites)
}
+func (i ConfigResource) siteMatchParams(req *restful.Request, resp *restful.Response) {
+ db, _ := models.GetDB()
+ defer db.Close()
+
+ id := req.PathParameter("site")
+ if id == "" {
+ return
+ }
+
+ var site models.Site
+ err := site.GetIfExist(id)
+ if err != nil {
+ log.Error(err)
+ return
+ }
+
+ var matchParams models.AltSrcMatchParams
+ matchParams.UnmarshalParams(site.MatchingParams)
+ resp.WriteHeaderAndEntity(http.StatusOK, matchParams)
+}
+func (i ConfigResource) saveSiteMatchParams(req *restful.Request, resp *restful.Response) {
+ db, _ := models.GetDB()
+ defer db.Close()
+ var r RequestSaveSiteMatchParams
+ if err := req.ReadEntity(&r); err != nil {
+ APIError(req, resp, http.StatusInternalServerError, err)
+ return
+ }
+
+ var site models.Site
+ err := site.GetIfExist(r.SiteId)
+ if err != nil {
+ log.Error(err)
+ return
+ }
+
+ json, _ := json.Marshal(r.MatchParams)
+ site.MatchingParams = string(json)
+ site.Save()
+
+ resp.WriteHeaderAndEntity(http.StatusOK, nil)
+}
+
func (i ConfigResource) saveOptionsWeb(req *restful.Request, resp *restful.Response) {
var r RequestSaveOptionsWeb
err := req.ReadEntity(&r)
@@ -404,6 +490,10 @@ func (i ConfigResource) saveOptionsAdvanced(req *restful.Request, resp *restful.
config.Config.Advanced.StashApiKey = r.StashApiKey
config.Config.Advanced.ScrapeActorAfterScene = r.ScrapeActorAfterScene
config.Config.Advanced.UseImperialEntry = r.UseImperialEntry
+ config.Config.Advanced.LinkScenesAfterSceneScraping = r.LinkScenesAfterSceneScraping
+ config.Config.Advanced.UseAltSrcInFileMatching = r.UseAltSrcInFileMatching
+ config.Config.Advanced.UseAltSrcInScriptFilters = r.UseAltSrcInScriptFilters
+ config.Config.Advanced.IgnoreReleasedBefore = r.IgnoreReleasedBefore
config.SaveConfig()
resp.WriteHeaderAndEntity(http.StatusOK, r)
@@ -468,7 +558,10 @@ func (i ConfigResource) listStorage(req *restful.Request, resp *restful.Response
(select sum(files.size) from files where files.volume_id = volumes.id) as total_size
from volumes order by last_scan desc;`).Scan(&vol)
- resp.WriteHeaderAndEntity(http.StatusOK, vol)
+ var out GetStorageResponse
+ out.Volumes = vol
+ out.MatchOhash = config.Config.Storage.MatchOhash
+ resp.WriteHeaderAndEntity(http.StatusOK, out)
}
func (i ConfigResource) addStorage(req *restful.Request, resp *restful.Response) {
@@ -849,6 +942,14 @@ func (i ConfigResource) saveOptionsTaskSchedule(req *restful.Request, resp *rest
config.Config.Cron.StashdbRescrapeSchedule.HourEnd = r.StashdbRescrapeHourEnd
config.Config.Cron.StashdbRescrapeSchedule.RunAtStartDelay = r.StashdbRescrapeStartDelay
+ config.Config.Cron.LinkScenesSchedule.Enabled = r.LinkScenesEnabled
+ config.Config.Cron.LinkScenesSchedule.HourInterval = r.LinkScenesHourInterval
+ config.Config.Cron.LinkScenesSchedule.UseRange = r.LinkScenesUseRange
+ config.Config.Cron.LinkScenesSchedule.MinuteStart = r.LinkScenesMinuteStart
+ config.Config.Cron.LinkScenesSchedule.HourStart = r.LinkScenesHourStart
+ config.Config.Cron.LinkScenesSchedule.HourEnd = r.LinkScenesHourEnd
+ config.Config.Cron.LinkScenesSchedule.RunAtStartDelay = r.LinkScenesStartDelay
+
config.SaveConfig()
resp.WriteHeaderAndEntity(http.StatusOK, r)
@@ -884,6 +985,7 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
r.Name = strings.TrimSpace(r.Name)
r.Company = strings.TrimSpace(r.Company)
r.Avatar = strings.TrimSpace(r.Avatar)
+ r.MasterSiteId = strings.TrimSpace(r.MasterSiteId)
if r.Company == "" {
r.Company = r.Name
}
@@ -913,12 +1015,13 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
scrapers[key][idx].Name = r.Name
scrapers[key][idx].Company = r.Company
scrapers[key][idx].AvatarUrl = r.Avatar
+ scrapers[key][idx].MasterSiteId = r.MasterSiteId
}
}
}
if !exists {
- scraper := config.ScraperConfig{URL: r.Url, Name: r.Name, Company: r.Company, AvatarUrl: r.Avatar}
+ scraper := config.ScraperConfig{URL: r.Url, Name: r.Name, Company: r.Company, AvatarUrl: r.Avatar, MasterSiteId: r.MasterSiteId}
switch match[3] {
case "povr":
scrapers["povr"] = append(scrapers["povr"], scraper)
@@ -940,3 +1043,16 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
resp.WriteHeader(http.StatusOK)
}
+func (i ConfigResource) saveOptionsStorage(req *restful.Request, resp *restful.Response) {
+ var r RequestSaveOptionsStorage
+ err := req.ReadEntity(&r)
+ if err != nil {
+ log.Error(err)
+ return
+ }
+
+ config.Config.Storage.MatchOhash = r.MatchOhash
+ config.SaveConfig()
+
+ resp.WriteHeaderAndEntity(http.StatusOK, r)
+}
diff --git a/pkg/api/scenes.go b/pkg/api/scenes.go
index 25edba5c0..4e50ac724 100644
--- a/pkg/api/scenes.go
+++ b/pkg/api/scenes.go
@@ -1,6 +1,7 @@
package api
import (
+ "encoding/json"
"fmt"
"net/http"
"strconv"
@@ -86,6 +87,14 @@ type ResponseSceneSearchValue struct {
FieldName string `json:"fieldName"`
FieldValue string `json:"fieldValue"`
}
+type ResponseGetAlternateSources struct {
+ Url string `json:"url"`
+ Icon string `json:"site_icon"`
+ ExternalSource string `json:"external_source"`
+ ExternalId string `json:"external_id"`
+ ExternalData string `json:"external_data"`
+}
+
type SceneResource struct{}
func (i SceneResource) WebService() *restful.WebService {
@@ -148,6 +157,10 @@ func (i SceneResource) WebService() *restful.WebService {
Metadata(restfulspec.KeyOpenAPITags, tags).
Writes(models.Scene{}))
+ ws.Route(ws.GET("/alternate_source/{scene-id}").To(i.getSceneAlternateSources).
+ Metadata(restfulspec.KeyOpenAPITags, tags).
+ Writes(ResponseGetAlternateSources{}))
+
return ws
}
@@ -347,6 +360,7 @@ func (i SceneResource) getFilters(req *restful.Request, resp *restful.Response)
outAttributes = append(outAttributes, "Cast 4")
outAttributes = append(outAttributes, "Cast 5")
outAttributes = append(outAttributes, "Cast 6+")
+ outAttributes = append(outAttributes, "No Actor/Cast")
outAttributes = append(outAttributes, "Flat video")
outAttributes = append(outAttributes, "FOV: 180°")
outAttributes = append(outAttributes, "FOV: 190°")
@@ -373,6 +387,11 @@ func (i SceneResource) getFilters(req *restful.Request, resp *restful.Response)
outAttributes = append(outAttributes, "Has AI Generated Script")
outAttributes = append(outAttributes, "Has Human Generated Script")
outAttributes = append(outAttributes, "Has Favourite Actor")
+ outAttributes = append(outAttributes, "Available from Alternate Sites")
+ outAttributes = append(outAttributes, "Available from POVR")
+ outAttributes = append(outAttributes, "Available from VRPorn")
+ outAttributes = append(outAttributes, "Available from SLR")
+ outAttributes = append(outAttributes, "Multiple Scenes Available at an Alternate Site")
type Results struct {
Result string
}
@@ -598,6 +617,28 @@ func (i SceneResource) searchSceneIndex(req *restful.Request, resp *restful.Resp
log.Error(err)
return
}
+ if strings.HasPrefix(q, "http") {
+ // if searching for a link, see if it is in the external ref table for scene alternate source
+ var extref models.ExternalReference
+ var scene models.Scene
+ splits := strings.Split(q, "?")
+ q = splits[0]
+
+ // see if the url matches a scrapped scene
+ scene.GetIfExistURL(q)
+ if scene.ID != 0 {
+ scenes = append(scenes, scene)
+ } else {
+ db.Preload("XbvrLinks").Where("(external_source like 'alternate scene %' or external_source = 'stashdb scene') and external_url = ?", q).First(&extref)
+ for _, link := range extref.XbvrLinks {
+ if link.InternalTable == "scenes" {
+ scene.GetIfExistByPK(link.InternalDbId)
+ scenes = append(scenes, scene)
+ }
+ }
+ }
+ }
+
defer idx.Bleve.Close()
query := bleve.NewQueryStringQuery(q)
@@ -938,3 +979,36 @@ func ProcessTagChanges(scene *models.Scene, tags *[]string, db *gorm.DB) {
}
}
}
+func (i SceneResource) getSceneAlternateSources(req *restful.Request, resp *restful.Response) {
+ var extref models.ExternalReferenceLink
+ var refs []models.ExternalReferenceLink
+ var ressults []ResponseGetAlternateSources
+ db, _ := models.GetDB()
+
+ if strings.Contains(req.PathParameter("scene-id"), "-") {
+ refs = extref.FindByInternalName("scenes", req.PathParameter("scene-id"))
+ } else {
+ id, err := strconv.Atoi(req.PathParameter("scene-id"))
+ if err != nil {
+ log.Error(err)
+ return
+ }
+ refs = extref.FindByInternalID("scenes", uint(id))
+ }
+
+ for _, ref := range refs {
+ var altscene models.SceneAlternateSource
+ var site models.Site
+
+ if ref.ExternalSource == "stashdb scene" {
+ ressults = append(ressults, ResponseGetAlternateSources{Url: ref.ExternalReference.ExternalURL, Icon: "https://docs.stashapp.cc/favicon.ico", ExternalSource: ref.ExternalReference.ExternalSource, ExternalId: ref.ExternalReference.ExternalId, ExternalData: ref.ExternalReference.ExternalData})
+ } else {
+ json.Unmarshal([]byte(ref.ExternalReference.ExternalData), &altscene)
+ site.GetIfExist(altscene.Scene.ScraperId)
+ ressults = append(ressults, ResponseGetAlternateSources{Url: ref.ExternalReference.ExternalURL, Icon: site.AvatarURL, ExternalSource: ref.ExternalReference.ExternalSource, ExternalId: ref.ExternalReference.ExternalId, ExternalData: ref.ExternalReference.ExternalData})
+ }
+ }
+ db.Close()
+
+ resp.WriteHeaderAndEntity(http.StatusOK, ressults)
+}
diff --git a/pkg/api/tasks.go b/pkg/api/tasks.go
index f1a090994..f3a119b56 100644
--- a/pkg/api/tasks.go
+++ b/pkg/api/tasks.go
@@ -100,6 +100,8 @@ func (i TaskResource) WebService() *restful.WebService {
ws.Route(ws.POST("/scrape-tpdb").To(i.scrapeTPDB).
Metadata(restfulspec.KeyOpenAPITags, tags))
+ ws.Route(ws.GET("/relink_alt_aource_scenes").To(i.relink_alt_aource_scenes).
+ Metadata(restfulspec.KeyOpenAPITags, tags))
return ws
}
@@ -173,10 +175,12 @@ func (i TaskResource) backupBundle(req *restful.Request, resp *restful.Response)
inclActors, _ := strconv.ParseBool(req.QueryParameter("inclActors"))
inclActorActions, _ := strconv.ParseBool(req.QueryParameter("inclActorActions"))
inclConfig, _ := strconv.ParseBool(req.QueryParameter("inclConfig"))
+ extRefSubset := req.QueryParameter("extRefSubset")
playlistId := req.QueryParameter("playlistId")
download := req.QueryParameter("download")
- bundle := tasks.BackupBundle(inclAllSites, onlyIncludeOfficalSites, inclScenes, inclFileLinks, inclCuepoints, inclHistory, inclPlaylists, inclActorAkas, inclTagGroups, inclVolumes, inclSites, inclActions, inclExtRefs, inclActors, inclActorActions, inclConfig, playlistId, "", "")
+ bundle := tasks.BackupBundle(inclAllSites, onlyIncludeOfficalSites, inclScenes, inclFileLinks, inclCuepoints, inclHistory, inclPlaylists,
+ inclActorAkas, inclTagGroups, inclVolumes, inclSites, inclActions, inclExtRefs, inclActors, inclActorActions, inclConfig, extRefSubset, playlistId, "", "")
if download == "true" {
resp.WriteHeaderAndEntity(http.StatusOK, ResponseBackupBundle{Response: "Ready to Download from http://xxx.xxx.xxx.xxx:9999/download/xbvr-content-bundle.json"})
} else {
@@ -226,3 +230,6 @@ func (i TaskResource) scrapeTPDB(req *restful.Request, resp *restful.Response) {
go tasks.ScrapeTPDB(strings.TrimSpace(r.ApiToken), strings.TrimSpace(r.SceneUrl))
}
}
+func (i TaskResource) relink_alt_aource_scenes(req *restful.Request, resp *restful.Response) {
+ go tasks.MatchAlternateSources()
+}
diff --git a/pkg/common/common.go b/pkg/common/common.go
index 972895de2..693b91486 100644
--- a/pkg/common/common.go
+++ b/pkg/common/common.go
@@ -12,15 +12,16 @@ var (
)
type EnvConfigSpec struct {
- Debug bool `envconfig:"DEBUG" default:"false"`
- DebugRequests bool `envconfig:"DEBUG_REQUESTS" default:"false"`
- DebugSQL bool `envconfig:"DEBUG_SQL" default:"false"`
- DebugWS bool `envconfig:"DEBUG_WS" default:"false"`
- UIUsername string `envconfig:"UI_USERNAME" required:"false"`
- UIPassword string `envconfig:"UI_PASSWORD" required:"false"`
- DatabaseURL string `envconfig:"DATABASE_URL" required:"false" default:""`
- WsAddr string `envconfig:"XBVR_WS_ADDR" required:"false" default:""`
- WebPort int `envconfig:"XBVR_WEB_PORT" required:"false" default:"0"`
+ Debug bool `envconfig:"DEBUG" default:"false"`
+ DebugRequests bool `envconfig:"DEBUG_REQUESTS" default:"false"`
+ DebugSQL bool `envconfig:"DEBUG_SQL" default:"false"`
+ DebugWS bool `envconfig:"DEBUG_WS" default:"false"`
+ UIUsername string `envconfig:"UI_USERNAME" required:"false"`
+ UIPassword string `envconfig:"UI_PASSWORD" required:"false"`
+ DatabaseURL string `envconfig:"DATABASE_URL" required:"false" default:""`
+ WsAddr string `envconfig:"XBVR_WS_ADDR" required:"false" default:""`
+ WebPort int `envconfig:"XBVR_WEB_PORT" required:"false" default:"0"`
+ DBConnectionPoolSize int `envconfig:"DB_CONNECTION_POOL_SIZE" required:"false" default:"0"`
}
var EnvConfig EnvConfigSpec
diff --git a/pkg/common/paths.go b/pkg/common/paths.go
index b206c5132..7e5b258b0 100644
--- a/pkg/common/paths.go
+++ b/pkg/common/paths.go
@@ -23,6 +23,7 @@ var ScriptHeatmapDir string
var MyFilesDir string
var DownloadDir string
var WebPort int
+var DBConnectionPoolSize int
func DirSize(path string) (int64, error) {
var size int64
@@ -51,6 +52,7 @@ func InitPaths() {
databaseurl := flag.String("database_url", "", "Optional: override default database path")
web_port := flag.Int("web_port", 0, "Optional: override default Web Page port 9999")
ws_addr := flag.String("ws_addr", "", "Optional: override default Websocket address from the default 0.0.0.0:9998")
+ db_connection_pool_size := flag.Int("db_connection_pool_size", 0, "Optional: sets a limit to the number of db connections while scraping")
flag.Parse()
@@ -113,6 +115,11 @@ func InitPaths() {
WsAddr = EnvConfig.WsAddr
}
}
+ if *db_connection_pool_size != 0 {
+ DBConnectionPoolSize = *db_connection_pool_size
+ } else {
+ DBConnectionPoolSize = EnvConfig.DBConnectionPoolSize
+ }
_ = os.MkdirAll(AppDir, os.ModePerm)
_ = os.MkdirAll(ImgDir, os.ModePerm)
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 1f0e7ddb6..3b8bfa2ea 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -2,6 +2,7 @@ package config
import (
"encoding/json"
+ "time"
"github.com/creasty/defaults"
@@ -47,13 +48,17 @@ type ObjectConfig struct {
IsAvailOpacity int `default:"40" json:"isAvailOpacity"`
} `json:"web"`
Advanced struct {
- ShowInternalSceneId bool `default:"false" json:"showInternalSceneId"`
- ShowHSPApiLink bool `default:"false" json:"showHSPApiLink"`
- ShowSceneSearchField bool `default:"false" json:"showSceneSearchField"`
- StashApiKey string `default:"" json:"stashApiKey"`
- ScrapeActorAfterScene bool `default:"true" json:"scrapeActorAfterScene"`
- UseImperialEntry bool `default:"false" json:"useImperialEntry"`
- ProgressTimeInterval int `default:"15" json:"progressTimeInterval"`
+ ShowInternalSceneId bool `default:"false" json:"showInternalSceneId"`
+ ShowHSPApiLink bool `default:"false" json:"showHSPApiLink"`
+ ShowSceneSearchField bool `default:"false" json:"showSceneSearchField"`
+ StashApiKey string `default:"" json:"stashApiKey"`
+ ScrapeActorAfterScene bool `default:"true" json:"scrapeActorAfterScene"`
+ UseImperialEntry bool `default:"false" json:"useImperialEntry"`
+ ProgressTimeInterval int `default:"15" json:"progressTimeInterval"`
+ LinkScenesAfterSceneScraping bool `default:"true" json:"linkScenesAfterSceneScraping"`
+ UseAltSrcInFileMatching bool `default:"true" json:"useAltSrcInFileMatching"`
+ UseAltSrcInScriptFilters bool `default:"true" json:"useAltSrcInScriptFilters"`
+ IgnoreReleasedBefore time.Time `json:"ignoreReleasedBefore"`
} `json:"advanced"`
Funscripts struct {
ScrapeFunscripts bool `default:"false" json:"scrapeFunscripts"`
@@ -152,7 +157,24 @@ type ObjectConfig struct {
HourEnd int `default:"23" json:"hourEnd"`
RunAtStartDelay int `default:"0" json:"runAtStartDelay"`
} `json:"stashdbRescrapeSchedule"`
+ LinkScenesSchedule struct {
+ Enabled bool `default:"false" json:"enabled"`
+ HourInterval int `default:"12" json:"hourInterval"`
+ UseRange bool `default:"false" json:"useRange"`
+ MinuteStart int `default:"0" json:"minuteStart"`
+ HourStart int `default:"0" json:"hourStart"`
+ HourEnd int `default:"23" json:"hourEnd"`
+ RunAtStartDelay int `default:"0" json:"runAtStartDelay"`
+ } `json:"linkScenesSchedule"`
} `json:"cron"`
+ Storage struct {
+ MatchOhash bool `default:"false" json:"match_ohash"`
+ } `json:"storage"`
+ ScraperSettings struct {
+ TMWVRNet struct {
+ TmwMembersDomain string `default:"members.tmwvrnet.com" json:"tmwMembersDomain"`
+ } `json:"tmwvrnet"`
+ } `json:"scraper_settings"`
}
var (
diff --git a/pkg/config/scraper_list.go b/pkg/config/scraper_list.go
index 3ebb82586..29296522d 100644
--- a/pkg/config/scraper_list.go
+++ b/pkg/config/scraper_list.go
@@ -33,12 +33,13 @@ type CustomScrapers struct {
VrphubScrapers []ScraperConfig `json:"vrphub"`
}
type ScraperConfig struct {
- ID string `json:"-"`
- URL string `json:"url"`
- Name string `json:"name"`
- Company string `json:"company"`
- AvatarUrl string `json:"avatar_url"`
- FileID string `json:"id,omitempty"`
+ ID string `json:"-"`
+ URL string `json:"url"`
+ Name string `json:"name"`
+ Company string `json:"company"`
+ AvatarUrl string `json:"avatar_url"`
+ FileID string `json:"id,omitempty"`
+ MasterSiteId string `json:"master_site_id,omitempty"`
}
var loadLock sync.Mutex
@@ -117,7 +118,7 @@ func CheckMatchingSite(findSite ScraperConfig, searchList []ScraperConfig) bool
if !strings.HasSuffix(s2, "/") {
s2 += "/"
}
- if s1 == s2 {
+ if s1 == s2 && customSite.MasterSiteId == findSite.MasterSiteId {
return true
}
}
diff --git a/pkg/config/scrapers.json b/pkg/config/scrapers.json
index f2a7dc0c2..d4bee433c 100644
--- a/pkg/config/scrapers.json
+++ b/pkg/config/scrapers.json
@@ -179,7 +179,7 @@
"url": "https://www.sexlikereal.com/studios/fuckpassvr",
"name": "FuckPassVR",
"company": "FuckPassVR",
- "avatar_url": "https://cdn-vr.sexlikereal.com/images/studio_creatives/logotypes/1/352/logo_crop_1635153994.png"
+ "avatar_url": "https://cdn-vr.sexlikereal.com/images/studio_creatives/logotypes/1/352/logo_crop_1635153994.png"
},
{
"url": "https://www.sexlikereal.com/studios/heathering",
@@ -197,7 +197,7 @@
"url": "https://www.sexlikereal.com/studios/jackandjillvr",
"name": "JackandJillVR",
"company": "JackandJillVR",
- "avatar_url": "https://cdn-vr.sexlikereal.com/images/studio_creatives/logotypes/1/367/logo_crop_1645997567.png"
+ "avatar_url": "https://cdn-vr.sexlikereal.com/images/studio_creatives/logotypes/1/367/logo_crop_1645997567.png"
},
{
"url": "https://www.sexlikereal.com/studios/jimmydraws",
diff --git a/pkg/externalreference/stashdb.go b/pkg/externalreference/stashdb.go
index 22b035bb8..149fa1c02 100644
--- a/pkg/externalreference/stashdb.go
+++ b/pkg/externalreference/stashdb.go
@@ -279,7 +279,10 @@ func UpdateXbvrActor(performer models.StashPerformer, xbvrActorID uint) {
changed := false
actor := models.Actor{ID: xbvrActorID}
- db.Where(&actor).First(&actor)
+ err := db.Where(&actor).First(&actor).Error
+ if err != nil {
+ return
+ }
if len(performer.Images) > 0 {
if actor.ImageUrl != performer.Images[0].URL && !actor.CheckForSetImage() {
@@ -532,7 +535,6 @@ func ReverseMatch() {
var data models.StashPerformer
json.Unmarshal([]byte(extref.ExternalData), &data)
UpdateXbvrActor(data, actor.ID)
- log.Info("match")
}
break sceneLoop
}
diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go
index fa029c252..4710c9296 100644
--- a/pkg/migrations/migrations.go
+++ b/pkg/migrations/migrations.go
@@ -760,13 +760,50 @@ func Migrate() {
ID: "0073-scene-_id-index-plus-columns_size_changes",
Migrate: func(tx *gorm.DB) error {
type Scene struct {
- SceneID string `gorm:"index" json:"scene_id" xbvrbackup:"scene_id"`
CoverURL string `gorm:"size:500" json:"cover_url" xbvrbackup:"cover_url"`
SceneURL string `gorm:"size:500" json:"scene_url" xbvrbackup:"scene_url"`
}
+
+ sql := `CREATE INDEX idx_scenes_scene_id ON scenes (scene_id)`
+ tx.Exec(sql)
return tx.AutoMigrate(&Scene{}).Error
},
},
+ {
+ ID: "0074-Limit-Scraper",
+ Migrate: func(tx *gorm.DB) error {
+ type Site struct {
+ LimitScraping bool `json:"limit_scraping" xbvrbackup:"limit_scraping"`
+ }
+ return tx.AutoMigrate(Site{}).Error
+ },
+ },
+ {
+ ID: "0076-Scene-Alt-Sources",
+ Migrate: func(tx *gorm.DB) error {
+ type Site struct {
+ MasterSiteID string `json:"master_site_id" xbvrbackup:"master_site_id"`
+ MatchingParams string `json:"matching_params" gorm:"size:1000" xbvrbackup:"matching_params"`
+ }
+ type ExternalReference struct {
+ UdfBool1 bool `json:"udf_bool1" xbvrbackup:"udf_bool1"` // user defined fields, use depends what type of data the extref is for.
+ UdfBool2 bool `json:"udf_bool2" xbvrbackup:"udf_bool2"`
+ UdfDatetime1 time.Time `json:"udf_datetime1" xbvrbackup:"udf_datetime1"`
+ }
+ type ExternalReferenceLink struct {
+ UdfDatetime1 time.Time `json:"udf_datetime1" xbvrbackup:"udf_datetime1"`
+ }
+ err := tx.AutoMigrate(Site{}).Error
+ if err != nil {
+ return err
+ }
+ err = tx.AutoMigrate(ExternalReferenceLink{}).Error
+ if err != nil {
+ return err
+ }
+ return tx.AutoMigrate(ExternalReference{}).Error
+ },
+ },
// ===============================================================================================
// Put DB Schema migrations above this line and migrations that rely on the updated schema below
@@ -1494,7 +1531,7 @@ func Migrate() {
}
// backup bundle
common.Log.Infof("Creating pre-migration backup, please waiit, backups can take some time on a system with a large number of scenes ")
- tasks.BackupBundle(true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, false, "0", "xbvr-premigration-bundle.json", "2")
+ tasks.BackupBundle(true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, false, "", "0", "xbvr-premigration-bundle.json", "2")
common.Log.Infof("Go to download/xbvr-premigration-bundle.json, or http://xxx.xxx.xxx.xxx:9999/download/xbvr-premigration-bundle.json if you need access to the backup")
var sites []models.Site
officalSiteChanges := []SiteChange{
@@ -1851,6 +1888,43 @@ func Migrate() {
return tx.Exec(sql).Error
},
},
+ {
+ ID: "0073-reset-RealJamVR-scenes-with-duped-actors",
+ Migrate: func(tx *gorm.DB) error {
+
+ rjn := [...]string{"realjam-vr-39859", "realjam-vr-39861", "realjam-vr-40044", "realjam-vr-40064", "porncorn-vr-39827", "porncorn-vr-39902", "porncorn-vr-40031", "porncorn-vr-39903"}
+ var scenes []models.Scene
+ err := tx.Where("studio = ?", "Real Jam Network").Find(&scenes).Error
+ if err != nil {
+ return err
+ }
+ for _, scene := range scenes {
+ for _, v := range rjn {
+ if scene.SceneID == v {
+ scene.NeedsUpdate = true
+ err = tx.Save(&scene).Error
+ if err != nil {
+ return err
+ }
+ // common.Log.Infof("Updated scene %s", scene.SceneID)
+ }
+ }
+ }
+ return nil
+ },
+ },
+ {
+ ID: "0075-Update-tmwvrnet-members",
+ Migrate: func(tx *gorm.DB) error {
+ sql := `update scenes set member_url = replace(replace(scene_url, 'https://tmwvrnet.com/trailers/', 'https://members.tmwvrnet.com/scenes/'), '.html', '_vids.html') where scene_url like 'https://tmwvrnet.com/trailers/%';`
+ err := tx.Exec(sql).Error
+ if err == nil {
+
+ err = tx.Exec(sql).Error
+ }
+ return err
+ },
+ },
})
if err := m.Migrate(); err != nil {
diff --git a/pkg/models/db.go b/pkg/models/db.go
index 341770afe..3927a6b9c 100644
--- a/pkg/models/db.go
+++ b/pkg/models/db.go
@@ -2,6 +2,7 @@ package models
import (
"strings"
+ "time"
"github.com/avast/retry-go/v4"
"github.com/jinzhu/gorm"
@@ -16,6 +17,7 @@ import (
var log = &common.Log
var dbConn *dburl.URL
var supportedDB = []string{"mysql", "sqlite3"}
+var commonConnection *gorm.DB
func parseDBConnString() {
var err error
@@ -78,6 +80,38 @@ func GetDB() (*gorm.DB, error) {
return db, nil
}
+func GetCommonDB() (*gorm.DB, error) {
+ if common.EnvConfig.DebugSQL {
+ log.Debug("Getting Common DB handle from ", common.GetCallerFunctionName())
+ }
+
+ var err error
+
+ if commonConnection != nil {
+ return commonConnection, nil
+ }
+ err = retry.Do(
+ func() error {
+ commonConnection, err = gorm.Open(dbConn.Driver, dbConn.DSN)
+ commonConnection.LogMode(common.EnvConfig.DebugSQL)
+ commonConnection.DB().SetConnMaxIdleTime(4 * time.Minute)
+ if common.DBConnectionPoolSize > 0 {
+ commonConnection.DB().SetMaxOpenConns(common.DBConnectionPoolSize)
+ }
+ if err != nil {
+ return err
+ }
+ return nil
+ },
+ )
+
+ if err != nil {
+ log.Fatal("Failed to connect to database ", err)
+ }
+
+ return commonConnection, nil
+}
+
// Lock functions
func CreateLock(lock string) {
@@ -127,4 +161,5 @@ func init() {
common.InitPaths()
common.InitLogging()
parseDBConnString()
+ GetCommonDB()
}
diff --git a/pkg/models/model_actor.go b/pkg/models/model_actor.go
index f7c37d1b3..03ab9965f 100644
--- a/pkg/models/model_actor.go
+++ b/pkg/models/model_actor.go
@@ -97,12 +97,11 @@ type ActorLink struct {
}
func (i *Actor) Save() error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var err error = retry.Do(
func() error {
- err := db.Save(&i).Error
+ err := commonDb.Save(&i).Error
if err != nil {
return err
}
@@ -118,8 +117,7 @@ func (i *Actor) Save() error {
}
func (i *Actor) CountActorTags() {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
type CountResults struct {
ID int
@@ -131,7 +129,7 @@ func (i *Actor) CountActorTags() {
var results []CountResults
- db.Model(&Actor{}).
+ commonDb.Model(&Actor{}).
Select("actors.id, count as existingcnt, count(*) cnt, sum(scenes.is_available ) is_available, avail_count as existingavail").
Group("actors.id").
Joins("join scene_cast on scene_cast.actor_id = actors.id").
@@ -141,7 +139,7 @@ func (i *Actor) CountActorTags() {
for i := range results {
var actor Actor
if results[i].Cnt != results[i].Existingcnt || results[i].IsAvailable != results[i].Existingavail {
- db.First(&actor, results[i].ID)
+ commonDb.First(&actor, results[i].ID)
actor.Count = results[i].Cnt
actor.AvailCount = results[i].IsAvailable
actor.Save()
@@ -170,11 +168,10 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList {
limit := r.Limit.OrElse(100)
offset := r.Offset.OrElse(0)
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var actors []Actor
- tx := db.Model(&actors)
+ tx := commonDb.Model(&actors)
var out ResponseActorList
@@ -504,10 +501,9 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList {
}
func (o *Actor) GetIfExist(id string) error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
- return db.
+ return commonDb.
Preload("Scenes", func(db *gorm.DB) *gorm.DB {
return db.Where("is_hidden = 0")
}).
@@ -515,10 +511,9 @@ func (o *Actor) GetIfExist(id string) error {
}
func (o *Actor) GetIfExistByPK(id uint) error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
- return db.
+ return commonDb.
Preload("Scenes", func(db *gorm.DB) *gorm.DB {
return db.Where("is_hidden = 0")
}).
@@ -526,10 +521,9 @@ func (o *Actor) GetIfExistByPK(id uint) error {
}
func (o *Actor) GetIfExistByPKWithSceneAvg(id uint) error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
- tx := db.Model(&Actor{})
+ tx := commonDb.Model(&Actor{})
tx = tx.Select(`actors.*,
(select AVG(s.star_rating) scene_avg from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 and is_hidden=0) as scene_rating_average`)
@@ -631,19 +625,17 @@ func addToStringArray(inputArray string, newValue string) (string, bool) {
func (a *Actor) CheckForSetImage() bool {
// check if the field was deleted by the user,
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var action ActionActor
- db.Where("source = 'edit_actor' and actor_id = ? and changed_column = 'image_url' and action_type = 'setimage'", a.ID).Order("ID desc").First(&action)
+ commonDb.Where("source = 'edit_actor' and actor_id = ? and changed_column = 'image_url' and action_type = 'setimage'", a.ID).Order("ID desc").First(&action)
return action.ID != 0
}
func (a *Actor) CheckForUserDeletes(fieldName string, newValue string) bool {
// check if the field was deleted by the user,
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var action ActionActor
- db.Where("source = 'edit_actor' and actor_id = ? and changed_column = ? and new_value = ?", a.ID, fieldName, newValue).Order("ID desc").First(&action)
+ commonDb.Where("source = 'edit_actor' and actor_id = ? and changed_column = ? and new_value = ?", a.ID, fieldName, newValue).Order("ID desc").First(&action)
if action.ID != 0 && action.ActionType == "delete" {
return true
}
diff --git a/pkg/models/model_aka.go b/pkg/models/model_aka.go
index 35b44b45f..dc2533db4 100644
--- a/pkg/models/model_aka.go
+++ b/pkg/models/model_aka.go
@@ -20,12 +20,11 @@ type Aka struct {
}
func (i *Aka) Save() error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
err := retry.Do(
func() error {
- err := db.Save(&i).Error
+ err := commonDb.Save(&i).Error
if err != nil {
return err
}
@@ -41,24 +40,22 @@ func (i *Aka) Save() error {
}
func (o *Aka) GetIfExistByPK(id uint) error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
- return db.
+ return commonDb.
Preload("Actors").
Where(&Aka{ID: id}).First(o).Error
}
func (o *Aka) UpdateAkaSceneCastRecords() {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
// Queries to update the scene_cast table for the aka actor are comlex but fast.
// Significating faster than iterating through the results of multiple simpler queries.
// The Raw Sql used is compatible between mysql & sqlite
// add missing scene_cast records for aka actors
- db.Exec(`
+ commonDb.Exec(`
insert into scene_cast
select distinct sc.scene_id, a.aka_actor_id
from akas a
@@ -69,7 +66,7 @@ func (o *Aka) UpdateAkaSceneCastRecords() {
`)
// delete scene_cast records for aka actors that have been removed
- db.Exec(`
+ commonDb.Exec(`
with SceneIds as (
select distinct a.id, sc.scene_id
from akas a
@@ -95,8 +92,7 @@ func (o *Aka) UpdateAkaSceneCastRecords() {
}
func (o *Aka) RefreshAkaActorNames() {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
type SortedList struct {
AkaActorId uint
@@ -105,7 +101,7 @@ func (o *Aka) RefreshAkaActorNames() {
var sortedList []SortedList
// this update the aka names by reordering the actor names based on descending count
- db.Raw(`
+ commonDb.Raw(`
with sorted as (
select a.aka_actor_id, a2.name, a2.count from akas a
join actor_akas aa on a.id =aa.aka_id
@@ -118,7 +114,7 @@ func (o *Aka) RefreshAkaActorNames() {
for _, listItem := range sortedList {
var actor Actor
actor.ID = listItem.AkaActorId
- db.Model(&actor).Where("name != ?", "aka:"+listItem.SortedName).Update("name", "aka:"+listItem.SortedName)
+ commonDb.Model(&actor).Where("name != ?", "aka:"+listItem.SortedName).Update("name", "aka:"+listItem.SortedName)
}
}
diff --git a/pkg/models/model_external_reference.go b/pkg/models/model_external_reference.go
index 9c779a3c3..e65a0694a 100644
--- a/pkg/models/model_external_reference.go
+++ b/pkg/models/model_external_reference.go
@@ -27,7 +27,11 @@ type ExternalReference struct {
ExternalURL string `json:"external_url" gorm:"size:1000" xbvrbackup:"external_url"`
ExternalDate time.Time `json:"external_date" xbvrbackup:"external_date"`
ExternalData string `json:"external_data" sql:"type:text;" xbvrbackup:"external_data"`
+ UdfBool1 bool `json:"udf_bool1" xbvrbackup:"udf_bool1"` // user defined fields, use depends what type of data the extref is for.
+ UdfBool2 bool `json:"udf_bool2" xbvrbackup:"udf_bool2"`
+ UdfDatetime1 time.Time `json:"udf_datetime1" xbvrbackup:"udf_datetime1"`
}
+
type ExternalReferenceLink struct {
ID uint `gorm:"primary_key" json:"id" xbvrbackup:"-"`
CreatedAt time.Time `json:"-" xbvrbackup:"created_at-"`
@@ -36,10 +40,11 @@ type ExternalReferenceLink struct {
InternalDbId uint `json:"internal_db_id" gorm:"index" xbvrbackup:"-"`
InternalNameId string `json:"internal_name_id" gorm:"index" xbvrbackup:"internal_name_id"`
- ExternalReferenceID uint `json:"external_reference_id" gorm:"index" xbvrbackup:"-"`
- ExternalSource string `json:"external_source" xbvrbackup:"-"`
- ExternalId string `json:"external_id" gorm:"index" xbvrbackup:"-"`
- MatchType int `json:"match_type" xbvrbackup:"match_type"`
+ ExternalReferenceID uint `json:"external_reference_id" gorm:"index" xbvrbackup:"-"`
+ ExternalSource string `json:"external_source" xbvrbackup:"-"`
+ ExternalId string `json:"external_id" gorm:"index" xbvrbackup:"-"`
+ MatchType int `json:"match_type" xbvrbackup:"match_type"`
+ UdfDatetime1 time.Time `json:"udf_datetime1" xbvrbackup:"udf_datetime1"`
ExternalReference ExternalReference `json:"external_reference" gorm:"foreignKey:ExternalReferenceId" xbvrbackup:"-"`
}
@@ -90,33 +95,46 @@ type SceneMatchRule struct {
}
func (o *ExternalReference) GetIfExist(id uint) error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
- return db.Preload("XbvrLinks").Where(&ExternalReference{ID: id}).First(o).Error
+ return commonDb.Preload("XbvrLinks").Where(&ExternalReference{ID: id}).First(o).Error
}
func (o *ExternalReference) FindExternalUrl(externalSource string, externalUrl string) error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
- return db.Preload("XbvrLinks").Where(&ExternalReference{ExternalSource: externalSource, ExternalURL: externalUrl}).First(o).Error
+ return commonDb.Preload("XbvrLinks").Where(&ExternalReference{ExternalSource: externalSource, ExternalURL: externalUrl}).First(o).Error
}
func (o *ExternalReference) FindExternalId(externalSource string, externalId string) error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
- return db.Preload("XbvrLinks").Where(&ExternalReference{ExternalSource: externalSource, ExternalId: externalId}).First(o).Error
+ return commonDb.Preload("XbvrLinks").Where(&ExternalReference{ExternalSource: externalSource, ExternalId: externalId}).First(o).Error
+}
+
+func (o *ExternalReferenceLink) FindByInternalID(internalTable string, internalId uint) []ExternalReferenceLink {
+ commonDb, _ := GetCommonDB()
+ var refs []ExternalReferenceLink
+ commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{InternalTable: internalTable, InternalDbId: internalId}).Find(&refs)
+ return refs
+}
+func (o *ExternalReferenceLink) FindByInternalName(internalTable string, internalName string) []ExternalReferenceLink {
+ commonDb, _ := GetCommonDB()
+ var refs []ExternalReferenceLink
+ commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{InternalTable: internalTable, InternalNameId: internalName}).Find(&refs)
+ return refs
+}
+func (o *ExternalReferenceLink) FindByExternaID(externalSource string, externalId string) {
+ commonDb, _ := GetCommonDB()
+ commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{ExternalSource: externalSource, ExternalId: externalId}).Find(&o)
}
func (o *ExternalReference) Save() {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
err := retry.Do(
func() error {
- err := db.Save(&o).Error
+ err := commonDb.Save(&o).Error
if err != nil {
return err
}
@@ -129,14 +147,17 @@ func (o *ExternalReference) Save() {
}
func (o *ExternalReference) Delete() {
- db, _ := GetDB()
- db.Delete(&o)
- db.Close()
+ commonDb, _ := GetCommonDB()
+ commonDb.Delete(&o)
+}
+
+func (o *ExternalReferenceLink) Delete() {
+ commonDb, _ := GetCommonDB()
+ commonDb.Delete(&o)
}
func (o *ExternalReference) AddUpdateWithUrl() {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
existingRef := ExternalReference{ExternalSource: o.ExternalSource, ExternalURL: o.ExternalURL}
existingRef.FindExternalUrl(o.ExternalSource, o.ExternalURL)
@@ -153,7 +174,7 @@ func (o *ExternalReference) AddUpdateWithUrl() {
err := retry.Do(
func() error {
- err := db.Save(&o).Error
+ err := commonDb.Save(&o).Error
if err != nil {
return err
}
@@ -166,8 +187,7 @@ func (o *ExternalReference) AddUpdateWithUrl() {
}
func (o *ExternalReference) AddUpdateWithId() {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
existingRef := ExternalReference{ExternalSource: o.ExternalSource, ExternalId: o.ExternalId}
existingRef.FindExternalId(o.ExternalSource, o.ExternalId)
@@ -184,7 +204,7 @@ func (o *ExternalReference) AddUpdateWithId() {
err := retry.Do(
func() error {
- err := db.Save(&o).Error
+ err := commonDb.Save(&o).Error
if err != nil {
return err
}
@@ -197,12 +217,11 @@ func (o *ExternalReference) AddUpdateWithId() {
}
func (o *ExternalReferenceLink) Save() {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
err := retry.Do(
func() error {
- err := db.Save(&o).Error
+ err := commonDb.Save(&o).Error
if err != nil {
return err
}
@@ -215,10 +234,9 @@ func (o *ExternalReferenceLink) Save() {
}
func (o *ExternalReferenceLink) Find(externalSource string, internalName string) error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
- return db.Where(&ExternalReferenceLink{ExternalSource: externalSource, InternalNameId: internalName}).First(o).Error
+ return commonDb.Where(&ExternalReferenceLink{ExternalSource: externalSource, InternalNameId: internalName}).First(o).Error
}
func FormatInternalDbId(input uint) string {
@@ -264,11 +282,10 @@ func (o *ExternalReference) DetermineActorScraperByUrl(url string) string {
}
func (o *ExternalReference) DetermineActorScraperBySiteId(siteId string) string {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var site Site
- db.Where("id = ?", siteId).First(&site)
+ commonDb.Where("id = ?", siteId).First(&site)
if site.Name == "" {
return siteId
}
@@ -304,8 +321,7 @@ func (config ActorScraperConfig) loadActorScraperRules() {
}
func (scrapeRules ActorScraperConfig) buildGenericActorScraperRules() {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var sites []Site
// To understand the regex used, sign up to chat.openai.com and just ask something like Explain (.*, )?(.*)$
@@ -357,7 +373,7 @@ func (scrapeRules ActorScraperConfig) buildGenericActorScraperRules() {
siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "aliases", Selector: `div[data-qa="model-info-aliases"] div.u-wh`})
scrapeRules.GenericActorScrapingConfig["slr-originals scrape"] = siteDetails
scrapeRules.GenericActorScrapingConfig["slr-jav-originals scrape"] = siteDetails
- db.Where("name like ?", "%SLR)").Find(&sites)
+ commonDb.Where("name like ?", "%SLR)").Find(&sites)
scrapeRules.GenericActorScrapingConfig["slr scrape"] = siteDetails
siteDetails = GenericScraperRuleSet{}
@@ -1042,7 +1058,7 @@ func (scrapeRules ActorScraperConfig) getCustomRules() {
Attribute: "attribute id you want, eg src for an image of href for a link",
PostProcessing: []PostProcessing{{
Function: "builtin function to apply to the extarcted text, eg RegexString to extract with regex, Parse Date, lbs to kg, see postProcessing function for options. You may specify multiple function, eg RegexString to extract a Date followed by Parse Date if not in the right format",
- Params: []string{`Paramerter depends on the functions requirements `},
+ Params: []string{`Parameter depends on the functions requirements `},
}},
})
exampleConfig.GenericActorScrapingConfig["example scrape"] = siteDetails
@@ -1080,8 +1096,7 @@ func (scrapeRules ActorScraperConfig) getCustomRules() {
}
func (scrapeRules ActorScraperConfig) getSiteUrlMatchingRules() {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var sites []Site
@@ -1191,7 +1206,7 @@ func (scrapeRules ActorScraperConfig) getSiteUrlMatchingRules() {
Rules: []SceneMatchRule{{XbvrField: "scene_url", XbvrMatch: `(lethalhardcorevr.com).*\/(\d{6,8})\/.*`, XbvrMatchResultPosition: 2, StashRule: `(lethalhardcorevr.com).*\/(\d{6,8})\/.*`, StashMatchResultPosition: 2}},
}
- db.Where(&Site{IsEnabled: true}).Order("id").Find(&sites)
+ commonDb.Where(&Site{IsEnabled: true}).Order("id").Find(&sites)
for _, site := range sites {
if _, found := scrapeRules.StashSceneMatching[site.ID]; !found {
if strings.HasSuffix(site.Name, "SLR)") {
diff --git a/pkg/models/model_scene.go b/pkg/models/model_scene.go
index 1ff13bb00..76970bf7a 100644
--- a/pkg/models/model_scene.go
+++ b/pkg/models/model_scene.go
@@ -34,12 +34,11 @@ type SceneCuepoint struct {
}
func (o *SceneCuepoint) Save() error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var err error = retry.Do(
func() error {
- err := db.Save(&o).Error
+ err := commonDb.Save(&o).Error
if err != nil {
return err
}
@@ -115,6 +114,8 @@ type Scene struct {
Description string `gorm:"-" json:"description" xbvrbackup:"-"`
Score float64 `gorm:"-" json:"_score" xbvrbackup:"-"`
+
+ AlternateSource []ExternalReferenceLink `json:"alternate_source" xbvrbackup:"-"`
}
type Image struct {
@@ -132,13 +133,18 @@ type VideoSource struct {
Quality string `json:"quality"`
}
+type Config struct {
+ Advanced struct {
+ UseAltSrcInFileMatching bool `json:"useAltSrcInFileMatching"`
+ } `json:"advanced"`
+}
+
func (i *Scene) Save() error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var err error = retry.Do(
func() error {
- err := db.Save(&i).Error
+ err := commonDb.Save(&i).Error
if err != nil {
return err
}
@@ -162,10 +168,9 @@ func (i *Scene) FromJSON(data []byte) error {
}
func (o *Scene) GetIfExist(id string) error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
- return db.
+ return commonDb.
Preload("Tags").
Preload("Cast").
Preload("Files").
@@ -175,10 +180,9 @@ func (o *Scene) GetIfExist(id string) error {
}
func (o *Scene) GetIfExistByPK(id uint) error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
- return db.
+ return commonDb.
Preload("Tags").
Preload("Cast").
Preload("Files").
@@ -188,10 +192,9 @@ func (o *Scene) GetIfExistByPK(id uint) error {
}
func (o *Scene) GetIfExistURL(u string) error {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
- return db.
+ return commonDb.
Preload("Tags").
Preload("Cast").
Preload("Files").
@@ -215,21 +218,19 @@ func (o *Scene) GetFunscriptTitle() string {
}
func (o *Scene) GetFiles() ([]File, error) {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var files []File
- db.Preload("Volume").Where(&File{SceneID: o.ID}).Find(&files)
+ commonDb.Preload("Volume").Where(&File{SceneID: o.ID}).Find(&files)
return files, nil
}
func (o *Scene) GetTotalWatchTime() int {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
totalResult := struct{ Total float64 }{}
- db.Raw(`select sum(duration) as total from histories where scene_id = ?`, o.ID).Scan(&totalResult)
+ commonDb.Raw(`select sum(duration) as total from histories where scene_id = ?`, o.ID).Scan(&totalResult)
return int(totalResult.Total)
}
@@ -240,14 +241,13 @@ func (o *Scene) GetVideoFiles() ([]File, error) {
}
func (o *Scene) GetVideoFilesSorted(sort string) ([]File, error) {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var files []File
if sort == "" {
- db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "video").Find(&files)
+ commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "video").Find(&files)
} else {
- db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "video").Order(sort).Find(&files)
+ commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "video").Order(sort).Find(&files)
}
return files, nil
@@ -259,35 +259,32 @@ func (o *Scene) GetScriptFiles() ([]File, error) {
}
func (o *Scene) GetScriptFilesSorted(sort string) ([]File, error) {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var files []File
if sort == "" {
- db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "script").Find(&files)
+ commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "script").Find(&files)
} else {
- db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "script").Order(sort).Find(&files)
+ commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "script").Order(sort).Find(&files)
}
return files, nil
}
func (o *Scene) GetHSPFiles() ([]File, error) {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var files []File
- db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "hsp").Find(&files)
+ commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "hsp").Find(&files)
return files, nil
}
func (o *Scene) GetSubtitlesFiles() ([]File, error) {
- db, _ := GetDB()
- defer db.Close()
+ commonDb, _ := GetCommonDB()
var files []File
- db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "subtitles").Find(&files)
+ commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "subtitles").Find(&files)
return files, nil
}
@@ -415,14 +412,7 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error {
var o Scene
db.Where(&Scene{SceneID: ext.SceneID}).FirstOrCreate(&o)
- o.NeedsUpdate = false
- o.EditsApplied = false
- o.SceneID = ext.SceneID
- o.ScraperId = ext.ScraperID
-
if o.Title != ext.Title {
- o.Title = ext.Title
-
// reset scriptfile.IsExported state on title change
scriptfiles, err := o.GetScriptFiles()
if err == nil {
@@ -435,6 +425,71 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error {
}
}
+ o.PopulateSceneFieldsFromExternal(db, ext)
+ var site Site
+ db.Where("id = ?", o.ScraperId).FirstOrInit(&site)
+ o.IsSubscribed = site.Subscribed
+ SaveWithRetry(db, &o)
+
+ // Clean & Associate Tags
+ var tags = o.Tags
+ db.Model(&o).Association("Tags").Clear()
+ for _, tag := range tags {
+ tmpTag := Tag{}
+ db.Where(&Tag{Name: tag.Name}).FirstOrCreate(&tmpTag)
+ db.Model(&o).Association("Tags").Append(tmpTag)
+ }
+
+ // Clean & Associate Actors
+ db.Model(&o).Association("Cast").Clear()
+ var tmpActor Actor
+ for _, name := range ext.Cast {
+ tmpActor = Actor{}
+ db.Where(&Actor{Name: strings.Replace(name, ".", "", -1)}).FirstOrCreate(&tmpActor)
+ saveActor := false
+ if ext.ActorDetails[name].ImageUrl != "" {
+ if tmpActor.ImageUrl == "" {
+ tmpActor.ImageUrl = ext.ActorDetails[name].ImageUrl
+ saveActor = true
+ }
+ if tmpActor.AddToImageArray(ext.ActorDetails[name].ImageUrl) {
+ saveActor = true
+ }
+ }
+ if ext.ActorDetails[name].ProfileUrl != "" {
+ if tmpActor.AddToActorUrlArray(ActorLink{Url: ext.ActorDetails[name].ProfileUrl, Type: ext.ActorDetails[name].Source}) {
+ saveActor = true
+ }
+ }
+ if saveActor {
+ tmpActor.Save()
+ }
+ db.Model(&o).Association("Cast").Append(tmpActor)
+ }
+ // delete any altrernate scene records, in case this scene was originally a linked scene
+ var extrefs []ExternalReference
+ db.Where("external_source like 'alternate scene %' and external_url = ?", o.SceneURL).Find(&extrefs)
+ for _, extref := range extrefs {
+ db.Where("external_reference_id = ?", extref.ID).Delete(&ExternalReferenceLink{})
+ db.Delete(&extref)
+ }
+
+ return nil
+}
+
+func (o *Scene) PopulateSceneFieldsFromExternal(db *gorm.DB, ext ScrapedScene) {
+ // this function is shared between scenes and alternate scenes,
+ // it should only setup values in the scene record from the scraped scene
+ // it should not update scene data, as that won't apply for alternate scene sources
+ if ext.SceneID == "" {
+ return
+ }
+
+ o.NeedsUpdate = false
+ o.EditsApplied = false
+ o.SceneID = ext.SceneID
+ o.ScraperId = ext.ScraperID
+ o.Title = ext.Title
o.SceneType = ext.SceneType
o.Studio = ext.Studio
o.Site = ext.Site
@@ -500,62 +555,58 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error {
var site Site
db.Where("id = ?", o.ScraperId).FirstOrInit(&site)
o.IsSubscribed = site.Subscribed
- SaveWithRetry(db, &o)
- // Clean & Associate Tags
- db.Model(&o).Association("Tags").Clear()
- var tmpTag Tag
+ var tags []Tag
for _, name := range ext.Tags {
tagClean := ConvertTag(name)
if tagClean != "" {
- tmpTag = Tag{}
- db.Where(&Tag{Name: tagClean}).FirstOrCreate(&tmpTag)
- db.Model(&o).Association("Tags").Append(tmpTag)
+ tags = append(tags, Tag{Name: tagClean})
}
}
+ o.Tags = tags
// Clean & Associate Actors
- db.Model(&o).Association("Cast").Clear()
+ var cast []Actor
var tmpActor Actor
for _, name := range ext.Cast {
tmpActor = Actor{}
db.Where(&Actor{Name: strings.Replace(name, ".", "", -1)}).FirstOrCreate(&tmpActor)
- saveActor := false
- if ext.ActorDetails[name].ImageUrl != "" {
- if tmpActor.ImageUrl == "" {
- tmpActor.ImageUrl = ext.ActorDetails[name].ImageUrl
- saveActor = true
- }
- if tmpActor.AddToImageArray(ext.ActorDetails[name].ImageUrl) {
- saveActor = true
- }
- // AddActionActor(name, ext.ActorDetails[name].Source, "add", "image_url", ext.ActorDetails[name].ImageUrl)
- }
- if ext.ActorDetails[name].ProfileUrl != "" {
- if tmpActor.AddToActorUrlArray(ActorLink{Url: ext.ActorDetails[name].ProfileUrl, Type: ext.ActorDetails[name].Source}) {
- saveActor = true
- }
- // AddActionActor(name, ext.ActorDetails[name].Source, "add", "image_url", ext.ActorDetails[name].ImageUrl)
- }
- if saveActor {
- tmpActor.Save()
- }
- db.Model(&o).Association("Cast").Append(tmpActor)
+ cast = append(cast, tmpActor)
}
-
- return nil
+ o.Cast = cast
}
func SceneUpdateScriptData(db *gorm.DB, ext ScrapedScene) {
- var o Scene
- o.GetIfExistByPK(ext.InternalSceneId)
-
- if o.ID != 0 {
- if o.ScriptPublished.IsZero() || o.HumanScript != ext.HumanScript || o.AiScript != ext.AiScript {
- o.ScriptPublished = time.Now()
- o.HumanScript = ext.HumanScript
- o.AiScript = ext.AiScript
- o.Save()
+ if ext.MasterSiteId == "" {
+ var o Scene
+ o.GetIfExistByPK(ext.InternalSceneId)
+
+ if o.ID != 0 {
+ if o.ScriptPublished.IsZero() || o.HumanScript != ext.HumanScript || o.AiScript != ext.AiScript {
+ o.ScriptPublished = time.Now()
+ o.HumanScript = ext.HumanScript
+ o.AiScript = ext.AiScript
+ o.Save()
+ }
+ }
+ } else {
+ var extref ExternalReference
+ extref.FindExternalId("alternate scene "+ext.ScraperID, ext.SceneID)
+ var externalData SceneAlternateSource
+ json.Unmarshal([]byte(extref.ExternalData), &externalData)
+ if extref.ID > 0 {
+ if externalData.Scene.ScriptPublished.IsZero() || externalData.Scene.HumanScript != ext.HumanScript || externalData.Scene.AiScript != ext.AiScript {
+ // set user defined fields for querying, rather than querying the json
+ extref.UdfDatetime1 = time.Now()
+ extref.UdfBool1 = ext.HumanScript
+ extref.UdfBool2 = ext.AiScript
+ externalData.Scene.ScriptPublished = extref.UdfDatetime1
+ externalData.Scene.HumanScript = ext.HumanScript
+ externalData.Scene.AiScript = ext.AiScript
+ newjson, _ := json.Marshal(externalData)
+ extref.ExternalData = string(newjson)
+ extref.Save()
+ }
}
}
}
@@ -672,6 +723,9 @@ func QuerySceneSummaries(r RequestSceneList) []SceneSummary {
}
func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) {
+ // get config, can't reference config directly due to circular package references
+ config := getConfig(db)
+
tx := db.Model(&Scene{})
if r.IsWatched.Present() {
@@ -760,6 +814,8 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) {
where = "is_subscribed = 1"
case "Rating":
where = "scenes.star_rating = " + value
+ case "No Actor/Cast":
+ where = "exists (select 1 from scenes s left join scene_cast sc on sc.scene_id =s.id where s.id=scenes.id and sc.scene_id is NULL)"
case "Cast 6+":
where = "exists (select 1 from scene_cast join actors on actors.id = scene_cast.actor_id where scene_cast.scene_id = scenes.id and actors.name not like 'aka:%' group by scene_cast.scene_id having count(*) > 5)"
case "Cast 1", "Cast 2", "Cast 3", "Cast 4", "Cast 5":
@@ -829,13 +885,38 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) {
case "VRPorn Scraper":
where = `scenes.scene_id like "vrporn-%"`
case "Has Script Download":
- where = "scenes.script_published > '0001-01-01 00:00:00+00:00'"
+ // querying the scenes in from alternate sources (stored in external_reference) has a performance impact, so it's user choice
+ if config.Advanced.UseAltSrcInFileMatching {
+ where = "(scenes.script_published > '0001-01-01 00:00:00+00:00' or (select distinct 1 from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and internal_db_id=scenes.id and er.udf_datetime1 > '0001-01-02'))"
+ } else {
+ where = "scenes.script_published > '0001-01-01 00:00:00+00:00'"
+ }
case "Has AI Generated Script":
- where = "scenes.ai_script = 1"
+ // querying the scenes in from alternate sources (stored in external_reference) has a performance impact, so it's user choice
+ if config.Advanced.UseAltSrcInFileMatching {
+ where = "(scenes.ai_script = 1 or (select distinct 1 from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and internal_db_id=scenes.id and JSON_EXTRACT(er.external_data, '$.scene.ai_script') = 1))"
+ } else {
+ where = "scenes.ai_script = 1"
+ }
case "Has Human Generated Script":
- where = "scenes.human_script = 1"
+ // querying the scenes in from alternate sources (stored in external_reference) has a performance impact, so it's user choice
+ if config.Advanced.UseAltSrcInFileMatching {
+ where = "(scenes.human_script = 1 or (select distinct 1 from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and internal_db_id=scenes.id and JSON_EXTRACT(er.external_data, '$.scene.human_script') = 1))"
+ } else {
+ where = "scenes.human_script = 1"
+ }
case "Has Favourite Actor":
where = "exists (select * from scene_cast join actors on actors.id=scene_cast.actor_id where actors.favourite=1 and scene_cast.scene_id=scenes.id)"
+ case "Available from POVR":
+ where = "exists (select 1 from external_reference_links where external_source like 'alternate scene %' and external_id like 'povr-%' and internal_db_id = scenes.id)"
+ case "Available from VRPorn":
+ where = "exists (select 1 from external_reference_links where external_source like 'alternate scene %' and external_id like 'vrporn-%' and internal_db_id = scenes.id)"
+ case "Available from SLR":
+ where = "exists (select 1 from external_reference_links where external_source like 'alternate scene %' and external_id like 'slr-%' and internal_db_id = scenes.id)"
+ case "Available from Alternate Sites":
+ where = "exists (select 1 from external_reference_links where external_source like 'alternate scene %' and internal_db_id = scenes.id)"
+ case "Multiple Scenes Available at an Alternate Site":
+ where = "exists (select 1 from external_reference_links where external_source like 'alternate scene %' and internal_db_id = scenes.id group by external_source having count(*)>1)"
}
if negate {
@@ -1048,7 +1129,16 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) {
case "scene_updated_desc":
tx = tx.Order("updated_at desc")
case "script_published_desc":
- tx = tx.Order("script_published desc")
+ // querying the scenes in from alternate sources (stored in external_reference) has a performance impact, so it's user choice
+ if config.Advanced.UseAltSrcInFileMatching {
+ tx = tx.Order(`
+ case when script_published > (select max(er.udf_datetime1) from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and erl.internal_db_id=scenes.id and er.external_source like 'alternate scene %')
+ then script_published
+ else (select max(er.udf_datetime1) from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and erl.internal_db_id=scenes.id and er.external_source like 'alternate scene %')
+ end desc`)
+ } else {
+ tx = tx.Order("script_published desc")
+ }
case "scene_id_desc":
tx = tx.Order("scene_id desc")
case "random":
@@ -1057,6 +1147,9 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) {
} else {
tx = tx.Order("random()")
}
+ case "alt_src_desc":
+ //tx = tx.Order(`(select max(er.external_date) from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and erl.internal_db_id=scenes.id and er.external_source like 'alternate scene %') desc`)
+ tx = tx.Order(`(select max(erl.udf_datetime1) from external_reference_links erl where erl.internal_table='scenes' and erl.internal_db_id=scenes.id and erl.external_source like 'alternate scene %') desc`)
default:
tx = tx.Order("release_date desc")
}
@@ -1101,3 +1194,13 @@ func setCuepointString(cuepoint string) string {
return "%" + cuepoint + "%"
}
}
+
+func getConfig(db *gorm.DB) Config {
+ var config Config
+
+ var kv KV
+ db.Where("`key`='config'").First(&kv)
+
+ json.Unmarshal([]byte(kv.Value), &config)
+ return config
+}
diff --git a/pkg/models/model_scene_alternate_source.go b/pkg/models/model_scene_alternate_source.go
new file mode 100644
index 000000000..78cdfa0c3
--- /dev/null
+++ b/pkg/models/model_scene_alternate_source.go
@@ -0,0 +1,109 @@
+package models
+
+import (
+ "encoding/json"
+ "strings"
+ "time"
+)
+
+// SceneCuepoint data model
+type SceneAlternateSource struct {
+ MasterSiteId string `json:"master_site_id"`
+ MatchAchieved int `json:"MatchAchieved"`
+ Query string `json:"query"` // ******* remove before go live, just to help testing query used for the scene
+ Scene Scene `json:"scene"`
+}
+type AltSrcMatchParams struct {
+ IgnoreReleasedBefore time.Time `json:"ignore_released_before"` // optionally don't check the released date, if before a certain date, this may be useful to only search on release dates after an inital load of scene on a site
+ DelayLinking int `json:"delay_linking"` // allows delaying linking from the release date. Useful for LethalHardcore which is often released on their site after SLR, etc, which would result in linking to another scene
+ ReprocessLinks int `json:"reprocess_links"` // will reprocess linking for the specified number of days. Another approach for LethalHardcore.
+ ReleasedMatchType string `json:"released_match_type"` // must, should, do not
+ ReleasedPrior int `json:"released_prior"` // match on where the scene release date within x days prior to the release date of the alternate scene, eg a studio may only release on other sites after 90 days
+ ReleasedAfter int `json:"released_after"` // do not match scenes befoew a certain date
+ DurationMatchType string `json:"duration_match_type"` // must, should, do not
+ DurationMin int `json:"duration_min"` // ignore if the scene duration is below a specific value, may be useful for trailers
+ DurationRangeLess int `json:"duration_range_less"` // match on where the scene duration is more than x seconds less than the duration of the alternate scene
+ DurationRangeMore int `json:"duration_range_more"` // match on where the scene duration is less than x seconds more than the duration of the alternate scene
+ CastMatchType string `json:"cast_match_type"` // should or do not. Note "must" match is not an option for cast members due to alises and sites using different names.
+ DescriptionMatchType string `json:"desc_match_type"` // should, do not. "Must" is not an option
+ BoostTitleExact float32 `json:"boost_title"` // boosting allows you control the revalant significant between search fields, default is 1. Boostig has no effect on "must" matches
+ BoostTitleAnyWords float32 `json:"boost_title_any_words"` // boosting allows you control the revalant significant between search fields, default is 1. Boostig has no effect on "must" matches
+ BoostReleased float32 `json:"boost_released"`
+ BoostCast float32 `json:"boost_cast"`
+ BoostDescription float32 `json:"boost_description"`
+}
+
+const defaultMatchType = "should"
+const releasedPrior = 14
+const releasedAfter = 7
+const durationMin = 3
+const boostTitleExact = 4.0
+const boostTitleAnyWords = 1.5
+const boostReleased = 2
+const boostCast = 2
+const boostDescription = 0.25
+
+func (p *AltSrcMatchParams) Default() {
+ p.ReleasedMatchType = defaultMatchType
+ p.ReleasedPrior = releasedPrior
+ p.ReleasedAfter = releasedAfter
+ p.DurationMatchType = defaultMatchType
+ p.DurationMin = durationMin
+ p.CastMatchType = defaultMatchType
+ p.DescriptionMatchType = defaultMatchType
+ p.BoostReleased = boostReleased
+ p.BoostTitleExact = boostTitleExact
+ p.BoostTitleAnyWords = boostTitleAnyWords
+ p.BoostCast = boostCast
+ p.BoostDescription = boostDescription
+ p.BoostReleased = boostReleased
+}
+
+func (p *AltSrcMatchParams) UnmarshalParams(jsonStr string) error {
+ if jsonStr == "" {
+ p.Default()
+ return nil
+ } else {
+ err := json.Unmarshal([]byte(jsonStr), &p)
+ if err != nil {
+ return err
+ }
+ if !strings.Contains(jsonStr, "released_match_type") {
+ p.ReleasedMatchType = defaultMatchType
+ }
+ if !strings.Contains(jsonStr, "released_prior") {
+ p.ReleasedPrior = releasedPrior
+ }
+ if !strings.Contains(jsonStr, "released_after") {
+ p.ReleasedAfter = releasedAfter
+ }
+ if !strings.Contains(jsonStr, "duration_match_type") {
+ p.DurationMatchType = defaultMatchType
+ }
+ if !strings.Contains(jsonStr, "duration_min") {
+ p.DurationMin = durationMin
+ }
+ if !strings.Contains(jsonStr, "cast_match_type") {
+ p.CastMatchType = defaultMatchType
+ }
+ if !strings.Contains(jsonStr, "desc_match_type") {
+ p.DescriptionMatchType = defaultMatchType
+ }
+ if !strings.Contains(jsonStr, "boost_title") {
+ p.BoostTitleExact = boostTitleExact
+ }
+ if !strings.Contains(jsonStr, "boost_title_any_words") {
+ p.BoostTitleAnyWords = boostTitleAnyWords
+ }
+ if !strings.Contains(jsonStr, "boost_released") {
+ p.BoostCast = boostReleased
+ }
+ if !strings.Contains(jsonStr, "boost_cast") {
+ p.BoostCast = boostCast
+ }
+ if !strings.Contains(jsonStr, "boost_description") {
+ p.BoostDescription = boostDescription
+ }
+ }
+ return nil
+}
diff --git a/pkg/models/model_scraper.go b/pkg/models/model_scraper.go
index d4435461a..6d5bf44ec 100644
--- a/pkg/models/model_scraper.go
+++ b/pkg/models/model_scraper.go
@@ -7,14 +7,15 @@ import (
var scrapers []Scraper
-type ScraperFunc func(*sync.WaitGroup, bool, []string, chan<- ScrapedScene, string, string) error
+type ScraperFunc func(*sync.WaitGroup, bool, []string, chan<- ScrapedScene, string, string, bool) error
type Scraper struct {
- ID string `json:"id"`
- Name string `json:"name"`
- AvatarURL string `json:"avaatarurl"`
- Domain string `json:"domain"`
- Scrape ScraperFunc `json:"-"`
+ ID string `json:"id"`
+ Name string `json:"name"`
+ AvatarURL string `json:"avaatarurl"`
+ Domain string `json:"domain"`
+ Scrape ScraperFunc `json:"-"`
+ MasterSiteId string `json:"master_site_id"`
}
type ScrapedScene struct {
@@ -46,6 +47,7 @@ type ScrapedScene struct {
InternalSceneId uint `json:"internal_id"`
ActorDetails map[string]ActorDetails `json:"actor_details"`
+ MasterSiteId string `json:"master_site_id"`
}
type ActorDetails struct {
@@ -78,12 +80,13 @@ func GetScrapers() []Scraper {
return scrapers
}
-func RegisterScraper(id string, name string, avatarURL string, domain string, f ScraperFunc) {
+func RegisterScraper(id string, name string, avatarURL string, domain string, f ScraperFunc, masterSiteId string) {
s := Scraper{}
s.ID = id
s.Name = name
s.AvatarURL = avatarURL
s.Domain = domain
s.Scrape = f
+ s.MasterSiteId = masterSiteId
scrapers = append(scrapers, s)
}
diff --git a/pkg/models/model_site.go b/pkg/models/model_site.go
index 78b2f0b07..8af24250d 100644
--- a/pkg/models/model_site.go
+++ b/pkg/models/model_site.go
@@ -8,14 +8,17 @@ import (
)
type Site struct {
- ID string `gorm:"primary_key" json:"id" xbvrbackup:"-"`
- Name string `json:"name" xbvrbackup:"name"`
- AvatarURL string `json:"avatar_url" xbvrbackup:"-"`
- IsBuiltin bool `json:"is_builtin" xbvrbackup:"-"`
- IsEnabled bool `json:"is_enabled" xbvrbackup:"is_enabled"`
- LastUpdate time.Time `json:"last_update" xbvrbackup:"-"`
- Subscribed bool `json:"subscribed" xbvrbackup:"subscribed"`
- HasScraper bool `gorm:"-" json:"has_scraper" xbvrbackup:"-"`
+ ID string `gorm:"primary_key" json:"id" xbvrbackup:"-"`
+ Name string `json:"name" xbvrbackup:"name"`
+ AvatarURL string `json:"avatar_url" xbvrbackup:"-"`
+ IsBuiltin bool `json:"is_builtin" xbvrbackup:"-"`
+ IsEnabled bool `json:"is_enabled" xbvrbackup:"is_enabled"`
+ LastUpdate time.Time `json:"last_update" xbvrbackup:"-"`
+ Subscribed bool `json:"subscribed" xbvrbackup:"subscribed"`
+ HasScraper bool `gorm:"-" json:"has_scraper" xbvrbackup:"-"`
+ LimitScraping bool `json:"limit_scraping" xbvrbackup:"limit_scraping"`
+ MasterSiteID string `json:"master_site_id" xbvrbackup:"master_site_id"`
+ MatchingParams string `json:"matching_params" gorm:"size:1000" xbvrbackup:"matching_params"`
}
func (i *Site) Save() error {
@@ -58,6 +61,7 @@ func InitSites() {
st.Name = scrapers[i].Name
st.AvatarURL = scrapers[i].AvatarURL
st.IsBuiltin = true
+ st.MasterSiteID = scrapers[i].MasterSiteId
st.Save()
}
}
diff --git a/pkg/scrape/baberoticavr.go b/pkg/scrape/baberoticavr.go
index eef0a17fa..08ce27227 100644
--- a/pkg/scrape/baberoticavr.go
+++ b/pkg/scrape/baberoticavr.go
@@ -16,7 +16,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func BaberoticaVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func BaberoticaVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "baberoticavr"
siteID := "BaberoticaVR"
diff --git a/pkg/scrape/badoink.go b/pkg/scrape/badoink.go
index aa5760b10..425161f60 100644
--- a/pkg/scrape/badoink.go
+++ b/pkg/scrape/badoink.go
@@ -23,7 +23,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, URL string, singeScrapeAdditionalInfo string) error {
+func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, URL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
@@ -31,8 +31,9 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
siteCollector := createCollector("badoinkvr.com", "babevr.com", "vrcosplayx.com", "18vr.com", "kinkvr.com")
trailerCollector := cloneCollector(sceneCollector)
- db, _ := models.GetDB()
- defer db.Close()
+ commonDb, _ := models.GetCommonDB()
+
+ var Has6K7K bool = false
sceneCollector.OnHTML(`html`, func(e *colly.HTMLElement) {
sc := models.ScrapedScene{}
@@ -80,6 +81,9 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
// Tags
e.ForEach(`a.video-tag`, func(id int, e *colly.HTMLElement) {
sc.Tags = append(sc.Tags, strings.TrimSpace(e.Text))
+ if strings.Contains(strings.TrimSpace(e.Text), "7K") {
+ Has6K7K = true
+ }
})
if scraperID == "vrcosplayx" {
sc.Tags = append(sc.Tags, "Cosplay", "Parody")
@@ -144,7 +148,18 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
fragmentName := strings.Split(fpName, "/")
baseName := fragmentName[len(fragmentName)-1]
- filenames := []string{"samsung_180_180x180_3dh", "oculus_180_180x180_3dh", "mobile_180_180x180_3dh", "7k_180_180x180_3dh", "5k_180_180x180_3dh", "4k_HEVC_180_180x180_3dh", "samsung_180_180x180_3dh_LR", "oculus_180_180x180_3dh_LR", "mobile_180_180x180_3dh_LR", "7k_180_180x180_3dh_LR", "5k_180_180x180_3dh_LR", "4k_HEVC_180_180x180_3dh_LR", "ps4_180_sbs", "ps4_pro_180_sbs"}
+ e.ForEach(`a.video-tag`, func(id int, e *colly.HTMLElement) {
+ sc.Tags = append(sc.Tags, strings.TrimSpace(e.Text))
+ if strings.Contains(strings.TrimSpace(e.Text), "7K") {
+ Has6K7K = true
+ }
+ })
+
+ filenames := []string{"samsung_180_180x180_3dh", "oculus_180_180x180_3dh", "mobile_180_180x180_3dh", "5k_180_180x180_3dh", "4k_HEVC_180_180x180_3dh", "samsung_180_180x180_3dh_LR", "oculus_180_180x180_3dh_LR", "mobile_180_180x180_3dh_LR", "5k_180_180x180_3dh_LR", "4k_HEVC_180_180x180_3dh_LR", "ps4_180_sbs", "ps4_pro_180_sbs"}
+
+ if Has6K7K {
+ filenames = []string{"7k_180_180x180_3dh", "6k_180_180x180_3dh", "5k_180_180x180_3dh", "4k_HEVC_180_180x180_3dh", "7k_180_180x180_3dh_LR", "6k_180_180x180_3dh_LR", "5k_180_180x180_3dh_LR", "4k_HEVC_180_180x180_3dh_LR", "samsung_180_180x180_3dh", "oculus_180_180x180_3dh", "mobile_180_180x180_3dh", "samsung_180_180x180_3dh_LR", "oculus_180_180x180_3dh_LR", "mobile_180_180x180_3dh_LR", "ps4_180_sbs", "ps4_pro_180_sbs"}
+ }
for i := range filenames {
filenames[i] = baseName + "_" + filenames[i] + ".mp4"
@@ -190,8 +205,10 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
})
siteCollector.OnHTML(`div.pagination a`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`main[data-page=VideoList] a.video-card-image-container`, func(e *colly.HTMLElement) {
@@ -212,7 +229,7 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
e.ForEach(`a[href^="/category/funscript"]`, func(id int, e *colly.HTMLElement) {
var existingScene models.Scene
- existingScene.GetIfExistURL(sceneURL)
+ commonDb.Where(&models.Scene{SceneURL: sceneURL}).First(&existingScene)
if existingScene.ID != 0 && existingScene.ScriptPublished.IsZero() {
var sc models.ScrapedScene
sc.InternalSceneId = existingScene.ID
@@ -241,24 +258,24 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
return nil
}
-func BadoinkVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return BadoinkSite(wg, updateSite, knownScenes, out, singleSceneURL, "badoinkvr", "BadoinkVR", "https://badoinkvr.com/vrpornvideos?order=newest", singeScrapeAdditionalInfo)
+func BadoinkVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return BadoinkSite(wg, updateSite, knownScenes, out, singleSceneURL, "badoinkvr", "BadoinkVR", "https://badoinkvr.com/vrpornvideos?order=newest", singeScrapeAdditionalInfo, limitScraping)
}
-func B18VR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return BadoinkSite(wg, updateSite, knownScenes, out, singleSceneURL, "18vr", "18VR", "https://18vr.com/vrpornvideos", singeScrapeAdditionalInfo)
+func B18VR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return BadoinkSite(wg, updateSite, knownScenes, out, singleSceneURL, "18vr", "18VR", "https://18vr.com/vrpornvideos?order=newest", singeScrapeAdditionalInfo, limitScraping)
}
-func VRCosplayX(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return BadoinkSite(wg, updateSite, knownScenes, out, singleSceneURL, "vrcosplayx", "VRCosplayX", "https://vrcosplayx.com/cosplaypornvideos", singeScrapeAdditionalInfo)
+func VRCosplayX(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return BadoinkSite(wg, updateSite, knownScenes, out, singleSceneURL, "vrcosplayx", "VRCosplayX", "https://vrcosplayx.com/cosplaypornvideos?order=newest", singeScrapeAdditionalInfo, limitScraping)
}
-func BabeVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return BadoinkSite(wg, updateSite, knownScenes, out, singleSceneURL, "babevr", "BabeVR", "https://babevr.com/vrpornvideos", singeScrapeAdditionalInfo)
+func BabeVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return BadoinkSite(wg, updateSite, knownScenes, out, singleSceneURL, "babevr", "BabeVR", "https://babevr.com/vrpornvideos?order=newest", singeScrapeAdditionalInfo, limitScraping)
}
-func KinkVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return BadoinkSite(wg, updateSite, knownScenes, out, singleSceneURL, "kinkvr", "KinkVR", "https://kinkvr.com/bdsm-vr-videos", singeScrapeAdditionalInfo)
+func KinkVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return BadoinkSite(wg, updateSite, knownScenes, out, singleSceneURL, "kinkvr", "KinkVR", "https://kinkvr.com/bdsm-vr-videos?order=newest", singeScrapeAdditionalInfo, limitScraping)
}
func init() {
diff --git a/pkg/scrape/caribbeancom.go b/pkg/scrape/caribbeancom.go
index 4857c59a3..82152d4c9 100644
--- a/pkg/scrape/caribbeancom.go
+++ b/pkg/scrape/caribbeancom.go
@@ -15,7 +15,7 @@ import (
"golang.org/x/text/language"
)
-func CariVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func CariVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "caribbeancomvr"
siteID := "CaribbeanCom VR"
@@ -125,9 +125,11 @@ func CariVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<
})
siteCollector.OnHTML(`.pagination-large .pagination__item[rel="next"]`, func(e *colly.HTMLElement) {
- // replace "all" with "vr" to allow for correct page navigation
- pageURL := strings.Replace(e.Request.AbsoluteURL(e.Attr("href")), "all", "vr", 1)
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ // replace "all" with "vr" to allow for correct page navigation
+ pageURL := strings.Replace(e.Request.AbsoluteURL(e.Attr("href")), "all", "vr", 1)
+ siteCollector.Visit(pageURL)
+ }
})
if singleSceneURL != "" {
diff --git a/pkg/scrape/czechvr.go b/pkg/scrape/czechvr.go
index 40c3b18a3..dc9b4ad4b 100644
--- a/pkg/scrape/czechvr.go
+++ b/pkg/scrape/czechvr.go
@@ -14,11 +14,10 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func CzechVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, nwID string, singeScrapeAdditionalInfo string) error {
+func CzechVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, nwID string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
- db, _ := models.GetDB()
- defer db.Close()
+ commonDb, _ := models.GetCommonDB()
sceneCollector := createCollector("www.czechvrnetwork.com")
siteCollector := createCollector("www.czechvrnetwork.com")
@@ -151,8 +150,10 @@ func CzechVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan
})
siteCollector.OnHTML(`div#StrankovaniDesktop span.stred a,div#StrankovaniDesktopHome span.stred a`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href") + "&sites=" + nwID)
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href") + "&sites=" + nwID)
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.postTag`, func(e *colly.HTMLElement) {
@@ -167,7 +168,7 @@ func CzechVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan
if config.Config.Funscripts.ScrapeFunscripts {
e.ForEach(`div.iconinteractive`, func(id int, e *colly.HTMLElement) {
var existingScene models.Scene
- existingScene.GetIfExistURL(sceneURL)
+ commonDb.Where(&models.Scene{SceneURL: sceneURL}).First(&existingScene)
if existingScene.ID != 0 && existingScene.ScriptPublished.IsZero() {
var sc models.ScrapedScene
sc.InternalSceneId = existingScene.ID
@@ -184,7 +185,7 @@ func CzechVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan
if singleSceneURL != "" {
sceneCollector.Visit(singleSceneURL)
} else {
- siteCollector.Visit("https://www.czechvrnetwork.com/vr-porn-videos&sites=" + nwID)
+ siteCollector.Visit("https://www.czechvrnetwork.com/vr-porn-videos&sort=date&sites=" + nwID)
}
if updateSite {
@@ -196,15 +197,15 @@ func CzechVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan
}
func addCZVRScraper(id string, name string, nwid string, avatarURL string) {
- registerScraper(id, name, avatarURL, "czechvrnetwork.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return CzechVR(wg, updateSite, knownScenes, out, singleSceneURL, id, name, nwid, singeScrapeAdditionalInfo)
+ registerScraper(id, name, avatarURL, "czechvrnetwork.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return CzechVR(wg, updateSite, knownScenes, out, singleSceneURL, id, name, nwid, singeScrapeAdditionalInfo, limitScraping)
})
}
func init() {
// scraper for scraping single scenes where only the url is provided
- registerScraper("czechvr-single_scene", "Czech VR - Other Studios", "", "czechvrnetwork.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return CzechVR(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "")
+ registerScraper("czechvr-single_scene", "Czech VR - Other Studios", "", "czechvrnetwork.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return CzechVR(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", limitScraping)
})
addCZVRScraper("czechvr", "Czech VR", "15", "https://www.czechvr.com/images/favicon/android-chrome-256x256.png")
addCZVRScraper("czechvrfetish", "Czech VR Fetish", "16", "https://www.czechvrfetish.com/images/favicon/android-chrome-256x256.png")
diff --git a/pkg/scrape/darkroomvr.go b/pkg/scrape/darkroomvr.go
index ff2a7e5f7..27050697d 100644
--- a/pkg/scrape/darkroomvr.go
+++ b/pkg/scrape/darkroomvr.go
@@ -13,7 +13,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func DarkRoomVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func DarkRoomVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "darkroomvr"
siteID := "DarkRoomVR"
@@ -114,8 +114,10 @@ func DarkRoomVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out c
})
siteCollector.OnHTML(`div.pagination a`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.video-card__item a[class=image-container]`, func(e *colly.HTMLElement) {
diff --git a/pkg/scrape/fuckpassvr.go b/pkg/scrape/fuckpassvr.go
index aa7afca2c..73be9dc36 100644
--- a/pkg/scrape/fuckpassvr.go
+++ b/pkg/scrape/fuckpassvr.go
@@ -16,7 +16,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func FuckPassVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func FuckPassVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "fuckpassvr-native"
siteID := "FuckPassVR"
@@ -121,6 +121,9 @@ func FuckPassVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out c
var page int64 = 1
var lastPage int64 = 1
+ if limitScraping {
+ lastPage = 1
+ }
if singleSceneURL != "" {
ctx := colly.NewContext()
diff --git a/pkg/scrape/genericactorscraper.go b/pkg/scrape/genericactorscraper.go
index 4b723ff3e..01327ea04 100644
--- a/pkg/scrape/genericactorscraper.go
+++ b/pkg/scrape/genericactorscraper.go
@@ -34,20 +34,19 @@ func GenericActorScrapers() {
tlog := log.WithField("task", "scrape")
tlog.Infof("Scraping Actor Details from Sites")
- db, _ := models.GetDB()
- defer db.Close()
+ commonDb, _ := models.GetCommonDB()
scraperConfig := models.BuildActorScraperRules()
var actors []models.Actor
- db.Preload("Scenes").
+ commonDb.Preload("Scenes").
Where("id =1").
Find(&actors)
sqlcmd := ""
var output []outputList
- switch db.Dialect().GetName() {
+ switch commonDb.Dialect().GetName() {
// gets the list of an actors Urls for scraper and join to external_reference_links to see if the haven't been linked
case "mysql":
sqlcmd = `
@@ -78,7 +77,7 @@ func GenericActorScrapers() {
processed := 0
lastMessage := time.Now()
- db.Raw(sqlcmd).Scan(&output)
+ commonDb.Raw(sqlcmd).Scan(&output)
var wg sync.WaitGroup
concurrentLimit := 10 // Maximum number of concurrent tasks
@@ -133,16 +132,15 @@ func processAuthorLink(row outputList, siteRules map[string]models.GenericScrape
func GenericSingleActorScraper(actorId uint, actorPage string) {
log.Infof("Scraping Actor Details from %s", actorPage)
- db, _ := models.GetDB()
- defer db.Close()
+ commonDb, _ := models.GetCommonDB()
var actor models.Actor
actor.ID = actorId
- db.Find(&actor)
+ commonDb.Find(&actor)
scraperConfig := models.BuildActorScraperRules()
var extRefLink models.ExternalReferenceLink
- db.Preload("ExternalReference").
+ commonDb.Preload("ExternalReference").
Where(&models.ExternalReferenceLink{ExternalId: actorPage, InternalDbId: actor.ID}).
First(&extRefLink)
@@ -160,16 +158,14 @@ func GenericActorScrapersBySite(site string) {
tlog := log.WithField("task", "scrape")
tlog.Infof("Scraping Actor Details from %s", site)
- db, _ := models.GetDB()
- defer db.Close()
-
+ commonDb, _ := models.GetCommonDB()
scraperConfig := models.BuildActorScraperRules()
er := models.ExternalReference{}
scrapeId := er.DetermineActorScraperBySiteId(site)
var actors []models.Actor
- db.Debug().Select("DISTINCT actors.*").
+ commonDb.Debug().Select("DISTINCT actors.*").
Joins("JOIN scene_cast ON scene_cast.actor_id = actors.id").
Joins("JOIN scenes ON scenes.id = scene_cast.scene_id").
Where("scenes.scraper_id = ?", site).
@@ -184,7 +180,7 @@ func GenericActorScrapersBySite(site string) {
// find the url for the actor for this site
var extreflink models.ExternalReferenceLink
- db.Where(`internal_table = 'actors' and internal_db_id = ? and external_source = ?`, actor.ID, scrapeId).First(&extreflink)
+ commonDb.Where(`internal_table = 'actors' and internal_db_id = ? and external_source = ?`, actor.ID, scrapeId).First(&extreflink)
for source, rule := range scraperConfig.GenericActorScrapingConfig {
if source == scrapeId {
applyRules(extreflink.ExternalId, scrapeId, rule, &actor, true)
@@ -278,9 +274,8 @@ func applyRules(actorPage string, source string, rules models.GenericScraperRule
var extref models.ExternalReference
var extreflink models.ExternalReferenceLink
- db, _ := models.GetDB()
- defer db.Close()
- db.Preload("ExternalReference").
+ commonDb, _ := models.GetCommonDB()
+ commonDb.Preload("ExternalReference").
Where(&models.ExternalReferenceLink{ExternalSource: source, InternalDbId: actor.ID}).
First(&extreflink)
extref = extreflink.ExternalReference
diff --git a/pkg/scrape/groobyvr.go b/pkg/scrape/groobyvr.go
index ad1006b35..62d7f467c 100644
--- a/pkg/scrape/groobyvr.go
+++ b/pkg/scrape/groobyvr.go
@@ -14,7 +14,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func GroobyVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func GroobyVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "groobyvr"
siteID := "GroobyVR"
@@ -115,8 +115,10 @@ func GroobyVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out cha
})
siteCollector.OnHTML(`div.pagination li a:not(.active)`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
if singleSceneURL != "" {
diff --git a/pkg/scrape/hologirlsvr.go b/pkg/scrape/hologirlsvr.go
index 66009b276..fba258474 100644
--- a/pkg/scrape/hologirlsvr.go
+++ b/pkg/scrape/hologirlsvr.go
@@ -11,7 +11,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func HoloGirlsVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func HoloGirlsVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "hologirlsvr"
siteID := "HoloGirlsVR"
diff --git a/pkg/scrape/lethalhardcorevr.go b/pkg/scrape/lethalhardcorevr.go
index 5c366b1a7..95cdce180 100644
--- a/pkg/scrape/lethalhardcorevr.go
+++ b/pkg/scrape/lethalhardcorevr.go
@@ -26,7 +26,7 @@ func isGoodTag(lookup string) bool {
return true
}
-func LethalHardcoreSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, URL string, singeScrapeAdditionalInfo string) error {
+func LethalHardcoreSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, URL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
@@ -138,8 +138,10 @@ func LethalHardcoreSite(wg *sync.WaitGroup, updateSite bool, knownScenes []strin
})
siteCollector.OnHTML(`div.pagination a`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.scene-list-item`, func(e *colly.HTMLElement) {
@@ -174,12 +176,12 @@ func LethalHardcoreSite(wg *sync.WaitGroup, updateSite bool, knownScenes []strin
return nil
}
-func LethalHardcoreVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return LethalHardcoreSite(wg, updateSite, knownScenes, out, singleSceneURL, "lethalhardcorevr", "LethalHardcoreVR", "https://lethalhardcorevr.com/lethal-hardcore-vr-scenes.html?studio=95595", singeScrapeAdditionalInfo)
+func LethalHardcoreVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return LethalHardcoreSite(wg, updateSite, knownScenes, out, singleSceneURL, "lethalhardcorevr", "LethalHardcoreVR", "https://lethalhardcorevr.com/lethal-hardcore-vr-scenes.html?studio=95595&sort=released", singeScrapeAdditionalInfo, limitScraping)
}
-func WhorecraftVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return LethalHardcoreSite(wg, updateSite, knownScenes, out, singleSceneURL, "whorecraftvr", "WhorecraftVR", "https://lethalhardcorevr.com/lethal-hardcore-vr-scenes.html?studio=95347", singeScrapeAdditionalInfo)
+func WhorecraftVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return LethalHardcoreSite(wg, updateSite, knownScenes, out, singleSceneURL, "whorecraftvr", "WhorecraftVR", "https://lethalhardcorevr.com/lethal-hardcore-vr-scenes.html?studio=95347&sort=released", singeScrapeAdditionalInfo, limitScraping)
}
func init() {
diff --git a/pkg/scrape/littlecaprice.go b/pkg/scrape/littlecaprice.go
index 569f0ef9c..8ec4b01c4 100644
--- a/pkg/scrape/littlecaprice.go
+++ b/pkg/scrape/littlecaprice.go
@@ -13,7 +13,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func LittleCaprice(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func LittleCaprice(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "littlecaprice"
siteID := "Little Caprice Dreams"
diff --git a/pkg/scrape/navr.go b/pkg/scrape/navr.go
index ce6e27a6d..8582c437b 100644
--- a/pkg/scrape/navr.go
+++ b/pkg/scrape/navr.go
@@ -14,7 +14,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func NaughtyAmericaVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func NaughtyAmericaVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "naughtyamericavr"
siteID := "NaughtyAmerica VR"
@@ -163,8 +163,10 @@ func NaughtyAmericaVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string,
})
siteCollector.OnHTML(`ul[class=pagination] li a`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div[class=site-list] div[class=scene-item] a.contain-img`, func(e *colly.HTMLElement) {
diff --git a/pkg/scrape/povr.go b/pkg/scrape/povr.go
index ed4c143a5..1d916cd84 100644
--- a/pkg/scrape/povr.go
+++ b/pkg/scrape/povr.go
@@ -15,7 +15,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func POVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string) error {
+func POVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string, limitScraping bool, masterSiteId string) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
@@ -29,6 +29,7 @@ func POVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
sc.Studio = company
sc.Site = siteID
sc.HomepageURL = strings.Split(e.Request.URL.String(), "?")[0]
+ sc.MasterSiteId = masterSiteId
if scraperID == "" {
// there maybe no site/studio if user is jusy scraping a scene url
@@ -123,15 +124,17 @@ func POVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
siteCollector.OnHTML(`div.thumbnail-wrap div.thumbnail a.thumbnail__link`, func(e *colly.HTMLElement) {
sceneURL := e.Request.AbsoluteURL(e.Attr("href"))
- // If scene exist in database, there's no need to scrape
+ // If scene exists in database, or the slternate source exists, there's no need to scrape
if !funk.ContainsString(knownScenes, sceneURL) && !strings.Contains(sceneURL, "/join") {
sceneCollector.Visit(sceneURL)
}
})
siteCollector.OnHTML(`div.pagination a[class="pagination__page next"]`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
if singleSceneURL != "" {
@@ -147,7 +150,7 @@ func POVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
return nil
}
-func addPOVRScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string) {
+func addPOVRScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string, masterSiteId string) {
suffixedName := name
siteNameSuffix := name
if custom {
@@ -156,21 +159,31 @@ func addPOVRScraper(id string, name string, company string, avatarURL string, cu
} else {
suffixedName += " (POVR)"
}
- registerScraper(id, suffixedName, avatarURL, "povr.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return POVR(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo)
- })
+ if avatarURL == "" {
+ avatarURL = "https://images.povr.com/img/povr/android-icon-192x192.png"
+ }
+
+ if masterSiteId == "" {
+ registerScraper(id, suffixedName, avatarURL, "povr.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return POVR(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, "")
+ })
+ } else {
+ registerAlternateScraper(id, suffixedName, avatarURL, "povr.com", masterSiteId, func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return POVR(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, masterSiteId)
+ })
+ }
}
func init() {
- registerScraper("povr-single_scene", "POVR - Other Studios", "", "povr.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return POVR(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo)
+ registerScraper("povr-single_scene", "POVR - Other Studios", "", "povr.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return POVR(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo, limitScraping, "")
})
var scrapers config.ScraperList
scrapers.Load()
for _, scraper := range scrapers.XbvrScrapers.PovrScrapers {
- addPOVRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL)
+ addPOVRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL, scraper.MasterSiteId)
}
for _, scraper := range scrapers.CustomScrapers.PovrScrapers {
- addPOVRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL)
+ addPOVRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL, scraper.MasterSiteId)
}
}
diff --git a/pkg/scrape/realitylovers.go b/pkg/scrape/realitylovers.go
index 7d897f509..23a0d5bfd 100644
--- a/pkg/scrape/realitylovers.go
+++ b/pkg/scrape/realitylovers.go
@@ -15,7 +15,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func RealityLoversSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, domain string, singeScrapeAdditionalInfo string) error {
+func RealityLoversSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, domain string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
@@ -127,6 +127,9 @@ func RealityLoversSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string
break
}
page++
+ if limitScraping {
+ break
+ }
// have seen instances of status 404, so make sure we don't span will calls
time.Sleep(time.Second)
}
@@ -148,12 +151,12 @@ func RealityLoversSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string
return nil
}
-func RealityLovers(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return RealityLoversSite(wg, updateSite, knownScenes, out, singleSceneURL, "realitylovers", "RealityLovers", "realitylovers.com", singeScrapeAdditionalInfo)
+func RealityLovers(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return RealityLoversSite(wg, updateSite, knownScenes, out, singleSceneURL, "realitylovers", "RealityLovers", "realitylovers.com", singeScrapeAdditionalInfo, limitScraping)
}
-func TSVirtualLovers(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return RealityLoversSite(wg, updateSite, knownScenes, out, singleSceneURL, "tsvirtuallovers", "TSVirtualLovers", "tsvirtuallovers.com", singeScrapeAdditionalInfo)
+func TSVirtualLovers(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return RealityLoversSite(wg, updateSite, knownScenes, out, singleSceneURL, "tsvirtuallovers", "TSVirtualLovers", "tsvirtuallovers.com", singeScrapeAdditionalInfo, limitScraping)
}
func init() {
diff --git a/pkg/scrape/realjamvr.go b/pkg/scrape/realjamvr.go
index aa162daad..913cc0b03 100644
--- a/pkg/scrape/realjamvr.go
+++ b/pkg/scrape/realjamvr.go
@@ -18,7 +18,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func RealJamSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, domain string, singeScrapeAdditionalInfo string) error {
+func RealJamSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, domain string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
@@ -68,7 +68,7 @@ func RealJamSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
// Cast
sc.ActorDetails = make(map[string]models.ActorDetails)
- e.ForEach(`div.scene-view a[href^='/actor/']`, func(id int, e *colly.HTMLElement) {
+ e.ForEach(`div.scene-view > a[href^='/actor/']`, func(id int, e *colly.HTMLElement) {
sc.Cast = append(sc.Cast, strings.TrimSpace(e.Text))
sc.ActorDetails[strings.TrimSpace(e.Text)] = models.ActorDetails{Source: sc.ScraperID + " scrape", ProfileUrl: e.Request.AbsoluteURL(e.Attr("href"))}
})
@@ -180,8 +180,10 @@ func RealJamSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
})
siteCollector.OnHTML(`a.page-link`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.panel a`, func(e *colly.HTMLElement) {
@@ -207,11 +209,11 @@ func RealJamSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
return nil
}
-func RealJamVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return RealJamSite(wg, updateSite, knownScenes, out, singleSceneURL, "realjamvr", "RealJam VR", "realjamvr.com", singeScrapeAdditionalInfo)
+func RealJamVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return RealJamSite(wg, updateSite, knownScenes, out, singleSceneURL, "realjamvr", "RealJam VR", "realjamvr.com", singeScrapeAdditionalInfo, limitScraping)
}
-func PornCornVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return RealJamSite(wg, updateSite, knownScenes, out, singleSceneURL, "porncornvr", "PornCorn VR", "porncornvr.com", singeScrapeAdditionalInfo)
+func PornCornVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return RealJamSite(wg, updateSite, knownScenes, out, singleSceneURL, "porncornvr", "PornCorn VR", "porncornvr.com", singeScrapeAdditionalInfo, limitScraping)
}
func init() {
diff --git a/pkg/scrape/scrape.go b/pkg/scrape/scrape.go
index 118556f6c..871b5feaa 100644
--- a/pkg/scrape/scrape.go
+++ b/pkg/scrape/scrape.go
@@ -84,7 +84,12 @@ func getScrapeCacheDir() string {
}
func registerScraper(id string, name string, avatarURL string, domain string, f models.ScraperFunc) {
- models.RegisterScraper(id, name, avatarURL, domain, f)
+ models.RegisterScraper(id, name, avatarURL, domain, f, "")
+}
+
+func registerAlternateScraper(id string, name string, avatarURL string, domain string, masterSiteId string, f models.ScraperFunc) {
+ // alternate scrapers are to scrape scenes available at other sites to match against a scenes from the studio's site, eg scrape VRHush scenes from SLR and match to scenes from VRHush
+ models.RegisterScraper(id, name, avatarURL, domain, f, masterSiteId)
}
func logScrapeStart(id string, name string) {
diff --git a/pkg/scrape/sexbabesvr.go b/pkg/scrape/sexbabesvr.go
index 27d14baa7..edf92dbe7 100644
--- a/pkg/scrape/sexbabesvr.go
+++ b/pkg/scrape/sexbabesvr.go
@@ -14,7 +14,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func SexBabesVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func SexBabesVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "sexbabesvr"
siteID := "SexBabesVR"
@@ -98,8 +98,10 @@ func SexBabesVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out c
})
siteCollector.OnHTML(`a.pagination__button`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.videos__content`, func(e *colly.HTMLElement) {
diff --git a/pkg/scrape/sinsvr.go b/pkg/scrape/sinsvr.go
index 7a97b4861..731399e12 100644
--- a/pkg/scrape/sinsvr.go
+++ b/pkg/scrape/sinsvr.go
@@ -15,7 +15,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func SinsVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func SinsVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "sinsvr"
siteID := "SinsVR"
@@ -120,9 +120,11 @@ func SinsVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<
})
siteCollector.OnHTML(`nav.pagination a`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- if !strings.Contains(pageURL, "/join") {
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ if !strings.Contains(pageURL, "/join") {
+ siteCollector.Visit(pageURL)
+ }
}
})
diff --git a/pkg/scrape/slrstudios.go b/pkg/scrape/slrstudios.go
index d6056f92c..ca7358cef 100644
--- a/pkg/scrape/slrstudios.go
+++ b/pkg/scrape/slrstudios.go
@@ -1,6 +1,7 @@
package scrape
import (
+ "encoding/json"
"html"
"regexp"
"strconv"
@@ -15,15 +16,23 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string) error {
+func absolutegallery(match string) string {
+ re := regexp.MustCompile(`(https:\/\/cdn-vr\.(sexlikereal|trannypornvr)\.com\/images\/\d+\/)vr-porn-[\w\-]+?-(\d+)-original(\.webp|\.jpg)`)
+ submatches := re.FindStringSubmatch(match)
+ if len(submatches) == 0 {
+ return match // no match, return original string
+ }
+ return submatches[1] + submatches[3] + "_o.jpg" // construct new string with desired format
+}
+
+func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string, limitScraping bool, masterSiteId string) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
sceneCollector := createCollector("www.sexlikereal.com")
siteCollector := createCollector("www.sexlikereal.com")
- db, _ := models.GetDB()
- defer db.Close()
+ commonDb, _ := models.GetCommonDB()
// RegEx Patterns
coverRegEx := regexp.MustCompile(`background(?:-image)?\s*?:\s*?url\s*?\(\s*?(.*?)\s*?\)`)
@@ -37,6 +46,7 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
sc.SceneType = "VR"
sc.Studio = company
sc.Site = siteID
+ sc.MasterSiteId = masterSiteId
if scraperID == "" {
// there maybe no site/studio if user is jusy scraping a scene url
e.ForEach(`div[data-qa="page-scene-studio-name"]`, func(id int, e *colly.HTMLElement) {
@@ -48,10 +58,8 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
studioId = strings.TrimSuffix(strings.ReplaceAll(studioId, "/studios/", ""), "/")
// see if we can find the site record, there may not be
- db, _ := models.GetDB()
- defer db.Close()
var site models.Site
- db.Where("id = ? or name like ? or (name = ? and name like 'SLR%')", studioId, sc.Studio+"%SLR)", sc.Studio).First(&site)
+ commonDb.Where("id = ? or name like ? or (name = ? and name like 'SLR%')", studioId, sc.Studio+"%SLR)", sc.Studio).First(&site)
if site.ID != "" {
sc.ScraperID = site.ID
}
@@ -65,24 +73,6 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
sc.SiteID = tmp[len(tmp)-1]
sc.SceneID = "slr-" + sc.SiteID
- // Cover
- coverURL := e.ChildAttr(`.splash-screen > img`, "src")
- if len(coverURL) > 0 {
- sc.Covers = append(sc.Covers, coverURL)
- } else {
- m := coverRegEx.FindStringSubmatch(strings.TrimSpace(e.ChildAttr(`.splash-screen`, "style")))
- if len(m) > 0 && len(m[1]) > 0 {
- sc.Covers = append(sc.Covers, m[1])
- }
- }
-
- // Gallery
- e.ForEach(`meta[name^="twitter:image"]`, func(id int, e *colly.HTMLElement) {
- if e.Attr("name") != "twitter:image" { // we need image1, image2...
- sc.Gallery = append(sc.Gallery, e.Request.AbsoluteURL(e.Attr("content")))
- }
- })
-
// Synopsis
sc.Synopsis = strings.TrimSpace(
e.DOM.Find(`div#tabs-about > div > div.u-px--four > div.u-wh`).First().Text())
@@ -140,6 +130,51 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
isTransScene := e.Request.Ctx.GetAny("isTransScene").(bool)
+ // Gallery
+ e.ForEach(`meta[name^="twitter:image"]`, func(id int, e *colly.HTMLElement) {
+ re := regexp.MustCompile(`(https:\/\/cdn-vr\.(sexlikereal|trannypornvr)\.com\/images\/\d+\/)vr-porn-[\w\-]+?-(\d+)-original(\.webp|\.jpg)`)
+ if e.Attr("name") != "twitter:image" { // we need image1, image2...
+ if !isTransScene {
+ sc.Gallery = append(sc.Gallery, re.ReplaceAllStringFunc(e.Request.AbsoluteURL(e.Attr("content")), absolutegallery))
+ } //else {
+ // sc.Gallery = append(sc.Gallery, e.Request.AbsoluteURL(e.Attr("content")))
+ //}
+ // Trans scenes currently do not scrape gallery images at all
+ // I'm not sure how to, since we're using "twitter:image" and there are none for trans scenes
+ // The URLs are available on the "Photos" tab and they can be re-written to redirect to original images similarly
+ // The RegEx will work for either
+ // i.e. "https://cdn-vr.trannypornvr.com/images/861/vr-porn-Welcome-Back-4570.webp" -> "https://cdn-vr.trannypornvr.com/images/861/4570_o.jpg"
+ }
+ })
+
+ // Cover
+ if !isTransScene {
+ coverURL := strings.Replace(gjson.Get(JsonMetadataA, "thumbnailUrl").String(), "app", "desktop", -1)
+ if len(coverURL) > 0 {
+ sc.Covers = append(sc.Covers, coverURL)
+ } else {
+ coverURL := e.ChildAttr(`.splash-screen > img`, "src")
+ if len(coverURL) > 0 {
+ sc.Covers = append(sc.Covers, coverURL)
+ } else {
+ m := coverRegEx.FindStringSubmatch(strings.TrimSpace(e.ChildAttr(`.c-webxr-splash-screen`, "style")))
+ if len(m) > 0 && len(m[1]) > 0 {
+ sc.Covers = append(sc.Covers, m[1])
+ }
+ }
+ }
+ } else {
+ tcoverURL := e.ChildAttr(`.splash-screen > img`, "src")
+ if len(tcoverURL) > 0 {
+ sc.Covers = append(sc.Covers, tcoverURL)
+ } else {
+ m := coverRegEx.FindStringSubmatch(strings.TrimSpace(e.ChildAttr(`.c-webxr-splash-screen`, "style")))
+ if len(m) > 0 && len(m[1]) > 0 {
+ sc.Covers = append(sc.Covers, m[1])
+ }
+ }
+ }
+
// straight and trans videos use a different page structure
if !isTransScene {
// Extract from JSON meta data
@@ -267,8 +302,10 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
})
siteCollector.OnHTML(`div.c-pagination ul li a`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.c-grid--scenes article`, func(e *colly.HTMLElement) {
@@ -281,8 +318,19 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
if config.Config.Funscripts.ScrapeFunscripts {
var existingScene models.Scene
- existingScene.GetIfExistURL(sceneURL)
- if existingScene.ID != 0 {
+
+ if masterSiteId == "" {
+ commonDb.Where(&models.Scene{SceneURL: sceneURL}).First(&existingScene)
+ } else {
+ // get the scene from the external_reference table
+ var extref models.ExternalReference
+ extref.FindExternalUrl("alternate scene "+scraperID, sceneURL)
+ var externalData models.SceneAlternateSource
+ json.Unmarshal([]byte(extref.ExternalData), &externalData)
+ existingScene = externalData.Scene
+ }
+
+ if existingScene.ID != 0 || masterSiteId != "" {
fleshlightBadge := false
aiBadge := false
multiBadge := false
@@ -313,16 +361,18 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
if existingScene.HumanScript != human || existingScene.AiScript != ai {
var sc models.ScrapedScene
sc.InternalSceneId = existingScene.ID
+ sc.SceneID = existingScene.SceneID
+ sc.ScraperID = scraperID
sc.HasScriptDownload = true
sc.OnlyUpdateScriptData = true
sc.HumanScript = human
sc.AiScript = ai
+ sc.MasterSiteId = masterSiteId
out <- sc
}
}
}
- // If scene exist in database, there's no need to scrape
if !funk.ContainsString(knownScenes, sceneURL) {
durationText := e.ChildText("div.c-grid-ratio-bottom.u-z--two")
m := durationRegExForSceneCard.FindStringSubmatch(durationText)
@@ -448,7 +498,7 @@ func appendFilenames(sc *models.ScrapedScene, siteID string, filenameRegEx *rege
}
}
-func addSLRScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string) {
+func addSLRScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string, masterSiteId string) {
suffixedName := name
siteNameSuffix := name
if custom {
@@ -464,23 +514,29 @@ func addSLRScraper(id string, name string, company string, avatarURL string, cus
avatarURL = "https://www.sexlikereal.com/s/refactor/images/favicons/android-icon-192x192.png"
}
- registerScraper(id, suffixedName, avatarURL, "sexlikereal.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return SexLikeReal(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo)
- })
+ if masterSiteId == "" {
+ registerScraper(id, suffixedName, avatarURL, "sexlikereal.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return SexLikeReal(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, "")
+ })
+ } else {
+ registerAlternateScraper(id, suffixedName, avatarURL, "sexlikereal.com", masterSiteId, func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return SexLikeReal(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, masterSiteId)
+ })
+ }
}
func init() {
var scrapers config.ScraperList
// scraper for single scenes with no existing scraper for the studio
- registerScraper("slr-single_scene", "SLR - Other Studios", "", "sexlikereal.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return SexLikeReal(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo)
+ registerScraper("slr-single_scene", "SLR - Other Studios", "", "sexlikereal.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return SexLikeReal(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo, limitScraping, "")
})
scrapers.Load()
for _, scraper := range scrapers.XbvrScrapers.SlrScrapers {
- addSLRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL)
+ addSLRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL, scraper.MasterSiteId)
}
for _, scraper := range scrapers.CustomScrapers.SlrScrapers {
- addSLRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL)
+ addSLRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL, scraper.MasterSiteId)
}
}
diff --git a/pkg/scrape/stashdb.go b/pkg/scrape/stashdb.go
index d9d73eb07..e0d59c7ad 100644
--- a/pkg/scrape/stashdb.go
+++ b/pkg/scrape/stashdb.go
@@ -145,7 +145,7 @@ func findStudio(studio string, field string) FindStudioResult {
// Define the variables needed for your query as a Go map
variables := `{"` + field + `": "` + studio + `"}`
- resp := callStashDb(query, variables)
+ resp := CallStashDb(query, variables)
var data FindStudioResult
json.Unmarshal(resp, &data)
return data
@@ -191,7 +191,7 @@ func getPerformersPage(studioId string, page int) QueryPerformerResult {
}
`
- resp := callStashDb(query, variables)
+ resp := CallStashDb(query, variables)
var data QueryPerformerResult
json.Unmarshal(resp, &data)
return data
@@ -214,7 +214,7 @@ func getScenes(studioId string, parentId string, tagId string) QueryScenesResult
} else {
variables = getStudioSceneQueryVariable(studioId, page, count)
}
- sceneList = getScenePage(variables)
+ sceneList = GetScenePage(variables)
nextList = sceneList
for len(nextList.Data.QueryScenes.Scenes) > 0 &&
len(sceneList.Data.QueryScenes.Scenes) < sceneList.Data.QueryScenes.Count && // {
@@ -225,7 +225,7 @@ func getScenes(studioId string, parentId string, tagId string) QueryScenesResult
} else {
variables = getStudioSceneQueryVariable(studioId, page, count)
}
- nextList = getScenePage(variables)
+ nextList = GetScenePage(variables)
sceneList.Data.QueryScenes.Scenes = append(sceneList.Data.QueryScenes.Scenes, nextList.Data.QueryScenes.Scenes...)
}
return sceneList
@@ -267,7 +267,7 @@ func getParentSceneQueryVariable(parentId string, tagId string, page int, count
}
// calls graphql scene query and return a list of scenes
-func getScenePage(variables string) QueryScenesResult {
+func GetScenePage(variables string) QueryScenesResult {
query := `
query queryScenes($input: SceneQueryInput!) {
queryScenes(input: $input) {
@@ -325,7 +325,7 @@ func getScenePage(variables string) QueryScenesResult {
`
// Define the variables needed for your query as a Go map
- resp := callStashDb(query, variables)
+ resp := CallStashDb(query, variables)
var data QueryScenesResult
json.Unmarshal(resp, &data)
return data
@@ -492,14 +492,15 @@ func getStashPerformer(performer string) FindPerformerResult {
// Define the variables needed for your query as a Go map
var data FindPerformerResult
variables := `{"id": "` + performer + `"}`
- resp := callStashDb(query, variables)
+ resp := CallStashDb(query, variables)
err := json.Unmarshal(resp, &data)
if err != nil {
log.Errorf("Eror extracting actor json")
}
return data
}
-func callStashDb(query string, rawVariables string) []byte {
+
+func CallStashDb(query string, rawVariables string) []byte {
var variables map[string]interface{}
json.Unmarshal([]byte(rawVariables), &variables)
diff --git a/pkg/scrape/stasyqvr.go b/pkg/scrape/stasyqvr.go
index 2e32b6f09..a634d00ea 100644
--- a/pkg/scrape/stasyqvr.go
+++ b/pkg/scrape/stasyqvr.go
@@ -13,7 +13,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func StasyQVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func StasyQVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "stasyqvr"
siteID := "StasyQVR"
@@ -109,8 +109,10 @@ func StasyQVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out cha
})
siteCollector.OnHTML(`div.pagination div.select-links a`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`section.grid div.grid-info-inner`, func(e *colly.HTMLElement) {
diff --git a/pkg/scrape/tmwvrnet.go b/pkg/scrape/tmwvrnet.go
index 210c4a1dc..17f826d59 100644
--- a/pkg/scrape/tmwvrnet.go
+++ b/pkg/scrape/tmwvrnet.go
@@ -10,10 +10,11 @@ import (
"github.com/mozillazg/go-slugify"
"github.com/nleeper/goment"
"github.com/thoas/go-funk"
+ "github.com/xbapps/xbvr/pkg/config"
"github.com/xbapps/xbvr/pkg/models"
)
-func TmwVRnet(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func TmwVRnet(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "tmwvrnet"
siteID := "TmwVRnet"
@@ -30,6 +31,8 @@ func TmwVRnet(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out cha
sc.Studio = "TeenMegaWorld"
sc.Site = siteID
sc.HomepageURL = strings.Split(e.Request.URL.String(), "?")[0]
+ sc.MembersUrl = strings.Replace(sc.HomepageURL, "https://tmwvrnet.com/trailers/", "https://"+config.Config.ScraperSettings.TMWVRNet.TmwMembersDomain+"/scenes/", 1)
+ sc.MembersUrl = strings.Replace(sc.MembersUrl, ".html", "_vids.html", 1)
// Date & Duration
e.ForEach(`.video-info-data`, func(id int, e *colly.HTMLElement) {
@@ -100,9 +103,11 @@ func TmwVRnet(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out cha
})
siteCollector.OnHTML(`a.pagination-element__link`, func(e *colly.HTMLElement) {
- if strings.Contains(e.Text, "Next") {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ if strings.Contains(e.Text, "Next") {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
}
})
diff --git a/pkg/scrape/tngf.go b/pkg/scrape/tngf.go
index 0f0d1aaa4..47273b6f4 100644
--- a/pkg/scrape/tngf.go
+++ b/pkg/scrape/tngf.go
@@ -13,7 +13,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func TNGFVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func TNGFVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "tonightsgirlfriend"
siteID := "Tonight's Girlfriend VR"
@@ -142,8 +142,10 @@ func TNGFVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<
})
siteCollector.OnHTML(`ul[class=pagination] a.page-link[rel="next"]`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.panel-body`, func(e *colly.HTMLElement) {
diff --git a/pkg/scrape/virtualporn.go b/pkg/scrape/virtualporn.go
index 37a044b2d..4edc86162 100644
--- a/pkg/scrape/virtualporn.go
+++ b/pkg/scrape/virtualporn.go
@@ -12,7 +12,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func VirtualPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func VirtualPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "bvr"
siteID := "VirtualPorn"
@@ -109,7 +109,9 @@ func VirtualPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
if sceneCnt > 0 {
pageCnt += 1
- siteCollector.Visit("https://virtualporn.com/videos/" + strconv.Itoa(pageCnt))
+ if !limitScraping {
+ siteCollector.Visit("https://virtualporn.com/videos/" + strconv.Itoa(pageCnt))
+ }
}
})
diff --git a/pkg/scrape/virtualrealporn.go b/pkg/scrape/virtualrealporn.go
index cef44c682..ecab19f92 100644
--- a/pkg/scrape/virtualrealporn.go
+++ b/pkg/scrape/virtualrealporn.go
@@ -17,7 +17,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func VirtualRealPornSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, URL string, singeScrapeAdditionalInfo string) error {
+func VirtualRealPornSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, URL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
page := 1
@@ -245,13 +245,15 @@ func VirtualRealPornSite(wg *sync.WaitGroup, updateSite bool, knownScenes []stri
})
siteCollector.OnHTML(`.searchBox option`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("data-url"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("data-url"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.videoListContainer.paginated`, func(e *colly.HTMLElement) {
e.ForEach(`a.w-portfolio-item-anchor`, func(id int, e *colly.HTMLElement) {
- if e.Request.URL.RawQuery == "videoPage="+strconv.Itoa(page) {
+ if e.Request.URL.RawQuery == "videoPage="+strconv.Itoa(page) && !limitScraping {
// found scenes on this page, get the next page of results
page++
siteCollector.Visit(fmt.Sprintf("%s?videoPage=%v", URL, page))
@@ -278,20 +280,20 @@ func VirtualRealPornSite(wg *sync.WaitGroup, updateSite bool, knownScenes []stri
return nil
}
-func VirtualRealPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VirtualRealPornSite(wg, updateSite, knownScenes, out, singleSceneURL, "virtualrealporn", "VirtualRealPorn", "https://virtualrealporn.com/", singeScrapeAdditionalInfo)
+func VirtualRealPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VirtualRealPornSite(wg, updateSite, knownScenes, out, singleSceneURL, "virtualrealporn", "VirtualRealPorn", "https://virtualrealporn.com/", singeScrapeAdditionalInfo, limitScraping)
}
-func VirtualRealTrans(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VirtualRealPornSite(wg, updateSite, knownScenes, out, singleSceneURL, "virtualrealtrans", "VirtualRealTrans", "https://virtualrealtrans.com/", singeScrapeAdditionalInfo)
+func VirtualRealTrans(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VirtualRealPornSite(wg, updateSite, knownScenes, out, singleSceneURL, "virtualrealtrans", "VirtualRealTrans", "https://virtualrealtrans.com/", singeScrapeAdditionalInfo, limitScraping)
}
-func VirtualRealAmateur(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VirtualRealPornSite(wg, updateSite, knownScenes, out, singleSceneURL, "virtualrealamateur", "VirtualRealAmateurPorn", "https://virtualrealamateurporn.com/", singeScrapeAdditionalInfo)
+func VirtualRealAmateur(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VirtualRealPornSite(wg, updateSite, knownScenes, out, singleSceneURL, "virtualrealamateur", "VirtualRealAmateurPorn", "https://virtualrealamateurporn.com/", singeScrapeAdditionalInfo, limitScraping)
}
-func VirtualRealGay(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VirtualRealPornSite(wg, updateSite, knownScenes, out, singleSceneURL, "virtualrealgay", "VirtualRealGay", "https://virtualrealgay.com/", singeScrapeAdditionalInfo)
+func VirtualRealGay(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VirtualRealPornSite(wg, updateSite, knownScenes, out, singleSceneURL, "virtualrealgay", "VirtualRealGay", "https://virtualrealgay.com/", singeScrapeAdditionalInfo, limitScraping)
}
-func VirtualRealPassion(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VirtualRealPornSite(wg, updateSite, knownScenes, out, singleSceneURL, "virtualrealpassion", "VirtualRealPassion", "https://virtualrealpassion.com/", singeScrapeAdditionalInfo)
+func VirtualRealPassion(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VirtualRealPornSite(wg, updateSite, knownScenes, out, singleSceneURL, "virtualrealpassion", "VirtualRealPassion", "https://virtualrealpassion.com/", singeScrapeAdditionalInfo, limitScraping)
}
func init() {
diff --git a/pkg/scrape/virtualtaboo.go b/pkg/scrape/virtualtaboo.go
index 6a5cd885f..a284eb3e6 100644
--- a/pkg/scrape/virtualtaboo.go
+++ b/pkg/scrape/virtualtaboo.go
@@ -14,7 +14,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func VirtualTaboo(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func VirtualTaboo(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "virtualtaboo"
siteID := "VirtualTaboo"
@@ -111,8 +111,10 @@ func VirtualTaboo(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
})
siteCollector.OnHTML(`ul.pagination a`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.video-card__item a[class=image-container]`, func(e *colly.HTMLElement) {
diff --git a/pkg/scrape/vr3000.go b/pkg/scrape/vr3000.go
index b2bb57801..52a32fb44 100644
--- a/pkg/scrape/vr3000.go
+++ b/pkg/scrape/vr3000.go
@@ -14,7 +14,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func VR3000(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func VR3000(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "vr3000"
siteID := "VR3000"
diff --git a/pkg/scrape/vrallure.go b/pkg/scrape/vrallure.go
index 759030761..cca50ba5d 100644
--- a/pkg/scrape/vrallure.go
+++ b/pkg/scrape/vrallure.go
@@ -18,7 +18,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func VRAllure(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func VRAllure(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "vrallure"
siteID := "VRAllure"
@@ -138,8 +138,10 @@ func VRAllure(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out cha
})
siteCollector.OnHTML(`ul.pagination li a`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.row h4.latest-scene-title a`, func(e *colly.HTMLElement) {
diff --git a/pkg/scrape/vrbangers.go b/pkg/scrape/vrbangers.go
index 7bb556b82..72d273001 100755
--- a/pkg/scrape/vrbangers.go
+++ b/pkg/scrape/vrbangers.go
@@ -15,7 +15,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func VRBangersSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, URL string) error {
+func VRBangersSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, URL string, limitScraping bool) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
@@ -156,9 +156,10 @@ func VRBangersSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, ou
})
siteCollector.OnHTML(`a.page-pagination__next`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
-
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
if singleSceneURL != "" {
@@ -174,20 +175,20 @@ func VRBangersSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, ou
return nil
}
-func VRBangers(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VRBangersSite(wg, updateSite, knownScenes, out, singleSceneURL, "vrbangers", "VRBangers", "https://vrbangers.com/")
+func VRBangers(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VRBangersSite(wg, updateSite, knownScenes, out, singleSceneURL, "vrbangers", "VRBangers", "https://vrbangers.com/", limitScraping)
}
-func VRBTrans(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VRBangersSite(wg, updateSite, knownScenes, out, singleSceneURL, "vrbtrans", "VRBTrans", "https://vrbtrans.com/")
+func VRBTrans(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VRBangersSite(wg, updateSite, knownScenes, out, singleSceneURL, "vrbtrans", "VRBTrans", "https://vrbtrans.com/", limitScraping)
}
-func VRBGay(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VRBangersSite(wg, updateSite, knownScenes, out, singleSceneURL, "vrbgay", "VRBGay", "https://vrbgay.com/")
+func VRBGay(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VRBangersSite(wg, updateSite, knownScenes, out, singleSceneURL, "vrbgay", "VRBGay", "https://vrbgay.com/", limitScraping)
}
-func VRConk(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VRBangersSite(wg, updateSite, knownScenes, out, singleSceneURL, "vrconk", "VRCONK", "https://vrconk.com/")
+func VRConk(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VRBangersSite(wg, updateSite, knownScenes, out, singleSceneURL, "vrconk", "VRCONK", "https://vrconk.com/", limitScraping)
}
-func BlowVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VRBangersSite(wg, updateSite, knownScenes, out, singleSceneURL, "blowvr", "BlowVR", "https://blowvr.com/")
+func BlowVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VRBangersSite(wg, updateSite, knownScenes, out, singleSceneURL, "blowvr", "BlowVR", "https://blowvr.com/", limitScraping)
}
func init() {
diff --git a/pkg/scrape/vrhush.go b/pkg/scrape/vrhush.go
index 9f3c5434c..564801e82 100644
--- a/pkg/scrape/vrhush.go
+++ b/pkg/scrape/vrhush.go
@@ -16,7 +16,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func VRHush(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func VRHush(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "vrhush"
siteID := "VRHush"
@@ -138,8 +138,10 @@ func VRHush(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<
siteCollector.OnHTML(`ul.pagination li`, func(e *colly.HTMLElement) {
if strings.Contains(e.Attr("class"), "next") && !strings.Contains(e.Attr("class"), "disabled") {
pageCnt += 1
- pageURL := e.Request.AbsoluteURL(`https://vrhush.com/scenes?page=` + fmt.Sprint(pageCnt) + `&order_by=publish_date&sort_by=desc`)
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(`https://vrhush.com/scenes?page=` + fmt.Sprint(pageCnt) + `&order_by=publish_date&sort_by=desc`)
+ siteCollector.Visit(pageURL)
+ }
}
})
diff --git a/pkg/scrape/vrlatina.go b/pkg/scrape/vrlatina.go
index 78b814bda..8dcf3c112 100644
--- a/pkg/scrape/vrlatina.go
+++ b/pkg/scrape/vrlatina.go
@@ -14,7 +14,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func VRLatina(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func VRLatina(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "vrlatina"
siteID := "VRLatina"
@@ -109,8 +109,10 @@ func VRLatina(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out cha
})
siteCollector.OnHTML(`div.pagination a`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.item-col.-video a`, func(e *colly.HTMLElement) {
diff --git a/pkg/scrape/vrphub.go b/pkg/scrape/vrphub.go
index 4b12ff1f8..21fb85cdb 100644
--- a/pkg/scrape/vrphub.go
+++ b/pkg/scrape/vrphub.go
@@ -29,7 +29,7 @@ func getVideoName(fileUrl string) (string, error) {
return filename, nil
}
-func VRPHub(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string, callback func(e *colly.HTMLElement, sc *models.ScrapedScene)) error {
+func VRPHub(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string, limitScraping bool, callback func(e *colly.HTMLElement, sc *models.ScrapedScene)) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
@@ -164,8 +164,10 @@ func VRPHub(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<
})
siteCollector.OnHTML(`div.page-nav a.page`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.td-main-content div.td-module-image-main a`, func(e *colly.HTMLElement) {
@@ -257,14 +259,14 @@ func addVRPHubScraper(id string, name string, company string, avatarURL string,
avatarURL = "https://cdn.vrphub.com/wp-content/uploads/2016/08/vrphubnew.png"
}
- registerScraper(id, suffixedName, avatarURL, "vrphub.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VRPHub(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, callback)
+ registerScraper(id, suffixedName, avatarURL, "vrphub.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VRPHub(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, callback)
})
}
func init() {
- registerScraper("vrphub-single_scene", "VRPHub - Other Studios", "", "vrphub.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VRPHub(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo, noop)
+ registerScraper("vrphub-single_scene", "VRPHub - Other Studios", "", "vrphub.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VRPHub(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo, limitScraping, noop)
})
var scrapers config.ScraperList
scrapers.Load()
diff --git a/pkg/scrape/vrporn.go b/pkg/scrape/vrporn.go
index b8b7a78f3..2de6631d8 100644
--- a/pkg/scrape/vrporn.go
+++ b/pkg/scrape/vrporn.go
@@ -14,7 +14,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string) error {
+func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string, limitScraping bool, masterSiteId string) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
@@ -37,6 +37,7 @@ func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<
sc.Studio = company
sc.Site = siteID
sc.HomepageURL = strings.Split(e.Request.URL.String(), "?")[0]
+ sc.MasterSiteId = masterSiteId
if scraperID == "" {
// there maybe no site/studio if user is jusy scraping a scene url
@@ -45,10 +46,9 @@ func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<
sc.Studio = strings.TrimSpace(e.Text)
sc.Site = sc.Studio
// see if we can find the site record, there may not be
- db, _ := models.GetDB()
- defer db.Close()
+ commonDb, _ := models.GetCommonDB()
var site models.Site
- db.Where("name like ?", sc.Studio+"%VRPorn) or id = ?", sc.Studio, studioId).First(&site)
+ commonDb.Where("name like ?", sc.Studio+"%VRPorn) or id = ?", sc.Studio, studioId).First(&site)
if site.ID != "" {
sc.ScraperID = site.ID
}
@@ -143,13 +143,15 @@ func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<
})
siteCollector.OnHTML(`div.pagination a.next`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`body.tax-studio article.post div.tube-post a`, func(e *colly.HTMLElement) {
sceneURL := e.Request.AbsoluteURL(e.Attr("href"))
- // If scene exists in database, there's no need to scrape
+ // If scene exists in database, or the slternate source exists, there's no need to scrape
if !funk.ContainsString(knownScenes, sceneURL) {
sceneCollector.Visit(sceneURL)
}
@@ -168,7 +170,7 @@ func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<
return nil
}
-func addVRPornScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string) {
+func addVRPornScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string, masterSiteId string) {
suffixedName := name
siteNameSuffix := name
if custom {
@@ -177,22 +179,32 @@ func addVRPornScraper(id string, name string, company string, avatarURL string,
} else {
suffixedName += " (VRPorn)"
}
- registerScraper(id, suffixedName, avatarURL, "vrporn.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VRPorn(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo)
- })
+ if avatarURL == "" {
+ avatarURL = "https://vrporn.com/assets/favicon.png"
+ }
+
+ if masterSiteId == "" {
+ registerScraper(id, suffixedName, avatarURL, "vrporn.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VRPorn(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, "")
+ })
+ } else {
+ registerAlternateScraper(id, suffixedName, avatarURL, "vrporn.com", masterSiteId, func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VRPorn(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, masterSiteId)
+ })
+ }
}
func init() {
- registerScraper("vrporn-single_scene", "VRPorn - Other Studios", "", "vrporn.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return VRPorn(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo)
+ registerScraper("vrporn-single_scene", "VRPorn - Other Studios", "", "vrporn.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return VRPorn(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo, limitScraping, "")
})
var scrapers config.ScraperList
scrapers.Load()
for _, scraper := range scrapers.XbvrScrapers.VrpornScrapers {
- addVRPornScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL)
+ addVRPornScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL, scraper.MasterSiteId)
}
for _, scraper := range scrapers.CustomScrapers.VrpornScrapers {
- addVRPornScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL)
+ addVRPornScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL, scraper.MasterSiteId)
}
}
diff --git a/pkg/scrape/vrsexygirlz.go b/pkg/scrape/vrsexygirlz.go
index b819da7d2..03019986d 100644
--- a/pkg/scrape/vrsexygirlz.go
+++ b/pkg/scrape/vrsexygirlz.go
@@ -12,7 +12,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func VRSexygirlz(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func VRSexygirlz(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "vrsexygirlz"
diff --git a/pkg/scrape/vrspy.go b/pkg/scrape/vrspy.go
index cc54eb0d6..8814560de 100755
--- a/pkg/scrape/vrspy.go
+++ b/pkg/scrape/vrspy.go
@@ -23,7 +23,7 @@ const (
baseURL = "https://" + domain
)
-func VRSpy(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singleScrapeAdditionalInfo string) error {
+func VRSpy(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singleScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
@@ -129,7 +129,9 @@ func VRSpy(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
siteCollector.OnHTML(`body`, func(e *colly.HTMLElement) {
e.ForEachWithBreak(`.video-section a.photo-preview`, func(id int, e *colly.HTMLElement) bool {
currentPage, _ := strconv.Atoi(e.Request.URL.Query().Get("page"))
- siteCollector.Visit(fmt.Sprintf("%s/videos?sort=new&page=%d", baseURL, currentPage+1))
+ if !limitScraping {
+ siteCollector.Visit(fmt.Sprintf("%s/videos?sort=new&page=%d", baseURL, currentPage+1))
+ }
return false
})
})
diff --git a/pkg/scrape/wetvr.go b/pkg/scrape/wetvr.go
index e6d864520..4de99ce98 100644
--- a/pkg/scrape/wetvr.go
+++ b/pkg/scrape/wetvr.go
@@ -13,7 +13,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func WetVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
+func WetVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
defer wg.Done()
scraperID := "wetvr"
siteID := "WetVR"
@@ -90,8 +90,10 @@ func WetVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
})
siteCollector.OnHTML(`ul a.page-link`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div:has(p:contains("Latest")) div[id^="r-"]`, func(e *colly.HTMLElement) {
diff --git a/pkg/scrape/zexywankitnow.go b/pkg/scrape/zexywankitnow.go
index 393e89445..e442f0c95 100644
--- a/pkg/scrape/zexywankitnow.go
+++ b/pkg/scrape/zexywankitnow.go
@@ -13,7 +13,7 @@ import (
"github.com/xbapps/xbvr/pkg/models"
)
-func TwoWebMediaSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, URL string) error {
+func TwoWebMediaSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, URL string, limitScraping bool) error {
defer wg.Done()
logScrapeStart(scraperID, siteID)
@@ -161,8 +161,10 @@ func TwoWebMediaSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string,
})
siteCollector.OnHTML(`ul.pagination a.page-link`, func(e *colly.HTMLElement) {
- pageURL := e.Request.AbsoluteURL(e.Attr("href"))
- siteCollector.Visit(pageURL)
+ if !limitScraping {
+ pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+ siteCollector.Visit(pageURL)
+ }
})
siteCollector.OnHTML(`div.container div.card > a`, func(e *colly.HTMLElement) {
@@ -177,7 +179,7 @@ func TwoWebMediaSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string,
if singleSceneURL != "" {
sceneCollector.Visit(singleSceneURL)
} else {
- siteCollector.Visit(URL)
+ siteCollector.Visit(URL + "?order=newest")
}
if updateSite {
@@ -187,12 +189,12 @@ func TwoWebMediaSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string,
return nil
}
-func WankitNowVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return TwoWebMediaSite(wg, updateSite, knownScenes, out, singleSceneURL, "wankitnowvr", "WankitNowVR", "https://wankitnowvr.com/videos/")
+func WankitNowVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return TwoWebMediaSite(wg, updateSite, knownScenes, out, singleSceneURL, "wankitnowvr", "WankitNowVR", "https://wankitnowvr.com/videos/", limitScraping)
}
-func ZexyVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error {
- return TwoWebMediaSite(wg, updateSite, knownScenes, out, singleSceneURL, "zexyvr", "ZexyVR", "https://zexyvr.com/videos/")
+func ZexyVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error {
+ return TwoWebMediaSite(wg, updateSite, knownScenes, out, singleSceneURL, "zexyvr", "ZexyVR", "https://zexyvr.com/videos/", limitScraping)
}
func init() {
diff --git a/pkg/server/cron.go b/pkg/server/cron.go
index 4be4407ec..001fd1964 100644
--- a/pkg/server/cron.go
+++ b/pkg/server/cron.go
@@ -17,6 +17,7 @@ var rescanTask cron.EntryID
var previewTask cron.EntryID
var actorScrapeTask cron.EntryID
var stashdbScrapeTask cron.EntryID
+var linkScenesTask cron.EntryID
func SetupCron() {
cronInstance = cron.New()
@@ -43,6 +44,10 @@ func SetupCron() {
log.Println(fmt.Sprintf("Setup Stashdb Rescrape Task %v", formatCronSchedule(config.CronSchedule(config.Config.Cron.StashdbRescrapeSchedule))))
stashdbScrapeTask, _ = cronInstance.AddFunc(formatCronSchedule(config.CronSchedule(config.Config.Cron.StashdbRescrapeSchedule)), stashdbRescrapeCron)
}
+ if config.Config.Cron.LinkScenesSchedule.Enabled {
+ log.Println(fmt.Sprintf("Setup Link Scenes Task %v", formatCronSchedule(config.CronSchedule(config.Config.Cron.LinkScenesSchedule))))
+ linkScenesTask, _ = cronInstance.AddFunc(formatCronSchedule(config.CronSchedule(config.Config.Cron.LinkScenesSchedule)), linkScenesCron)
+ }
cronInstance.Start()
go tasks.CalculateCacheSizes()
@@ -62,12 +67,16 @@ func SetupCron() {
if config.Config.Cron.StashdbRescrapeSchedule.RunAtStartDelay > 0 {
time.AfterFunc(time.Duration(config.Config.Cron.StashdbRescrapeSchedule.RunAtStartDelay)*time.Minute, stashdbRescrapeCron)
}
+ if config.Config.Cron.LinkScenesSchedule.RunAtStartDelay > 0 {
+ time.AfterFunc(time.Duration(config.Config.Cron.LinkScenesSchedule.RunAtStartDelay)*time.Minute, linkScenesCron)
+ }
log.Println(fmt.Sprintf("Next Rescrape Task at %v", cronInstance.Entry(rescrapTask).Next))
log.Println(fmt.Sprintf("Next Rescan Task at %v", cronInstance.Entry(rescanTask).Next))
log.Println(fmt.Sprintf("Next Preview Generation Task at %v", cronInstance.Entry(previewTask).Next))
log.Println(fmt.Sprintf("Next Actor Rescripe Task at %v", cronInstance.Entry(actorScrapeTask).Next))
log.Println(fmt.Sprintf("Next Stashdb Rescrape Task at %v", cronInstance.Entry(stashdbScrapeTask).Next))
+ log.Println(fmt.Sprintf("Next Link Scenes Task at %v", cronInstance.Entry(linkScenesTask).Next))
}
func scrapeCron() {
@@ -96,6 +105,13 @@ func stashdbRescrapeCron() {
log.Println(fmt.Sprintf("Next Stashdb Rescrape Task at %v", cronInstance.Entry(rescrapTask).Next))
}
+func linkScenesCron() {
+ if !session.HasActiveSession() {
+ tasks.MatchAlternateSources()
+ }
+ log.Println(fmt.Sprintf("Next Link Scenes Task at %v", cronInstance.Entry(rescrapTask).Next))
+}
+
var previewGenerateInProgress = false
func generatePreviewCron() {
diff --git a/pkg/tasks/alternate_scene_source.go b/pkg/tasks/alternate_scene_source.go
new file mode 100644
index 000000000..6760dc817
--- /dev/null
+++ b/pkg/tasks/alternate_scene_source.go
@@ -0,0 +1,280 @@
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/blevesearch/bleve/v2"
+ "github.com/jinzhu/gorm"
+ "github.com/xbapps/xbvr/pkg/config"
+ "github.com/xbapps/xbvr/pkg/models"
+)
+
+func AddAlternateSceneSource(db *gorm.DB, scrapedScene models.ScrapedScene) {
+ var extref models.ExternalReference
+ source := "alternate scene " + scrapedScene.ScraperID
+ extref.FindExternalId(source, scrapedScene.SceneID)
+ extref.ExternalId = scrapedScene.SceneID
+ extref.ExternalSource = source
+ extref.ExternalURL = scrapedScene.HomepageURL
+
+ var scene models.Scene
+ scene.PopulateSceneFieldsFromExternal(db, scrapedScene)
+ extref.ExternalDate = scene.ReleaseDate
+
+ // strip out other actor columns, it makes the data too large and we only need the name
+ var newCastList []models.Actor
+ for _, actor := range scene.Cast {
+ newCastList = append(newCastList, models.Actor{ID: actor.ID, Name: actor.Name})
+ }
+ scene.Cast = newCastList
+ var data models.SceneAlternateSource
+ data.MasterSiteId = scrapedScene.MasterSiteId
+ data.Scene = scene
+ data.MatchAchieved = -1
+ jsonData, _ := json.Marshal(data)
+ extref.ExternalData = string(jsonData)
+ extref.UdfBool1 = scene.HumanScript
+ extref.UdfBool2 = scene.AiScript
+ extref.UdfDatetime1 = scene.ScriptPublished
+ extref.Save()
+}
+
+func MatchAlternateSources() {
+ tlog := log.WithField("task", "scrape")
+ tlog.Info("Matching scenes from alternate sources")
+ commonDb, _ := models.GetCommonDB()
+
+ var unmatchedScenes []models.ExternalReference
+
+ commonDb.Joins("Left JOIN external_reference_links erl on erl.external_reference_id = external_references.id").
+ Where("external_references.external_source like 'alternate scene %' and erl.external_reference_id is NULL").
+ Find(&unmatchedScenes)
+
+ // check for scenes that should be relinked based on the reprocess_links param
+ var sites []models.Site
+ commonDb.Where("matching_params<> '' and JSON_EXTRACT(matching_params, '$.reprocess_links')>0").Find(&sites)
+ for _, site := range sites {
+ var reprocessList []models.ExternalReference
+ var matchParams models.AltSrcMatchParams
+ matchParams.UnmarshalParams(site.MatchingParams)
+ reprocessDate := time.Now().AddDate(0, 0, matchParams.ReprocessLinks*-1)
+ // ignore match_type 99999, they are manually match and shouldn't change
+ commonDb.Preload("XbvrLinks").
+ Joins("Join external_reference_links on external_reference_links.external_reference_id = external_references.id").
+ Where("external_references.external_source = ? and external_references.external_date > ? and external_reference_links.match_type not in (-1, 99999)", "alternate scene "+site.ID, reprocessDate).Find(&reprocessList)
+ unmatchedScenes = append(unmatchedScenes, reprocessList...)
+
+ }
+ lastProgressUpdate := time.Now()
+ for cnt, altsource := range unmatchedScenes {
+ if time.Since(lastProgressUpdate) > time.Duration(config.Config.Advanced.ProgressTimeInterval)*time.Second {
+ tlog.Infof("Matching alternate scene sources %v of %v", cnt, len(unmatchedScenes))
+ lastProgressUpdate = time.Now()
+ }
+ var unmatchedSceneData models.SceneAlternateSource
+ var possiblematchs []models.Scene
+ var site models.Site
+ var matchParams models.AltSrcMatchParams
+ json.Unmarshal([]byte(altsource.ExternalData), &unmatchedSceneData)
+ site.GetIfExist(strings.TrimPrefix(altsource.ExternalSource, "alternate scene "))
+ err := matchParams.UnmarshalParams(site.MatchingParams)
+ if err != nil {
+ tlog.Infof("Cannot read Matching parameters for %s %s", site.Name, err)
+ continue
+ }
+ if matchParams.DelayLinking > 0 {
+ skipDate := altsource.ExternalDate.AddDate(0, 0, matchParams.DelayLinking)
+ if skipDate.After(time.Now()) {
+ continue
+ }
+ }
+
+ if !matchParams.IgnoreReleasedBefore.IsZero() {
+ if unmatchedSceneData.Scene.ReleaseDate.Before(matchParams.IgnoreReleasedBefore) {
+ continue
+ }
+ } else {
+ if !config.Config.Advanced.IgnoreReleasedBefore.IsZero() {
+ if unmatchedSceneData.Scene.ReleaseDate.Before(config.Config.Advanced.IgnoreReleasedBefore) {
+ continue
+ }
+ }
+ }
+
+ tmpTitle := strings.ReplaceAll(strings.ReplaceAll(unmatchedSceneData.Scene.Title, " and ", "&"), "+", "&")
+ for _, char := range `'- :!?"/.,@()–` {
+ tmpTitle = strings.ReplaceAll(tmpTitle, string(char), "")
+ }
+ commonDb.Preload("Cast").Where(`replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(title,'''',''),'-',''),' ',''),':',''),'!',''),'?',''),'"',''),'/',''),'.',''),',',''),' and ','&'),'@',''),')',''),'(',''), '+','&'),'–','') like ? and scraper_id = ?`,
+ tmpTitle, unmatchedSceneData.MasterSiteId).Find(&possiblematchs)
+
+ if len(possiblematchs) == 1 {
+ found := false
+ // check not already linked, if we update unnessarily, it will mess up the sort by "Released on Alternate Sites" sort option
+ for _, link := range altsource.XbvrLinks {
+ if link.InternalDbId == possiblematchs[0].ID {
+ found = true
+ break
+ }
+ }
+ if !found {
+ UpdateLinks(commonDb, altsource.ID, models.ExternalReferenceLink{InternalTable: "scenes", InternalDbId: possiblematchs[0].ID, InternalNameId: possiblematchs[0].SceneID,
+ ExternalReferenceID: altsource.ID, ExternalSource: altsource.ExternalSource, ExternalId: altsource.ExternalId, MatchType: 10000, UdfDatetime1: time.Now()})
+ }
+ } else {
+ // build the search query based on the sites matching params
+ ignoreChars := `+-=&|> 0 {
+ sitename = masterSite.Name[:strings.Index(masterSite.Name, `(`)-1]
+ }
+
+ q := fmt.Sprintf(`+site:"%s" title:"%s"^%v`, sitename, title, matchParams.BoostTitleExact)
+ for _, word := range strings.Fields(title) {
+ q = fmt.Sprintf(`%s title:%s^%v`, q, word, matchParams.BoostTitleAnyWords)
+ }
+
+ if matchParams.CastMatchType == "should" {
+ for _, actor := range unmatchedSceneData.Scene.Cast {
+ q = fmt.Sprintf(`%s cast:"%s"^%v`, q, actor.Name, matchParams.BoostCast)
+ // split to indivual words as well, eg handles cases like "someone baberoticvr" or one site only using a first name
+ // but lower the boosting a bit
+ words := strings.Fields(actor.Name)
+ for _, word := range words {
+ q = fmt.Sprintf(`%s cast:%s^%0.2f`, q, word, matchParams.BoostCast*0.75)
+ }
+ }
+ }
+
+ if matchParams.DurationMatchType != "do not" {
+ // note: no boosting for duration, doesn't appear to work except on ranges, where it must be within the range anyway
+ if matchParams.DurationMatchType == "must" {
+ matchParams.DurationRangeLess = 0
+ matchParams.DurationRangeMore = 0
+ }
+ if unmatchedSceneData.Scene.Duration >= matchParams.DurationMin {
+ if matchParams.DurationRangeLess+matchParams.DurationRangeMore > 1 {
+ q = fmt.Sprintf("%s duration: %v duration:>=%v duration:<=%v", q, unmatchedSceneData.Scene.Duration, unmatchedSceneData.Scene.Duration-matchParams.DurationRangeLess, unmatchedSceneData.Scene.Duration+matchParams.DurationRangeMore)
+ } else {
+ q = fmt.Sprintf("%s duration: %v", q, unmatchedSceneData.Scene.Duration)
+ }
+ }
+ }
+
+ if matchParams.ReleasedMatchType != "do not" {
+ if matchParams.ReleasedMatchType == "must" {
+ matchParams.ReleasedPrior = 0
+ matchParams.ReleasedAfter = 0
+ }
+ if matchParams.IgnoreReleasedBefore.IsZero() || unmatchedSceneData.Scene.ReleaseDate.After(matchParams.IgnoreReleasedBefore.AddDate(0, 0, -1)) {
+ prefix := ""
+ if matchParams.ReleasedMatchType == "must" {
+ prefix = "+"
+ }
+ q = fmt.Sprintf(`%s %sreleased:>="%s"^%v %sreleased:<="%s"^%v`, q,
+ prefix, unmatchedSceneData.Scene.ReleaseDate.AddDate(0, 0, matchParams.ReleasedPrior*-1).Format("2006-01-02"), matchParams.BoostReleased,
+ prefix, unmatchedSceneData.Scene.ReleaseDate.AddDate(0, 0, matchParams.ReleasedAfter).Format("2006-01-02"), matchParams.BoostReleased)
+ }
+ }
+
+ if matchParams.DescriptionMatchType == "should" { // "must" is not an option for description
+ words := strings.Fields(desc)
+ for _, word := range words {
+ if strings.TrimSpace(word) != "" {
+ q = fmt.Sprintf("%s description: %v^%v", q, strings.TrimSpace(word), matchParams.BoostDescription)
+ }
+ }
+ }
+
+ query := bleve.NewQueryStringQuery(q)
+
+ searchRequest := bleve.NewSearchRequest(query)
+ searchRequest.Fields = []string{"title", "cast", "site", "description", "released"}
+ searchRequest.IncludeLocations = true
+ searchRequest.From = 0
+ searchRequest.Size = 25
+ searchRequest.SortBy([]string{"-_score"})
+
+ searchResults, err := AltSourceSearch(searchRequest)
+ if err != nil {
+ log.Error(err)
+ log.Error(q)
+ continue
+ }
+
+ var extdata models.SceneAlternateSource
+ // remove saving the query later, keep for debug purposes
+ json.Unmarshal([]byte(altsource.ExternalData), &extdata)
+ extdata.Query = q
+ newjson, _ := json.Marshal(extdata)
+ altsource.ExternalData = string(newjson)
+ tmpAltSource := altsource
+ tmpAltSource.XbvrLinks = nil
+ // save the updated query used, but don't update the links
+ tmpAltSource.Save()
+ if len(searchResults.Hits) > 0 {
+ var scene models.Scene
+ scene.GetIfExist(searchResults.Hits[0].ID)
+ if scene.ID > 0 {
+ // check not already linked, if we update unnessarily, it will mess up the sort by "Released on Alternate Sites" sort option
+ found := false
+ for _, link := range altsource.XbvrLinks {
+ if link.InternalDbId == scene.ID {
+ found = true
+ break
+ }
+ }
+ if !found {
+ UpdateLinks(commonDb, altsource.ID, models.ExternalReferenceLink{InternalTable: "scenes", InternalDbId: scene.ID, InternalNameId: scene.SceneID,
+ ExternalReferenceID: altsource.ID, ExternalSource: altsource.ExternalSource, ExternalId: altsource.ExternalId, MatchType: int(searchResults.Hits[0].Score), UdfDatetime1: time.Now()})
+ }
+ }
+ }
+ }
+ }
+ tlog.Info("Completed Matching scenes from alternate sources")
+}
+func AltSourceSearch(searchRequest *bleve.SearchRequest) (*bleve.SearchResult, error) {
+ // open and close the search for each search, this stops the search function from locking users out of searching
+ idx, err := NewIndex("scenes")
+ if err != nil {
+ return nil, err
+ }
+ defer idx.Bleve.Close()
+ return idx.Bleve.Search(searchRequest)
+}
+func UpdateLinks(db *gorm.DB, externalreference_id uint, newLink models.ExternalReferenceLink) {
+ var extref models.ExternalReference
+ extref.GetIfExist(externalreference_id)
+ exists := false
+
+ db.Where("external_source = ? and external_reference_id = ? and internal_db_id <> ?").Find(&extref)
+ for _, link := range extref.XbvrLinks {
+ if link.InternalDbId == newLink.InternalDbId {
+ exists = true
+ } else {
+ link.Delete()
+ }
+ }
+ if !exists {
+ newLink.Save()
+ }
+
+}
diff --git a/pkg/tasks/content.go b/pkg/tasks/content.go
index 4607e8a50..3a7e920ff 100644
--- a/pkg/tasks/content.go
+++ b/pkg/tasks/content.go
@@ -55,22 +55,23 @@ type BackupActionActor struct {
ActionActors []models.ActionActor `xbvrbackup:"action_actors"`
}
type BackupContentBundle struct {
- Timestamp time.Time `xbvrbackup:"timestamp"`
- BundleVersion string `xbvrbackup:"bundleVersion"`
- Volumne []models.Volume `xbvrbackup:"volumes"`
- Playlists []models.Playlist `xbvrbackup:"playlists"`
- Sites []models.Site `xbvrbackup:"sites"`
- Scenes []models.Scene `xbvrbackup:"scenes"`
- FilesLinks []BackupFileLink `xbvrbackup:"sceneFileLinks"`
- Cuepoints []BackupSceneCuepoint `xbvrbackup:"sceneCuepoints"`
- History []BackupSceneHistory `xbvrbackup:"sceneHistory"`
- Actions []BackupSceneAction `xbvrbackup:"actions"`
- Akas []models.Aka `xbvrbackup:"akas"`
- TagGroups []models.TagGroup `xbvrbackup:"tagGroups"`
- ExternalRefs []models.ExternalReference `xbvrbackup:"externalReferences"`
- Actors []models.Actor `xbvrbackup:"actors"`
- ActionActors []BackupActionActor `xbvrbackup:"actionActors"`
- Kvs []models.KV `xbvrbackup:"config"`
+ Timestamp time.Time `xbvrbackup:"timestamp"`
+ BundleVersion string `xbvrbackup:"bundleVersion"`
+ Volumne []models.Volume `xbvrbackup:"volumes"`
+ Playlists []models.Playlist `xbvrbackup:"playlists"`
+ Sites []models.Site `xbvrbackup:"sites"`
+ Scenes []models.Scene `xbvrbackup:"scenes"`
+ FilesLinks []BackupFileLink `xbvrbackup:"sceneFileLinks"`
+ Cuepoints []BackupSceneCuepoint `xbvrbackup:"sceneCuepoints"`
+ History []BackupSceneHistory `xbvrbackup:"sceneHistory"`
+ Actions []BackupSceneAction `xbvrbackup:"actions"`
+ Akas []models.Aka `xbvrbackup:"akas"`
+ TagGroups []models.TagGroup `xbvrbackup:"tagGroups"`
+ ExternalRefs []models.ExternalReference `xbvrbackup:"externalReferences"`
+ ManualSceneMatches []models.ExternalReference `xbvrbackup:"manualSceneMatches"`
+ Actors []models.Actor `xbvrbackup:"actors"`
+ ActionActors []BackupActionActor `xbvrbackup:"actionActors"`
+ Kvs []models.KV `xbvrbackup:"config"`
}
type RequestRestore struct {
InclAllSites bool `json:"allSites"`
@@ -91,6 +92,7 @@ type RequestRestore struct {
InclActors bool `json:"inclActors"`
InclActorActions bool `json:"inclActorActions"`
InclConfig bool `json:"inclConfig"`
+ ExtRefSubset string `json:"extRefSubset"`
}
func CleanTags() {
@@ -104,15 +106,14 @@ func runScrapers(knownScenes []string, toScrape string, updateSite bool, collect
scrapers := models.GetScrapers()
var sites []models.Site
- db, _ := models.GetDB()
+ commonDb, _ := models.GetCommonDB()
if toScrape == "_all" {
- db.Find(&sites)
+ commonDb.Find(&sites)
} else if toScrape == "_enabled" {
- db.Where(&models.Site{IsEnabled: true}).Find(&sites)
+ commonDb.Where(&models.Site{IsEnabled: true}).Find(&sites)
} else {
- db.Where(&models.Site{ID: toScrape}).Find(&sites)
+ commonDb.Where(&models.Site{ID: toScrape}).Find(&sites)
}
- db.Close()
var wg sync.WaitGroup
@@ -121,7 +122,7 @@ func runScrapers(knownScenes []string, toScrape string, updateSite bool, collect
for _, scraper := range scrapers {
if site.ID == scraper.ID {
wg.Add(1)
- go scraper.Scrape(&wg, updateSite, knownScenes, collectedScenes, singleSceneURL, singeScrapeAdditionalInfo)
+ go scraper.Scrape(&wg, updateSite, knownScenes, collectedScenes, singleSceneURL, singeScrapeAdditionalInfo, site.LimitScraping)
}
}
}
@@ -130,7 +131,7 @@ func runScrapers(knownScenes []string, toScrape string, updateSite bool, collect
for _, scraper := range scrapers {
if toScrape == scraper.ID {
wg.Add(1)
- go scraper.Scrape(&wg, updateSite, knownScenes, collectedScenes, singleSceneURL, singeScrapeAdditionalInfo)
+ go scraper.Scrape(&wg, updateSite, knownScenes, collectedScenes, singleSceneURL, singeScrapeAdditionalInfo, false)
}
}
} else {
@@ -151,25 +152,29 @@ func sceneSliceAppender(collectedScenes *[]models.ScrapedScene, scenes <-chan mo
func sceneDBWriter(wg *sync.WaitGroup, i *uint64, scenes <-chan models.ScrapedScene, processedScenes *[]models.ScrapedScene, lock *sync.Mutex) {
defer wg.Done()
- db, _ := models.GetDB()
- defer db.Close()
+ commonDb, _ := models.GetCommonDB()
for scene := range scenes {
if os.Getenv("DEBUG") != "" {
log.Printf("Saving %v", scene.SceneID)
}
if scene.OnlyUpdateScriptData {
if config.Config.Funscripts.ScrapeFunscripts {
- models.SceneUpdateScriptData(db, scene)
+ models.SceneUpdateScriptData(commonDb, scene)
}
} else {
- models.SceneCreateUpdateFromExternal(db, scene)
+ if scene.MasterSiteId == "" {
+ models.SceneCreateUpdateFromExternal(commonDb, scene)
+ } else {
+ AddAlternateSceneSource(commonDb, scene)
+ }
+ }
+ if scene.MasterSiteId == "" {
+ // Add the processed scene to the list to re/index
+ lock.Lock()
+ *processedScenes = append(*processedScenes, scene)
+ lock.Unlock()
+ atomic.AddUint64(i, 1)
}
- // Add the processed scene to the list to re/index
- lock.Lock()
- *processedScenes = append(*processedScenes, scene)
- lock.Unlock()
-
- atomic.AddUint64(i, 1)
if os.Getenv("DEBUG") != "" {
log.Printf("Saved %v", scene.SceneID)
}
@@ -261,9 +266,8 @@ func ReapplyEdits() {
func ScrapeSingleScene(toScrape string, singleSceneURL string, singeScrapeAdditionalInfo string) models.Scene {
var newScene models.Scene
Scrape(toScrape, singleSceneURL, singeScrapeAdditionalInfo)
- db, _ := models.GetDB()
- defer db.Close()
- db.
+ commonDb, _ := models.GetCommonDB()
+ commonDb.
Preload("Tags").
Preload("Cast").
Preload("Files").
@@ -286,9 +290,10 @@ func Scrape(toScrape string, singleSceneURL string, singeScrapeAdditionalInfo st
// Get all known scenes
var scenes []models.Scene
- db, _ := models.GetDB()
- db.Find(&scenes)
- db.Close()
+ var extrefs []models.ExternalReference
+ commonDb, _ := models.GetCommonDB()
+ commonDb.Find(&scenes)
+ commonDb.Where("external_source like 'alternate scene %'").Find(&extrefs)
var knownScenes []string
for i := range scenes {
@@ -296,6 +301,9 @@ func Scrape(toScrape string, singleSceneURL string, singeScrapeAdditionalInfo st
knownScenes = append(knownScenes, scenes[i].SceneURL)
}
}
+ for i := range extrefs {
+ knownScenes = append(knownScenes, extrefs[i].ExternalURL)
+ }
collectedScenes := make(chan models.ScrapedScene, 250)
var sceneCount uint64
@@ -338,6 +346,9 @@ func Scrape(toScrape string, singleSceneURL string, singeScrapeAdditionalInfo st
ReapplyEdits()
IndexScrapedScenes(&processedScenes)
+ if config.Config.Advanced.LinkScenesAfterSceneScraping {
+ MatchAlternateSources()
+ }
tlog.Infof("Scraped %v new scenes in %s",
sceneCount,
@@ -522,7 +533,7 @@ func ImportBundleV1(bundleData ContentBundle) {
}
-func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bool, inclFileLinks bool, inclCuepoints bool, inclHistory bool, inclPlaylists bool, InclActorAkas bool, inclTagGroups bool, inclVolumes bool, inclSites bool, inclActions bool, inclExtRefs bool, inclActors bool, inclActorActions bool, inclConfig bool, playlistId string, outputBundleFilename string, version string) string {
+func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bool, inclFileLinks bool, inclCuepoints bool, inclHistory bool, inclPlaylists bool, InclActorAkas bool, inclTagGroups bool, inclVolumes bool, inclSites bool, inclActions bool, inclExtRefs bool, inclActors bool, inclActorActions bool, inclConfig bool, extRefSubset string, playlistId string, outputBundleFilename string, version string) string {
var out BackupContentBundle
var content []byte
exportCnt := 0
@@ -664,9 +675,15 @@ func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bo
}
var externalReferences []models.ExternalReference
+ var filteredxternalReferences []models.ExternalReference
if inclExtRefs {
lastMessage := time.Now()
- db.Order("external_source").Order("id").Find(&externalReferences)
+ switch extRefSubset {
+ case "":
+ db.Order("external_source").Order("external_source").Order("external_id").Find(&externalReferences)
+ case "manual_matched", "deleted_match":
+ db.Where("external_source like 'alternate scene %'").Order("external_source").Order("external_id").Find(&externalReferences)
+ }
recCnt := 0
for idx, ref := range externalReferences {
if time.Since(lastMessage) > time.Duration(config.Config.Advanced.ProgressTimeInterval)*time.Second {
@@ -674,10 +691,28 @@ func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bo
lastMessage = time.Now()
}
var links []models.ExternalReferenceLink
- db.Where("external_reference_id = ?", ref.ID).Find(&links)
- externalReferences[idx].XbvrLinks = links
+ switch extRefSubset {
+ case "", "all":
+ db.Where("external_reference_id = ?", ref.ID).Order("external_source").Order("external_id").Find(&links)
+ externalReferences[idx].XbvrLinks = links
+ case "manual_matched":
+ db.Where("external_reference_id = ? and match_type=99999", ref.ID).Order("external_source").Order("external_id").Find(&links)
+ if len(links) > 0 {
+ externalReferences[idx].XbvrLinks = links
+ filteredxternalReferences = append(filteredxternalReferences, externalReferences[idx])
+ }
+ case "deleted_match":
+ db.Where("external_reference_id = ? and match_type=-1 and internal_name_id='deleted'", ref.ID).Order("external_source").Order("external_id").Find(&links)
+ if len(links) > 0 {
+ externalReferences[idx].XbvrLinks = links
+ filteredxternalReferences = append(filteredxternalReferences, externalReferences[idx])
+ }
+ }
recCnt += 1
}
+ if extRefSubset != "" {
+ externalReferences = filteredxternalReferences
+ }
tlog.Infof("Reading %v of %v external references", recCnt, len(externalReferences))
}
@@ -843,7 +878,7 @@ func RestoreBundle(request RequestRestore) {
tagGroup.UpdateSceneTagRecords()
}
if request.InclExternalRefs {
- RestoreExternalRefs(bundleData.ExternalRefs, request.Overwrite, db)
+ RestoreExternalRefs(bundleData.ExternalRefs, request.Overwrite, request.ExtRefSubset, db)
}
if request.InclActors {
RestoreActors(bundleData.Actors, request.Overwrite, db)
@@ -1314,7 +1349,7 @@ func RenameTags() {
}
}
-func RestoreExternalRefs(extRefs []models.ExternalReference, overwrite bool, db *gorm.DB) {
+func RestoreExternalRefs(extRefs []models.ExternalReference, overwrite bool, extRefSubset string, db *gorm.DB) {
tlog := log.WithField("task", "scrape")
tlog.Infof("Restoring External References")
@@ -1326,6 +1361,35 @@ func RestoreExternalRefs(extRefs []models.ExternalReference, overwrite bool, db
lastMessage = time.Now()
}
+ // if we specified a filter check if we should import
+ switch extRefSubset {
+ case "manual_matched":
+ if !strings.HasPrefix(extRef.ExternalSource, "alternate scene") {
+ continue
+ }
+ skip := true
+ for _, link := range extRef.XbvrLinks {
+ if link.MatchType == 99999 {
+ skip = false
+ }
+ }
+ if skip {
+ continue
+ }
+ case "deleted_match":
+ if !strings.HasPrefix(extRef.ExternalSource, "alternate scene") {
+ continue
+ }
+ skip := true
+ for _, link := range extRef.XbvrLinks {
+ if link.MatchType == -1 && link.InternalNameId == "deleted" {
+ skip = false
+ }
+ }
+ if skip {
+ continue
+ }
+ }
var found models.ExternalReference
db.Preload("XbvrLinks").Where(&models.ExternalReference{ExternalSource: extRef.ExternalSource, ExternalId: extRef.ExternalId}).First(&found)
@@ -1345,12 +1409,15 @@ func RestoreExternalRefs(extRefs []models.ExternalReference, overwrite bool, db
// case "sites":
// extRef.XbvrLinks[idx].InternalDbId = link.InternalNameId
case "scenes":
- var scene models.Scene
- scene.GetIfExist(link.InternalNameId)
- if scene.ID == 0 {
- continue
+ if link.InternalNameId != "deleted" {
+ // if the name is deleted, we don't need the scene id
+ var scene models.Scene
+ scene.GetIfExist(link.InternalNameId)
+ if scene.ID == 0 {
+ continue
+ }
+ extRef.XbvrLinks[idx].InternalDbId = scene.ID
}
- extRef.XbvrLinks[idx].InternalDbId = scene.ID
case "actors":
var actor models.Actor
db.Where("name = ?", link.InternalNameId).First(&actor)
diff --git a/pkg/tasks/volume.go b/pkg/tasks/volume.go
index b483cd453..4c9def118 100644
--- a/pkg/tasks/volume.go
+++ b/pkg/tasks/volume.go
@@ -19,8 +19,10 @@ import (
"github.com/sirupsen/logrus"
"github.com/thoas/go-funk"
"github.com/xbapps/xbvr/pkg/common"
+ "github.com/xbapps/xbvr/pkg/config"
"github.com/xbapps/xbvr/pkg/ffprobe"
"github.com/xbapps/xbvr/pkg/models"
+ "github.com/xbapps/xbvr/pkg/scrape"
)
var allowedVideoExt = []string{".mp4", ".avi", ".wmv", ".mpeg4", ".mov", ".mkv"}
@@ -59,6 +61,7 @@ func RescanVolumes(id int) {
// Match Scene to File
var files []models.File
var scenes []models.Scene
+ var extrefs []models.ExternalReference
tlog.Infof("Matching Scenes to known filenames")
db.Model(&models.File{}).Where("files.scene_id = 0").Find(&files)
@@ -79,11 +82,79 @@ func RescanVolumes(id int) {
if err != nil {
log.Error(err, " when matching "+unescapedFilename)
}
-
+ if len(scenes) == 0 && config.Config.Advanced.UseAltSrcInFileMatching {
+ // check if the filename matches in external_reference record
+
+ db.Preload("XbvrLinks").Where("external_source like 'alternate scene %' and external_data LIKE ? OR external_data LIKE ? OR external_data LIKE ?", `%"`+filename+`%`, `%"`+filename2+`%`, `%"`+filename3+`%`).Find(&extrefs)
+ if len(extrefs) == 1 {
+ if len(extrefs[0].XbvrLinks) == 1 {
+ // the scene id will be the Internal DB Id from the associated link
+ var scene models.Scene
+ scene.GetIfExistByPK(extrefs[0].XbvrLinks[0].InternalDbId)
+ // Add File to the list of Scene filenames
+ var pfTxt []string
+ err = json.Unmarshal([]byte(scene.FilenamesArr), &pfTxt)
+ if err != nil {
+ continue
+ }
+ pfTxt = append(pfTxt, files[i].Filename)
+ tmp, err := json.Marshal(pfTxt)
+ if err == nil {
+ scene.FilenamesArr = string(tmp)
+ }
+ scene.Save()
+ scenes = append(scenes, scene)
+ }
+ }
+ }
if len(scenes) == 1 {
files[i].SceneID = scenes[0].ID
files[i].Save()
scenes[0].UpdateStatus()
+ } else {
+ if config.Config.Storage.MatchOhash && config.Config.Advanced.StashApiKey != "" {
+ hash := files[i].OsHash
+ if len(hash) < 16 {
+ // the has in xbvr is sometiomes < 16 pad with zeros
+ paddingLength := 16 - len(hash)
+ hash = strings.Repeat("0", paddingLength) + hash
+ }
+ queryVariable := `
+ {"input":{
+ "fingerprints": {
+ "value": "` + hash + `",
+ "modifier": "INCLUDES"
+ },
+ "page": 1
+ }
+ }`
+ // call Stashdb graphql searching for os_hash
+ stashMatches := scrape.GetScenePage(queryVariable)
+ for _, match := range stashMatches.Data.QueryScenes.Scenes {
+ if match.ID != "" {
+ var externalRefLink models.ExternalReferenceLink
+ db.Where(&models.ExternalReferenceLink{ExternalSource: "stashdb scene", ExternalId: match.ID}).First(&externalRefLink)
+ if externalRefLink.ID != 0 {
+ files[i].SceneID = externalRefLink.InternalDbId
+ files[i].Save()
+ var scene models.Scene
+ scene.GetIfExistByPK(externalRefLink.InternalDbId)
+
+ // add filename tyo the array
+ var pfTxt []string
+ json.Unmarshal([]byte(scene.FilenamesArr), &pfTxt)
+ pfTxt = append(pfTxt, files[i].Filename)
+ tmp, _ := json.Marshal(pfTxt)
+ scene.FilenamesArr = string(tmp)
+ scene.Save()
+ models.AddAction(scene.SceneID, "match", "filenames_arr", scene.FilenamesArr)
+
+ scene.UpdateStatus()
+ log.Infof("File %s matched to Scene %s matched using stashdb hash %s", path.Base(files[i].Filename), scene.SceneID, hash)
+ }
+ }
+ }
+ }
}
if (i % 50) == 0 {
diff --git a/ui/src/QuickFind.vue b/ui/src/QuickFind.vue
index 2b22d6d1d..17ea9ff6b 100644
--- a/ui/src/QuickFind.vue
+++ b/ui/src/QuickFind.vue
@@ -87,16 +87,23 @@ export default {
computed: {
isActive: {
get () {
- const out = this.$store.state.overlay.showQuickFind
+ if (this.queryString!=null && this.queryString!="") {
+ this.getAsyncData(this.queryString)
+ }
+ const out = this.$store.state.overlay.quickFind.show
if (out === true) {
this.$nextTick(() => {
this.$refs.autocompleteInput.$refs.input.focus()
+ if (this.$store.state.overlay.quickFind.searchString !=null && this.$store.state.overlay.quickFind.searchString!="") {
+ this.queryString = this.$store.state.overlay.quickFind.searchString
+ this.$store.state.overlay.quickFind.searchString = null
+ }
})
}
return out
},
set (values) {
- this.$store.state.overlay.showQuickFind = values
+ this.$store.state.overlay.quickFind.show = values
}
}
},
@@ -153,12 +160,20 @@ export default {
}
},
showSceneDetails (scene) {
- if (this.$router.currentRoute.name !== 'scenes') {
- this.$router.push({ name: 'scenes' })
- }
this.$store.commit('overlay/hideQuickFind')
- this.data = []
- this.$store.commit('overlay/showDetails', { scene })
+ if (this.$store.state.overlay.quickFind.displaySelectedScene) {
+ if (this.$router.currentRoute.name !== 'scenes') {
+ this.$router.push({ name: 'scenes' })
+ }
+ this.$store.commit('overlay/hideQuickFind')
+ this.data = []
+ this.$store.commit('overlay/showDetails', { scene })
+ } else {
+ // don't display the scene, just pass the selected scene back in the $store.state and close
+ this.$store.state.overlay.quickFind.selectedScene = scene
+ this.$store.commit('overlay/hideQuickFind')
+ this.data = []
+ }
},
searchPrefix(prefix) {
let textbox = this.$refs.autocompleteInput.$refs.input.$refs.input
diff --git a/ui/src/components/EditButton.vue b/ui/src/components/EditButton.vue
index 1066328c2..fa843e355 100644
--- a/ui/src/components/EditButton.vue
+++ b/ui/src/components/EditButton.vue
@@ -1,7 +1,7 @@
+ @click="editScene(item)"
+ :title="item.id == 0 ? 'Display scene details': 'Edit scene details'">
{{ $t("Matching parameters")}}: {{ site.name }} Selection Criteria Release Date Searching Title Searching Duration Searching Cast Searching Description Searching
+ Linking Scenes will not start after the Time Window Ends +
+{{ $t('Edit scene details') }}
+{{ this.scene.id == 0 ? $t('Display scene details') : $t('Edit scene details') }}