new source: torrents-paradise API

README.org

@@ -11,10 +11,14 @@
 - a simple *GUI*.
 - The *[[https://github.com/vindarel/cl-torrents-web][Web UI]]* works too.
 
-  We currently  scrape [[http://1337x.to][1337x.to]]  (since v0.9)  and [[https://www.torrentdownloads.me][torrentdownloads.me]]
-  (v0.10) and present the results sorted by seeders.
+  We currently  scrape [[http://1337x.to][1337x.to]]  (since v0.9), [[https://www.torrentdownloads.me][torrentdownloads.me]]
+  (v0.10) and [[https://torrent-paradise.ml][torrents-paradise]] (v0.11) and present the results sorted by seeders.
 
-  Download for GNU/Linux 64 bits: see [[https://gitlab.com/vindarel/cl-torrents/tags][Releases]] (/23MB, self-contained executable/).
+  Download for GNU/Linux 64 bits: see [[https://gitlab.com/vindarel/cl-torrents/tags][Releases]] (/23MB, self-contained
+  executable/).
+
+NOTE: torrents-paradise has a rich collection, a simple website as in KISS, it works on IPFS and has
+it offers an API… frankly, I want to use its web service and click on its adds.
 
 #+html: <p align="center"><img src="assets/img-colored-results.png" /></p>
 
@@ -311,14 +315,18 @@ You  can  copy the  example  below  in =~/.torrents.lisp=  and  re-run
 
 [[https://gitlab.com/vindarel/cl-torrents/tags][https://gitlab.com/vindarel/cl-torrents/tags]]
 
-*** dev (upcoming v0.11)
+*** WIP
+- added:  a simple  Tk GUI  (search,  open in  browser, download  with
+  torrent desktop client).
+
+*** v0.12
+- june, 2021: added [[https://torrent-paradise.ml][torrents-paradise]] API.
 
-- november, 2019: added connection to transmission-remote (not in
-  Quicklisp yet)
+*** v0.11
+- november, 2019: added connection to transmission-remote
+- added rarbg scraper (needs new tests)
 - june, 2019: changed the location of the lisp init file from
   =~/.torrents.lisp= to =~/.config/torrents.lisp=.
-- added:  a simple  Tk GUI  (search,  open in  browser, download  with
-  torrent desktop client).
 - added:  a =filter=  command,  to only  display  results whose  title
   contains a given  string. It reduces the TAB-completion  of ids (but
   doesn't constrain it).
@@ -381,12 +389,12 @@ cache-p = true
 
 * Dev
 
-We use our [[https://github.com/vindarel/replic/][replic]] library to automatically build the readline repl.
-
 Clone cl-torrents in =~/quicklisp/local-projects=,
 
 now you can =(ql:quickload :torrents)=.
 
+We use our [[https://github.com/vindarel/replic/][replic]] library to automatically build the readline repl.
+
 Unit tests:
 
 : make test
@@ -395,6 +403,9 @@ End-to-end tests (checking that the websites respond and our scrapers still work
 
 : make test-end2end
 
+Try the Ltk GUI: load =gui-tk.lisp= and run =(main)=.
+
+
 * Resources
 
   Don't miss these good resources:
@@ -408,6 +419,10 @@ End-to-end tests (checking that the websites respond and our scrapers still work
 
 ** Tutorial
 
+  *update, 2021*: this tutorial is one of the first things I wrote
+  when discovering CL and it probably bitrot a bit. I'd recommend the
+  Cookbook now as I ported most of its useful content there.
+
    Writing   a   little   web   scraper   like   this   one   is   not
    difficult. However, I had to spend  some time to find out the right
    libraries and resources.  It is also  not trivial at first to start

src/gui-tk/gui-tk.lisp

@@ -6,6 +6,12 @@
         :nodgui)
   (:export :main))
 
+;;;
+;;; TODO: pre-select the search field
+;;; TODO: search on Enter key event.
+;;; etc
+;;;
+
 (in-package :torrents-tk)
 
 (defparameter +results-columns-width+ '(498 80 80 80 150)

src/models.lisp

@@ -10,6 +10,7 @@
            :size
            :size-unit
            :format-size
+           :magnet-link
            :source))
 (in-package :torrents.models)
 
@@ -32,6 +33,9 @@
    (size-unit
     :initarg :size-unit :initform nil
     :accessor size-unit)
+   (magnet-link
+    :initarg :magnet-link :initform nil
+    :accessor magnet-link)
    (source
     :initarg :source :initform nil
     :accessor source)
@@ -68,7 +72,8 @@
             (seeders it)
             (source it))))
 
-(defun make-torrent (&key title href seeders leechers size source size-unit)
+(defun make-torrent (&key title href seeders leechers size source size-unit
+                       magnet-link)
   (assert title)
   (assert source)
   (assert href)
@@ -79,4 +84,5 @@
                  :leechers leechers
                  :size size
                  :size-unit size-unit
+                 :magnet-link magnet-link
                  :source source))

src/torrents-paradise.lisp

@@ -0,0 +1,86 @@
+(in-package :cl-user)
+(defpackage torrents.torrents-paradise
+  (:use :cl)
+  (:import-from :torrents.models
+                :make-torrent)
+  (:import-from :access
+                :access)
+  (:import-from :torrents.utils
+                :colorize-all-keywords
+                :keyword-color-pairs
+                :exit
+                :sublist)
+  (:export :torrents)
+  (:documentation "Search on torrents-paradise API."))
+
+(in-package :torrents.torrents-paradise)
+
+#|
+
+https://torrent-paradise.ml/api/search?q=matrix
+
+Results seem to be sorted by seeders.
+
+Results are a list of this:
+
+ {
+    "id": "c7c2c0f0082a26409170cf2d437ef3c941f40906",  <-- base of magnet link
+    "text": "The.Matrix.Reloaded.ReTaiL.2003.BDRip.XviD.DuaL-PpB",
+    "len": 2368649977,
+    "s": 0,
+    "l": 0
+  },
+
+|#                                        ;
+                                        ;
+(defparameter *source-name* "torrents-paradise" ;
+"Human readable name of this source.")  ;
+
+(defparameter *source-name-short* "paradise"
+  "Short name to display data, for example in a 80-chars width window.")
+
+(defparameter *search-url* "https://torrent-paradise.ml/api/search?q={KEYWORDS}"
+  "Base search API end point. KEYWORDS to be replaced by the search terms (a string with +-separated words).")
+
+(defparameter *max-results* 20
+  "Max number of results to return. To not clutter our output.")
+
+(defun request (url)
+  "Wrapper around dex:get. Fetch an url."
+  (dex:get url))
+
+(defun torrents (words &key (stream t))
+  "Search torrents."
+  (format stream "searching '~a' on ~a…" words *source-name*)
+  (handler-case
+      (let* ((terms (if (listp words)
+                        words
+                        ;; The main gives words as a list,
+                        ;; the user at the Slime REPL one string.
+                        (str:words words)))
+             (query (str:join "+" terms))
+             (search-url (str:replace-all "{KEYWORDS}" query *search-url*))
+             (req (request search-url))
+             (data (jojo:parse req))
+             (res (subseq data 0 *max-results*))
+             (toret (map 'list (lambda (node)
+                                 (make-torrent
+                                  :title (access node "text")
+                                  :href (format nil "torrent URL not available on ~a" *source-name*)
+                                  :seeders (access node "s")
+                                  :leechers (access node "l")
+                                  :magnet-link (get-magnet-link (access node "id"))
+                                  :source :paradise))
+                         res)))
+        (format stream " found ~a results.~&" (length data))
+        toret)
+    (error ()
+      (format stream " no results.~&"))))
+
+(defun get-magnet-link (id)
+  "From this ID, construct the magnet link."
+  ;; See the sources
+  ;; https://github.com/urbanguacamole/torrent-paradise/blob/19224e723c5b6c9a34009c8f3d75434412700231/website/resultpage/index.html
+  (str:concat "magnet:?xt=urn:btih:"
+              id
+              "&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.internetwarriors.net%3A1337"))

src/torrents.lisp

@@ -279,14 +279,22 @@
   "Get the html page of the given url. Mocked in unit tests."
   (dex:get url))
 
+(defun looks-like-magnet-link-p (s)
+  (and (str:non-blank-string-p s)
+       (str:starts-with-p "magnet" s)))
+
 (defun magnet-link-from (torrent)
   "Extract the magnet link from a `torrent' result.
 
    Return the first href of the page that starts with 'magnet'."
-  (let* ((url (href torrent))
-         (html (request-details url))
-         (parsed (plump:parse html)))
-    (find-magnet-link parsed)))
+  (cond
+    ((looks-like-magnet-link-p (magnet-link torrent))
+     (magnet-link torrent))
+    (t
+     (let* ((url (href torrent))
+            (html (request-details url))
+            (parsed (plump:parse html)))
+       (find-magnet-link parsed)))))
 
 (defun magnet (index)
   "Search the magnet from last search's `index''s result."

torrents.asd

@@ -13,8 +13,10 @@
   :license "MIT"
   :depends-on (
                :replic
-               :cl-transmission         ;; june 2019: not in Quicklisp
+               :cl-transmission
+               :access
                :dexador
+               :jonathan
                :plump
                :str
                :lparallel
@@ -36,8 +38,9 @@
                  (:file "kat")
                  (:file "1337")
                  (:file "downloadsme")
+                 (:file "torrents-paradise")
                  (:file "torrents")
-                 (:file "transmission-remote")
+                 ;; (:file "transmission-remote")
                  (:file "commands")
                  (:file "config"))))
   ;; build executable with asdf:make :torrents.