Squashed commit of the following:

commit 80f511d Merge: 3e50cc0 3ddd833 Author: jimchamp <28732543+jimchamp@users.noreply.github.com> Date: Thu Sep 26 11:04:06 2024 -0700 Merge pull request internetarchive#9902 from cdrini/feature/toc-authors-etc Add authors, subtitle, and description to TOC on books page commit 3ddd833 Author: Drini Cami <cdrini@gmail.com> Date: Mon Sep 23 21:37:01 2024 +0200 Remove unused TableOfContents argument, highlighting commit 8a03b7e Author: Drini Cami <cdrini@gmail.com> Date: Mon Sep 23 21:32:51 2024 +0200 Display author, subtitle, description of TOC on books page commit 3e50cc0 Author: Benjamin Deitch <131627264+benbdeitch@users.noreply.github.com> Date: Thu Sep 26 08:56:46 2024 -0400 Implementing Bulk Search v2 Designs (internetarchive#9851) commit b428d1f Author: Scott Barnes <scottreidbarnes@gmail.com> Date: Wed Sep 25 13:15:45 2024 -0700 Nginx: add rate limiting (internetarchive#9904) * Nginx: add rate limiting See https://blog.nginx.org/blog/rate-limiting-nginx and https://nginx.org/en/docs/http/ngx_http_limit_req_module.html Adds comments in docker compose + web w/ instructions for re keeping nginx and docker replicas in sync --------- Co-authored-by: Mek <michael.karpeles@gmail.com>
d-dovale · Sep 26, 2024 · 5972578 · 5972578
1 parent 8305d1c
commit 5972578
Show file tree

Hide file tree

Showing 19 changed files with 452 additions and 116 deletions.
diff --git a/compose.production.yaml b/compose.production.yaml
@@ -73,6 +73,8 @@ services:
       - ../olsystem:/olsystem
       - /1:/1
     deploy:
+      # Note: the replicas here must be kept in sync with the `upstream covers_backend`
+      # value in `docker/covers_nginx.conf`.
       replicas: 2
 
   covers_nginx:

diff --git a/docker/covers_nginx.conf b/docker/covers_nginx.conf
@@ -16,6 +16,15 @@ server {
     ssl_prefer_server_ciphers on;
 }
 
+# Docker's internal load balancing ends up with unbalanced connections eventually.
+# This must be kept in sync with the `replicas` value in `compose.production.yaml`
+# for the `covers` service.
+upstream covers_backend {
+  least_conn;
+  server openlibrary-covers-1:7075;
+  server openlibrary-covers-2:7075;
+}
+
 server {
       listen 80;
       listen 443;
@@ -25,8 +34,15 @@ server {
 
       keepalive_timeout 5;
 
+      # Return 429 errors as JSON.
+      error_page 429 = @429;
+      location @429 {
+        default_type application/json;
+        return 429 '{"status": 429, "message": "Too Many Requests. Please email us at info@archive.org"}';
+      }
+
       location / {
-        proxy_pass http://covers:7075;
+        proxy_pass http://covers_backend;
         proxy_set_header Host $http_host;
 
         # Gunicorn takes IP from this header
@@ -37,8 +53,17 @@ server {
         proxy_set_header X-Scheme $scheme;
 
         if ($http_user_agent ~ (Bytespider) ) {
-           return 429;
+           return 444;
         }
+
+        if ($http_user_agent ~ (CloudFront) ) {
+           return 444;
+        }
+
+
+        # Covers rate limit.
+        limit_req zone=cover_limit burst=400 nodelay;
+        limit_req_status 429;
       }
 
       location ^~ /.well-known/acme-challenge/ {

diff --git a/docker/nginx.conf b/docker/nginx.conf
@@ -11,7 +11,7 @@ error_log  /var/log/nginx/error.log;
 pid        /var/run/nginx.pid;
 
 events {
-    worker_connections  1024;
+    worker_connections  2048;
     # multi_accept on;
 }
 
@@ -44,6 +44,25 @@ http {
     # Black-listed IPs
     include /olsystem/etc/nginx/deny.conf;
 
+    # Rate limiting: https://nginx.org/en/docs/http/ngx_http_limit_req_module.html
+    # No rate limit when IP obfuscation is not applied, as every IP is 255.0.0.0.
+    # These rules only do anything if invoked, e.g., in web_nginx.conf.
+    # TLDR: these rules can be disabled in `docker/web_nginx.conf`
+    # and `docker/covers_nginx.conf`.
+    geo $should_apply_limit {
+      255.0.0.0 0;
+      default 1;
+    }
+
+    map $should_apply_limit $rate_limit_key {
+      0 '';
+      1 $binary_remote_addr;
+    }
+
+    limit_req_zone $rate_limit_key zone=web_limit:10m rate=200r/m;
+    # Set a more permissive limit for covers because some pages might load 20+ covers.
+    limit_req_zone $rate_limit_key zone=cover_limit:10m rate=400r/m;
+
     # Things are mounted into here by the docker compose file
     include /etc/nginx/sites-enabled/*;
 }
diff --git a/docker/web_nginx.conf b/docker/web_nginx.conf
@@ -64,6 +64,14 @@ server {
     if ($api_call = "http:noapi") {
         rewrite ^(.*)$ https://$http_host$1 last;
     }
+
+    # Return 429 errors as JSON.
+    error_page 429 = @429;
+    location @429 {
+      default_type application/json;
+      return 429 '{"status": 429, "message": "Too Many Requests. Consider using https://openlibrary.org/developers/dumps."}';
+    }
+
     location / {
         proxy_pass http://webnodes;
         proxy_set_header Host $http_host;
@@ -76,8 +84,13 @@ server {
         proxy_set_header X-Scheme $scheme;
 
         if ($http_user_agent ~ (Bytespider) ) {
-           return 429;
+           return 444;
         }
+
+
+        # Web rate limit.
+        limit_req zone=web_limit burst=200 nodelay;
+        limit_req_status 429;
     }
 
     location ^~ /.well-known/acme-challenge/ {

diff --git a/openlibrary/components/BulkSearch.vue b/openlibrary/components/BulkSearch.vue
@@ -1,5 +1,5 @@
 <template>
-    <div>
+    <div class="ol-bulk-search">
         <BulkSearchControls :bulkSearchState="bulkSearchState" />
         <hr />
         <MatchTable v-if="bulkSearchState.matchedBooks.length" :bulkSearchState="bulkSearchState"  :listUrl = bulkSearchState.listUrl />
@@ -22,3 +22,19 @@ export default {
     }
 }
 </script>
+
+<style>
+.ol-bulk-search{
+    font-family: -apple-system,BlinkMacSystemFont,"Segoe UI","Noto Sans",Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji";
+}
+
+button, select, input{
+    font-family:inherit;
+}
+button, textarea{
+    font-size:inherit;
+}
+select, input, textarea{
+    padding:6px;
+}
+</style>
diff --git a/openlibrary/components/BulkSearch/components/BookCard.vue b/openlibrary/components/BulkSearch/components/BookCard.vue
@@ -31,7 +31,7 @@ export default {
 }</script>
 
 
-<style>
+<style lang="less">
 @keyframes pulse {
   0% {
     opacity: 0;
@@ -147,7 +147,7 @@ export default {
     margin-top: 4px;
     padding-top: 4px;
     color: #555;
-    border-top: 1px dotted;
+    border-top: 1px solid #D8D8D8;
     flex: 1;
   }
 

diff --git a/openlibrary/components/BulkSearch/components/BulkSearchControls.vue b/openlibrary/components/BulkSearch/components/BulkSearchControls.vue
@@ -2,40 +2,74 @@
 
 
 <template>
-    <details open class="bulk-search-controls">
-        <summary>Input</summary>
+    <div class="bulk-search-controls">
+
         <div>
             <p v-if="showColumnHint">Please include a header row. Supported columns include: "Title", "Author".</p>
-            <textarea v-model="bulkSearchState.inputText"></textarea>
+
+            <select class = "sampleBar" v-model="selectedValue">
+                <option v-for="sample in sampleData" :value = "sample.text" :key = "sample.source"> {{sample.name}} </option>
+            </select>
+            <textarea v-model="bulkSearchState.inputText" placeholder = "Enter your books..."></textarea>
             <br />
-            <label>Format: <select v-model="bulkSearchState._activeExtractorIndex">
-                    <option v-for="extractor, index in bulkSearchState.extractors" :value = "index" :key="index">
-                        {{ extractor.label }}
-                    </option>
-                </select></label>
-            <label v-if="this.showApiKey">OpenAI API Key:
-                <input v-if="showPassword" type="password" @click="togglePasswordVisibility()" v-model="bulkSearchState.extractionOptions.openaiApiKey" />
-
-                <input v-else type="text" @click="togglePasswordVisibility" v-model="bulkSearchState.extractionOptions.openaiApiKey" />
-            </label>
-            <label>Sample Data:
-                <select v-model="selectedValue">
-                    <option v-for="sample in sampleData" :value = "sample.text" :key = "sample.source"> {{sample.name}} </option>
-                </select>
-            </label>
-            <label>
-                <input v-model="bulkSearchState.matchOptions.includeAuthor" type="checkbox" /> Use author in
-                search query
-            </label>
-            <br>
-            <button @click="extractBooks" :disabled="loadingExtractedBooks">{{ extractBooksText }}</button>
-            <button @click="matchBooks" :disabled="loadingMatchedBooks">{{ matchBooksText }}</button>
+            <div class = "progressCarousel">
+                <div class="progressCard">
+                    <div class = "numeral">1</div>
+                    <div class = "info">
+                        <div class = "heading">
+                            <h3> Extract Books</h3>
+                            <p><i>How to convert your books above into structured information, like title and author.</i></p>
+                        </div>
+                        <label><strong> Extractor</strong> <br> <select v-model="bulkSearchState._activeExtractorIndex">
+                            <option v-for="extractor, index in bulkSearchState.extractors" :value = "index" :key="index">
+                                {{ extractor.label }}
+                            </option>
+                        </select></label>
+
+                        <label v-if="this.showApiKey"><strong >OpenAI API Key</strong> <br>
+                            <input class="api-key-bar" v-if="showPassword" placeholder= "Enter your OpenAI API key to use this feature." type="password" @click="togglePasswordVisibility()" v-model="bulkSearchState.extractionOptions.openaiApiKey" />
+
+                            <input class="api-key-bar" v-else type="text" placeholder= "OpenAI API key here...." @click="togglePasswordVisibility" v-model="bulkSearchState.extractionOptions.openaiApiKey" />
+                        </label>
+
+
+                        <button @click="extractBooks" :disabled="loadingExtractedBooks">{{ extractBooksText }}</button>
+                    </div>
+                </div>
+                <div class = "progressCard" :class="{ progressCardDisabled: matchBooksDisabled}">
+                    <div class = "numeral">2</div>
+                    <div class="info">
+                        <div class="heading">
+                            <h3>Match Books</h3>
+                            <p><i>Once structured data has been found, it's time to match it to a book in OpenLibrary!</i></p>
+                        </div>
+                        <label><strong>Options</strong> <br>
+                            <input v-model="bulkSearchState.matchOptions.includeAuthor" type="checkbox" /> Use author in
+                            search query
+                        </label>
+                        <button @click="matchBooks" :disabled="loadingMatchedBooks || matchBooksDisabled">{{ matchBooksText }}</button>
+                    </div>
+                </div>
+                <div class = "progressCard" :class="{ progressCardDisabled: createListDisabled }">
+                    <div class = "numeral">3</div>
+                    <div class="info">
+                            <div class="heading">
+                                <h3 class="heading">Save your Matches</h3>
+                                <p class="heading"><i> Now that you've found your books, why not save them to your reading log? Or a list?</i></p>
+                            </div>
+                        <a :href="bulkSearchState.listUrl" target="_blank" id="listMakerLink"><button :disabled="createListDisabled">Add to list</button></a>
+                    </div>
+                </div>
+            </div>
+
+
+
         </div>
         <div v-if="bulkSearchState.errorMessage">
             <p v-for="error in bulkSearchState.errorMessage" :key="error">
                 {{ error }}</p>
         </div>
-    </details>
+    </div>
 </template>
 
 <script>
@@ -54,6 +88,8 @@ export default {
             sampleData: sampleData,
             loadingExtractedBooks: false,
             loadingMatchedBooks: false,
+            matchBooksDisabled: true,
+            createListDisabled: true,
         }
     },
     watch: {
@@ -90,6 +126,8 @@ export default {
             const extractedData = await this.bulkSearchState.activeExtractor.run(this.bulkSearchState.extractionOptions, this.bulkSearchState.inputText)
             this.bulkSearchState.matchedBooks = extractedData
             this.loadingExtractedBooks = false
+            this.matchBooksDisabled = false;
+            this.createListDisabled = true;
         },
         async matchBooks() {
             const fetchSolrBook = async function (book, matchOptions) {
@@ -104,6 +142,7 @@ export default {
                 bookMatch.solrDocs = await fetchSolrBook(bookMatch.extractedBook, this.bulkSearchState.matchOptions)
             }
             this.loadingMatchedBooks = false
+            this.createListDisabled = false
         },
 
     }
@@ -119,10 +158,87 @@ label input {
     flex: 1;
 }
 
+.sampleBar{
+    float:right;
+    margin-bottom: 5px;
+}
 textarea {
     width: 100%;
+    height: 120px;
     display: flex;
     resize: vertical;
     box-sizing: border-box;
 }
+.progressCarousel{
+    display:flex;
+    overflow-x:scroll;
+    column-gap:10px;
+}
+.progressCard{
+    background-color:#C7E3FC;
+    padding: 16px;
+    width: min(450px, 66vw);
+    height:fit-content;
+    border-radius:30px;
+    display:flex;
+    column-gap:16px;
+    flex-shrink:0;
+    .info{
+        display:flex;
+        flex-direction:column;
+        row-gap:10px;
+        .heading{
+            color:#0376B8;
+            h3{
+                margin:0px;
+            }
+            p{
+                margin:0px;
+            }
+        }
+        select{
+            width: 100%;
+        }
+        button{
+            background-color:#0376B8;
+            color:white;
+            border-radius:4px;
+            box-shadow: none;
+            border:none;
+            padding: 0.5rem;
+            transition:  background-color 0.2s;
+            min-width:140px;
+            align-self:center;
+            &:not([disabled]) {
+                cursor:pointer;
+                &:hover{
+                    background-color:#014c78;
+                }
+            }
+        }
+    }
+    .numeral{
+        border-radius: 50%;
+        height:48px;
+        width:48px;
+        background-color:white;
+        color:#0376B8;
+        font-weight:bold;
+        justify-content:center;
+        flex-shrink: 0;
+        display:flex;
+        align-items:center;
+        font-size:24px;
+    }
+}
+
+.progressCardDisabled{
+    opacity:50%;
+}
+
+
+.api-key-bar{
+    width:100%;
+    box-sizing:border-box;
+}
 </style>