downloader: switch back to surt downloads, properly sorted

update to warcio 1.4.5, generate 'sha256:' instead of 'sha-256:' prefix in warc digests
recorder: don't attach to service_worker targets, only do behavior insert in page/iframe targets, fixes #33
update to latest ruffle, replaywebpage 1.4.2, wabac.js 2.7.3
bump to 0.6.6

package.json

@@ -1,14 +1,14 @@
 {
   "name": "archiveweb.page",
-  "version": "0.6.5",
+  "version": "0.6.6",
   "main": "index.js",
   "description": "Create Web Archives directly in your browser",
   "repository": "https://github.com/webrecorder/archiveweb.page",
   "author": "Webrecorder Software",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.13.0",
-    "@webrecorder/wabac": "^2.7.1",
+    "@webrecorder/wabac": "^2.7.3",
     "browsertrix-behaviors": "^0.2.1",
     "bulma": "^0.9.2",
     "flexsearch": "^0.6.32",
@@ -21,14 +21,14 @@
     "lodash": "^4.17.20",
     "node-fetch": "^2.6.1",
     "pretty-bytes": "^5.3.0",
-    "replaywebpage": "^1.4.1",
+    "replaywebpage": "^1.4.2",
     "uuid": "^8.3.2",
-    "warcio": "^1.4.4"
+    "warcio": "^1.4.5"
   },
   "devDependencies": {
     "copy-webpack-plugin": "^6.4.0",
     "css-loader": "^3.5.3",
-    "electron": "12.0.5",
+    "electron": "^12.0.7",
     "electron-builder": "^22.9.1",
     "electron-notarize": "^1.0.0",
     "generate-json-webpack-plugin": "1.0.0",

src/downloader.js

@@ -8,7 +8,7 @@ import { v5 as uuidv5 } from "uuid";
 
 import { createSHA256 } from "hash-wasm";
 
-import { WARCRecord, WARCSerializer } from "warcio";
+import { getSurt, WARCRecord, WARCSerializer } from "warcio";
 
 import { getTSMillis, getStatusText } from "@webrecorder/wabac/src/utils";
 
@@ -158,6 +158,22 @@ class Downloader
     } else {
       this.resources = await this.db.db.getAll("resources");  
     }
+
+    this.resources.sort((a, b) => {
+      if (!a.surt) {
+        a.surt = getSurt(a.url);
+      }
+
+      if (!b.surt) {
+        b.surt = getSurt(b.url);
+      }
+
+      if (a.surt == b.surt) {
+        return 0;
+      }
+
+      return a.surt < b.surt ? -1 : 1;
+    });
   }
 
   async queueWARC(controller, filename, sizeCallback) {
@@ -328,7 +344,7 @@ class Downloader
     const getCDX = (resource, filename, raw) => {
 
       const data = {
-        //url: resource.url,
+        url: resource.url,
         digest: resource.digest,
         mime: resource.mime,
         offset: resource.offset,
@@ -350,10 +366,7 @@ class Downloader
         data.method = resource.method;
       }
 
-      //const surt = getSurt(resource.url);
-      const url = resource.url;
-
-      const cdx = `${url} ${resource.timestamp} ${JSON.stringify(data)}\n`;
+      const cdx = `${resource.surt} ${resource.timestamp} ${JSON.stringify(data)}\n`;
 
       if (!raw) {
         return cdx;

src/ext/browser-recorder.js

@@ -213,9 +213,9 @@ class BrowserRecorder extends Recorder {
     let writtenSize = 0;
     const payloadSize = data.payload.length;
 
-    await this.db.initing;
-
     try {
+      await this.db.initing;
+
       if (await this.db.addResource(data)) {
         writtenSize = payloadSize;
       }

src/recorder.js

@@ -81,8 +81,6 @@ class Recorder {
     this.behaviorState = BEHAVIOR_WAIT_LOAD;
     this.behaviorData = null;
     this.autorun = false;
-
-    this._bgFetchId = setInterval(() => this.doBackgroundFetch(), 10000);
   }
 
   setAutoRunBehavior(autorun) {
@@ -157,6 +155,7 @@ class Recorder {
   async _stop(domNodes = null) {
     clearInterval(this._updateStatusId);
     clearInterval(this._loopId);
+    clearInterval(this._bgFetchId);
 
     this.flushPending();
     this.running = false;
@@ -194,6 +193,8 @@ class Recorder {
     this._updateStatusId = setInterval(() => this.updateStatus(), 1000);
 
     this._loopId = setInterval(() => this.updateLoop(), 10000);
+
+    this._bgFetchId = setInterval(() => this.doBackgroundFetch(), 10000);
   }
 
   updateLoop() {
@@ -420,15 +421,30 @@ class Recorder {
       try {
         this.sessionSet.add(params.sessionId);
 
-        await this.sessionInit(sessions);
+        const type = params.targetInfo.type;
+
+        const allowAttach = type !== "service_worker";
+
+        if (allowAttach) {
+          await this.sessionInit(sessions);
+        }
 
         if (params.waitingForDebugger) {
           await this.send("Runtime.runIfWaitingForDebugger", null, sessions);
         }
 
-        console.log("Target Attached: " + params.targetInfo.type + " " + params.targetInfo.url + " " + params.sessionId);
+        if (allowAttach) {
+          console.log("Target Attached: " + type + " " + params.targetInfo.url + " " + params.sessionId);
 
-        await this._doInjectIframe(sessions);
+          if (type === "page" || type === "iframe") {
+            await this._doInjectIframe(sessions);
+          }
+        } else {
+          console.log("Not allowed attach for: " + type + " " + params.targetInfo.url + " " + params.sessionId);
+
+          const params2 = this.flatMode ? {sessionId: params.sessionId} : {targetId: params.targetInfo.targetId};
+          await this.send("Runtime.runIfWaitingForDebugger", params2, sessions);
+        }
 
       } catch (e) {
         console.log(e);
@@ -771,7 +787,7 @@ class Recorder {
   }
 
   async updatePage(sessions) {
-    console.log("updatePage", this.pageInfo);
+    //console.log("updatePage", this.pageInfo);
 
     if (!this.pageInfo) {
       console.warn("no page info!");

wr-ext/manifest.json

@@ -1,7 +1,7 @@
 {
   "name": "Webrecorder ArchiveWeb.page",
   "description": "Create high-fidelity web archives directly in your browser",
-  "version": "0.6.5",
+  "version": "0.6.6",
   "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self'",
   "permissions": [
     "debugger",