Merge branch 'master' into ipfs-bug-fix

# Conflicts:
#	examples/dweb_transport_ipfs_bundled.js

README.md

@@ -2,83 +2,125 @@
 
 Welcome to the Internet Archive's Decentralized Wed (Dweb) libraries. 
 
-## Running the examples
-Once the source is checked out, you should be able to open any of the files:
-[example_block.html](examples/example_block.html);
-[example_smartdict.html](examples/example_smartdict.html); 
-[example_list.html](examples/example_list.html); 
-or [objbrowser.html](examples/objbrowser.html); 
-directly in your browser.
+VERBOSE
 
-IMPORTANT - DO THIS ON CHROME NOT ON FIREFOX - SEE "Major Issues"
+## Running the examples
+The examples can run either from the [dweb.me/examples](https://dweb.me/examples) server, 
+or once the source is checked out, locally from your file system.
+
+- example_block.html: [IPFS](https://dweb.me/examples/example_block.html)
+    or [HTTP](https://dweb.me/examples/example_block.html?transport=HTTP)
+- example_smartdict.html: [IPFS](https://dweb.me/examples/example_smartdict.html)
+    or [HTTP](https://dweb.me/examples/example_smartdict.html?transport=HTTP); 
+- example_list.html: [IPFS](https://dweb.me/examples/example_list.html)
+    or [HTTP](https://dweb.me/examples/example_list.html?transport=HTTP)
+- example_academic.html: [IPFS](https://dweb.me/examples/example_academic.html)
+    or [HTTP](https://dweb.me/examples/example_academic.html?transport=HTTP)
+- example_keys.html: [IPFS](https://dweb.me/examples/example_keys.html)
+    or [HTTP](https://dweb.me/examples/example_keys.html?transport=HTTP); 
+- [objbrowser.html](https://dweb.me/examples/objbrowser.html); 
+
+**Browser Support**: This should work on Chrome and Firefox (Safari doesn't support many newer features), 
+see below for IPFS bugs, 
+
+**Verbosity**: You can get debugging output by appending verbose=true to the URLs, 
+this shows up in your console and also (for HTTP) in our server logs.
 
 ###BLOCK example
-In your browser, open the file:  examples/example_block.html  
-Type some text into the editor and hit Save  
-A hash should appear below.  
-If it doesn't then open the browser console (e.g. Firefox/tools/Web Developer/Web Console)  
-Click "FetchIt" and the data should be returned.
+- In your browser, open examples/example_block.html: 
+[IPFS](https://dweb.me/examples/example_block.html) 
+or [HTTP](https://dweb.me/examples/example_block.html?transport=HTTP)
+- Type some text into the editor and hit Save  
+- A hash should appear below.  
+- If it doesn't then run with the verbose argument 
+[IPFS](https://dweb.me/examples/example_block.html?verbose=true) 
+or [HTTP](https://dweb.me/examples/example_block.html?transport=HTTP&verbose=true) 
+and open the browser console (e.g. Firefox/tools/Web Developer/Web Console)  
+- Click "FetchIt" and the data should be returned.
 
 ###SMART DICT example
-In your browser, open the file:  examples/example_smartdict.html  
-Type some text into the name, and a HTML color nmae into the color (e.g. "red") and hit Save  
-A hash should appear below.  
-If it doesn't then open the browser console (e.g. Firefox/tools/Web Developer/Web Console)  
-Click "FetchIt" and the data should be returned and displayed.  
-Hover over "Object Browser" to see the structure of the object.
+- In your browser, open example_smartdict.html
+[IPFS](https://dweb.me/examples/example_smartdict.html)
+or [HTTP](https://dweb.me/examples/example_smartdict.html?transport=HTTP);
+- Type some text into the name, and a HTML color nmae into the color (e.g. "red") and hit Save  
+- A hash should appear below.  
+- Click "FetchIt" and the data should be returned and displayed.  
+- Hover over "Object Browser" to see the structure of the object.
 
 ###COMMON LIST example
-In your browser, open the file:  examples/example_smartdict.html  
-Click New and enter a name for your list  
-A blank list should appear along with the name and hashes (retrieved from Dweb)  
-Enter something in the text field and hit Send  
-The item should be announced to the list and appear in the text field above.
-
-The link icons next to the private hash can be opened on another machine and gives 
+- In your browser, open the file:  example_list.html:
+[IPFS](https://dweb.me/examples/example_list.html)
+or [HTTP](https://dweb.me/examples/example_list.html?transport=HTTP)
+- Click New and enter a name for your list  
+- A blank list should appear along with the name and hashes (retrieved from Dweb)  
+- Enter something in the text field and hit Send  
+- The item should be announced to the list and appear in the text field above.
+- The link icons next to the private hash can be opened on another machine and gives 
 the user ability to also write to the list.
-
-The link icon next to the public hash will only give them the ability to display the list.
-
-Hover over "Object Browser" to see the structure of the object.
+- The link icon next to the public hash will only give them the ability to display the list.
+- Hover over "Object Browser" to see the structure of the object.
+
+###ACADEMIC DOCS example
+
+This is a work in progress, dependent on the incompleteness of both the Academic Document virtual collection at Archive.org and 
+the bugs/issues in IPFS.
+
+- In your browser, open the file:  example_academic.html:
+[IPFS](https://dweb.me/examples/example_academic.html)
+or [HTTP](https://dweb.me/examples/example_academic.html?transport=HTTP)
+- Enter a search term 
+(use just one word, as there are problems with multi-word search)
+- A list of papers should be returned, along with their DOI.
+- Clicking on a DOI will find metadata on it. 
+(Currently (24Oct2017) we don't have most of those DOI's, so you probably won't see a location)
+- Instead try DOI: 10.1001/jama.2009.1064 or DOI: 10.1002/asjc.98
+- As you search for these DOI's the paper is pushed into our contenthash server, and IPFS.
+- You should see metadata on that paper, and a list of ways to receive it.
+- The first three fetch from: the Archive's contenthash server; and from two IPFS http gateways.
+- The last link fetches directly in the browser without coming to the Archive or any other single point of failure.
+- (Unfortunately there is currently (24Oct2017) a problem with the IPFS API which means only one of those two DOI's above will work)
 
 ###AUTHENTICATION example
-In your browser, open the file:  examples/example_keys.html  
-Click on the "KeyChain icon"  
-Click on Register  
-Choose a name for your first keychain, remember exactly how you spelled and capitalised it.  
-Choose a long and complex passphrase that is easy for you to remember and hard for others to guess, ideally include numbers and punctuation, but you'll need to remember this exactly.  
+- In your browser, open the file:  examples/example_keys.html:
+[IPFS](https://dweb.me/examples/example_keys.html)
+or [HTTP](https://dweb.me/examples/example_keys.html?transport=HTTP)
+- Click on the "KeyChain icon"  
+- Click on Register  
+- Choose a name for your first keychain, remember exactly how you spelled and capitalised it.  
+- Choose a long and complex passphrase that is easy for you to remember and hard for others to guess, ideally include numbers and punctuation, but you'll need to remember this exactly.  
 Note there is no way to change a name or password later, since there is no central authority to change them with.  
-Your name should appear next to the keyhain icon.  
-Click on your name.  
-A box should appear showing that you have no keys.  
-Click on New Key, give it a name (which you dont have to remember) and click Generate  
-The new key should show up.  
-Click on "New Access Control List, give it a name (which you dont have to remember) and click Generate  
-...  This example is still being written, and will be expanded here.   
-Click on the Key - you should get a prompt which you can copy the URL out of.  
+- Your name should appear next to the keyhain icon, click on it.
+- A box should appear showing that you have no keys.  
+- Click on New Key, give it a name (which you dont have to remember) and click Generate  
+- The new key should show up, Click on it. 
+- Copy and Paste the "url" of the hash (including the full string from http: or ipfs:)
+- Click on "New Access Control List", give it a name (which you dont have to remember) and click Generate
+- Click on "new key"
+- Give it a name (you don't have to remember it); and `paste the URL
+- You have now created a Lock, and given yourself access to it. 
+
+Further examples will demonstrate using the lock.
 
 ## Installing a compilable version
-If you haven't already, then install npm from [https://nodejs.org/en/download]  
-And on a Mac you'll need Xcode from the App store.  
-Then install the dependencies: ```> npm install --dev```
-
-Note that this gets a forked version of libsodium-wrappers from (Mitra's repository)[https://github.com/mitra42/libsodium.js], 
+- Checkout the repository
+- If you haven't already, then install [npm](https://nodejs.org/en/download)
+- And on a Mac you'll need Xcode from the App store. 
+- Then install the dependencies: ```> npm install --dev```
+- Note that this gets a forked version of libsodium-wrappers from [Mitra's repository][https://github.com/mitra42/libsodium.js], 
 as the current libsodium-wrappers release doesn't have urlsafebase54.
-
-Often the first run of ```> npm install --dev``` generates a lot of warnings and a second, 
+- Often the first run of ```> npm install --dev``` generates a lot of warnings and a second, 
 virtually clean run gives more confidence that the install worked.
+- Now compile the javascript library for the browser: ```> npm run bundle_transport_ipfs```
+- If this worked without errors, try the node specific test. ```> npm run test```
+- This should start a IPFS instance, and generate some messages ending in "delaying 10 secs" and "Completed test".
+- It will leave the IPFS instance running and usually will need a Ctrl-C to exit.
 
-Now compile the javascript library for the browser: ```> npm run bundle_transport_ipfs```
-
-If this worked without errors, try the node specific test. ```> npm run test```
-
-This should start a IPFS instance, and generate some messages ending in "delaying 10 secs" and "Completed test".
-It will leave the IPFS instance running and usually will need a Ctrl-C to exit.
-
-##Major Issues
+##Major Browser Issues
 
-Please not there is an issue with IPFS on some Firefox versions (seen on 54.0.1, not on 49.0.2 for example) that is currently leaking Threads and slowing the machine down
-drastically. This is being explored!  Use it on Chrome for now, and expect it to crash every 5 minutes.
+Please not there is an issue with IPFS on some Firefox versions (seen on 54.0.1, not on 49.0.2 for example)
+that is currently leaking Threads and slowing the machine down drastically. This is being explored!  
+Use it on Chrome for now, and expect it to crash every 5 minutes. 
+The HTTP versions don't have this problem, but also don't support live notification of changes.
 
 ##See also:
 

examples/example_keys.html

@@ -100,7 +100,8 @@
             hide('aclnew_div');
             let dict = form2dict("newaclform"); //name
             let kc = document.getElementById('keylist_header').source;  // The KeyChain being added to.
-            return Dweb.AccessControlList.p_new({name: dict.name, _acl: kc}, true, {keygen: true}, verbose, null, kc );    //(data, master, key, verbose, options, kc)
+            return Dweb.AccessControlList.p_new({name: dict.name, _acl: kc}, true, {keygen: true}, verbose, null, kc )    //(data, master, key, verbose, options, kc)
+            .then((acl) => _showkeyorlock(acl)); // Put in UI, as listmonitor return rejected as duplicate
         }
         function p_key_click(el) {
             if (verbose) console.log("p_key_click ---");
@@ -131,7 +132,7 @@
 
     </script>
 </head>
-<body>
+<body class="exampleskeys">
     <div class="floatright">
         <div id="status">Starting</div>
         <ul id="kclist_ul"><!-- List of all existing keychains-->

examples/example_styles.css

@@ -68,3 +68,5 @@
 .examplesacademic #metadataresults [name=doi_org_metadata_title] {font-weight: bold;}
 .examplesacademic #searchresults li {text-align: left; margin-bottom: 10px;}
 .examplesacademic #searchresults span {display: inline-block;}/*Try and keep each part together*/
+
+.exampleskeys .inline_ul {display: grid;  grid-template-columns: repeat(auto-fit, minmax(7em, 1fr));  grid-auto-flow: dense;}

examples/htmlutils.js

@@ -113,7 +113,7 @@ function addtemplatedchild(el, ...dict) {
 
     el: An HTML element, or a string with the id of one.
     html: html to add under outerelement
-    dict: Dictionary with parameters to replace in html, it looks for nodes with name="xxx" and replaces text inside it with dict[xxx]
+    dict: Dictionary with parameters to replace in html, it looks for nodes with name="xyz" and replaces text inside it with dict[xyz]
     */
     el = (typeof(el) === "string") ? document.getElementById(el) : el;
     let el_li = el.getElementsByClassName("template")[0].cloneNode(true);   // Copy first child with class=Template
@@ -131,7 +131,7 @@ function addhtml(el, htmleach, dict) { //TODO merge into addtemplatechild - note
 
     el: An HTML element, or a string with the id of one.
     html: html to add under outerelement
-    dict: Dictionary with parameters to replace in html, it looks for nodes with name="xxx" and replaces text inside it with dict[xxx]
+    dict: Dictionary with parameters to replace in html, it looks for nodes with name="xyz" and replaces text inside it with dict[xyz]
     */
     el = (typeof(el) === "string") ? document.getElementById(el) : el;
     let el_li = document.createElement('div');   // usually a 'li' but could be a 'div'

examples/index.html

@@ -9,6 +9,7 @@
     <li>Block - edit, store, retrieve <a href="./example_block.html">IPFS</a> <a href="./example_block.html?transport=HTTP">HTTP</a></li>
     <li>Smart Dictionary - edit, store, retrieve <a href="./example_smartdict.html">IPFS</a> <a href="./example_smartdict.html?transport=HTTP">HTTP</a></li>
     <li>List - create, add to, retrieve <a href="./example_list.html">IPFS</a> <a href="./example_list.html?transport=HTTP">HTTP</a></li>
+    <li>Academic Documents search <a href="./example_academic.html">IPFS</a> <a href="./example_academic.html?transport=HTTP">HTTP</a></li>
     <li>Keys & Authentication - create manage etc <a href="./example_keys.html">IPFS</a> <a href="./example_keys.html?transport=HTTP">HTTP</a></li>
 </ul>
 

js/Block.js

@@ -37,7 +37,7 @@ class Block extends Transportable {
             blk.p_store(verbose)                            // Store it to transport
             .then(() => Block.p_fetch(blk._url, verbose))
             .then((blk2) => {
-                console.assert(blk2._data.toString() === blk._data, "Block should survive round trip");
+                if (blk2._data.toString() !== blk._data) throw new CodingError("Block should survive round trip");
                 resolve(blk2);
             })
             .catch((err) => { console.log("Block Test failed", err); reject(err); })

js/CommonList.js

@@ -169,8 +169,8 @@ class CommonList extends SmartDict {
         }
     }
 
-    publicurl() { console.assert(false, "XXX Undefined function CommonList.publicurl"); }   // For access via web
-    privateurl() { console.assert(false, "XXX Undefined function CommonList.privateurl"); }   // For access via web
+    publicurl() { throw new Dweb.errors.ToBeImplementedError("Undefined function CommonList.publicurl"); }   // For access via web
+    privateurl() { throw new Dweb.errors.ToBeImplementedError("Undefined function CommonList.privateurl"); }   // For access via web
 
     p_push(obj, verbose ) {
         /*
@@ -208,7 +208,7 @@ class CommonList extends SmartDict {
         if (!url) throw new Dweb.errors.CodingError("Empty url is a coding error");
         if (!this._master) throw new Dweb.errors.ForbiddenError("Must be master to sign something");
         let sig = Dweb.Signature.sign(this, url, verbose); //returns a new Signature
-        console.assert(sig.signature, "Must be a signature");
+        if (!sig.signature) throw new CodingError("Must be a signature");
         return sig
     }
     p_add(sig, verbose) {
@@ -235,10 +235,8 @@ class CommonList extends SmartDict {
     // ----- Listener interface ----- see https://developer.mozilla.org/en-US/docs/Web/API/EventTarget for the pattern
 
     addEventListener(type, callback) {
-        console.log("XXX@CL.addEventListener",type);
         if (!(type in this._listeners)) this._listeners[type] = [];
         this._listeners[type].push(callback);
-        console.log("XXX@CL.addEventListener done")
     }
 
     removeEventListener(type, callback) {
@@ -252,7 +250,7 @@ class CommonList extends SmartDict {
         }
     }
     dispatchEvent(event) {
-        console.log("XXX@CL.dispatchEvent",event);
+        console.log("CL.dispatchEvent",event);
         if (!(event.type in this._listeners)) return true;
         let stack = this._listeners[event.type];
         console.log("THIS=",this, "event.target=",event.target);

js/Errors.js

@@ -9,6 +9,14 @@ class ToBeImplementedError extends Error {
 }
 errors.ToBeImplementedError = ToBeImplementedError;
 
+class ObsoleteError extends Error {
+    constructor(message) {
+        super("Obsolete: " + message);
+        this.name = "ObsoleteError"
+    }
+}
+errors.ObsoleteError = ObsoleteError;
+
 //TODO TransportError is wanted in TransportHTTP but its out of scope there. Think about moving to Transport class
 class TransportError extends Error {
     constructor(message) {
@@ -38,6 +46,15 @@ class EncryptionError extends Error {
 }
 errors.EncryptionError = EncryptionError;
 
+// Use this something that should have been signed isn't - this is externally signed, i.e. a data rather than coding error
+class SigningError extends Error {
+    constructor(message) {
+        super(message || "Signing Error");
+        this.name = "SigningError"
+    }
+}
+errors.SigningError = SigningError;
+
 class ForbiddenError extends Error {
     constructor(message) {
         super(message || "Forbidden failure");

js/KeyChain.js

@@ -202,12 +202,12 @@ class KeyChain extends CommonList {
     .then(() => sb.p_store(verbose))
     .then(() => {
             let mvk = KeyChain.mykeys(Dweb.KeyPair);
-            console.assert(mvk[0].name === vkpname, "Should find viewerkeypair stored above");
+            if (mvk[0].name !== vkpname) throw new CodingError("Should find viewerkeypair stored above");
         if (verbose) console.log("KEYCHAIN 6: Check can fetch and decrypt - should use viewerkeypair stored above");
         return Dweb.SmartDict.p_fetch(sb._url, verbose); // Will be StructuredBlock, fetched and decrypted
     })
     .then((sb2) => {
-            console.assert(sb2.data === qbf, "Data should survive round trip");
+            if (sb2.data !== qbf) throw new CodingError("Data should survive round trip");
         if (verbose) console.log("KEYCHAIN 7: Check can store content via an MB");
         //MB.new(acl, contentacl, name, _allowunsafestore, content, signandstore, verbose)
     })
@@ -220,7 +220,7 @@ class KeyChain extends CommonList {
     .then((newpublicmb) => mb = newpublicmb)
     .then(() => mb.p_list_then_current(verbose))
     .then(() => {
-            console.assert(mb.content() === qbf, "Data should round trip through ACL");
+            if (mb.content() !== qbf) throw new CodingError("Data should round trip through ACL");
     })
 
     .then(() => {