Bug 340: Content hash is out of spec in announcements (#343)

# Problem

Content watcher bug #340  in calculating `contentHash`,

## Details
- [x] Updated content-watcher to calculate correct contentHash
- [x] Updated content-watcher 
- [x] Update app template (if needed): Note: social app template simply
uses contentHash as key and do not really do any hash validation so no
updates necessary

Closes: #340


# Solution

What I/we did to solve this problem

Fixed by following DSNP spec and hashing the bytes of content in line
with DSNP

---------

Co-authored-by: Joe Caputo <joseph.caputo@amplica.io>
Co-authored-by: Wil Wade <wil.wade@amplica.io>

services/content-publishing/apps/worker/src/request_processor/dsnp.announcement.processor.ts

@@ -179,7 +179,10 @@ export class DsnpAnnouncementProcessor {
       attachment: attachments,
     });
     const noteString = JSON.stringify(note);
-    const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(noteString));
+    const toUint8Array = new TextEncoder();
+    const encoded = toUint8Array.encode(noteString);
+
+    const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(encoded));
     const ipfsUrl = this.formIpfsUrl(cid);
     return [cid, ipfsUrl, hash];
   }
@@ -397,8 +400,11 @@ export class DsnpAnnouncementProcessor {
       icon: attachments,
       tag: this.prepareTags(content.profile.tag),
     };
+    const toUint8Array = new TextEncoder();
     const profileString = JSON.stringify(profileActivity);
-    const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(profileString));
+    const profileEncoded = toUint8Array.encode(profileString);
+
+    const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(profileEncoded));
     return createProfile(dsnpUserId, this.formIpfsUrl(cid), hash);
   }
 

services/content-publishing/libs/common/src/utils/ipfs.client.spec.ts

@@ -0,0 +1,43 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { IpfsService } from './ipfs.client';
+import { ConfigService } from '#libs/config';
+import { Logger } from '@nestjs/common';
+
+jest.mock('axios');
+
+describe('IpfsService Tests', () => {
+    let service: IpfsService;
+    let configService: ConfigService;
+    let logger: Logger;
+
+    beforeEach(async () => {
+      const module: TestingModule = await Test.createTestingModule({
+        providers: [
+          IpfsService,
+          {
+            provide: ConfigService,
+            useValue: {
+              ipfsEndpoint: 'http://localhost:5001',
+              ipfsBasicAuthUser: '',
+              ipfsBasicAuthSecret: '',
+              ipfsGatewayUrl: 'http://localhost:8080/ipfs/[CID]',
+            },
+          },
+        ],
+      }).compile();
+
+      service = module.get<IpfsService>(IpfsService);
+      configService = module.get<ConfigService>(ConfigService);
+      logger = new Logger(IpfsService.name);
+    });
+
+
+    it('should be defined', () => {
+        expect(service).toBeDefined();
+    });
+
+    it("hashes blake2b correctly ABC", async () => {
+        const mb = await service.ipfsHashBuffer(Buffer.from("abc"));
+        expect(mb).toMatch("bciqlu6awx6hqdt7kifaubxs5vyrchmadmgrzmf32ts2bb73b6iablli");
+    });
+});

services/content-publishing/libs/common/src/utils/ipfs.client.ts

@@ -5,10 +5,9 @@ import axios from 'axios';
 import FormData from 'form-data';
 import { extension as getExtension } from 'mime-types';
 import { CID } from 'multiformats/cid';
-import { blake2b256 as hasher } from '@multiformats/blake2/blake2b';
-import { create } from 'multiformats/hashes/digest';
+import { sha256 } from "multiformats/hashes/sha2";
 import { randomUUID } from 'crypto';
-import { base58btc } from 'multiformats/bases/base58';
+import { base32 } from 'multiformats/bases/base32';
 import { ConfigService } from '#libs/config';
 
 export interface FilePin {
@@ -127,11 +126,12 @@ export class IpfsService {
     return response && response.data && JSON.stringify(response.data).indexOf(v0Cid) >= 0;
   }
 
-  private async ipfsHashBuffer(fileBuffer: Buffer): Promise<string> {
+  public async ipfsHashBuffer(fileBuffer: Buffer): Promise<string> {
+    // Hash with sha256
+    // Encode with base32
     this.logger.debug(`Hashing file buffer with length: ${fileBuffer.length}`);
-    const hashed = await hasher.digest(fileBuffer);
-    const hash = create(hasher.code, hashed.bytes);
-    return base58btc.encode(hash.bytes);
+    const hash = await sha256.digest(fileBuffer);
+    return base32.encode(hash.bytes);
   }
 
   public ipfsUrl(cid: string): string {

services/content-watcher/libs/common/src/ipfs/ipfs.dsnp.ts

@@ -98,7 +98,7 @@ export class IPFSContentProcessor extends BaseConsumer {
       if (isBroadcast(mapRecord)) {
         announcementResponse.announcement = {
           fromId: mapRecord.fromId,
-          contentHash: bases.base16.encode(mapRecord.contentHash as never),
+          contentHash: mapRecord.contentHash,
           url: mapRecord.url,
           announcementType: mapRecord.announcementType,
         };
@@ -108,7 +108,7 @@ export class IPFSContentProcessor extends BaseConsumer {
         announcementResponse.announcement = {
           fromId: mapRecord.fromId,
           targetAnnouncementType: mapRecord.targetAnnouncementType,
-          targetContentHash: bases.base58btc.encode(mapRecord.targetContentHash as any),
+          targetContentHash: mapRecord.targetContentHash,
           announcementType: mapRecord.announcementType,
         };
         queue = this.tombstoneQueue;
@@ -129,7 +129,7 @@ export class IPFSContentProcessor extends BaseConsumer {
           announcementType: mapRecord.announcementType,
           url: mapRecord.url,
           inReplyTo: mapRecord.inReplyTo,
-          contentHash: bases.base58btc.encode(mapRecord.contentHash as any),
+          contentHash: mapRecord.contentHash,
         };
         queue = this.replyQueue;
         typeName = 'Reply';
@@ -138,7 +138,7 @@ export class IPFSContentProcessor extends BaseConsumer {
           fromId: mapRecord.fromId,
           announcementType: mapRecord.announcementType,
           url: mapRecord.url,
-          contentHash: bases.base58btc.encode(mapRecord.contentHash as any),
+          contentHash: mapRecord.contentHash,
         };
         queue = this.profileQueue;
         typeName = 'Profile';
@@ -147,9 +147,9 @@ export class IPFSContentProcessor extends BaseConsumer {
           fromId: mapRecord.fromId,
           announcementType: mapRecord.announcementType,
           url: mapRecord.url,
-          contentHash: bases.base58btc.encode(mapRecord.contentHash as any),
+          contentHash: mapRecord.contentHash,
           targetAnnouncementType: mapRecord.targetAnnouncementType,
-          targetContentHash: bases.base58btc.encode(mapRecord.targetContentHash as any),
+          targetContentHash: mapRecord.targetContentHash,
         };
         queue = this.updateQueue;
         typeName = 'Update';

services/content-watcher/libs/common/src/utils/ipfs.client.ts

@@ -5,11 +5,10 @@ import axios from 'axios';
 import FormData from 'form-data';
 import { extension as getExtension } from 'mime-types';
 import { CID } from 'multiformats/cid';
-import { blake2b256 as hasher } from '@multiformats/blake2/blake2b';
-import { create } from 'multiformats/hashes/digest';
+import { sha256 } from 'multiformats/hashes/sha2';
 import { randomUUID } from 'crypto';
-import { base58btc } from 'multiformats/bases/base58';
 import { AppConfigService } from '../config/config.service';
+import { base32 } from 'multiformats/bases/base32';
 
 export interface FilePin {
   cid: string;
@@ -127,11 +126,12 @@ export class IpfsService {
     return response && response.data && JSON.stringify(response.data).indexOf(v0Cid) >= 0;
   }
 
-  private async ipfsHashBuffer(fileBuffer: Buffer): Promise<string> {
+  public async ipfsHashBuffer(fileBuffer: Buffer): Promise<string> {
+    // Hash with sha256
+    // Encode with base32
     this.logger.debug(`Hashing file buffer with length: ${fileBuffer.length}`);
-    const hashed = await hasher.digest(fileBuffer);
-    const hash = create(hasher.code, hashed.bytes);
-    return base58btc.encode(hash.bytes);
+    const hash = await sha256.digest(fileBuffer);
+    return base32.encode(hash.bytes);
   }
 
   public ipfsUrl(cid: string): string {