Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bug 340: Content hash is out of spec in announcements #343

Merged
merged 10 commits into from
Aug 7, 2024
Merged

Bug 340: Content hash is out of spec in announcements #343

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
aea2ea6
use text encoder for content and reutrn bytes of hash itself encoded
saraswatpuneet Aug 6, 2024
7c141b5
fix content watcher
saraswatpuneet Aug 6, 2024
a4b388e
fix profile hash as well
saraswatpuneet Aug 6, 2024
a0ba7b1
add unit test to verify correct hash is being generated
saraswatpuneet Aug 6, 2024
8e8d4ae
fix DSNP hashes on watcher side
saraswatpuneet Aug 6, 2024
8bff997
Update services/content-publishing/libs/common/src/utils/ipfs.client.ts
saraswatpuneet Aug 7, 2024
07d5a76
Merge branch 'main' into bug_340_hashes
wilwade Aug 7, 2024
47f806c
use sha256 and base32 encode
saraswatpuneet Aug 7, 2024
12882b5
update duplicate implementation
saraswatpuneet Aug 7, 2024
41899e8
update test
saraswatpuneet Aug 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions services/content-publishing/apps/worker/src/request_processor/dsnp.announcement.processor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,10 @@ export class DsnpAnnouncementProcessor {
attachment: attachments,
});
const noteString = JSON.stringify(note);
const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(noteString));
const toUint8Array = new TextEncoder();
const encoded = toUint8Array.encode(noteString);

const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(encoded));
const ipfsUrl = this.formIpfsUrl(cid);
return [cid, ipfsUrl, hash];
}
Expand Down Expand Up @@ -397,8 +400,11 @@ export class DsnpAnnouncementProcessor {
icon: attachments,
tag: this.prepareTags(content.profile.tag),
};
const toUint8Array = new TextEncoder();
const profileString = JSON.stringify(profileActivity);
const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(profileString));
const profileEncoded = toUint8Array.encode(profileString);

const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(profileEncoded));
saraswatpuneet marked this conversation as resolved.
Show resolved Hide resolved
return createProfile(dsnpUserId, this.formIpfsUrl(cid), hash);
}

Expand Down
43 changes: 43 additions & 0 deletions services/content-publishing/libs/common/src/utils/ipfs.client.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import { Test, TestingModule } from '@nestjs/testing';
import { IpfsService } from './ipfs.client';
import { ConfigService } from '#libs/config';
import { Logger } from '@nestjs/common';

jest.mock('axios');

describe('IpfsService Tests', () => {
let service: IpfsService;
let configService: ConfigService;
let logger: Logger;

beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
IpfsService,
{
provide: ConfigService,
useValue: {
ipfsEndpoint: 'http://localhost:5001',
ipfsBasicAuthUser: '',
ipfsBasicAuthSecret: '',
ipfsGatewayUrl: 'http://localhost:8080/ipfs/[CID]',
},
},
],
}).compile();

service = module.get<IpfsService>(IpfsService);
configService = module.get<ConfigService>(ConfigService);
logger = new Logger(IpfsService.name);
});


it('should be defined', () => {
expect(service).toBeDefined();
});

it("hashes blake2b correctly ABC", async () => {
const mb = await service.ipfsHashBuffer(Buffer.from("abc"));
expect(mb).toMatch("bciqlu6awx6hqdt7kifaubxs5vyrchmadmgrzmf32ts2bb73b6iablli");
});
});
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

picked up an old test https://github.com/LibertyDSNP/activity-content/blob/main/src/test/hash.test.ts

14 changes: 7 additions & 7 deletions services/content-publishing/libs/common/src/utils/ipfs.client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@ import axios from 'axios';
import FormData from 'form-data';
import { extension as getExtension } from 'mime-types';
import { CID } from 'multiformats/cid';
import { blake2b256 as hasher } from '@multiformats/blake2/blake2b';
import { create } from 'multiformats/hashes/digest';
import { sha256 } from "multiformats/hashes/sha2";
import { randomUUID } from 'crypto';
import { base58btc } from 'multiformats/bases/base58';
import { base32 } from 'multiformats/bases/base32';
import { ConfigService } from '#libs/config';

export interface FilePin {
Expand Down Expand Up @@ -127,11 +126,12 @@ export class IpfsService {
return response && response.data && JSON.stringify(response.data).indexOf(v0Cid) >= 0;
}

private async ipfsHashBuffer(fileBuffer: Buffer): Promise<string> {
public async ipfsHashBuffer(fileBuffer: Buffer): Promise<string> {
// Hash with sha256
// Encode with base32
this.logger.debug(`Hashing file buffer with length: ${fileBuffer.length}`);
const hashed = await hasher.digest(fileBuffer);
const hash = create(hasher.code, hashed.bytes);
saraswatpuneet marked this conversation as resolved.
Show resolved Hide resolved
return base58btc.encode(hash.bytes);
const hash = await sha256.digest(fileBuffer);
return base32.encode(hash.bytes);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

according to dsnp spec (to be updated) base32(multihash)

}

public ipfsUrl(cid: string): string {
Expand Down
12 changes: 6 additions & 6 deletions services/content-watcher/libs/common/src/ipfs/ipfs.dsnp.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ export class IPFSContentProcessor extends BaseConsumer {
if (isBroadcast(mapRecord)) {
announcementResponse.announcement = {
fromId: mapRecord.fromId,
contentHash: bases.base16.encode(mapRecord.contentHash as never),
saraswatpuneet marked this conversation as resolved.
Show resolved Hide resolved
contentHash: mapRecord.contentHash,
url: mapRecord.url,
announcementType: mapRecord.announcementType,
};
Expand All @@ -108,7 +108,7 @@ export class IPFSContentProcessor extends BaseConsumer {
announcementResponse.announcement = {
fromId: mapRecord.fromId,
targetAnnouncementType: mapRecord.targetAnnouncementType,
targetContentHash: bases.base58btc.encode(mapRecord.targetContentHash as any),
targetContentHash: mapRecord.targetContentHash,
announcementType: mapRecord.announcementType,
};
queue = this.tombstoneQueue;
Expand All @@ -129,7 +129,7 @@ export class IPFSContentProcessor extends BaseConsumer {
announcementType: mapRecord.announcementType,
url: mapRecord.url,
inReplyTo: mapRecord.inReplyTo,
contentHash: bases.base58btc.encode(mapRecord.contentHash as any),
contentHash: mapRecord.contentHash,
};
queue = this.replyQueue;
typeName = 'Reply';
Expand All @@ -138,7 +138,7 @@ export class IPFSContentProcessor extends BaseConsumer {
fromId: mapRecord.fromId,
announcementType: mapRecord.announcementType,
url: mapRecord.url,
contentHash: bases.base58btc.encode(mapRecord.contentHash as any),
contentHash: mapRecord.contentHash,
};
queue = this.profileQueue;
typeName = 'Profile';
Expand All @@ -147,9 +147,9 @@ export class IPFSContentProcessor extends BaseConsumer {
fromId: mapRecord.fromId,
announcementType: mapRecord.announcementType,
url: mapRecord.url,
contentHash: bases.base58btc.encode(mapRecord.contentHash as any),
contentHash: mapRecord.contentHash,
targetAnnouncementType: mapRecord.targetAnnouncementType,
targetContentHash: bases.base58btc.encode(mapRecord.targetContentHash as any),
targetContentHash: mapRecord.targetContentHash,
};
queue = this.updateQueue;
typeName = 'Update';
Expand Down
14 changes: 7 additions & 7 deletions services/content-watcher/libs/common/src/utils/ipfs.client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@ import axios from 'axios';
import FormData from 'form-data';
import { extension as getExtension } from 'mime-types';
import { CID } from 'multiformats/cid';
import { blake2b256 as hasher } from '@multiformats/blake2/blake2b';
import { create } from 'multiformats/hashes/digest';
import { sha256 } from 'multiformats/hashes/sha2';
import { randomUUID } from 'crypto';
import { base58btc } from 'multiformats/bases/base58';
import { AppConfigService } from '../config/config.service';
import { base32 } from 'multiformats/bases/base32';

export interface FilePin {
cid: string;
Expand Down Expand Up @@ -127,11 +126,12 @@ export class IpfsService {
return response && response.data && JSON.stringify(response.data).indexOf(v0Cid) >= 0;
}

private async ipfsHashBuffer(fileBuffer: Buffer): Promise<string> {
public async ipfsHashBuffer(fileBuffer: Buffer): Promise<string> {
// Hash with sha256
// Encode with base32
this.logger.debug(`Hashing file buffer with length: ${fileBuffer.length}`);
const hashed = await hasher.digest(fileBuffer);
const hash = create(hasher.code, hashed.bytes);
return base58btc.encode(hash.bytes);
const hash = await sha256.digest(fileBuffer);
return base32.encode(hash.bytes);
}

public ipfsUrl(cid: string): string {
Expand Down
Loading