chore: (breaking changes) use rust "ipfs" by default (#3250)

* chore: change default rust "ipfs" rpc port to 5101

* chore: replace CERAMIC_RECON_MODE with IPFS_FLAVOR and access from EnvironmentUtils

we no longer have rust ceramic without recon mode, so we don't need both

* chore: update tests to use common recon/v4 setting

* chore: prettier

* chore: rely on more ceramic one defaults and use stdout/err from parent

We bind all IPs on the host so it works in docker and generally as expected.
This feels a bit janky and not in one place still. We really don't pipe down rust options, so it mostly relies on env vars and tries to keep the behavior the same for tests with temp dirs and inmemory, running from the cli, or running in docker

* chore: set path to rust-ceramic vars in dockerfile

can be overriden via `docker run -e CERAMIC_ONE_PATH=... -e CERAMIC_ONE_STORE_DIR=...`

* chore: remove CERAMIC_RECON_MODE from docs/test set up

* chore: fix lint error and unnecessary switch branch

* chore: use default IPFS config in tests

we only override a portion which was invalid, now we just use env vars/default rust

* chore: override metrics and swarm addresses for tests so they pass

this has the unfortunate side effect of using a temp directory if network or store env vars aren't set that gets cleaned up, but fixing that will require making the tests explicit about what they want. should happen next though.

* chore: specify CERAMIC_ONE_* variables in both layers of dockerfile

we sometimes run the js-ceramic layer directly and this makes it start up without external config

.circleci/config.yml

@@ -68,7 +68,6 @@ jobs:
     environment:
       IPFS_FLAVOR: rust
       CERAMIC_ONE_PATH: /usr/local/bin/ceramic-one
-      CERAMIC_RECON_MODE: 'true'
     steps:
       - build-and-test
       - run:

DEVELOPMENT.md

@@ -36,7 +36,7 @@ If you only want to test a specific package just `cd` into the specific package
 
 If you want to run the tests in recon mode, you must specify the ceramic-one path, the IPFS flavor as `rust`, and indicate that ceramic will run in recon mode on:
 ```
-CERAMIC_ONE_PATH=<PATH> CERAMIC_RECON_MODE=true IPFS_FLAVOR=rust npm test
+CERAMIC_ONE_PATH=<PATH> IPFS_FLAVOR=rust npm test
 ```
 
 ## Debugging and Local Development
@@ -74,13 +74,13 @@ This repo uses lerna to make releases of all packages which have been changed. T
 After merging the changes you want to release into the relevant branch (main or release-candidate), you should build and verify the docker image before triggering the release to make sure nothing is broken.
 
 First, install Dagger:
-`curl -L https://dl.dagger.io/dagger/install.sh`
+`curl -L https://dl.dagger.io/dagger/install.sh | DAGGER_VERSION=0.2.36 sh`
 
 Then, in `js-ceramic` root:
 ```
 dagger project init
 dagger project update
-dagger project update github.com/3box/pipeline-tools@v0.2.0
+dagger project update "github.com/3box/pipeline-tools/ci"
 dagger do verify --log-format=plain -p cue.mod/pkg/github.com/3box/pipeline-tools/ci/plans/ceramic.cue
 dagger do testJs --log-format=plain -p cue.mod/pkg/github.com/3box/pipeline-tools/ci/plans/ceramic.cue
 dagger do testGo --log-format=plain -p cue.mod/pkg/github.com/3box/pipeline-tools/ci/plans/ceramic.cue

Dockerfile.daemon

@@ -25,12 +25,14 @@ ARG GIT_COMMIT_HASH=docker
 
 RUN lerna run build
 
+ENV CERAMIC_ONE_PATH=/usr/local/bin/ceramic-one CERAMIC_ONE_STORE_DIR=~/.ceramic-one
+
 EXPOSE 7007
 
 ENTRYPOINT ["./packages/cli/bin/ceramic.js", "daemon"]
 
 FROM ceramic as composedb
 
 ARG COMPOSEDB_VERSION=latest
-
+ENV CERAMIC_ONE_PATH=/usr/local/bin/ceramic-one CERAMIC_ONE_STORE_DIR=~/.ceramic-one
 RUN npm install --location=global @composedb/cli@${COMPOSEDB_VERSION}

packages/cli/src/__tests__/ceramic-daemon.test.ts

@@ -22,7 +22,11 @@ import { makeDID } from './make-did.js'
 import { makeCeramicCore } from './make-ceramic-core.js'
 import { makeCeramicDaemon } from './make-ceramic-daemon.js'
 import { DID } from 'dids'
-import { CommonTestUtils as TestUtils } from '@ceramicnetwork/common-test-utils'
+import {
+  CommonTestUtils as TestUtils,
+  describeIfV3,
+  testIfV3,
+} from '@ceramicnetwork/common-test-utils'
 import { EventSource } from 'cross-eventsource'
 import { AggregationDocument, JsonAsString } from '@ceramicnetwork/codecs'
 import { Model, ModelDefinition } from '@ceramicnetwork/stream-model'
@@ -31,10 +35,6 @@ import { decode } from 'codeco'
 
 const seed = 'SEED'
 
-// Should  pass on v4 if updated from TileDocument
-const describeIfV3 = process.env.CERAMIC_RECON_MODE ? describe.skip : describe
-const testIfV3 = process.env.CERAMIC_RECON_MODE ? test.skip : test
-
 describe('Ceramic interop: core <> http-client', () => {
   jest.setTimeout(30000)
   let ipfs: IpfsApi

packages/cli/src/__tests__/ceramic-error.test.ts

@@ -11,6 +11,7 @@ import { TileDocument } from '@ceramicnetwork/stream-tile'
 import * as path from 'path'
 import * as fs from 'fs'
 import { DaemonConfig, StateStoreMode } from '../daemon-config.js'
+import { testIfV3 } from '@ceramicnetwork/common-test-utils'
 
 const TOPIC = `/${random.randomString(10)}`
 const SEED = 'Hello, crypto!'
@@ -29,7 +30,6 @@ function safeRead(filepath: string): string {
   }
 }
 // TODO: Tests marked with IfV3 should passed in V' if updated from TileDocuments to Models/MIDs
-const testIfV3 = process.env.CERAMIC_RECON_MODE ? test.skip : test
 
 beforeAll(async () => {
   ipfs = await createIPFS()

packages/cli/src/__tests__/ceramic-multi-daemon.test.ts

@@ -11,6 +11,7 @@ import { createIPFS, swarmConnect } from '@ceramicnetwork/ipfs-daemon'
 import { makeDID } from './make-did.js'
 import { DaemonConfig } from '../daemon-config.js'
 import type { DID } from 'dids'
+import { describeIfV3 } from '@ceramicnetwork/common-test-utils'
 
 const seed = 'SEED'
 const TOPIC = '/ceramic'
@@ -41,7 +42,6 @@ const makeCeramicCore = async (ipfs: IpfsApi, stateStoreDirectory: string): Prom
 }
 
 // Should  pass on v4 if updated from TileDocument
-const describeIfV3 = process.env.CERAMIC_RECON_MODE ? describe.skip : describe
 
 describeIfV3('Ceramic interop between multiple daemons and http clients', () => {
   jest.setTimeout(20000)

packages/cli/src/daemon-config.ts

@@ -407,7 +407,6 @@ export class DaemonMetricsConfig {
    */
   @jsonMember(Number, { name: 'metrics-publish-interval-ms' })
   metricsPublishIntervalMS?: number
-
 }
 
 /**

packages/common-test-utils/src/index.ts

@@ -5,6 +5,7 @@ import type { StreamState, Stream } from '@ceramicnetwork/common'
 import {
   AdminApi,
   AnchorStatus,
+  EnvironmentUtils,
   EventType,
   RunningStateLike,
   SignatureStatus,
@@ -13,8 +14,8 @@ import {
 import first from 'it-first'
 import { BaseTestUtils } from '@ceramicnetwork/base-test-utils'
 
-export const testIfV3 = process.env['CERAMIC_RECON_MODE'] ? test.skip : test
-export const describeIfV3 = process.env['CERAMIC_RECON_MODE'] ? describe.skip : describe
+export const testIfV3 = EnvironmentUtils.useRustCeramic() ? test.skip : test
+export const describeIfV3 = EnvironmentUtils.useRustCeramic() ? describe.skip : describe
 
 class FakeRunningState extends BehaviorSubject<StreamState> implements RunningStateLike {
   readonly id: StreamID

packages/common/src/index.ts

@@ -6,6 +6,7 @@ export * from './stream.js'
 export * from './stream-reader.js'
 export * from './stream-state-loader.js'
 export * from './stream-writer.js'
+export * from './utils/environment-utils.js'
 export * from './utils/fetch-json.js'
 export * from './utils/stream-utils.js'
 export * from './utils/accountid-utils.js'

packages/common/src/utils/environment-utils.ts

@@ -0,0 +1,11 @@
+/**
+ * Environment related utils, that is information about the mode and system we're operating in.
+ */
+export class EnvironmentUtils {
+  /**
+   * Returns whether or not we're running using rust-ceramic
+   */
+  static useRustCeramic(): boolean {
+    return process.env.IPFS_FLAVOR !== 'go'
+  }
+}

packages/common/src/utils/stream-utils.ts

@@ -14,6 +14,7 @@ import {
   SignedCommitContainer,
   StreamState,
 } from '../stream.js'
+import { EnvironmentUtils } from '../utils/environment-utils.js'
 import type { DagJWS } from 'dids'
 import { CommitID, StreamID, StreamType } from '@ceramicnetwork/streamid'
 import { CID } from 'multiformats/cid'
@@ -282,7 +283,7 @@ export class StreamUtils {
     if (StreamUtils.isSignedCommit(commit)) {
       const block = await ipfs.block.get(toCID((commit as DagJWS).link), {
         // @ts-ignore
-        offline: process.env.CERAMIC_RECON_MODE == 'true',
+        offline: EnvironmentUtils.useRustCeramic(),
       })
       return {
         jws: commit as DagJWS,

packages/core/src/__tests__/caip10-link.test.ts

@@ -7,6 +7,7 @@ import MockDate from 'mockdate'
 import type { Ceramic } from '../ceramic.js'
 import { createIPFS } from '@ceramicnetwork/ipfs-daemon'
 import { createCeramic } from './create-ceramic.js'
+import { describeIfV3 } from '@ceramicnetwork/common-test-utils'
 
 const DID_USED = 'did:3:bafysdfwefwe'
 const LEGACY_ACCOUNT = '0x8fe2c4516e920425e177658aaac451ca0463ed69@eip155:1337'
@@ -40,7 +41,6 @@ const EMPTY_DID_PROOF = {
 }
 
 // These tests are not expected to run in v4 mode
-const describeIfV3 = process.env.CERAMIC_RECON_MODE ? describe.skip : describe
 
 describe('Ceramic API', () => {
   jest.setTimeout(60000)

packages/core/src/__tests__/ceramic-anchor.test.ts

@@ -6,7 +6,7 @@ import { createIPFS, swarmConnect } from '@ceramicnetwork/ipfs-daemon'
 import { TileDocument } from '@ceramicnetwork/stream-tile'
 import { InMemoryAnchorService } from '../anchor/memory/in-memory-anchor-service.js'
 import { createCeramic as vanillaCreateCeramic } from './create-ceramic.js'
-import { CommonTestUtils as TestUtils } from '@ceramicnetwork/common-test-utils'
+import { CommonTestUtils as TestUtils, describeIfV3 } from '@ceramicnetwork/common-test-utils'
 
 const SEED = '6e34b2e1a9624113d81ece8a8a22e6e97f0e145c25c1d4d2d0e62753b4060c83'
 
@@ -18,7 +18,6 @@ const createCeramic = async (ipfs: IpfsApi, anchorManual: boolean): Promise<Cera
 }
 
 // these should pass in v4 mode when recon integrated (need to be updated from TileDocument)
-const describeIfV3 = process.env.CERAMIC_RECON_MODE ? describe.skip : describe
 
 describeIfV3('Ceramic anchoring', () => {
   jest.setTimeout(60000)

packages/core/src/__tests__/ceramic-api.test.ts

@@ -11,7 +11,13 @@ import {
 import tmp from 'tmp-promise'
 import type { Ceramic } from '../ceramic.js'
 import { TileDocument } from '@ceramicnetwork/stream-tile'
-import { AnchorStatus, IpfsApi, StreamUtils, LoggerProvider } from '@ceramicnetwork/common'
+import {
+  AnchorStatus,
+  IpfsApi,
+  StreamUtils,
+  LoggerProvider,
+  EnvironmentUtils,
+} from '@ceramicnetwork/common'
 import { Utils as CoreUtils } from '@ceramicnetwork/core'
 import { CommitID, StreamID } from '@ceramicnetwork/streamid'
 import cloneDeep from 'lodash.clonedeep'
@@ -21,7 +27,7 @@ import { ModelInstanceDocument } from '@ceramicnetwork/stream-model-instance'
 import { Model, ModelDefinition } from '@ceramicnetwork/stream-model'
 import type { AddOperation } from 'fast-json-patch'
 import { InMemoryAnchorService } from '../anchor/memory/in-memory-anchor-service.js'
-import { CommonTestUtils } from '@ceramicnetwork/common-test-utils'
+import { CommonTestUtils, describeIfV3 } from '@ceramicnetwork/common-test-utils'
 
 /**
  * Generates string of particular size in bytes
@@ -38,7 +44,6 @@ const generateStringOfSize = (size): string => {
   return random_data.join('')
 }
 // Should  pass on v4 if updated from TileDocument
-const describeIfV3 = process.env.CERAMIC_RECON_MODE ? describe.skip : describe
 
 describe('Ceramic API', () => {
   jest.setTimeout(1000 * 30)
@@ -320,7 +325,7 @@ describe('Ceramic API', () => {
       // Stream. Once from loading the genesis commit, a second from applying the handling the
       // create. In v4 mode and going forward, there is only 1 call into the index since we don't
       // update persisted state on stream loads.
-      const NUM_INDEX_CALLS_PER_STREAM_CREATE = process.env.CERAMIC_RECON_MODE ? 1 : 2
+      const NUM_INDEX_CALLS_PER_STREAM_CREATE = EnvironmentUtils.useRustCeramic() ? 1 : 2
 
       expect(addIndexSpy).toBeCalledTimes(NUM_INDEX_CALLS_PER_STREAM_CREATE)
       const midMetadata = { model: model.id }
@@ -352,7 +357,7 @@ describe('Ceramic API', () => {
       // Stream. Once from loading the genesis commit, a second from applying the handling the
       // create. In v4 mode and going forward, there is only 1 call into the index since we don't
       // update persisted state on stream loads.
-      const NUM_INDEX_CALLS_PER_STREAM_CREATE = process.env.CERAMIC_RECON_MODE ? 1 : 2
+      const NUM_INDEX_CALLS_PER_STREAM_CREATE = EnvironmentUtils.useRustCeramic() ? 1 : 2
 
       expect(addIndexSpy).toBeCalledTimes(NUM_INDEX_CALLS_PER_STREAM_CREATE)
       const midMetadata = { model: model.id }
@@ -379,7 +384,7 @@ describe('Ceramic API', () => {
       // Stream. Once from loading the genesis commit, a second from applying the handling the
       // create. In v4 mode and going forward, there is only 1 call into the index since we don't
       // update persisted state on stream loads.
-      const NUM_INDEX_CALLS_PER_STREAM_CREATE = process.env.CERAMIC_RECON_MODE ? 1 : 2
+      const NUM_INDEX_CALLS_PER_STREAM_CREATE = EnvironmentUtils.useRustCeramic() ? 1 : 2
 
       expect(addIndexSpy).toBeCalledTimes(NUM_INDEX_CALLS_PER_STREAM_CREATE)
       const midMetadata = { model: model.id }
@@ -412,7 +417,7 @@ describe('Ceramic API', () => {
       // Stream. Once from loading the genesis commit, a second from applying the handling the
       // create. In v4 mode and going forward, there is only 1 call into the index since we don't
       // update persisted state on stream loads.
-      const NUM_INDEX_CALLS_PER_STREAM_CREATE = process.env.CERAMIC_RECON_MODE ? 1 : 2
+      const NUM_INDEX_CALLS_PER_STREAM_CREATE = EnvironmentUtils.useRustCeramic() ? 1 : 2
 
       expect(addIndexSpy).toBeCalledTimes(NUM_INDEX_CALLS_PER_STREAM_CREATE)
       const midMetadata = { model: model.id }

packages/core/src/__tests__/ceramic-feed.test.ts

@@ -38,18 +38,8 @@ describe('Ceramic feed', () => {
   let ceramic1: Ceramic
   let ceramic2: Ceramic
   beforeEach(async () => {
-    ipfs1 = await createIPFS({
-      rust: {
-        type: 'binary',
-        network: Networks.INMEMORY,
-      },
-    })
-    ipfs2 = await createIPFS({
-      rust: {
-        type: 'binary',
-        network: Networks.INMEMORY,
-      },
-    })
+    ipfs1 = await createIPFS()
+    ipfs2 = await createIPFS()
     ceramic1 = await createCeramic(ipfs1)
     ceramic2 = await createCeramic(ipfs2)
     await swarmConnect(ipfs2, ipfs1)

packages/core/src/__tests__/ceramic-pinning.test.ts

@@ -3,7 +3,7 @@ import { Ceramic, VersionInfo } from '../ceramic.js'
 import { Ed25519Provider } from 'key-did-provider-ed25519'
 import tmp from 'tmp-promise'
 import { IpfsApi, SyncOptions } from '@ceramicnetwork/common'
-import { CommonTestUtils as TestUtils } from '@ceramicnetwork/common-test-utils'
+import { CommonTestUtils as TestUtils, describeIfV3 } from '@ceramicnetwork/common-test-utils'
 import * as u8a from 'uint8arrays'
 import { createIPFS } from '@ceramicnetwork/ipfs-daemon'
 import { TileDocument } from '@ceramicnetwork/stream-tile'
@@ -84,7 +84,6 @@ async function createDeterministicStream(
 }
 
 // should pass on v4 if updated from tile document (possibly with adjustments for recon)
-const describeIfV3 = process.env.CERAMIC_RECON_MODE ? describe.skip : describe
 
 describeIfV3('Ceramic stream pinning', () => {
   jest.setTimeout(60000)

packages/core/src/__tests__/ceramic.test.ts

@@ -14,7 +14,7 @@ import { StreamID, CommitID } from '@ceramicnetwork/streamid'
 import { createIPFS, swarmConnect, withFleet } from '@ceramicnetwork/ipfs-daemon'
 import type { Ceramic } from '../ceramic.js'
 import { createCeramic as vanillaCreateCeramic } from './create-ceramic.js'
-import { CommonTestUtils as TestUtils } from '@ceramicnetwork/common-test-utils'
+import { CommonTestUtils as TestUtils, describeIfV3 } from '@ceramicnetwork/common-test-utils'
 
 const TEST_TIMEOUT = 1000 * 60 * 12 // 12 minutes
 
@@ -34,7 +34,6 @@ function expectEqualStates(a: StreamState, b: StreamState) {
 }
 
 // These tests are expected to pass when running in V4 mode when recon is integrated if updated from tile documents
-const describeIfV3 = process.env.CERAMIC_RECON_MODE ? describe.skip : describe
 
 describeIfV3('IPFS caching', () => {
   let ipfs: IpfsApi

packages/core/src/__tests__/dispatcher-mock-ipfs.test.ts

@@ -2,7 +2,7 @@ import { expect, jest, it, describe, beforeEach, afterEach } from '@jest/globals
 import { Dispatcher } from '../dispatcher.js'
 import { CID } from 'multiformats/cid'
 import { StreamID } from '@ceramicnetwork/streamid'
-import { EventType, StreamState, IpfsApi } from '@ceramicnetwork/common'
+import { EventType, StreamState, IpfsApi, EnvironmentUtils } from '@ceramicnetwork/common'
 import { CommonTestUtils as TestUtils, testIfV3 } from '@ceramicnetwork/common-test-utils'
 import { serialize, MsgType } from '../pubsub/pubsub-message.js'
 import { Repository } from '../state-management/repository.js'
@@ -56,7 +56,7 @@ const mock_ipfs = {
 
 const carFactory = new CARFactory()
 
-const isV3 = !process.env.CERAMIC_RECON_MODE
+const isV3 = !EnvironmentUtils.useRustCeramic()
 
 describe('Dispatcher with mock ipfs', () => {
   let dispatcher: Dispatcher

packages/core/src/__tests__/dispatcher-real-ipfs.test.ts

@@ -2,7 +2,7 @@ import { jest } from '@jest/globals'
 import { Dispatcher } from '../dispatcher.js'
 import { CID } from 'multiformats/cid'
 import { IpfsApi } from '@ceramicnetwork/common'
-import { CommonTestUtils as TestUtils } from '@ceramicnetwork/common-test-utils'
+import { CommonTestUtils as TestUtils, testIfV3 } from '@ceramicnetwork/common-test-utils'
 import { createIPFS } from '@ceramicnetwork/ipfs-daemon'
 import { StreamID } from '@ceramicnetwork/streamid'
 import { createDispatcher } from './create-dispatcher.js'
@@ -14,7 +14,6 @@ const FAKE_STREAM_ID = StreamID.fromString(
 )
 
 // A different scenario will likely apply in V4 based on changes to loading flow
-const testIfV3 = process.env.CERAMIC_RECON_MODE ? test.skip : test
 
 describe('Dispatcher with real ipfs over http', () => {
   jest.setTimeout(1000 * 30)

packages/core/src/__tests__/state-manager.test.ts

@@ -20,7 +20,7 @@ import { createCeramic } from './create-ceramic.js'
 import { Ceramic } from '../ceramic.js'
 import { TileDocument } from '@ceramicnetwork/stream-tile'
 import { InMemoryAnchorService } from '../anchor/memory/in-memory-anchor-service.js'
-import { CommonTestUtils as TestUtils } from '@ceramicnetwork/common-test-utils'
+import { CommonTestUtils as TestUtils, describeIfV3 } from '@ceramicnetwork/common-test-utils'
 
 const INITIAL_CONTENT = { abc: 123, def: 456 }
 const STRING_MAP_SCHEMA = {
@@ -33,7 +33,6 @@ const STRING_MAP_SCHEMA = {
 }
 
 //should pass in v4 if updated from TileDocument
-const describeIfV3 = process.env.CERAMIC_RECON_MODE ? describe.skip : describe
 
 describeIfV3('anchor', () => {
   let realHandleTip