Crust service and use the latest docker images

README.md

@@ -19,9 +19,10 @@ Official crust node service for running crust protocol.
 
 ### Install lib
 ```shell
+sudo apt install -y git jq curl wget build-essential kmod
 curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun
 sudo apt install docker-compose
-sudo apt install jq
+sudo apt install -y linux-headers-`uname -r`
 ```
 
 ### Install sgx driver

config.yaml

@@ -1,3 +1,12 @@
+## node configurations
+node:
+  ## the type of crust, for example: authority, full, light
+  chain: "authority"
+  ## enable or disable sworker
+  sworker: "enable"
+  ## enable or disable karst
+  karst: "disable"
+
 ## identity configurations
 identity:
   ## the backup of chain controller account
@@ -8,7 +17,7 @@ identity:
 ## chain configurations
 chain:
   ## chain data will be saved in this path, need absolute path
-  base_path: "/crust/chain"
+  base_path: "/opt/crust/data/chain"
   ## the name of chain node
   name: "vnode"
   ## the port
@@ -26,11 +35,11 @@ api:
 ## sworker configurations
 sworker:
   ## sworker base data will be saved in this path, need absolute path
-  base_path: "/crust/sworker"
+  base_path: "/opt/crust/data/sworker"
   ## the srd data will be saved in those paths, need absolute path
   srd_paths:
-    - "/crust/srd1"
-    - "/crust/srd2"
+    - "/opt/crust/data/sworker/srd1"
+    - "/opt/crust/data/sworker/srd2"
   ## the number of srd in GB, input must be an integer greater than 0
   srd_init_capacity: 1
   ## the port
@@ -39,9 +48,8 @@ sworker:
 ## karst configurations
 karst:
   ## karst data will be saved in this path, need absolute path
-  base_path: "/crust/karst"
+  base_path: "/opt/crust/data/karst"
   ## the thacker address
-  tracker_addrs:
-    - "127.0.0.1:22122"
+  tracker_addrs: "127.0.0.1:22122"
   ## the port
   port: 17000

generator/config-gen/chain-config.gen.js

@@ -11,16 +11,23 @@ async function genChainConfig(config, outputCfg) {
 }
 
 async function genChainComposeConfig(config) {
-  const args = [
+  let args = [
     `--base-path ${config.chain.base_path}`,
     '--chain maxwell',
-    '--validator',
     `--port ${config.chain.port}`,
     `--name ${config.chain.name}`,
     `--rpc-port ${config.chain.rpc_port}`,
     `--ws-port ${config.chain.ws_port}`,
-    '--pruning archive'
-  ].join(' ')
+  ]
+
+  if (config.node.chain == "authority") {
+    args.push('--validator', '--pruning archive')
+  } else if (config.node.chain == "light") {
+    args.push('--light')
+  }
+
+  args=args.join(' ')
+
   return {
     image: 'crustio/crust:latest',
     network_mode: 'host',
@@ -46,7 +53,7 @@ function getSharedChainConfig(config) {
 function getSharedChainConfigForKarst(config) {
   return {
     ...config.identity,
-    base_url: `127.0.0.1:${config.api.port}/api/v1`,
+    base_url: `127.0.0.1:${config.api.port}`,
     address: config.identity.backup.address,
     backup: JSON.stringify(config.identity.backup),
   }

generator/config-gen/index.js

@@ -55,6 +55,9 @@ async function genConfig(config, outputOpts) {
   let outputs = []
   const { baseDir } = outputOpts
   for (const cg of configGenerators) {
+    if (!config[cg.name]) {
+      continue
+    }
     logger.info('generating config for %s', cg.name)
     const ret = await cg.configFunc(config, outputOpts)
     await writeConfig(path.join(baseDir, cg.to), ret.config)
@@ -76,6 +79,9 @@ async function genComposeConfig(config) {
   }
 
   for (const cg of configGenerators) {
+    if (!config[cg.name]) {
+      continue
+    }
     logger.info('generating compose config for %s', cg.name)
     const cfg = await cg.composeFunc(config)
     output = {

generator/config-gen/karst-config.gen.js

@@ -4,14 +4,20 @@ const { getSharedChainConfigForKarst } = require('./chain-config.gen')
 async function genKarstConfig(config, outputCfg) {
   const karstConfig = {
     base_path: config.karst.base_path,
-    base_url: `0.0.0.0:${config.karst.port}`,
+    port: config.karst.port,
+    debug: true,
     crust: getSharedChainConfigForKarst(config),
-    fastdfs: {
-      max_conns: 100,
-      tracker_addrs: config.karst.tracker_addrs,
+    sworker: {
+      base_url: `127.0.0.1:${config.sworker.port}`
     },
-    log_level: 'debug',
-    tee_base_url: `127.0.0.1:${config.sworker.port}/api/v0`,
+    file_system: {
+      fastdfs: {
+        tracker_addrs: config.karst.tracker_addrs
+      },
+      ipfs: {
+        base_url: ""
+      }
+    }
   }
   const basePaths = _.isEmpty(config.karst.base_path) ? [] : [{
     required: true,

generator/config-gen/sworker-config.gen.js

@@ -3,10 +3,15 @@ const { createDir, writeConfig, } = require('../utils')
 const { getSharedChainConfig } = require('./chain-config.gen')
 
 async function genSworkerConfig(config, outputCfg) {
+  karstPort = 17000
+  if (config.karst) {
+    karstPort = config.karst.port
+  }
+
   const sworkerConfig = {
     ..._.omit(config.sworker, ['port']),
     base_url: `http://0.0.0.0:${config.sworker.port}/api/v0`,
-    karst_url: `ws://127.0.0.1:${config.karst.port}/api/v0/node/data`,
+    karst_url: `ws://127.0.0.1:${karstPort}/api/v0/node/data`,
     chain: getSharedChainConfig(config),
   }
   const srdPaths = _.map(config.sworker.srdPaths, (p) => ({
@@ -32,7 +37,7 @@ async function genSworkerComposeConfig(config) {
     './sworker:/config'
   ]
 
-  if (config.karst.base_path) {
+  if (config.karst) {
     tempVolumes.push(`${config.karst.base_path}:${config.karst.base_path}`)
   }
 

generator/index.js

@@ -4,7 +4,7 @@ const fs = require('fs-extra')
 const path = require('path')
 const yaml = require('js-yaml')
 const { genConfig, genComposeConfig } = require('./config-gen')
-const { configSchema } = require('./schema')
+const { getConfigSchema } = require('./schema')
 const { validate } = require('./config-validator')
 const { logger } = require('./logger')
 const { inspectKey } = require('./key-utils')
@@ -16,13 +16,18 @@ async function loadConfig(file) {
   logger.debug('loading config file: %s', file)
   const c = await fs.readFile('config.yaml', 'utf8')
   const config = yaml.safeLoad(c)
+  const configSchema = getConfigSchema(config)
   const value = await configSchema.validateAsync(config, {
     allowUnknown: true,
+    stripUnknown: true,
   })
-  logger.debug('got config: %o', value)
-  const keyInfo = await inspectKey(value.identity.backup.address)
-  logger.info('key info: %o', keyInfo)
-  value.identity.account_id = keyInfo.accountId
+  logger.debug('get config: %o', value)
+
+  if (value.node.sworker == "enable") {
+    const keyInfo = await inspectKey(value.identity.backup.address)
+    logger.info('key info: %o', keyInfo)
+    value.identity.account_id = keyInfo.accountId
+  }
 
   const data = await genConfig(value, {
     baseDir: '.tmp',

generator/schema/api.schema.js

@@ -1,7 +1,7 @@
 const Joi = require('joi')
 
 const apiSchema = Joi.object({
-  port: Joi.number().port().default(5666),
+  port: Joi.number().port().default(56666),
 })
 
 module.exports = {

generator/schema/identity.schema.js

@@ -7,7 +7,7 @@ const backupSchema = Joi.object({
   encoded: Joi.string().required(),
   encoding: Joi.object({
     content: Joi.array().items(Joi.string()).min(1).required(),
-    type: Joi.string().required(),
+    type: Joi.array().items(Joi.string()).required(),
     version: Joi.string().required(),
   }).required(),
   meta: Joi.object({

generator/schema/index.js

@@ -5,15 +5,29 @@ const { chainSchema } = require('./chain.schema')
 const { karstSchema } = require('./karst.schema')
 const { identitySchema } = require('./identity.schema')
 const { sworkerSchema } = require('./sworker.schema')
+const { nodeSchema } = require('./node.schema')
 
-const configSchema = Joi.object({
-  identity: identitySchema.required(),
-  chain: chainSchema.required(),
-  api: apiSchema.required(),
-  sworker: sworkerSchema.required(),
-  karst: karstSchema.required(),
-})
+function getConfigSchema(config) {
+  sMap = {
+    node: nodeSchema.required(),
+    chain: chainSchema.required(),
+  }
+
+  if (config.node.sworker != "enable") {
+    return Joi.object(sMap)
+  }
+  sMap["api"] = apiSchema.required()
+  sMap["identity"] = identitySchema.required()
+  sMap["sworker"] = sworkerSchema.required()
+
+  if (config.node.karst != "enable") {
+    return Joi.object(sMap)
+  }
+  sMap["karst"] = karstSchema.required()
+
+  return Joi.object(sMap)
+}
 
 module.exports = {
-  configSchema,
+  getConfigSchema,
 }

generator/schema/karst.schema.js

@@ -3,7 +3,7 @@ const Joi = require('joi')
 
 const karstSchema = Joi.object({
   base_path: Joi.string().required(),
-  tracker_addrs: Joi.array().items(Joi.string()),
+  tracker_addrs: Joi.string().required(),
   port: Joi.number().port().default(17000),
 })
 

generator/schema/node.schema.js

@@ -0,0 +1,11 @@
+const Joi = require('joi')
+
+const nodeSchema = Joi.object({
+  chain: Joi.string().valid('authority', 'full', 'light').required(),
+  sworker: Joi.string().valid('enable', 'disable').required(),
+  karst: Joi.string().valid('enable', 'disable').required(),
+})
+
+module.exports = {
+  nodeSchema,
+}

install.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+basedir=$(cd `dirname $0`;pwd)
+scriptdir=$basedir/scripts
+installdir=/opt/crust/crust-node
+source $scriptdir/utils.sh
+
+if [ $(id -u) -ne 0 ]; then
+    log_err "Please run with sudo!"
+    exit 1
+fi
+
+log_info "------------Apt update--------------"
+apt-get update
+if [ $? -ne 0 ]; then
+    log_err "Apt update failed"
+    exit 1
+fi
+
+log_info "------------Install depenencies--------------"
+apt install -y git jq curl wget build-essential kmod linux-headers-`uname -r`
+
+if [ $? -ne 0 ]; then
+    log_err "Install libs failed"
+    exit 1
+fi
+
+docker-compose -v
+if [ $? -ne 0 ]; then
+    curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun
+    if [ $? -ne 0 ]; then
+        log_err "Install docker failed"
+        exit 1
+    fi
+    apt install docker-compose
+    if [ $? -ne 0 ]; then
+        log_err "Install docker compose failed"
+        exit 1
+    fi
+fi
+
+log_info "-------Download crust docker images----------"
+res=0
+docker pull crustio/config-generator
+res=$(($?|$res))
+docker pull crustio/crust
+res=$(($?|$res))
+docker pull crustio/crust-api
+res=$(($?|$res))
+docker pull crustio/crust-sworker
+res=$(($?|$res))
+docker pull crustio/karst
+res=$(($?|$res))
+if [ $res -ne 0 ]; then
+    log_err "Install docker failed"
+    exit 1
+fi
+
+log_info "--------------Install crust node-------------"
+
+echo "uninstall old crust node"
+./scripts/uninstall.sh
+
+echo "Install crust node data"
+mkdir -p $installdir
+cp -r scripts $installdir/
+cp config.yaml $installdir/
+
+echo "Install crust service"
+cp services/crust.service /lib/systemd/system/
+systemctl daemon-reload
+
+log_success "------------Install success-------------"