first commit

.eslintrc.json

@@ -0,0 +1,14 @@
+{
+  "extends": ["prettier"],
+  "plugins": ["prettier"],
+  "rules": {
+    "prettier/prettier": "error"
+  },
+  "parserOptions": {
+    "ecmaVersion": 2018,
+    "sourceType": "module",
+    "ecmaFeatures": {
+      "jsx": true
+    }
+  }
+}

.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+static/
+yarn-error.log

.prettierrc

@@ -0,0 +1,5 @@
+{
+  "semi": false,
+  "singleQuote": true,
+  "trailingComma": "all"
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Vitor Capretz (capretzvitor@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,48 @@
+# background-jobs
+
+This package provides a queue manager and a UI to inspect jobs. The manager is based on [bull](https://github.com/OptimalBits/bull) and it depends on redis.
+
+## Architecture
+
+Before using background-jobs, please consider if it is the right solution for your problem. Here's a bunch of cool things it can do:
+
+- Run arbitrary or scheduled (like in cron) jobs in the background.
+- Set prios, retry, pause, resume and limit the rate of your jobs.
+- Concurrently run in a sandboxed process that will not crash your app.
+- Render a UI in your app with all the info you'll need to manage the jobs.
+
+Most background jobs tools depend on multiple processes to work. This means you would need to use something like a Procfile to keep your server and all your workers running. background-jobs will use your node service process as the manager of executions and (if you so wish) spawn child processes in your instances to deal with the workload.
+
+On the other hand this is also one of its disadvantages. That is, you probably should not use background-jobs if:
+
+- you can't afford your server to be doing something else than serving requests.
+  - in this case, consider having a separate service to be a queue manager, or lambdas.
+- you need to aggressively scale your workers independently of your instances.
+  - in this case, probably lambdas.
+- all you need is a simple cron job (UI is just a luxury).
+  - use a cron job. perhaps [node-cron](https://www.npmjs.com/package/node-cron)?
+
+## Hello world
+
+```js
+const { createQueues, UI } = require("background-jobs");
+
+const queues = createQueues(redisConfig); // sets up your queues
+
+const helloQueue = queues.add("helloQueue"); // adds a queue
+helloQueue.process(async job => {
+  console.log(`Hello ${job.data.hello}`);
+}); // defines how the queue works
+
+helloQueue.add({ hello: "world" }); // adds a job to the queue
+
+app.use("/queues", UI); // serves the UI at /queues, don't forget authentication!
+```
+
+## Further ref
+
+For further ref, please check [bull's docs](https://github.com/OptimalBits/bull). Appart from the way you config and start your UI, this library doesn't hijack bull's way of working.
+
+## UI
+
+![UI](./shot.png)

example.js

@@ -0,0 +1,43 @@
+const { createQueues, UI } = require('./')
+const app = require('express')()
+
+const sleep = t => new Promise(resolve => setTimeout(resolve, t * 1000))
+
+const redisOptions = {
+  redis: {
+    port: 6379,
+    host: 'localhost',
+    password: '',
+    tls: false,
+  },
+}
+
+const run = () => {
+  const queues = createQueues(redisOptions)
+
+  const example = queues.add('example')
+
+  example.process(async job => {
+    for (let i = 0; i <= 100; i++) {
+      await sleep(Math.random())
+      job.progress(i)
+      if (Math.random() * 200 < 1) throw new Error(`Random error ${i}`)
+    }
+  })
+
+  app.use('/add', (req, res) => {
+    example.add({ title: req.query.title })
+    res.json({ ok: true })
+  })
+
+  app.use('/ui', UI)
+  app.listen(3000, () => {
+    console.log('Running on 3000...')
+    console.log('For the UI, open http://localhost:3000/ui')
+    console.log('Make sure Redis is running on port 6379 by default')
+    console.log('To populate the queue, run:')
+    console.log('  curl http://localhost:3000/add?title=Example')
+  })
+}
+
+run()

index.js

@@ -0,0 +1,41 @@
+const Queue = require('bull')
+const express = require('express')
+const bodyParser = require('body-parser')
+const router = require('express-async-router').AsyncRouter()
+const path = require('path')
+
+const queues = {}
+
+function UI() {
+  const app = express()
+
+  app.locals.queues = queues
+
+  app.set('view engine', 'ejs')
+  app.set('views', `${__dirname}/ui`)
+
+  router.use('/static', express.static(path.join(__dirname, './static')))
+  router.get('/queues', require('./routes/queues'))
+  router.put('/queues/:queueName/retry', require('./routes/retryAll'))
+  router.put('/queues/:queueName/:id/retry', require('./routes/retryJob'))
+  router.get('/', require('./routes/index'))
+
+  app.use(bodyParser.json())
+  app.use(router)
+
+  return app
+}
+
+module.exports = {
+  UI: UI(),
+  createQueues: redis => {
+    return {
+      add: name => {
+        const queue = new Queue(name, redis)
+        queues[name] = queue
+
+        return queue
+      },
+    }
+  },
+}

package.json

@@ -0,0 +1,39 @@
+{
+  "name": "background-jobs",
+  "description": "Background jobs inspector",
+  "main": "index.js",
+  "private": false,
+  "version": "0.1.0",
+  "license": "MIT",
+  "dependencies": {
+    "body-parser": "^1.17.2",
+    "bull": "3.6.0",
+    "ejs": "^2.6.1",
+    "express": "^4.15.2",
+    "express-async-router": "^0.1.15",
+    "pretty-bytes": "^5.1.0",
+    "ramda": "^0.26.1"
+  },
+  "scripts": {
+    "build": "NODE_ENV=production webpack",
+    "start:example": "node example.js",
+    "start": "yarn watch & yarn start:example",
+    "watch": "NODE_ENV=production webpack --watch"
+  },
+  "devDependencies": {
+    "babel-preset-react-app": "^7.0.2",
+    "css-loader": "^2.1.1",
+    "eslint": "^6.2.1",
+    "eslint-config-prettier": "^6.1.0",
+    "eslint-plugin-prettier": "^3.1.0",
+    "prettier": "^1.18.2",
+    "react": "^16.8.6",
+    "react-dev-utils": "^8.0.0",
+    "react-dom": "^16.8.6",
+    "react-json-syntax-highlighter": "^0.2.0",
+    "style-loader": "^0.23.1",
+    "webpack": "^4.29.6",
+    "webpack-cli": "^3.3.0",
+    "webpack-manifest-plugin": "^2.0.4"
+  }
+}

routes/getDataForQeues.js

@@ -0,0 +1,73 @@
+const { pick, isEmpty } = require('ramda')
+
+const metrics = [
+  'redis_version',
+  'used_memory',
+  'mem_fragmentation_ratio',
+  'connected_clients',
+  'blocked_clients',
+]
+
+async function getStats(queue) {
+  await queue.client.info()
+
+  const validMetrics = pick(metrics, queue.client.serverInfo)
+  validMetrics.total_system_memory =
+    queue.client.serverInfo.total_system_memory ||
+    queue.client.serverInfo.maxmemory
+
+  return validMetrics
+}
+
+const formatJob = job => {
+  return {
+    id: job.id,
+    timestamp: job.timestamp,
+    processedOn: job.processedOn,
+    finishedOn: job.finishedOn,
+    progress: job._progress,
+    attempts: job.attemptsMade,
+    failedReason: job.failedReason,
+    stacktrace: job.stacktrace,
+    opts: job.opts,
+    data: job.data,
+  }
+}
+
+const statuses = [
+  'active',
+  'waiting',
+  'completed',
+  'failed',
+  'delayed',
+  'paused',
+]
+
+module.exports = async function getDataForQeues({ queues, query = {} }) {
+  if (isEmpty(queues)) {
+    return { stats: {}, queues: [] }
+  }
+
+  const pairs = Object.entries(queues)
+
+  const counts = await Promise.all(
+    pairs.map(async ([name, queue]) => {
+      const counts = await queue.getJobCounts()
+
+      let jobs = []
+      if (name) {
+        const status = query[name] === 'latest' ? statuses : query[name]
+        jobs = await queue.getJobs(status, 0, 10)
+      }
+
+      return {
+        name,
+        counts,
+        jobs: jobs.map(formatJob),
+      }
+    }),
+  )
+  const stats = await getStats(pairs[0][1])
+
+  return { stats, queues: counts }
+}

routes/index.js

@@ -0,0 +1,3 @@
+module.exports = async (req, res) => {
+  res.render('index', { basePath: req.baseUrl })
+}

routes/retryAll.js

@@ -0,0 +1,22 @@
+module.exports = async function handler(req, res) {
+  try {
+    const { queueName } = req.params
+    const { queues } = req.app.locals
+
+    const queue = queues[queueName]
+    if (!queue) {
+      return res.status(404).send({ error: 'queue not found' })
+    }
+
+    const jobs = await queue.getJobs('failed')
+    await Promise.all(jobs.map(job => job.retry()))
+
+    return res.sendStatus(200)
+  } catch (e) {
+    const body = {
+      error: 'queue error',
+      details: e.stack,
+    }
+    return res.status(500).send(body)
+  }
+}

routes/retryJob.js

@@ -0,0 +1,27 @@
+module.exports = async function retryJob(req, res) {
+  try {
+    const { queues } = req.app.locals
+    const { queueName, id } = req.params
+
+    const queue = queues[queueName]
+
+    if (!queue) {
+      return res.status(404).send({ error: 'queue not found' })
+    }
+
+    const job = await queue.getJob(id)
+
+    if (!job) {
+      return res.status(404).send({ error: 'job not found' })
+    }
+
+    await job.retry()
+    return res.sendStatus(204)
+  } catch (e) {
+    const body = {
+      error: 'queue error',
+      details: e.stack,
+    }
+    return res.status(500).send(body)
+  }
+}

ui/components/App.js

@@ -0,0 +1,40 @@
+import React from 'react'
+import Queue from './Queue'
+import RedisStats from './RedisStats'
+import Header from './Header'
+import useStore from './hooks/useStore'
+
+export default function App({ basePath }) {
+  const {
+    state,
+    selectedStatuses,
+    setSelectedStatuses,
+    retryJob,
+    retryAll,
+  } = useStore(basePath)
+
+  return (
+    <>
+      <Header />
+      <main>
+        {state.loading ? (
+          'Loading...'
+        ) : (
+          <>
+            <RedisStats stats={state.data.stats} />
+            {state.data.queues.map(queue => (
+              <Queue
+                queue={queue}
+                key={queue.name}
+                selectedStatus={selectedStatuses[queue.name]}
+                selectStatus={setSelectedStatuses}
+                retryJob={retryJob(queue.name)}
+                retryAll={retryAll(queue.name)}
+              />
+            ))}
+          </>
+        )}
+      </main>
+    </>
+  )
+}