Merge remote-tracking branch 'upstream/master'

.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+- package-ecosystem: npm
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10
+  ignore:
+  - dependency-name: karma
+    versions:
+    - 6.0.0
+    - 6.1.0
+    - 6.1.2
+    - 6.2.0
+  - dependency-name: semver
+    versions:
+    - 7.3.4
+  - dependency-name: style-loader
+    versions:
+    - 1.2.1
+  - dependency-name: express-validator
+    versions:
+    - 6.9.2

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 3.6.1 (2021-03-11)
+
+### Fixes
+
+- Revert adbkit upgrade
+
+## 3.6.0 (2021-03-09)
+
+### Fixes
+
+- Fix AAB installation fails when /tmp is on different mountpoint than storage-temp save directory [#139](https://github.com/DeviceFarmer/stf/pull/139)
+
+### Enhancements
+
+- Update dependency versions
+
 ## 3.5.0 (2020-10-18)
 
 ### Fixes

Dockerfile

@@ -32,7 +32,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     tar -xJf node-v*.tar.xz --strip-components 1 -C /usr/local && \
     rm node-v*.tar.xz && \
     su stf-build -s /bin/bash -c '/usr/local/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js install' && \
-    apt-get -y install libzmq3-dev libprotobuf-dev git graphicsmagick openjdk-8-jdk yasm && \
+    apt-get -y install libzmq3-dev libprotobuf-dev git graphicsmagick openjdk-8-jdk yasm cmake && \
     apt-get clean && \
     rm -rf /var/cache/apt/* /var/lib/apt/lists/* && \
     mkdir /tmp/bundletool && \

Dockerfile-debian-armhf

@@ -45,7 +45,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     npm config set unsafe-perm true && \
     npm install -g npm && \
     echo 'npm cache clean --force' | su stf -s /bin/bash && \
-    echo 'npm install --no-optional --loglevel http' | su stf -s /bin/bash && \
+    echo 'npm install --no-optional --legacy-peer-deps --loglevel http' | su stf -s /bin/bash && \
     echo '--- Assembling app' && \
     echo 'npm pack' | su stf -s /bin/bash && \
     tar xzf devicefarmer-stf-*.tgz --strip-components 1 -C /app && \

Dockerfile-debian-x86_64

@@ -34,16 +34,15 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
       --shell /usr/sbin/nologin \
       stf && \
     su stf -s /bin/bash -c '/usr/local/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js install' && \
-    apt-get -y install --no-install-recommends libzmq3-dev libprotobuf-dev git graphicsmagick yasm && \
+    apt-get -y install --no-install-recommends libzmq3-dev libprotobuf-dev git graphicsmagick yasm cmake && \
     echo '--- Building app' && \
     mkdir -p /app && \
     chown -R stf:stf /tmp/build && \
     set -x && \
     cd /tmp/build && \
     export PATH=$PWD/node_modules/.bin:$PATH && \
     sed -i'' -e '/phantomjs/d' package.json && \
-    npm install -g npm && \
-    echo 'npm install --no-optional --loglevel http' | su stf -s /bin/bash && \
+    echo 'npm install --no-optional --loglevel http --legacy-peer-deps' | su stf -s /bin/bash && \
     echo '--- Assembling app' && \
     echo 'npm pack' | su stf -s /bin/bash && \
     tar xzf devicefarmer-stf-*.tgz --strip-components 1 -C /app && \

README.md

@@ -112,6 +112,7 @@ As the product has evolved from an internal tool running in our internal network
 * [Node.js](https://nodejs.org/) 8.x **required** (some dependencies don't support newer versions)
 * [ADB](http://developer.android.com/tools/help/adb.html) properly set up
 * [RethinkDB](http://rethinkdb.com/) >= 2.2
+* [CMake](https://cmake.org/) >= 3.9 (for [node-jpeg-turbo](https://github.com/julusian/node-jpeg-turbo#readme))
 * [GraphicsMagick](http://www.graphicsmagick.org/) (for resizing screenshots)
 * [ZeroMQ](http://zeromq.org/) libraries installed
 * [Protocol Buffers](https://github.com/google/protobuf) libraries installed
@@ -123,12 +124,13 @@ Note that you need these dependencies even if you've installed STF directly from
 On Mac OS, you can use [homebrew](http://brew.sh/) to install most of the dependencies:
 
 ```bash
-brew install rethinkdb graphicsmagick zeromq protobuf yasm pkg-config
+brew install rethinkdb graphicsmagick zeromq protobuf yasm pkg-config cmake
 ```
 
 On Windows you're on your own. In theory you might be able to get STF installed via [Cygwin](https://www.cygwin.com/) or similar, but we've never tried. In principle we will not provide any Windows installation support, but please do send a documentation pull request if you figure out what to do.
 
-We also provide a [Docker](http://docker.com/) container in the [Docker Hub](https://hub.docker.com/) as [openstf/stf](https://registry.hub.docker.com/u/openstf/stf/). You can use our [Dockerfile](Dockerfile) as guidance if you'd prefer to do the installation yourself.
+We also provide a [Docker](http://docker.com/) container in the [Docker Hub](https://hub.docker.com/) as [devicefarmer/stf](https://registry.hub.docker.com/u/devicefarmer/stf/). You can use our [Dockerfile](Dockerfile) as guidance if you'd prefer to do the installation yourself.
+An example standalone [docker-compose.yaml](docker-compose.yaml) file is also provided.
 
 You should now be ready to [build](#building) or [run](#running) STF.
 

bower.json

@@ -25,7 +25,7 @@
     "angular-hotkeys": "chieffancypants/angular-hotkeys#~1.6.0",
     "angular-borderlayout": "git://github.com/filearts/angular-borderlayout.git#7c9716aebd9260763f798561ca49d6fbfd4a5c67",
     "angular-ui-bootstrap": "~1.1.1",
-    "ng-context-menu": "AdiDahan/ng-context-menu#~1.0.5",
+    "ng-context-menu": "swimlane/ng-context-menu#~1.0.1",
     "components-font-awesome": "~4.5.0",
     "epoch": "~0.8.4",
     "ng-epoch": "~1.0.7",

doc/DEPLOYMENT.md

@@ -87,14 +87,15 @@ Requires=docker.service
 [Service]
 TimeoutStartSec=0
 Restart=always
+RestartSec=3
 ExecStartPre=/usr/bin/docker pull sorccu/adb:latest
 ExecStartPre=-/usr/bin/docker kill %p
 ExecStartPre=-/usr/bin/docker rm %p
 ExecStart=/usr/bin/docker run --rm \
   --name %p \
   --privileged \
+  -p 5037:5037 \
   -v /dev/bus/usb:/dev/bus/usb \
-  --net host \
   sorccu/adb:latest
 ExecStop=/usr/bin/docker exec %p adb kill-server
 ```
@@ -407,14 +408,14 @@ The provider unit connects to ADB and start worker processes for each device. It
 
 The name of the provider shows up in the device list, making it easier to see where the physical devices are located. In this configuration the name is set to the hostname.
 
-Note that the provider needs to be able to manage a certain port range, so `--net host` is required until Docker makes it easier to work with ranges. The ports are used for internal services and the screen capturing WebSocket.
-
 This is a template unit, meaning that you'll need to start it with an instance identifier. In this example configuration the identifier is used to specify the provider ID, which can then be matched against in the [nginx](http://nginx.org/) configuration later on. The ID should be unique and persistent. This is only one way to set things up, you may choose to do things differently if it seems sketchy.
 
 Note that you cannot have more than one provider unit running on the same host, as they would compete over which one gets to control the devices. In the future we might add a negotiation protocol to allow for relatively seamless upgrades.
 
 Furthermore, if you're using a self-signed cert, you may have to add `-e "NODE_TLS_REJECT_UNAUTHORIZED=0"` to the `docker run` command. Don't forget to end the line with `\`.
 
+`--net host` is no longer required since newer Docker versions allow port ranges to be specified.
+
 ```ini
 [Unit]
 Description=STF provider
@@ -425,12 +426,14 @@ BindsTo=adbd.service
 EnvironmentFile=/etc/environment
 TimeoutStartSec=0
 Restart=always
+RestartSec=3
 ExecStartPre=/usr/bin/docker pull openstf/stf:latest
 ExecStartPre=-/usr/bin/docker kill %p-%i
 ExecStartPre=-/usr/bin/docker rm %p-%i
 ExecStart=/usr/bin/docker run --rm \
   --name %p-%i \
-  --net host \
+  --link adbd:adbd \
+  -p 15000-25000:15000-25000 \
   openstf/stf:latest \
   stf provider \
     --name "%H/%i" \
@@ -441,7 +444,8 @@ ExecStart=/usr/bin/docker run --rm \
     --min-port=15000 \
     --max-port=25000 \
     --heartbeat-interval 10000 \
-    --screen-ws-url-pattern "wss://stf.example.org/d/%i/<%= serial %>/<%= publicPort %>/"
+    --screen-ws-url-pattern "wss://stf.example.org/d/%i/<%= serial %>/<%= publicPort %>/" \
+    --adb-host adbd
 ExecStop=-/usr/bin/docker stop -t 10 %p-%i
 ```
 

docker-compose.yaml

@@ -0,0 +1,36 @@
+version: "3"
+
+services:
+  rethinkdb:
+    container_name: rethinkdb
+    image: rethinkdb:2.3
+    restart: unless-stopped
+    volumes:
+        - "rethinkdb-data:/data"
+    command: "rethinkdb --bind all --cache-size 2048"
+
+  adb:
+    container_name: adb
+    image: devicefarmer/adb:latest
+    restart: unless-stopped
+    volumes: 
+      - "/dev/bus/usb:/dev/bus/usb"
+    privileged: true
+
+  stf:
+    container_name: stf
+    image: devicefarmer/stf
+    ports:
+      - "7100:7100"
+      - "7110:7110"
+      - "7400-7500:7400-7500"
+    environment:
+      - TZ='America/Los_Angeles'
+      - RETHINKDB_PORT_28015_TCP=tcp://rethinkdb:28015
+      - STF_ADMIN_EMAIL=<YOUR_EMAIL>
+      - STF_ADMIN_NAME=<YOUR_NAME>
+    restart: unless-stopped
+    command: stf local --adb-host adb --public-ip YOUR_IP --provider-min-port 7400 --provider-max-port 7500
+
+volumes:
+  rethinkdb-data: {}

lib/cli/device/index.js

@@ -97,6 +97,12 @@ module.exports.builder = function(yargs) {
     , type: 'number'
     , default: process.env.SCREEN_JPEG_QUALITY || 80
     })
+    .option('screen-grabber', {
+      describe: 'The tool to be used for screen capture. ' +
+        'Value must be either: minicap-bin (default) or minicap-apk'
+    , type: 'string'
+    , default: process.env.SCREEN_GRABBER || 'minicap-bin'
+    })
     .option('screen-ping-interval', {
       describe: 'The interval at which to send ping messages to keep the ' +
         'screen WebSocket alive.'
@@ -159,6 +165,7 @@ module.exports.handler = function(argv) {
   , adbHost: argv.adbHost
   , adbPort: argv.adbPort
   , screenJpegQuality: argv.screenJpegQuality
+  , screenGrabber: argv.screenGrabber
   , screenPingInterval: argv.screenPingInterval
   , screenPort: argv.screenPort
   , screenWsUrlPattern: argv.screenWsUrlPattern

lib/cli/provider/index.js

@@ -116,6 +116,12 @@ module.exports.builder = function(yargs) {
     , type: 'number'
     , default: process.env.SCREEN_JPEG_QUALITY || 80
     })
+    .option('screen-grabber', {
+      describe: 'The tool to be used for screen capture. ' +
+        'Value must be either: minicap-bin (default) or minicap-apk'
+    , type: 'string'
+    , default: process.env.SCREEN_GRABBER || 'minicap-bin'
+    })
     .option('screen-ping-interval', {
       describe: 'The interval at which to send ping messages to keep the ' +
         'screen WebSocket alive.'

lib/units/api/controllers/devices.js

@@ -459,6 +459,32 @@ function removeOriginGroupDevice(req, res) {
   apiutil.redirectApiWrapper('serial', removeOriginGroupDevices, req, res)
 }
 
+function putDeviceInfoBySerial(req, res) {
+  const serial = req.swagger.params.serial.value
+  const body = req.swagger.params.device.value.device
+  dbapi.loadDeviceBySerial(serial)
+    .then((data) => {
+      if (!data) {
+        return apiutil.respond(res, 404, `Not Found (${serial})`)
+      }
+      var updates = []
+      // Update fields based on given body
+      if (_.has(body, 'note')) {
+        updates.push(dbapi.setDeviceNote(serial, body.note))
+      }
+      if (updates.length === 0) {
+        return apiutil.respond(res, 400, 'No content to update')
+      }
+      return Promise.all(updates)
+        .then(function() {
+            apiutil.respond(res, 200)
+        })
+    })
+    .catch(function(err) {
+      apiutil.internalError(res, 'Failed to update device: ', err.stack)
+    })
+}
+
 function deleteDevices(req, res) {
   const serials = apiutil.getBodyParameter(req.body, 'serials')
   const target = apiutil.getQueryParameter(req.swagger.params.redirected) ? 'device' : 'devices'
@@ -509,12 +535,17 @@ function deleteDevices(req, res) {
   })
 }
 
+function putDeviceBySerial(req, res) {
+  apiutil.redirectApiWrapper('serial', putDeviceInfoBySerial, req, res)
+}
+
 function deleteDevice(req, res) {
   apiutil.redirectApiWrapper('serial', deleteDevices, req, res)
 }
 
 module.exports = {
   getDevices: getDevices
+, putDeviceBySerial: putDeviceBySerial
 , getDeviceBySerial: getDeviceBySerial
 , getDeviceGroups: getDeviceGroups
 , getDeviceBookings: getDeviceBookings