fix ci

.github/actions/compile/action.yml

@@ -0,0 +1,79 @@
+name: 'Compile libfibre'
+description: 'Compiles libfibre for the specified target'
+inputs:
+  target:
+    description: |
+      The build target on which the compiled libfibre is supposed to run.
+      Supported values are:
+        - local (the runner must be ubuntu-latest)
+    required: true
+    default: local
+runs:
+  using: "composite"
+  steps:
+    - name: Install Prerequisites
+      shell: bash
+      run: |
+        UNAME=$(uname)
+
+        if [ "$UNAME" == "Linux" ]; then
+          echo "installing tup..."
+          if ! (apt-cache search tup | grep "^tup - "); then
+            sudo add-apt-repository ppa:jonathonf/tup
+            sudo apt-get update
+          fi
+          sudo apt-get install tup
+
+          echo "installing interface_generator dependencies..."
+          sudo apt install python3 python3-yaml python3-jinja2 python3-jsonschema
+
+          echo "installing fibre-cpp build dependencies..."
+          sudo apt-get install libusb-1.0-0-dev
+
+        elif [ "$UNAME" == "Darwin" ]; then
+          echo "installing tup..."
+          brew cask install osxfuse && brew install tup
+
+          echo "installing interface_generator dependencies..."
+          pip3 install PyYAML Jinja2 jsonschema
+
+          echo "installing fibre-cpp build dependencies..."
+          # libusb already installed
+          #brew install libusb
+          pkg-config libusb-1.0 --cflags
+
+        elif [ "$UNAME" == "CYGWIN*" ] || [ "$UNAME" == "MINGW*" ] ; then
+          # # TODO: fix this
+          # echo "installing tup..."
+          # Invoke-WebRequest -Uri "http://gittup.org/tup/win32/tup-latest.zip" -OutFile ".\tup-latest.zip"
+          # Expand-Archive ".\tup-latest.zip" -DestinationPath ".\tup-latest" -Force
+          # echo "$(Resolve-Path .)\tup-latest" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
+          # echo "installing interface_generator dependencies..."
+          # pip install PyYAML Jinja2 jsonschema
+
+          # echo "installing fibre-cpp build dependencies..."
+          # Invoke-WebRequest -Uri "https://github.com/libusb/libusb/releases/download/v1.0.23/libusb-1.0.23.7z" -OutFile ".\libusb-1.0.23.7z"
+          # & "$env:ProgramFiles\7-Zip\7z.exe" x -o".\libusb-1.0.23" ".\libusb-1.0.23.7z" -r ;
+          # #Expand-Archive ".\libusb-1.0.23.7z" -DestinationPath ".\libusb-1.0.23" -Force
+          # dir
+          # echo 'CONFIG_LDFLAGS="./libusb-1.0.23/MinGW64/static/libusb-1.0.a"' | Out-File -FilePath "tup.config" -Encoding utf8 -Append
+          echo "not implemented"
+          return 1
+
+        else
+          echo "Unknown operating system: $UNAME"
+          return 1
+        fi
+
+    - name: Compile
+      shell: bash
+      run: |
+        tup init
+        tup generate ./tup_build.sh
+        ./tup_build.sh
+
+#    # This is required to make the action repeatable
+#    - name: Clean up
+#      shell: bash
+#      run: git clean -dffx

.github/workflows/nightly.yaml

@@ -0,0 +1,110 @@
+name: Nightly Releases
+
+# TODO: use schedule
+on:
+  push:
+    branches: [fix-ci]
+
+jobs:
+  create_release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Prepare Release
+        id: release_specs
+        run: |
+          BODY="This prerelease was autogenerated from the \`devel\` branch on $(date --iso). \
+          It should not be expected to be stable. \
+          If some binaries are missing then it is probably because the build \
+          failed for those binaries."
+
+          # Escape BODY string to set it as GitHub Actions output
+          BODY="${BODY//'%'/'%25'}"
+          BODY="${BODY//$'\n'/'%0A'}"
+          BODY="${BODY//$'\r'/'%0D'}"
+
+          echo "::set-output name=title::Nightly Build $(date --iso)"
+          echo "::set-output name=body::$BODY"
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ steps.release_specs.outputs.title }}
+          body: ${{ steps.release_specs.outputs.body }}
+          draft: true
+          prerelease: true
+
+      - name: Print URL
+        run: |
+          echo "URL: ${{ steps.create_release.outputs.upload_url }}"
+          echo "ref: ${{ github.ref }}"
+
+  create_assets:
+    needs: create_release
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {os: ubuntu-latest, target: local}
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/compile
+        with:
+          target: ${{ matrix.local }}
+
+      - name: Upload Binary
+        shell: bash
+        run: |
+          COMMIT="$GITHUB_SHA"
+          echo "commit hash: $COMMIT"
+
+          RELEASES="$(curl --silent --show-error --fail \
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            "https://api.github.com/repos/samuelsadok/fibre/releases")"
+
+          upload_url="$(echo "$RELEASES" | jq -r '.[] | select(.target_commitish == "'$COMMIT'") | .upload_url')"
+          echo "upload_url: $upload_url"
+
+          BINARY="$(ls cpp | grep "libfibre-" | head)"
+          
+          URL="$(echo "$upload_url" | sed 's/assets.*$/assets/')?name=$BINARY"
+          echo "uploading $BINARY to $URL" >&2
+
+          curl --silent --show-error --fail \
+            -H "Content-Type: application/octet-stream" \
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            --data-binary @"cpp/$BINARY" \
+            "$URL"
+
+  finalize_release:
+    needs: create_assets
+    runs-on: ubuntu-latest
+    steps:
+      - name: Publish Release
+        shell: bash
+        run: |
+          COMMIT="$GITHUB_SHA"
+          echo "commit hash: $COMMIT"
+
+          RELEASES="$(curl --silent --show-error --fail \
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            "https://api.github.com/repos/samuelsadok/fibre/releases")"
+
+          id="$(echo "$RELEASES" | jq -r '.[] | select(.target_commitish == "'$COMMIT'") | .id')"
+          echo "release id: $id"
+
+          curl --silent --show-error --fail \
+            -X PATCH \
+            -H "Accept: application/vnd.github.v3+json" \
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -d '{"draft":false}' \
+            "https://api.github.com/repos/samuelsadok/fibre/releases/$id"
+
+          # TODO: delete old nightly releases

.github/workflows/compile.yaml → .github/workflows/tests.yaml

@@ -11,27 +11,26 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-16.04, ubuntu-latest, windows-latest, macOS-latest]
+        include:
+          - {os: ubuntu-latest, target: local}
 
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
-
-      - name: Compile
-        run: |
-          tup init
-          tup generate ./tup_build.sh
-          ./tup_build.sh
+      - uses: ./.github/actions/compile
+        with:
+          target: ${{ matrix.local }}
 
   formatting:
     strategy:
       fail-fast: false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04 # need a recent clang-format
     steps:
       - uses: actions/checkout@v2
 
       - name: Check C++ Formatting
         run: |
+          clang-format --version
           # TODO: we run this only on one file for now until we have it properly configured
           clang-format -style=file --Werror --dry-run test/test_server.cpp
 

cpp/README.md

@@ -37,7 +37,7 @@ Refer to
   4. Navigate to this directory and run `make`
 
 ### Ubuntu
-  1. `sudo apt-get libusb-1.0-dev`
+  1. `sudo apt-get install libusb-1.0-0-dev`
   2. Navigate to this directory and run `make`
 
 ### macOS

cpp/Tupfile.lua

@@ -4,20 +4,6 @@ tup.include('package.lua')
 CFLAGS = {'-fPIC -std=c++11 -DFIBRE_COMPILE'}
 LDFLAGS = {'-static-libstdc++'}
 
--- Runs the specified shell command immediately (not as part of the dependency
--- graph).
--- Returns the values (return_code, stdout) where stdout has the trailing new
--- line removed.
-function run_now(command)
-    local handle
-    handle = io.popen(command)
-    local output = handle:read("*a")
-    local rc = {handle:close()}
-    if not rc[1] then
-        error("failed to invoke "..command)
-    end
-    return string.sub(output, 0, -2)
-end
 
 if tup.getconfig("CC") == "" then
     CXX = 'clang++'
@@ -95,7 +81,7 @@ pkg = get_fibre_package({
     enable_tcp_client_backend=get_bool_config("ENABLE_TCP_CLIENT_BACKEND", true),
     enable_libusb_backend=get_bool_config("ENABLE_LIBUSB_BACKEND", true),
     allow_heap=true,
-    use_pkgconf=get_bool_config("USE_PKGCONF", true)
+    use_pkgconf=get_bool_config("USE_PKGCONF", nil)
 })
 
 CFLAGS += pkg.cflags

cpp/package.lua

@@ -3,15 +3,20 @@
 -- Returns a table that contains the Fibre code files and the flags required to
 -- compile and link those files.
 --
--- args: A dictionary containing the fibre options. Possible keys are:
---       enable_server, enable_client, enable_event_loop, allow_heap, enable_libusb, max_log_verbosity
---       The meaning of these options is documented in README.md.
---       In addition:
---          pkgconfig: nil, a boolean or a string. If a string is provided, it
---                     names the "pkgconfig" binary that this function may
---                     invoke to find build dependencies. nil or a boolean of
---                     true is equal to providing "pkgconf". A boolean of false
---                     means the function may return an incomplete list of compile flags.
+-- args: A dictionary containing the fibre options. Refer to the Compile Options
+--       in README.md for a list of available options. For example the option
+--       `FIBRE_ENABLE_SERVER` maps to the argument `args.enable_server`.
+-- In addition:
+-- args.pkgconf: Controls the use of the pkgconf or pkg-config utility that
+--     shall be used to locate build dependencies. Can be one of the following:
+--        - A string: Use the binary provided by the string. Fail if it doesn't
+--          exist.
+--        - true: Use "pkgconf" and fall back to "pkg-config" if "pkgconf"
+--          doesn't exist. Fail if both don't exist.
+--        - false: Don't use pkg-config. The user is responsible of determining
+--          the required compile and link flags.
+--        - nil: Try both "pkgconf" and "pkg-config". If both don't exist fall
+--          back to a hardcoded list of well-known settings.
 --
 -- Returns: A dictionary with the following items:
 --  code_files: A list of strings that name the C++ code files to be compiled.
@@ -26,27 +31,38 @@
 function get_fibre_package(args)
     pkg = {
         code_files = {
-            --'libfibre.cpp',
             'fibre.cpp',
-            --'legacy_protocol.cpp',
             'channel_discoverer.cpp',
         },
         include_dirs = {'include'},
         cflags = {},
         ldflags = {},
     }
 
+    -- Select a pkgconf function
     if args.pkgconf == true or args.pkgconf == nil then
-        pkgconf_file = 'pkgconf'
-    elseif args.pkgconf then
-        pkgconf_file = args.pkgconf
-    end
-
-    function pkgconf(lib)
-        if pkgconf_file then
-            pkg.cflags += run_now(pkgconf_file..' '..lib..' --cflags')
-            pkg.ldflags += run_now(pkgconf_file..' '..lib..' --libs')
+        -- Autodetect pkgconf
+        if test_pkgconf('pkgconf') then
+            print("using pkgconf")
+            pkgconf_file = 'pkgconf'
+            pkgconf = real_pkgconf
+        elseif test_pkgconf('pkg-config') then
+            print("using pkg-config")
+            pkgconf_file = 'pkg-config'
+            pkgconf = real_pkgconf
+        elseif args.pkgconf == nil then
+            print("using hardcoded pkgconf")
+            pkgconf = hardcoded_pkgconf
+        else
+            error("couldn't find pkgconf nor pkg-config")
         end
+
+    elseif args.pkgconf == false then
+        pkgconf_file = nil
+        pkgconf = null_pkgconf
+    else
+        pkgconf_file = args.pkgconf
+        pkgconf = real_pkgconf
     end
 
     pkg.cflags += '-DFIBRE_ENABLE_SERVER='..(args.enable_server and '1' or '0')
@@ -61,7 +77,7 @@ function get_fibre_package(args)
 
     if args.enable_libusb_backend then
         pkg.code_files += 'platform_support/libusb_transport.cpp'
-        pkgconf("libusb-1.0")
+        pkgconf(pkg, "libusb-1.0")
 
         -- TODO: only add pthread on linux and windows
         pkg.ldflags += '-lpthread'
@@ -97,8 +113,27 @@ function run_now(command)
     handle = io.popen(command)
     local output = handle:read("*a")
     local rc = {handle:close()}
-    if not rc[1] then
-        error("failed to invoke "..command)
-    end
-    return string.sub(output, 0, -2)
+    return string.sub(output, 0, -2), rc[1]
+end
+
+function test_pkgconf(name)
+    local str, rc = run_now(name.." --version 2>&1 >/dev/null")
+    return rc
+end
+
+function real_pkgconf(pkg, lib)
+    pkg.cflags += run_now(pkgconf_file..' '..lib..' --cflags')
+    pkg.ldflags += run_now(pkgconf_file..' '..lib..' --libs')
 end
+
+function null_pkgconf(pkg, lib)
+    -- don't do anything
+end
+
+function hardcoded_pkgconf(pkg, lib)
+    libs = {
+        ['libusb-1.0'] = {cflags = {}, ldflags = {}},
+    }
+    tup.append_table(pkg.cflags, libs[lib].cflags)
+    tup.append_table(pkg.ldflags, libs[lib].ldflags)
+end