Merge pull request #389 from berty/dev/moul/sign-app-to-ipa

.gitignore

@@ -1,3 +1,4 @@
+.zsign_cache
 vendor/
 coverage.txt
 packrd/

Dockerfile

@@ -1,3 +1,10 @@
+# zsign builder
+FROM            alpine:3.11 as zsign-build
+RUN             apk add --no-cache --virtual .build-deps git g++ openssl-dev libgcc libstdc++ zip unzip
+RUN             git clone https://github.com/zhlynn/zsign
+WORKDIR         zsign
+RUN             g++ ./*.cpp common/*.cpp -lcrypto -O3 -o zsign
+
 # web build
 FROM            node:10 as web-build
 WORKDIR         /app
@@ -26,7 +33,8 @@ RUN             make install
 
 # minimalist runtime
 FROM alpine:3.11
-RUN             apk add --update --no-cache ca-certificates
+RUN             apk add --update --no-cache ca-certificates libstdc++ unzip zip
 COPY            --from=go-build /go/bin/yolo /bin/
+COPY            --from=zsign-build zsign/zsign /bin/
 ENTRYPOINT      ["yolo"]
 EXPOSE          8000

deployments/yolo.berty.io/.env.example

@@ -0,0 +1,12 @@
+GITHUB_TOKEN=
+CIRCLE_TOKEN=
+HOSTNAME=
+BUILDKITE_TOKEN=
+BASIC_AUTH_PASSWORD=
+BINTRAY_TOKEN=
+BINTRAY_USERNAME=
+BEARER_SECRETKEY=
+AUTH_SALT=
+IOS_PASS=
+IOS_PROV=
+IOS_PRIVKEY=

deployments/yolo.berty.io/docker-compose.yml

@@ -5,18 +5,33 @@ services:
     image: bertytech/yolo:latest
     restart: unless-stopped
     network_mode: bridge
+    working_dir: /tmp
     volumes:
       - ./data:/data
+      - ~/codesign:/codesign:ro
     expose:
       - 8000
     environment:
-      - BUILDKITE_TOKEN=${YOLO_BUILDKITE_TOKEN}
-      - CIRCLE_TOKEN=${YOLO_CIRCLE_TOKEN}
-      - GITHUB_TOKEN=${YOLO_GITHUB_TOKEN}
-      - BINTRAY_TOKEN=${YOLO_BINTRAY_TOKEN}
-      - BINTRAY_USERNAME=${YOLO_BINTRAY_USERNAME}
-      - BEARER_SECRETKEY=${YOLO_BEARER_SECRETKEY}
-    command: -v server --cors-allowed-origins="*" --max-builds=30 --db-path=/data/yolo.sqlite --basic-auth-password="${YOLO_BASIC_AUTH_PASSWORD}" --request-timeout=10s --shutdown-timeout=11s --http-cache-path=/data/httpcache --artifacts-cache-path=/data/artifacts-cache
+      - BUILDKITE_TOKEN
+      - CIRCLE_TOKEN
+      - GITHUB_TOKEN
+      - BINTRAY_TOKEN
+      - BINTRAY_USERNAME
+      - BEARER_SECRETKEY
+      - BASIC_AUTH_PASSWORD
+      - IOS_PASS
+      - IOS_PROV
+      - IOS_PRIVKEY
+    command:
+      - -v
+      - server
+      - --cors-allowed-origins=*
+      - --max-builds=30
+      - --db-path=/data/yolo.sqlite
+      - --request-timeout=10s
+      - --shutdown-timeout=11s
+      - --http-cache-path=/data/httpcache
+      - --artifacts-cache-path=/data/artifacts-cache
     labels:
       - 'com.centurylinklabs.watchtower.enable=true'
       # traefik specific labels
@@ -28,5 +43,5 @@ services:
       - 'traefik.http.routers.yolo.tls.certresolver=cf'
       - 'traefik.http.routers.yolo.tls.domains[0].main=berty.io'
       - 'traefik.http.routers.yolo.tls.domains[0].sans=yolo.berty.io'
-
       - 'traefik.http.services.yolo.loadbalancer.server.port=8000'
+      - 'traefik.http.services.yolo.loadbalancer.healthcheck.port=9090'

go/cmd/yolo/main.go

@@ -71,6 +71,9 @@ func yolo(args []string) error {
 		httpCachePath      string
 		realm              string
 		once               bool
+		iosPrivkeyPath     string
+		iosProvPath        string
+		iosPrivkeyPass     string
 	)
 	var (
 		rootFlagSet   = flag.NewFlagSet("yolo", flag.ExitOnError)
@@ -102,6 +105,9 @@ func yolo(args []string) error {
 	serverFlagSet.StringVar(&authSalt, "auth-salt", "", "salt used to generate authentication tokens at the end of the URLs")
 	serverFlagSet.StringVar(&httpCachePath, "http-cache-path", "", "if set, will cache http client requests")
 	serverFlagSet.BoolVar(&once, "once", false, "just run workers once")
+	serverFlagSet.StringVar(&iosPrivkeyPath, "ios-privkey", "", "iOS signing: path to private key or p12 file (PEM or DER format)")
+	serverFlagSet.StringVar(&iosProvPath, "ios-prov", "", "iOS signing: path to mobile provisioning profile")
+	serverFlagSet.StringVar(&iosPrivkeyPass, "ios-pass", "", "iOS signing: password for private key or p12 file")
 	storeFlagSet.StringVar(&dbStorePath, "db-path", ":memory:", "DB Store path")
 	storeFlagSet.BoolVar(&withPreloading, "with-preloading", false, "with auto DB preloading")
 
@@ -180,6 +186,9 @@ func yolo(args []string) error {
 				AuthSalt:           authSalt,
 				DevMode:            devMode,
 				ArtifactsCachePath: artifactsCachePath,
+				IOSPrivkeyPath:     iosPrivkeyPath,
+				IOSProvPath:        iosProvPath,
+				IOSPrivkeyPass:     iosPrivkeyPass,
 			})
 			if err != nil {
 				return err

go/pkg/yolosvc/api_download.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 	"strconv"
@@ -21,9 +22,12 @@ import (
 	"github.com/go-chi/chi"
 	"go.uber.org/zap"
 	"google.golang.org/grpc/codes"
+	"moul.io/u"
 )
 
 func (svc *service) ArtifactDownloader(w http.ResponseWriter, r *http.Request) {
+	// FIXME: if caching enabled, lock by artifact ID
+
 	id := chi.URLParam(r, "artifactID")
 	var artifact yolopb.Artifact
 	err := svc.db.First(&artifact, "ID = ?", id).Error
@@ -32,6 +36,119 @@ func (svc *service) ArtifactDownloader(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	switch ext := filepath.Ext(artifact.LocalPath); ext {
+	case ".unsigned-ipa", ".dummy-signed-ipa":
+		if !u.CommandExists("zsign") {
+			httpError(w, fmt.Errorf("missing signing binary"), codes.Internal)
+			return
+		}
+		if svc.iosPrivkeyPath == "" || svc.iosProvPath == "" {
+			httpError(w, fmt.Errorf("missing iOS signing configuration"), codes.InvalidArgument)
+			return
+		}
+		if !u.FileExists(svc.iosPrivkeyPath) || !u.FileExists(svc.iosProvPath) {
+			httpError(w, fmt.Errorf("invalid iOS signing configuration"), codes.InvalidArgument)
+			return
+		}
+
+		// send some headers early, to make loading icon appearing soon on the iOS device
+		{
+			filename := strings.TrimSuffix(path.Base(artifact.LocalPath), ext) + ".ipa"
+			w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename))
+			if artifact.MimeType != "" {
+				w.Header().Add("Content-Type", artifact.MimeType)
+			}
+		}
+
+		// FIXME: cache file + send content-length
+
+		err := svc.signAndStreamIPA(artifact, w)
+		if err != nil {
+			httpError(w, err, codes.Internal)
+		}
+	default:
+		base := path.Base(artifact.LocalPath)
+		w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=%s", base))
+		if artifact.FileSize > 0 {
+			w.Header().Add("Content-Length", fmt.Sprintf("%d", artifact.FileSize))
+		}
+		if artifact.MimeType != "" {
+			w.Header().Add("Content-Type", artifact.MimeType)
+		}
+
+		err := svc.artifactToStream(artifact, w)
+		if err != nil {
+			httpError(w, err, codes.Internal)
+		}
+	}
+}
+
+func (svc *service) signAndStreamIPA(artifact yolopb.Artifact, w io.Writer) error {
+	// sign ipa
+	var signed string
+	{
+		tempdir, err := ioutil.TempDir("", "yolo")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(tempdir)
+
+		// write unsigned-file to tempdir
+		unsigned := filepath.Join(tempdir, "unsigned.ipa")
+		signed = filepath.Join(tempdir, "signed.ipa")
+		f, err := os.OpenFile(unsigned, os.O_RDWR|os.O_CREATE, 0755)
+		if err != nil {
+			return err
+		}
+		err = svc.artifactToStream(artifact, f)
+		if err != nil {
+			return err
+		}
+		if err := f.Close(); err != nil {
+			return err
+		}
+
+		// zsign the archive
+		zsignArgs := []string{
+			"-k", svc.iosPrivkeyPath,
+			"-m", svc.iosProvPath, // should be retrieved from the artifact archive directly
+			"-o", signed,
+			"-z", "1",
+		}
+		if svc.iosPrivkeyPass != "" {
+			zsignArgs = append(zsignArgs,
+				"-p", svc.iosPrivkeyPass,
+			)
+		}
+		zsignArgs = append(zsignArgs, unsigned)
+		cmd := exec.Command("zsign", zsignArgs...)
+		svc.logger.Info("zsign", zap.Strings("args", zsignArgs))
+		cmd.Stderr = os.Stderr
+		cmd.Stdout = os.Stdout
+		err = cmd.Run()
+		if err != nil {
+			return err
+		}
+	}
+
+	// send the signed iPA
+	{
+		f, err := os.Open(signed)
+		if err != nil {
+			return err
+		}
+
+		// content-length
+
+		_, err = io.Copy(w, f)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (svc *service) artifactToStream(artifact yolopb.Artifact, w io.Writer) error {
 	cache := filepath.Join(svc.artifactsCachePath, artifact.ID)
 	// download missing cache
 	if svc.artifactsCachePath != "" {
@@ -40,54 +157,44 @@ func (svc *service) ArtifactDownloader(w http.ResponseWriter, r *http.Request) {
 			err := svc.artifactDownloadToFile(&artifact, cache)
 			if err != nil {
 				svc.artifactsCacheMutex.Unlock()
-				httpError(w, err, codes.Internal)
-				return
+				return err
 			}
 		}
 		svc.artifactsCacheMutex.Unlock()
 	}
 
-	base := path.Base(artifact.LocalPath)
-	w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=%s", base))
-	if artifact.FileSize > 0 {
-		w.Header().Add("Content-Length", fmt.Sprintf("%d", artifact.FileSize))
-	}
-	if artifact.MimeType != "" {
-		w.Header().Add("Content-Type", artifact.MimeType)
-	}
-
 	// save download
 	now := time.Now()
 	download := yolopb.Download{
 		HasArtifactID: artifact.ID,
 		CreatedAt:     &now,
 		// FIXME: user agent for analytics?
 	}
-	err = svc.db.Create(&download).Error
+	err := svc.db.Create(&download).Error
 	if err != nil {
-		svc.logger.Warn("add download entry", zap.Error(err))
+		svc.logger.Warn("failed to add download log entry", zap.Error(err))
 	}
 
 	if svc.artifactsCachePath != "" {
 		// send cache
 		f, err := os.Open(cache)
 		if err != nil {
-			httpError(w, err, codes.Internal)
-			return
+			return err
 		}
 		defer f.Close()
 		_, err = io.Copy(w, f)
 		if err != nil {
-			httpError(w, err, codes.Internal)
+			return err
 		}
 	} else {
 		// proxy
 		ctx := context.Background()
 		err = svc.artifactDownloadFromProvider(ctx, &artifact, w)
 		if err != nil {
-			httpError(w, err, codes.Internal)
+			return err
 		}
 	}
+	return nil
 }
 
 func (svc *service) artifactDownloadToFile(artifact *yolopb.Artifact, dest string) error {