WIP: Ship Gopkg.toml via Godeps.json conversion

Dockerfile

@@ -36,6 +36,7 @@ ADD _output/publishing-bot /publishing-bot
 ADD _output/collapsed-kube-commit-mapper /collapsed-kube-commit-mapper
 ADD _output/sync-tags /sync-tags
 ADD _output/init-repo /init-repo
+ADD _output/godep-to-gopkg /godep-to-gopkg
 ADD artifacts/scripts/ /publish_scripts
 
 CMD ["/publishing-bot", "--dry-run", "--token-file=/token"]

Gopkg.toml

@@ -44,3 +44,7 @@
 [prune]
   go-tests = true
   unused-packages = true
+
+[[constraint]]
+  name = "github.com/pelletier/go-toml"
+  version = "1.1.0"

Makefile

@@ -21,6 +21,7 @@ build:
 	$(call build_cmd,publishing-bot)
 	$(call build_cmd,sync-tags)
 	$(call build_cmd,init-repo)
+	$(call build_cmd,godep-to-gopkg)
 .PHONY: build
 
 build-image: build

artifacts/scripts/construct.sh

@@ -37,8 +37,8 @@ set -o nounset
 set -o pipefail
 set -o xtrace
 
-if [ ! $# -eq 10 ]; then
-    echo "usage: $0 repo src_branch dst_branch dependent_k8s.io_repos required_packages kubernetes_remote subdirectory source_repo_org source_repo_name is_library"
+if [ ! $# -eq 12 ]; then
+    echo "usage: $0 repo src_branch dst_branch dependent_k8s.io_repos required_packages kubernetes_remote subdirectory source_repo_org source_repo_name target_repo_org is_library recursive_delete_pattern"
     exit 1
 fi
 
@@ -64,10 +64,14 @@ SOURCE_REPO_NAME="${9}"
 
 shift 9
 
+# target repository organization name (e.g. openshift)
+TARGET_REPO_ORG="${1}"
 # If ${REPO} is a library
-IS_LIBRARY="${1}"
+IS_LIBRARY="${2}"
+# A ls-files pattern like "*/BUILD *.ext pkg/foo.go Makefile"
+RECURSIVE_DELETE_PATTERN="${3}"
 
-readonly SRC_BRANCH DST_BRANCH DEPS SOURCE_REMOTE SOURCE_REPO_ORG SOURCE_REPO_NAME SUBDIR IS_LIBRARY
+readonly SRC_BRANCH DST_BRANCH DEPS SOURCE_REMOTE SOURCE_REPO_ORG SOURCE_REPO_NAME TARGET_REPO_ORG SUBDIR IS_LIBRARY
 
 SCRIPT_DIR=$(dirname "${BASH_SOURCE}")
 source "${SCRIPT_DIR}"/util.sh
@@ -94,7 +98,12 @@ fi
 
 # sync_repo cherry-picks the commits that change
 # k8s.io/kubernetes/staging/src/k8s.io/${REPO} to the ${DST_BRANCH}
-sync_repo "${SOURCE_REPO_ORG}" "${SOURCE_REPO_NAME}" "${SUBDIR}" "${SRC_BRANCH}" "${DST_BRANCH}" "${SOURCE_REMOTE}" "${DEPS}" "${REQUIRED}" "${IS_LIBRARY}"
+sync_repo "${SOURCE_REPO_ORG}" "${SOURCE_REPO_NAME}" "${TARGET_REPO_ORG}" "${SUBDIR}" "${SRC_BRANCH}" "${DST_BRANCH}" "${SOURCE_REMOTE}" "${DEPS}" "${REQUIRED}" "${IS_LIBRARY}" "${RECURSIVE_DELETE_PATTERN}"
+
+ALTERNATIVE_SOURCE=""
+if [ "${SOURCE_REPO_ORG}" != "${TARGET_REPO_ORG}" ]; then
+   ALTERNATIVE_SOURCE="github.com/${TARGET_REPO_ORG}/"
+fi
 
 # add tags
 EXTRA_ARGS=()
@@ -106,5 +115,7 @@ chmod +x ${PUSH_SCRIPT}
            --source-remote upstream --source-branch "${SRC_BRANCH}" \
            --push-script ${PUSH_SCRIPT} \
            --dependencies "${DEPS}" \
+           --required "${REQUIRED}" \
+           --alternative-source "${ALTERNATIVE_SOURCE}" \
            -alsologtostderr \
            "${EXTRA_ARGS[@]-}"

artifacts/scripts/util.sh

@@ -126,13 +126,19 @@ sync_repo() {
     # subdirectory in k8s.io/kubernetes, e.g., staging/src/k8s.io/apimachinery
     local source_repo_org="${1}"
     local source_repo_name="${2}"
-    local subdirectory="${3}"
-    local src_branch="${4}"
-    local dst_branch="${5}"
-    local kubernetes_remote="${6}"
-    local deps="${7:-""}"
-    local required_packages="${8:-""}"
-    local is_library="${9}"
+    local target_repo_org="${3}"
+    local subdirectory="${4}"
+    local src_branch="${5}"
+    local dst_branch="${6}"
+    local kubernetes_remote="${7}"
+    local deps="${8:-""}"
+    local required_packages="${9:-""}"
+
+    shift 9
+
+    local is_library="${1}"
+    local recursive_delete_pattern="${2}"
+
     local commit_msg_tag="${source_repo_name^}-commit"
     readonly subdirectory src_branch dst_branch kubernetes_remote deps is_library
 
@@ -168,7 +174,7 @@ sync_repo() {
     local f_mainline_commits=""
     if [ "${new_branch}" = "true" ] && [ "${src_branch}" = master ]; then
         # new master branch
-        filter-branch "${commit_msg_tag}" "${subdirectory}" ${src_branch} filtered-branch
+        filter-branch "${commit_msg_tag}" "${subdirectory}" "${recursive_delete_pattern}" ${src_branch} filtered-branch
 
         # find commits on the main line (will mostly be merges, but could be non-merges if filter-branch dropped
         # the corresponding fast-forward merge and left the feature branch commits)
@@ -196,7 +202,7 @@ sync_repo() {
             git branch -f filtered-branch-base ${k_branch_point_commit} >/dev/null
 
             echo "Rewriting upstream branch ${src_branch} to only include commits for ${subdirectory}."
-            filter-branch "${commit_msg_tag}" "${subdirectory}" filtered-branch filtered-branch-base
+            filter-branch "${commit_msg_tag}" "${subdirectory}" "${recursive_delete_pattern}" filtered-branch filtered-branch-base
 
             # for a new branch that is not master: map filtered-branch-base to our ${dst_branch} as ${dst_branch_point_commit}
             local k_branch_point_commit=$(kube-commit ${commit_msg_tag} filtered-branch-base) # k_branch_point_commit will probably different thanthe k_branch_point_commit
@@ -223,7 +229,7 @@ sync_repo() {
             git branch -f filtered-branch-base ${k_base_merge} >/dev/null
 
             echo "Rewriting upstream branch ${src_branch} to only include commits for ${subdirectory}."
-            filter-branch "${commit_msg_tag}" "${subdirectory}" filtered-branch filtered-branch-base
+            filter-branch "${commit_msg_tag}" "${subdirectory}" "${recursive_delete_pattern}" filtered-branch filtered-branch-base
         fi
 
         # find commits on the main line (will mostly be merges, but could be non-merges if filter-branch dropped
@@ -243,6 +249,19 @@ sync_repo() {
         git commit -q -m "sync: remove kubernetes-sha"
     fi
 
+    # remove existing recursive-delete-pattern files. After a first removal commit, the filter-branch command
+    # will filter them out from upstream commits.
+    if [ -n "${recursive_delete_pattern}" ]; then
+        local split_recursive_delete_pattern
+        read -r -a split_recursive_delete_pattern <<< "${recursive_delete_pattern}"
+        git rm -q --ignore-unmatch -r "${split_recursive_delete_pattern[@]}"
+        git add -u
+        if ! git-index-clean; then
+            echo "Deleting files recursively: ${recursive_delete_pattern}"
+            git commit -m "sync: initially remove files ${recursive_delete_pattern}"
+        fi
+    fi
+
     local dst_old_head=$(git rev-parse HEAD) # will be the initial commit for new branch
 
     # apply all PRs
@@ -312,7 +331,7 @@ sync_repo() {
             local dst_new_merge=$(GIT_COMMITTER_DATE="${date}" GIT_AUTHOR_DATE="${date}" git commit-tree -p ${dst_merge_point_commit} -p ${dst_parent2} -m "$(commit-message ${k_pending_merge_commit}; echo; echo "${commit_msg_tag}: ${k_pending_merge_commit}")" HEAD^{tree})
             # no amend-godeps needed here: because the merge-commit was dropped, both parents had the same tree, i.e. Godeps.json did not change.
             git reset -q --hard ${dst_new_merge}
-            fix-godeps "${deps}" "${required_packages}" "${is_library}" ${dst_needs_godeps_update} true "${commit_msg_tag}"
+            fix-godeps "${deps}" "${required_packages}" "${is_library}" "${dst_needs_godeps_update}" true "${commit_msg_tag}" "${source_repo_org}" "${target_repo_org}"
             dst_needs_godeps_update=false
             dst_merge_point_commit=$(git rev-parse HEAD)
         fi
@@ -357,7 +376,7 @@ sync_repo() {
 
             # if there is no pending merge commit, update Godeps.json because this could be a target of tag
             if [ -z "${k_pending_merge_commit}" ]; then
-                fix-godeps "${deps}" "${required_packages}" "${is_library}" ${dst_needs_godeps_update} true ${commit_msg_tag}
+                fix-godeps "${deps}" "${required_packages}" "${is_library}" "${dst_needs_godeps_update}" true "${commit_msg_tag}" "${source_repo_org}" "${target_repo_org}"
                 dst_needs_godeps_update=false
                 dst_merge_point_commit=$(git rev-parse HEAD)
             fi
@@ -460,7 +479,7 @@ sync_repo() {
             # we would end up with "base B + change B" which misses the change A changes.
             amend-godeps-at ${f_mainline_commit}
 
-            fix-godeps "${deps}" "${required_packages}" "${is_library}" ${dst_needs_godeps_update} true ${commit_msg_tag}
+            fix-godeps "${deps}" "${required_packages}" "${is_library}" "${dst_needs_godeps_update}" true "${commit_msg_tag}" "${source_repo_org}" "${target_repo_org}"
             dst_needs_godeps_update=false
             dst_merge_point_commit=$(git rev-parse HEAD)
         fi
@@ -473,10 +492,10 @@ sync_repo() {
     #       output depends on upstream's HEAD.
     echo "Fixing up godeps after a complete sync"
     if [ $(git rev-parse HEAD) != "${dst_old_head}" ] || [ "${new_branch}" = "true" ]; then
-        fix-godeps "${deps}" "${required_packages}" "${is_library}" true true ${commit_msg_tag}
+        fix-godeps "${deps}" "${required_packages}" "${is_library}" true true "${commit_msg_tag}" "${source_repo_org}" "${target_repo_org}"
     else
         # update godeps without squashing because it would mutate a published commit
-        fix-godeps "${deps}" "${required_packages}" "${is_library}" true false ${commit_msg_tag}
+        fix-godeps "${deps}" "${required_packages}" "${is_library}" true false "${commit_msg_tag}" "${source_repo_org}" "${target_repo_org}"
     fi
 
     # create look-up file for collapsed upstream commits
@@ -533,8 +552,19 @@ function commit-subject() {
 function filter-branch() {
     local commit_msg_tag="${1}"
     local subdirectory="${2}"
+    local recursive_delete_pattern="${3}"
     echo "Running git filter-branch ..."
-    git filter-branch -f --msg-filter 'awk 1 && echo && echo "'"${commit_msg_tag}"': ${GIT_COMMIT}"' --subdirectory-filter "${subdirectory}" -- ${3} ${4} >/dev/null
+    local index_filter=""
+    if [ -n "${recursive_delete_pattern}" ]; then
+        local patterns=()
+        local p=""
+        index_filter="git rm -q --cached --ignore-unmatch -r"
+        IFS=" " read -ra patterns <<<"${recursive_delete_pattern}"
+        for p in "${patterns[@]}"; do
+            index_filter+=" '${p}'"
+        done
+    fi
+    git filter-branch -f --index-filter "${index_filter}" --msg-filter 'awk 1 && echo && echo "'"${commit_msg_tag}"': ${GIT_COMMIT}"' --subdirectory-filter "${subdirectory}" -- ${4} ${5} >/dev/null
 }
 
 function is-merge() {
@@ -624,6 +654,9 @@ function fix-godeps() {
     local needs_godeps_update="${4}"
     local squash="${5:-true}"
     local commit_msg_tag="${6}"
+    local source_repo_org="${7}"
+    local target_repo_org="${8}"
+
     local dst_old_commit=$(git rev-parse HEAD)
     if [ "${needs_godeps_update}" = true ]; then
         # run godeps restore+save
@@ -636,6 +669,23 @@ function fix-godeps() {
         update-deps-in-godep-json "${deps}" "${is_library}" "${commit_msg_tag}"
     fi
 
+    # update golang/dep Gopkg.toml
+    if [ -f Godeps/Godeps.json ]; then
+        echo "Converting Godeps/Godeps.json to Gopkg.toml"
+        local alternative_source=""
+        if [ "${source_repo_org}" != "${target_repo_org}" ]; then
+            alternative_source="github.com/${target_repo_org}/"
+        fi
+        /godep-to-gopkg --dependencies "${deps}" --required "${required_packages}" --alternative-source "${alternative_source}"
+        git add Gopkg.toml
+    elif [ -f Gopkg.toml ]; then
+        git rm -f Gopkg.toml
+    fi
+    if ! git-index-clean; then
+        echo "Committing Gopkg.toml"
+        git commit -q -m "sync: update Gopkg.toml"
+    fi
+
     # remove vendor/ on non-master branches for libraries
     if [ "$(git rev-parse --abbrev-ref HEAD)" != master ] && [ -d vendor/ ] && [ "${is_library}" = "true" ]; then
         echo "Removing vendor/ on non-master branch because this is a library"

cmd/godep-to-gopkg/main.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/golang/glog"
+	"k8s.io/publishing-bot/pkg/dependency"
+	"k8s.io/publishing-bot/pkg/dependency/dep"
+)
+
+func Usage() {
+	fmt.Fprintf(os.Stderr, `Convert Godeps/Godeps.json into Gopkg.toml.
+
+Usage: %s --source-remote <remote> --source-branch <source-branch>
+          [--dependencies <repo-name>[:branch]]
+          [--required <pkg>,...]
+          [--alternative-source <pkg>]
+`, os.Args[0])
+	flag.PrintDefaults()
+}
+
+func main() {
+	dependencies := flag.String("dependencies", "", "comma-separated list of repo:branch pairs of dependencies")
+	required := flag.String("required", "", "comma-separated list of Golang packages that are required")
+	alternativeSource := flag.String("alternative-source", "", "a package org like github.com/sttts to be used as alternative source of the dependencies")
+
+	flag.Usage = Usage
+	flag.Parse()
+
+	var requiredPkgs []string
+	if len(*required) > 0 {
+		requiredPkgs = strings.Split(*required, ",")
+	}
+
+	dependentRepos, err := dependency.ParseDependencies(*dependencies)
+	if err != nil {
+		glog.Fatalf("Failed to parse dependencies %q: %v", *dependencies, err)
+	}
+
+	if err := dep.GodepToGopkg(dependentRepos, requiredPkgs, *alternativeSource); err != nil {
+		glog.Fatal(err)
+	}
+}

cmd/init-repo/main.go

@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	depCommit   = "7c44971bbb9f0ed87db40b601f2d9fe4dffb750d"
+	depCommit   = "37d9ea0ac16f0e0a05afc3b60e1ac8c364b6c329"
 	godepCommit = "tags/v79"
 )
 

cmd/publishing-bot/config/rules.go

@@ -64,6 +64,9 @@ type RepositoryRules struct {
 	SkippedSourceBranches []string         `yaml:"skip-source-branches"`
 	SkipGodeps            bool             `yaml:"skip-godeps"`
 	Rules                 []RepositoryRule `yaml:"rules"`
+
+	// ls-files patterns like: */BUILD *.ext pkg/foo.go Makefile
+	RecursiveDeletePatterns []string `yaml:"recursive-delete-patterns"`
 }
 
 // LoadRules loads the repository rules either from the remote HTTP location or

cmd/publishing-bot/publisher.go

@@ -178,9 +178,11 @@ func (p *PublisherMunger) construct() error {
 				strings.Join(branchRule.RequiredPackages, ":"),
 				sourceRemote,
 				branchRule.Source.Dir,
+				p.config.SourceOrg,
 				p.config.SourceRepo,
-				p.config.SourceRepo,
-				fmt.Sprintf("%v", repoRule.Library))
+				p.config.TargetOrg,
+				fmt.Sprintf("%v", repoRule.Library),
+				strings.Join(p.reposRules.RecursiveDeletePatterns, " "))
 			if p.reposRules.SkipGodeps {
 				cmd.Env = append(os.Environ(), "PUBLISHER_BOT_SKIP_GODEPS=true")
 			}

cmd/publishing-bot/publisher_logger.go

@@ -34,7 +34,7 @@ import (
 )
 
 type plog struct {
-	buf         *bytes.Buffer
+	buf *bytes.Buffer
 }
 
 func NewPublisherLog(buf *bytes.Buffer) *plog {