Merge pull request #18 from jterry75/update_publish_cri

Merge remote-tracking branch 'origin/master' into publish-cri

.golangci.yml

@@ -15,7 +15,9 @@ linters:
     - errcheck
 
 run:
+  deadline: 2m
   skip-dirs:
     - api
     - design
     - docs
+    - docs/man

.mailmap

@@ -1,47 +1,49 @@
-Abhinandan Prativadi <abhi@docker.com> abhi <abhi@docker.com>
-Abhinandan Prativadi <abhi@docker.com> Abhinandan Prativadi <aprativadi@gmail.com>
-Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp> Akihiro Suda <suda.akihiro@lab.ntt.co.jp>
-Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp> Akihiro Suda <suda.kyoto@gmail.com>
-Andrei Vagin <avagin@virtuozzo.com> Andrei Vagin <avagin@openvz.org>
-Andrey Kolomentsev <andrey.kolomentsev@gmail.com> akolomentsev <andrey.kolomentsev@gmail.com>
-Brent Baude <bbaude@redhat.com> baude <bbaude@redhat.com>
-Carlos Eduardo <me@carlosedp.com> CarlosEDP <me@carlosedp.com>
-Eric Ren <renzhen.rz@alibaba-linux.com> renzhen.rz <renzhen.rz@alibaba-inc.com>
-Frank Yang <yyb196@gmail.com> frank yang <yyb196@gmail.com>
-Georgia Panoutsakopoulou <gpanoutsak@gmail.com> gpanouts <gpanoutsak@gmail.com>
-Guangming Wang <guangming.wang@daocloud.io> ethan <guangming.wang@daocloud.io>
-Haiyan Meng <haiyanmeng@google.com> haiyanmeng <haiyanmeng@google.com>
-Jian Liao <jliao@alauda.io> liaojian <liaojian@Dabllo.local>
-Jian Liao <jliao@alauda.io> liaoj <jliao@alauda.io>
-Ji'an Liu <anthonyliu@zju.edu.cn> ZeroMagic <anthonyliu@zju.edu.cn>
-Jie Zhang <iamkadisi@163.com> kadisi <iamkadisi@163.com>
-John Howard <john.howard@microsoft.com> John Howard <jhoward@microsoft.com>
-John Howard <john.howard@microsoft.com> John Howard <jhowardmsft@users.noreply.github.com>
-Julien Balestra <julien.balestra@datadoghq.com> JulienBalestra <julien.balestra@datadoghq.com>
-Justin Cormack <justin.cormack@docker.com> Justin Cormack <justin@specialbusservice.com>
-Justin Terry <juterry@microsoft.com> Justin <jterry75@users.noreply.github.com>
-Justin Terry <juterry@microsoft.com> Justin Terry (VM) <juterry@microsoft.com>
-Kenfe-Mickaël Laventure <mickael.laventure@gmail.com> Kenfe-Mickael Laventure <mickael.laventure@gmail.com>
-Kevin Xu <cming.xu@gmail.com> kevin.xu <cming.xu@gmail.com>
-Lantao Liu <lantaol@google.com> Lantao Liu <taotaotheripper@gmail.com>
-Lifubang <lifubang@aliyun.com> Lifubang <lifubang@acmcoder.com>
-Lu Jingxiao <lujingxiao@huawei.com> l00397676 <lujingxiao@huawei.com>
-Maksym Pavlenko <makpav@amazon.com> Maksym Pavlenko <pavlenko.maksym@gmail.com>
-Mark Gordon <msg555@gmail.com> msg555 <msg555@gmail.com>
-Michael Katsoulis <michaelkatsoulis88@gmail.com> MichaelKatsoulis <michaelkatsoulis88@gmail.com>
-Mike Brown <brownwm@us.ibm.com> Mike Brown <mikebrow@users.noreply.github.com>
-Phil Estes <estesp@gmail.com> Phil Estes <estesp@linux.vnet.ibm.com>
-Rui Cao <ruicao@alauda.io> ruicao <ruicao@alauda.io>
-Stephen J Day <stevvooe@gmail.com> Stephen Day <stephen.day@getcruise.com>
-Stephen J Day <stevvooe@gmail.com> Stephen Day <stevvooe@users.noreply.github.com>
-Stephen J Day <stevvooe@gmail.com> Stephen J Day <stephen.day@docker.com>
-Sudeesh John <sudeesh@linux.vnet.ibm.com> sudeesh john <sudeesh@linux.vnet.ibm.com>
-Su Fei  <fesu@ebay.com> fesu <fesu@ebay.com>
-Tõnis Tiigi <tonistiigi@gmail.com> Tonis Tiigi <tonistiigi@gmail.com>
-Wei Fu <fuweid89@gmail.com> Wei Fu <fhfuwei@163.com>
-Xiaodong Zhang <a4012017@sina.com> nashasha1 <a4012017@sina.com>
-Xuean Yan <yan.xuean@zte.com.cn> yanxuean <yan.xuean@zte.com.cn>
-Yuxing Liu <starnop@163.com> Starnop <starnop@163.com>
-zhenguang zhu <zhengguang.zhu@daocloud.io> dzzg <zhengguang.zhu@daocloud.io>
-zhoulin xie <zhoulin.xie@daocloud.io> JoeWrightss <42261994+JoeWrightss@users.noreply.github.com>
-zhoulin xie <zhoulin.xie@daocloud.io> JoeWrightss <zhoulin.xie@daocloud.io>
+Abhinandan Prativadi <abhi@docker.com>
+Abhinandan Prativadi <abhi@docker.com> <aprativadi@gmail.com>
+Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp> <suda.akihiro@lab.ntt.co.jp>
+Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp> <suda.kyoto@gmail.com>
+Andrei Vagin <avagin@virtuozzo.com> <avagin@openvz.org>
+Andrey Kolomentsev <andrey.kolomentsev@gmail.com>
+Brent Baude <bbaude@redhat.com>
+Carlos Eduardo <me@carlosedp.com> <me@carlosedp.com>
+Eric Ren <renzhen.rz@alibaba-linux.com> <renzhen.rz@alibaba-inc.com>
+Frank Yang <yyb196@gmail.com>
+Georgia Panoutsakopoulou <gpanoutsak@gmail.com>
+Guangming Wang <guangming.wang@daocloud.io>
+Haiyan Meng <haiyanmeng@google.com>
+Jian Liao <jliao@alauda.io>
+Jian Liao <jliao@alauda.io> <liaojian@Dabllo.local>
+Ji'an Liu <anthonyliu@zju.edu.cn>
+Jie Zhang <iamkadisi@163.com>
+John Howard <john.howard@microsoft.com> <jhoward@microsoft.com>
+John Howard <john.howard@microsoft.com> <jhowardmsft@users.noreply.github.com>
+Julien Balestra <julien.balestra@datadoghq.com>
+Justin Cormack <justin.cormack@docker.com> <justin@specialbusservice.com>
+Justin Terry <juterry@microsoft.com>
+Justin Terry <juterry@microsoft.com> <jterry75@users.noreply.github.com>
+Kenfe-Mickaël Laventure <mickael.laventure@gmail.com>
+Kevin Xu <cming.xu@gmail.com>
+Lantao Liu <lantaol@google.com> <taotaotheripper@gmail.com>
+Lifubang <lifubang@aliyun.com> <lifubang@acmcoder.com>
+Lu Jingxiao <lujingxiao@huawei.com>
+Maksym Pavlenko <makpav@amazon.com> <pavlenko.maksym@gmail.com>
+Mark Gordon <msg555@gmail.com>
+Michael Katsoulis <michaelkatsoulis88@gmail.com>
+Mike Brown <brownwm@us.ibm.com> <mikebrow@users.noreply.github.com>
+Nishchay Kumar <mrawesomenix@gmail.com>
+Phil Estes <estesp@gmail.com> <estesp@linux.vnet.ibm.com>
+Rui Cao <ruicao@alauda.io> <ruicao@alauda.io>
+Stephen J Day <stevvooe@gmail.com> <stephen.day@getcruise.com>
+Stephen J Day <stevvooe@gmail.com> <stevvooe@users.noreply.github.com>
+Stephen J Day <stevvooe@gmail.com> <stephen.day@docker.com>
+Sudeesh John <sudeesh@linux.vnet.ibm.com>
+Su Fei  <fesu@ebay.com> <fesu@ebay.com>
+Tõnis Tiigi <tonistiigi@gmail.com>
+Wei Fu <fuweid89@gmail.com> <fhfuwei@163.com>
+Xiaodong Zhang <a4012017@sina.com>
+Xuean Yan <yan.xuean@zte.com.cn>
+Yuxing Liu <starnop@163.com>
+zhenguang zhu <zhengguang.zhu@daocloud.io>
+zhongming chang<zhongming.chang@daocloud.io>
+zhoulin xie <zhoulin.xie@daocloud.io>
+zhoulin xie <zhoulin.xie@daocloud.io> <42261994+JoeWrightss@users.noreply.github.com>

.travis.yml

@@ -1,4 +1,4 @@
-dist: xenial
+dist: bionic
 sudo: required
 # setup travis so that we can run containers for integration tests
 services:
@@ -13,11 +13,19 @@ go:
   - "1.12.x"
 
 env:
-  - TRAVIS_GOOS=linux TEST_RUNTIME=io.containerd.runc.v1 TRAVIS_CGO_ENABLED=1
-  - TRAVIS_GOOS=linux TEST_RUNTIME=io.containerd.runc.v2 TRAVIS_CGO_ENABLED=1
-  - TRAVIS_GOOS=linux TEST_RUNTIME=io.containerd.runtime.v1.linux TRAVIS_CGO_ENABLED=1
+  - TRAVIS_GOOS=linux TEST_RUNTIME=io.containerd.runc.v1 TRAVIS_CGO_ENABLED=1 TRAVIS_DISTRO=bionic
+  - TRAVIS_GOOS=linux TEST_RUNTIME=io.containerd.runc.v2 TRAVIS_CGO_ENABLED=1 TRAVIS_DISTRO=bionic
+  - TRAVIS_GOOS=linux TEST_RUNTIME=io.containerd.runtime.v1.linux TRAVIS_CGO_ENABLED=1 TRAVIS_DISTRO=bionic
   - TRAVIS_GOOS=darwin TRAVIS_CGO_ENABLED=0
 
+matrix:
+  include:
+    # Skip testing previous LTS (Xenial / Ubuntu 16.04 LTS) on pull requests
+    - if: type != pull_request
+      os: linux
+      dist: xenial
+      env: TRAVIS_GOOS=linux TEST_RUNTIME=io.containerd.runc.v2 TRAVIS_CGO_ENABLED=1 TRAVIS_DISTRO=xenial
+
 go_import_path: github.com/containerd/containerd
 
 addons:
@@ -31,10 +39,9 @@ addons:
       - python-minimal
       - libcap-dev
       - libaio-dev
-      - libprotobuf-c0-dev
+      - libprotobuf-c-dev
       - libprotobuf-dev
       - socat
-      - libseccomp-dev
 
 before_install:
   - uname -r
@@ -48,6 +55,7 @@ install:
   - go get -u github.com/vbatts/git-validation
   - go get -u github.com/kunalkushwaha/ltag
   - go get -u github.com/LK4D4/vndr
+  - if [ "$TRAVIS_GOOS" = "linux" ]; then sudo PATH=$PATH GOPATH=$GOPATH script/setup/install-seccomp ; fi
   - if [ "$TRAVIS_GOOS" = "linux" ]; then sudo PATH=$PATH GOPATH=$GOPATH script/setup/install-runc ; fi
   - if [ "$TRAVIS_GOOS" = "linux" ]; then sudo PATH=$PATH GOPATH=$GOPATH script/setup/install-cni ; fi
   - if [ "$TRAVIS_GOOS" = "linux" ]; then sudo PATH=$PATH GOPATH=$GOPATH script/setup/install-critools ; fi
@@ -99,16 +107,25 @@ after_success:
 
 before_deploy:
   - make release
+  - if [ "$TRAVIS_GOOS" = "linux" ]; then make cri-release; fi
 
 deploy:
-  provider: releases
-  api_key:
-    secure: HO+WSIVVUMMsbU74x+YyFsTP3ahqnR4xjwKAziedJ5lZXKJszQBhiYTFmcTeVBoouNjTISd07GQzpoLChuGC20U3+1NbT+CkK8xWR/x1ao2D3JY3Ds6AD9ubWRNWRLptt/xOn5Vq3F8xZyUYchwvDMl4zKCuTKxQGVdHKsINb2DehKcP5cVL6MMvqzEdfj2g99vqXAqs8uuo6dOmvxmHV43bfzDaAJSabjZZs6TKlWTqCQMet8uxyx2Dmjl2lxLwdqv12oJdrszacasn41NYuEyHI2bXyef1mhWGYN4n9bU/Y5winctZ8DOSOZvYg/2ziAaUN0+CTn1IESwVesrPz23P2Sy7wdLxu8dSIZ2yUHl7OsA5T5a5rDchAGguRVNBWvoGtuepEhdRacxTQUo1cMFZsEXjgRKKjdfc1emYQPVdN8mBv8GJwndty473ZXdvFt5R0kNVFtvWuYCa6UYJD2cKrsPSAfbZCDC/LiR3FOoTaUPMZUVkR2ACEO7Dn4+KlmBajqT40Osk/A7k1XA/TzVhMIpLtE0Vk2DfPmGsjCv8bC+MFd+R2Sc8SFdE92oEWRdoPQY5SxMYQtGxA+cbKVlT1kSw6y80yEbx5JZsBnT6+NTHwmDO3kVU9ztLdawOozTElKNAK8HoAyFmzIZ3wL64oThuDrv/TUuY8Iyn814=
-  file_glob: true
-  file:
-    - releases/*.tar.gz
-    - releases/*.tar.gz.sha256sum
-  skip_cleanup: true
-  on:
-    repo: containerd/containerd
-    tags: true
+  - provider: releases
+    api_key:
+      secure: HO+WSIVVUMMsbU74x+YyFsTP3ahqnR4xjwKAziedJ5lZXKJszQBhiYTFmcTeVBoouNjTISd07GQzpoLChuGC20U3+1NbT+CkK8xWR/x1ao2D3JY3Ds6AD9ubWRNWRLptt/xOn5Vq3F8xZyUYchwvDMl4zKCuTKxQGVdHKsINb2DehKcP5cVL6MMvqzEdfj2g99vqXAqs8uuo6dOmvxmHV43bfzDaAJSabjZZs6TKlWTqCQMet8uxyx2Dmjl2lxLwdqv12oJdrszacasn41NYuEyHI2bXyef1mhWGYN4n9bU/Y5winctZ8DOSOZvYg/2ziAaUN0+CTn1IESwVesrPz23P2Sy7wdLxu8dSIZ2yUHl7OsA5T5a5rDchAGguRVNBWvoGtuepEhdRacxTQUo1cMFZsEXjgRKKjdfc1emYQPVdN8mBv8GJwndty473ZXdvFt5R0kNVFtvWuYCa6UYJD2cKrsPSAfbZCDC/LiR3FOoTaUPMZUVkR2ACEO7Dn4+KlmBajqT40Osk/A7k1XA/TzVhMIpLtE0Vk2DfPmGsjCv8bC+MFd+R2Sc8SFdE92oEWRdoPQY5SxMYQtGxA+cbKVlT1kSw6y80yEbx5JZsBnT6+NTHwmDO3kVU9ztLdawOozTElKNAK8HoAyFmzIZ3wL64oThuDrv/TUuY8Iyn814=
+    file_glob: true
+    file:
+      - releases/*.tar.gz
+      - releases/*.tar.gz.sha256sum
+    skip_cleanup: true
+    on:
+      repo: containerd/containerd
+      tags: true
+      condition: $TRAVIS_GOOS = linux
+  - provider: script
+    script: bash script/release/deploy-cri
+    skip_cleanup: true
+    on:
+      repo: containerd/containerd
+      tags: true
+      condition: $TRAVIS_GOOS = linux

Makefile

@@ -114,14 +114,14 @@ BINARIES=$(addprefix bin/,$(COMMANDS))
 TESTFLAGS ?= $(TESTFLAGS_RACE)
 TESTFLAGS_PARALLEL ?= 8
 
-.PHONY: clean all AUTHORS build binaries test integration generate protos checkprotos coverage ci check help install uninstall vendor release mandir install-man
+.PHONY: clean all AUTHORS build binaries test integration generate protos checkprotos coverage ci check help install uninstall vendor release mandir install-man genman
 .DEFAULT: default
 
 all: binaries
 
 check: proto-fmt ## run all linters
 	@echo "$(WHALE) $@"
-	golangci-lint run
+	GOGC=75 golangci-lint run
 
 ci: check binaries checkprotos coverage coverage-integration ## to be used by the CI
 
@@ -203,6 +203,9 @@ man: mandir $(addprefix man/,$(MANPAGES))
 mandir:
 	@mkdir -p man
 
+genman: FORCE
+	go run cmd/gen-manpages/main.go man/
+
 man/%: docs/man/%.md FORCE
 	@echo "$(WHALE) $<"
 	go-md2man -in "$<" -out "$@"
@@ -216,14 +219,22 @@ install-man:
 	@echo "$(WHALE) $@"
 	$(foreach manpage,$(addprefix man/,$(MANPAGES)), $(call installmanpage,$(manpage),$(subst .,,$(suffix $(manpage))),$(notdir $(manpage))))
 
-release: $(BINARIES)
+releases/$(RELEASE).tar.gz: $(BINARIES)
 	@echo "$(WHALE) $@"
 	@rm -rf releases/$(RELEASE) releases/$(RELEASE).tar.gz
 	@install -d releases/$(RELEASE)/bin
 	@install $(BINARIES) releases/$(RELEASE)/bin
-	@cd releases/$(RELEASE) && tar -czf ../$(RELEASE).tar.gz *
+	@tar -czf releases/$(RELEASE).tar.gz -C releases/$(RELEASE) bin
+	@rm -rf releases/$(RELEASE)
+
+release: $(BINARIES) releases/$(RELEASE).tar.gz
+	@echo "$(WHALE) $@"
 	@cd releases && sha256sum $(RELEASE).tar.gz >$(RELEASE).tar.gz.sha256sum
 
+cri-release: $(BINARIES) releases/$(RELEASE).tar.gz
+	@echo "$(WHALE) $@"
+	@VERSION=$(VERSION:v%=%) script/release/release-cri
+
 clean: ## clean up binaries
 	@echo "$(WHALE) $@"
 	@rm -f $(BINARIES)

archive/tar.go

@@ -504,6 +504,12 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e
 
 		hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
 
+		// truncate timestamp for compatibility. without PAX stdlib rounds timestamps instead
+		hdr.Format = tar.FormatPAX
+		hdr.ModTime = hdr.ModTime.Truncate(time.Second)
+		hdr.AccessTime = time.Time{}
+		hdr.ChangeTime = time.Time{}
+
 		name := p
 		if strings.HasPrefix(name, string(filepath.Separator)) {
 			name, err = filepath.Rel(string(filepath.Separator), name)

cio/io_unix.go

@@ -72,17 +72,19 @@ func copyIO(fifos *FIFOSet, ioset *Streams) (*cio, error) {
 	}
 
 	var wg = &sync.WaitGroup{}
-	wg.Add(1)
-	go func() {
-		p := bufPool.Get().(*[]byte)
-		defer bufPool.Put(p)
-
-		io.CopyBuffer(ioset.Stdout, pipes.Stdout, *p)
-		pipes.Stdout.Close()
-		wg.Done()
-	}()
+	if fifos.Stdout != "" {
+		wg.Add(1)
+		go func() {
+			p := bufPool.Get().(*[]byte)
+			defer bufPool.Put(p)
+
+			io.CopyBuffer(ioset.Stdout, pipes.Stdout, *p)
+			pipes.Stdout.Close()
+			wg.Done()
+		}()
+	}
 
-	if !fifos.Terminal {
+	if !fifos.Terminal && fifos.Stderr != "" {
 		wg.Add(1)
 		go func() {
 			p := bufPool.Get().(*[]byte)

client.go

@@ -99,6 +99,12 @@ func New(address string, opts ...ClientOpt) (*Client, error) {
 		c.runtime = defaults.DefaultRuntime
 	}
 
+	if copts.defaultPlatform != nil {
+		c.platform = copts.defaultPlatform
+	} else {
+		c.platform = platforms.Default()
+	}
+
 	if copts.services != nil {
 		c.services = *copts.services
 	}
@@ -193,6 +199,7 @@ type Client struct {
 	conn      *grpc.ClientConn
 	runtime   string
 	defaultns string
+	platform  platforms.MatchComparer
 	connector func() (*grpc.ClientConn, error)
 }
 
@@ -294,6 +301,7 @@ type RemoteContext struct {
 	PlatformMatcher platforms.MatchComparer
 
 	// Unpack is done after an image is pulled to extract into a snapshotter.
+	// It is done simultaneously for schema 2 images when they are pulled.
 	// If an image is not unpacked on pull, it can be unpacked any time
 	// afterwards. Unpacking is required to run an image.
 	Unpack bool
@@ -332,9 +340,8 @@ type RemoteContext struct {
 	// MaxConcurrentDownloads is the max concurrent content downloads for each pull.
 	MaxConcurrentDownloads int
 
-	// AppendDistributionSourceLabel allows fetcher to add distribute source
-	// label for each blob content, which doesn't work for legacy schema1.
-	AppendDistributionSourceLabel bool
+	// AllMetadata downloads all manifests and known-configuration files
+	AllMetadata bool
 }
 
 func defaultRemoteContext() *RemoteContext {

client_opts.go

@@ -26,11 +26,12 @@ import (
 )
 
 type clientOpts struct {
-	defaultns      string
-	defaultRuntime string
-	services       *services
-	dialOptions    []grpc.DialOption
-	timeout        time.Duration
+	defaultns       string
+	defaultRuntime  string
+	defaultPlatform platforms.MatchComparer
+	services        *services
+	dialOptions     []grpc.DialOption
+	timeout         time.Duration
 }
 
 // ClientOpt allows callers to set options on the containerd client
@@ -55,6 +56,14 @@ func WithDefaultRuntime(rt string) ClientOpt {
 	}
 }
 
+// WithDefaultPlatform sets the default platform matcher on the client
+func WithDefaultPlatform(platform platforms.MatchComparer) ClientOpt {
+	return func(c *clientOpts) error {
+		c.defaultPlatform = platform
+		return nil
+	}
+}
+
 // WithDialOpts allows grpc.DialOptions to be set on the connection
 func WithDialOpts(opts []grpc.DialOption) ClientOpt {
 	return func(c *clientOpts) error {
@@ -195,11 +204,10 @@ func WithMaxConcurrentDownloads(max int) RemoteOpt {
 	}
 }
 
-// WithAppendDistributionSourceLabel allows fetcher to add distribute source
-// label for each blob content, which doesn't work for legacy schema1.
-func WithAppendDistributionSourceLabel() RemoteOpt {
+// WithAllMetadata downloads all manifests and known-configuration files
+func WithAllMetadata() RemoteOpt {
 	return func(_ *Client, c *RemoteContext) error {
-		c.AppendDistributionSourceLabel = true
+		c.AllMetadata = true
 		return nil
 	}
 }

client_test.go

@@ -287,9 +287,6 @@ func TestImagePullSomePlatforms(t *testing.T) {
 	count := 0
 	for _, manifest := range manifests {
 		children, err := images.Children(ctx, cs, manifest)
-		if err != nil {
-			t.Fatal(err)
-		}
 
 		found := false
 		for _, matcher := range m {
@@ -315,6 +312,8 @@ func TestImagePullSomePlatforms(t *testing.T) {
 				}
 				ra.Close()
 			}
+		} else if err == nil {
+			t.Fatal("manifest should not have pulled children content")
 		}
 	}