Merge pull request #16 from ambarve/shim_panic_log

Log shim panic logs before shim cleanup

pkg/ioutil/ioutil.go

@@ -0,0 +1,45 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ioutil
+
+import (
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+// LimitedRead reads at max `readLimitBytes` bytes from the file at path `filePath`. If the file has
+// more than `readLimitBytes` bytes of data then first `readLimitBytes` will be returned.
+func LimitedRead(filePath string, readLimitBytes int64) ([]byte, error) {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return nil, errors.Wrap(err, "limited read failed to open file")
+	}
+	defer f.Close()
+	if fi, err := f.Stat(); err == nil {
+		if fi.Size() < readLimitBytes {
+			readLimitBytes = fi.Size()
+		}
+		buf := make([]byte, readLimitBytes)
+		_, err := f.Read(buf)
+		if err != nil {
+			return []byte{}, errors.Wrap(err, "limited read failed during file read")
+		}
+		return buf, nil
+	}
+	return []byte{}, errors.Wrap(err, "limited read failed during file stat")
+}

runtime/v2/binary.go

@@ -167,7 +167,7 @@ func (b *binary) Delete(ctx context.Context) (*runtime.Exit, error) {
 	if err := response.Unmarshal(out.Bytes()); err != nil {
 		return nil, err
 	}
-	if err := b.bundle.Delete(); err != nil {
+	if err := b.bundle.Delete(ctx); err != nil {
 		return nil, err
 	}
 	return &runtime.Exit{

runtime/v2/bundle.go

@@ -24,8 +24,10 @@ import (
 	"path/filepath"
 
 	"github.com/containerd/containerd/identifiers"
+	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/namespaces"
+	cioutil "github.com/containerd/containerd/pkg/ioutil"
 	"github.com/pkg/errors"
 )
 
@@ -115,16 +117,32 @@ type Bundle struct {
 }
 
 // Delete a bundle atomically
-func (b *Bundle) Delete() error {
+func (b *Bundle) Delete(ctx context.Context) error {
 	work, werr := os.Readlink(filepath.Join(b.Path, "work"))
 	rootfs := filepath.Join(b.Path, "rootfs")
+
+	// on windows hcsshim writes panic logs in the bundle directory in a file named
+	// "panic.log" log those messages (if any).
+	// Read only upto 1MB worth of data from this file. If the file is larger
+	// than that, log that.
+	readLimit := int64(1024 * 1024) // 1MB
+	logBytes, err := cioutil.LimitedRead(filepath.Join(b.Path, "panic.log"), readLimit)
+	if err == nil && len(logBytes) > 0 {
+		if int64(len(logBytes)) == readLimit {
+			log.G(ctx).Warnf("shim panic log file %s is larger than 1MB, logging only first 1MB", filepath.Join(b.Path, "panic.log"))
+		}
+		log.G(ctx).WithField("log", string(logBytes)).Warn("found shim panic logs during delete")
+	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
+		log.G(ctx).WithError(err).Warn("failed to open shim panic log")
+	}
+
 	if err := mount.UnmountAll(rootfs, 0); err != nil {
 		return errors.Wrapf(err, "unmount rootfs %s", rootfs)
 	}
 	if err := os.Remove(rootfs); err != nil && !os.IsNotExist(err) {
 		return errors.Wrap(err, "failed to remove bundle rootfs")
 	}
-	err := atomicDelete(b.Path)
+	err = atomicDelete(b.Path)
 	if err == nil {
 		if werr == nil {
 			return atomicDelete(work)

runtime/v2/manager.go

@@ -127,7 +127,7 @@ func (m *TaskManager) Create(ctx context.Context, id string, opts runtime.Create
 	}
 	defer func() {
 		if err != nil {
-			bundle.Delete()
+			bundle.Delete(ctx)
 		}
 	}()
 	topts := opts.TaskOptions
@@ -246,12 +246,12 @@ func (m *TaskManager) loadTasks(ctx context.Context) error {
 		// fast path
 		bf, err := ioutil.ReadDir(bundle.Path)
 		if err != nil {
-			bundle.Delete()
+			bundle.Delete(ctx)
 			log.G(ctx).WithError(err).Errorf("fast path read bundle path for %s", bundle.Path)
 			continue
 		}
 		if len(bf) == 0 {
-			bundle.Delete()
+			bundle.Delete(ctx)
 			continue
 		}
 		container, err := m.container(ctx, id)
@@ -260,7 +260,7 @@ func (m *TaskManager) loadTasks(ctx context.Context) error {
 			if err := mount.UnmountAll(filepath.Join(bundle.Path, "rootfs"), 0); err != nil {
 				log.G(ctx).WithError(err).Errorf("forceful unmount of rootfs %s", id)
 			}
-			bundle.Delete()
+			bundle.Delete(ctx)
 			continue
 		}
 		binaryCall := shimBinary(ctx, bundle, container.Runtime.Name, m.containerdAddress, m.containerdTTRPCAddress, m.events, m.tasks)

runtime/v2/shim.go

@@ -241,7 +241,7 @@ func (s *shim) Delete(ctx context.Context) (*runtime.Exit, error) {
 		log.G(ctx).WithField("id", s.ID()).WithError(err).Error("failed to shutdown shim")
 	}
 	s.Close()
-	if err := s.bundle.Delete(); err != nil {
+	if err := s.bundle.Delete(ctx); err != nil {
 		log.G(ctx).WithField("id", s.ID()).WithError(err).Error("failed to delete bundle")
 	}
 	if shimErr != nil {