Skip to content

Commit

Permalink
feat(storage): retain file permission and owner when adding via confi…
Browse files Browse the repository at this point in the history 
…gMap

Signed-off-by: Smuu <18609909+Smuu@users.noreply.github.com>
  • Loading branch information
@smuu
smuu committed Dec 4, 2024
1 parent ad86999 commit 02ac775
Show file tree
Hide file tree
Showing 7 changed files with 126 additions and 72 deletions.
9 changes: 6 additions & 3 deletions pkg/instance/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package instance
import (
"context"
"fmt"
"os"
"path/filepath"
"sync"

Expand All @@ -14,8 +15,6 @@ import (
"github.com/celestiaorg/knuu/pkg/container"
)

const buildDirBase = "/tmp/knuu"

type build struct {
instance *Instance
imageName string
Expand Down Expand Up @@ -226,7 +225,11 @@ func getImageRegistry(imageName string) (string, error) {

// getBuildDir returns the build directory for the instance
func (b *build) getBuildDir() string {
return filepath.Join(buildDirBase, b.instance.name)
tmpDir, err := os.MkdirTemp("", "knuu-build-*")
if err != nil {
return ""
}
return filepath.Join(tmpDir, b.instance.name)
}

// addFileToBuilder adds a file to the builder
Expand Down
2 changes: 2 additions & 0 deletions pkg/instance/errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,9 @@ var (
ErrCreatingDirectory = errors.New("CreatingDirectory", "error creating directory")
ErrFailedToCreateDestFile = errors.New("FailedToCreateDestFile", "failed to create destination file '%s'")
ErrFailedToOpenSrcFile = errors.New("FailedToOpenSrcFile", "failed to open source file '%s'")
ErrFailedToGetSrcFileInfo = errors.New("FailedToGetSrcFileInfo", "failed to get source file info for")
ErrFailedToCopyFile = errors.New("FailedToCopyFile", "failed to copy from source '%s' to destination '%s'")
ErrFailedToSetPermissions = errors.New("FailedToSetPermissions", "failed to set permissions for destination file")
ErrSrcDoesNotExistOrIsDirectory = errors.New("SrcDoesNotExistOrIsDirectory", "src '%s' does not exist or is a directory")
ErrInvalidFormat = errors.New("InvalidFormat", "invalid format")
ErrFailedToConvertToInt64 = errors.New("FailedToConvertToInt64", "failed to convert to int64")
Expand Down
1 change: 0 additions & 1 deletion pkg/instance/execution.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,6 @@ func (e *execution) prepareReplicaSetConfig() k8s.ReplicaSetConfig {
Name: e.instance.name,
Labels: e.Labels(),
ServiceAccountName: e.instance.name,
FsGroup: e.instance.storage.fsGroup,
ContainerConfig: containerConfig,
SidecarConfigs: sidecarConfigs,
}
Expand Down
12 changes: 5 additions & 7 deletions pkg/instance/resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,13 +89,12 @@ func (r *resources) deployStorage(ctx context.Context) error {
return ErrDeployingVolumeForInstance.WithParams(r.instance.name).Wrap(err)
}
}
if len(r.instance.storage.files) == 0 {
return nil
if len(r.instance.storage.files) != 0 {
if err := r.instance.storage.deployFiles(ctx); err != nil {
return ErrDeployingFilesForInstance.WithParams(r.instance.name).Wrap(err)
}
}

if err := r.instance.storage.deployFiles(ctx); err != nil {
return ErrDeployingFilesForInstance.WithParams(r.instance.name).Wrap(err)
}
return nil
}

Expand Down Expand Up @@ -123,8 +122,7 @@ func (r *resources) destroyResources(ctx context.Context) error {
}

if len(r.instance.storage.files) != 0 {
err := r.instance.storage.destroyFiles(ctx)
if err != nil {
if err := r.instance.storage.destroyFiles(ctx); err != nil {
return ErrDestroyingFilesForInstance.WithParams(r.instance.name).Wrap(err)
}
}
Expand Down
53 changes: 30 additions & 23 deletions pkg/instance/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ type storage struct {
instance *Instance
volumes []*k8s.Volume
files []*k8s.File
fsGroup int64
}

func (i *Instance) Storage() *storage {
Expand All @@ -41,7 +40,7 @@ func (s *storage) AddFile(src string, dest string, chown string) error {
return err
}

dstPath, err := s.copyFileToBuildDir(src, dest)
buildDirPath, err := s.copyFileToBuildDir(src, dest)
if err != nil {
return err
}
Expand All @@ -51,7 +50,7 @@ func (s *storage) AddFile(src string, dest string, chown string) error {
s.instance.build.addFileToBuilder(src, dest, chown)
return nil
case StateCommitted, StateStopped:
return s.addFileToInstance(dstPath, dest, chown)
return s.addFileToInstance(buildDirPath, dest, chown)
}

s.instance.Logger.WithFields(logrus.Fields{
Expand Down Expand Up @@ -249,46 +248,55 @@ func (s *storage) copyFileToBuildDir(src, dest string) (string, error) {
return "", ErrCreatingDirectory.Wrap(err)
}

dst, err := os.Create(dstPath)
if err != nil {
return "", ErrFailedToCreateDestFile.WithParams(dstPath).Wrap(err)
}
defer dst.Close()

srcFile, err := os.Open(src)
if err != nil {
return "", ErrFailedToOpenSrcFile.WithParams(src).Wrap(err)
}
defer srcFile.Close()

srcInfo, err := srcFile.Stat()
if err != nil {
return "", ErrFailedToGetSrcFileInfo.WithParams(src).Wrap(err)
}

dst, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY, srcInfo.Mode().Perm())
if err != nil {
return "", ErrFailedToCreateDestFile.WithParams(dstPath).Wrap(err)
}
defer dst.Close()

if _, err := io.Copy(dst, srcFile); err != nil {
return "", ErrFailedToCopyFile.WithParams(src, dstPath).Wrap(err)
}

// Ensure the destination file has the same permissions as the source file
if err := os.Chmod(dstPath, srcInfo.Mode().Perm()); err != nil {
return "", ErrFailedToSetPermissions.WithParams(dstPath).Wrap(err)
}

return dstPath, nil
}

func (s *storage) addFileToInstance(dstPath, dest, chown string) error {
srcInfo, err := os.Stat(dstPath)
func (s *storage) addFileToInstance(srcPath, dest, chown string) error {
srcInfo, err := os.Stat(srcPath)
if os.IsNotExist(err) || srcInfo.IsDir() {
return ErrSrcDoesNotExistOrIsDirectory.WithParams(dstPath).Wrap(err)
return ErrSrcDoesNotExistOrIsDirectory.WithParams(srcPath).Wrap(err)
}

file := s.instance.K8sClient.NewFile(dstPath, dest)
// get the permission of the src file
permission := fmt.Sprintf("%o", srcInfo.Mode().Perm())

parts := strings.Split(chown, ":")
if len(parts) != 2 {
return ErrInvalidFormat
return ErrInvalidFormat.WithParams(chown)
}

group, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return ErrFailedToConvertToInt64.Wrap(err)
for _, part := range parts {
if _, err := strconv.ParseInt(part, 10, 64); err != nil {
return ErrFailedToConvertToInt64.WithParams(part).Wrap(err)
}
}
file := s.instance.K8sClient.NewFile(srcPath, dest, chown, permission)

if s.fsGroup != 0 && s.fsGroup != group {
return ErrAllFilesMustHaveSameGroup
}
s.fsGroup = group
s.files = append(s.files, file)
return nil
}
Expand Down Expand Up @@ -439,6 +447,5 @@ func (s *storage) clone() *storage {
instance: nil,
volumes: volumesCopy,
files: filesCopy,
fsGroup: s.fsGroup,
}
}
119 changes: 82 additions & 37 deletions pkg/k8s/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,6 @@ const (
// knuuPath is the path where the knuu volume is mounted
knuuPath = "/knuu"

// 0777 is used so that the files are usable by any user in the container without needing to change permissions
defaultFileModeForVolume = 0777

podFilesConfigmapNameSuffix = "-config"

initContainerNameSuffix = "-init"
Expand Down Expand Up @@ -71,8 +68,10 @@ type Volume struct {
}

type File struct {
Source string
Dest string
Source string
Dest string
Chown string
Permission string
}

// DeployPod creates a new pod in the namespace that k8s client is initiate with if it doesn't already exist.
Expand Down Expand Up @@ -101,10 +100,12 @@ func (c *Client) NewVolume(path string, size resource.Quantity, owner int64) *Vo
}
}

func (c *Client) NewFile(source, dest string) *File {
func (c *Client) NewFile(source, dest, chown, permission string) *File {
return &File{
Source: source,
Dest: dest,
Source: source,
Dest: dest,
Chown: chown,
Permission: permission,
}
}

Expand Down Expand Up @@ -381,6 +382,16 @@ func buildPodVolumes(name string, volumesAmount, filesAmount int) []v1.Volume {
podVolumes = append(podVolumes, podVolume)
}

if volumesAmount == 0 && filesAmount != 0 {
podVolume := v1.Volume{
Name: name,
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
}
podVolumes = append(podVolumes, podVolume)
}

if filesAmount != 0 {
podFiles := v1.Volume{
Name: name + podFilesConfigmapNameSuffix,
Expand All @@ -389,7 +400,7 @@ func buildPodVolumes(name string, volumesAmount, filesAmount int) []v1.Volume {
LocalObjectReference: v1.LocalObjectReference{
Name: name,
},
DefaultMode: ptr.To[int32](defaultFileModeForVolume),
DefaultMode: ptr.To[int32](0600),
},
},
}
Expand All @@ -414,25 +425,39 @@ func buildContainerVolumes(name string, volumes []*Volume, files []*File) []v1.V
)
}

var containerFiles []v1.VolumeMount

for n, file := range files {
shouldAddFile := true
for _, volume := range volumes {
if strings.HasPrefix(file.Dest, volume.Path) {
shouldAddFile = false
break
}
if len(volumes) == 0 && len(files) != 0 {
uniquePaths := make(map[string]bool)
for _, file := range files {
uniquePaths[filepath.Dir(file.Dest)] = true
}
if shouldAddFile {
containerFiles = append(containerFiles, v1.VolumeMount{
Name: name + podFilesConfigmapNameSuffix,
MountPath: file.Dest,
SubPath: fmt.Sprintf("%d", n),
for path := range uniquePaths {
containerVolumes = append(containerVolumes, v1.VolumeMount{
Name: name,
MountPath: path,
SubPath: strings.TrimPrefix(path, "/"),
})
}
}

var containerFiles []v1.VolumeMount

// for n, file := range files {
// shouldAddFile := true
// for _, volume := range volumes {
// if strings.HasPrefix(file.Dest, volume.Path) {
// shouldAddFile = false
// break
// }
// }
// if shouldAddFile {
// containerFiles = append(containerFiles, v1.VolumeMount{
// Name: name + podFilesConfigmapNameSuffix,
// MountPath: file.Dest,
// SubPath: fmt.Sprintf("%d", n),
// })
// }
// }

return append(containerVolumes, containerFiles...)
}

Expand All @@ -442,11 +467,25 @@ func buildInitContainerVolumes(name string, volumes []*Volume, files []*File) []
return []v1.VolumeMount{} // return empty slice if no volumes are specified
}

containerVolumes := []v1.VolumeMount{
{
var containerVolumes []v1.VolumeMount
if len(volumes) != 0 {
containerVolumes = append(containerVolumes, v1.VolumeMount{
Name: name,
MountPath: knuuPath, // set the path to "/knuu" as per the requirements
},
MountPath: knuuPath,
})
}
if len(volumes) == 0 && len(files) != 0 {
uniquePaths := make(map[string]bool)
for _, file := range files {
uniquePaths[filepath.Dir(file.Dest)] = true
}
for path := range uniquePaths {
containerVolumes = append(containerVolumes, v1.VolumeMount{
Name: name,
MountPath: knuuPath + path,
SubPath: strings.TrimPrefix(path, "/"),
})
}
}

var containerFiles []v1.VolumeMount
Expand Down Expand Up @@ -483,26 +522,33 @@ func (c *Client) buildInitContainerCommand(volumes []*Volume, files []*File) []s
cmds = append(cmds, parentDirCmd)
dirsProcessed[folder] = true
}
copyFileToKnuu := fmt.Sprintf("cp %s %s && ", file.Dest, filepath.Join(knuuPath, file.Dest))
cmds = append(cmds, copyFileToKnuu)
chown := file.Chown
permission := file.Permission
addFileToKnuu := fmt.Sprintf("cp %s %s && ", file.Dest, filepath.Join(knuuPath, file.Dest))
if chown != "" {
addFileToKnuu += fmt.Sprintf("chown %s %s && ", chown, filepath.Join(knuuPath, file.Dest))
}
if permission != "" {
addFileToKnuu += fmt.Sprintf("chmod %s %s && ", permission, filepath.Join(knuuPath, file.Dest))
}
cmds = append(cmds, addFileToKnuu)
}

// for each volume, copy the contents of the volume to the knuu volume
for i, volume := range volumes {
for _, volume := range volumes {
knuuVolumePath := fmt.Sprintf("%s%s", knuuPath, volume.Path)
cmd := fmt.Sprintf("if [ -d %s ] && [ \"$(ls -A %s)\" ]; then mkdir -p %s && cp -r %s/* %s && chown -R %d:%d %s",
volume.Path, volume.Path, knuuVolumePath, volume.Path,
knuuVolumePath, volume.Owner, volume.Owner, knuuVolumePath)
if i < len(volumes)-1 {
cmd += " ;fi && "
} else {
cmd += " ;fi"
}
cmd += " ;fi && "
cmds = append(cmds, cmd)
}

fullCommand := strings.Join(cmds, "")
commands = append(commands, fullCommand)
if strings.HasSuffix(fullCommand, " && ") {
commands[len(commands)-1] = strings.TrimSuffix(commands[len(commands)-1], " && ")
}

c.logger.WithField("command", fullCommand).Debug("init container command")
return commands
Expand Down Expand Up @@ -541,7 +587,7 @@ func prepareContainer(config ContainerConfig) v1.Container {

// prepareInitContainers creates a slice of v1.Container as init containers.
func (c *Client) prepareInitContainers(config ContainerConfig, init bool) []v1.Container {
if !init || len(config.Volumes) == 0 {
if !init || (len(config.Volumes) == 0 && len(config.Files) == 0) {
return nil
}

Expand All @@ -566,7 +612,6 @@ func preparePodVolumes(config ContainerConfig) []v1.Volume {
func (c *Client) preparePodSpec(spec PodConfig, init bool) v1.PodSpec {
podSpec := v1.PodSpec{
ServiceAccountName: spec.ServiceAccountName,
SecurityContext: &v1.PodSecurityContext{FSGroup: &spec.FsGroup},
InitContainers: c.prepareInitContainers(spec.ContainerConfig, init),
Containers: []v1.Container{prepareContainer(spec.ContainerConfig)},
Volumes: preparePodVolumes(spec.ContainerConfig),
Expand Down
Loading

0 comments on commit 02ac775

Please sign in to comment.