From ae33d83c7ba2344e085816f4071b0763a4a1336e Mon Sep 17 00:00:00 2001 From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com> Date: Sat, 18 Apr 2020 19:48:51 +0530 Subject: [PATCH] Kelp UI: refactor os paths used to accommodate for 260 character file path limit in windows (#406) * 1 - convertUnixPathToNative + test * 2 - MakeFromUnixPath + test * 3 - checkGoosAllowed -> skipIfGoosNotAllowed in tests * 4 - remove unused RelFromBase * 5 - rework folder structure to use ~/.kelp 1. ops now lives in ~/.kelp and is renamed to bot_data 2. /logs is now renamed to ui_logs 3. renamed path variables in APIServer to be botConfigsPath and botLogsPath * 6 - manually expand ~ for kelp directory * 7 - add comment about 260 character limit on windows * 8 - MakeFromNativePath + test * 9 - usr.HomeDir is in native form so make OSPath from native path * 10 - fix MakeFromNativePath so it sets native and unix path correctly * 11 - replace MakeFromUnixPath usage with a simple ospath.Join call * 12 - set working directory to ~/.kelp and refactor paths to be saved and used via kos this should give us the smallest path lengths on windows * 13 - rename kelpBinName string to kelpBinPath *OSPath since we now run from the .kelp dir * 14 - add comments on why we use ~/.kelp as the working directory instead of the binary directory as the wd --- cmd/server_amd64.go | 48 ++++---- gui/backend/api_server.go | 33 +++--- gui/backend/autogenerate_bot.go | 4 +- gui/backend/delete_bot.go | 2 +- gui/backend/generate_bot_name.go | 2 +- gui/backend/get_bot_config.go | 4 +- gui/backend/get_bot_info.go | 2 +- gui/backend/list_bots.go | 2 +- gui/backend/start_bot.go | 6 +- gui/backend/upsert_bot_config.go | 4 +- support/kelpos/kelpos.go | 51 +++++++-- support/kelpos/ospath.go | 37 ++++-- support/kelpos/ospath_test.go | 186 ++++++++++++++++++++----------- support/kelpos/process.go | 4 +- 14 files changed, 244 insertions(+), 141 deletions(-) diff --git a/cmd/server_amd64.go b/cmd/server_amd64.go index 41ff59f3b..00c2e59dd 100644 --- a/cmd/server_amd64.go +++ b/cmd/server_amd64.go @@ -38,9 +38,8 @@ import ( "github.com/stellar/kelp/support/utils" ) -const kelpPrefsDirectory = ".kelp" const kelpAssetsPath = "/assets" -const logsDir = "/logs" +const uiLogsDir = "/ui_logs" const vendorDirectory = "/vendor" const trayIconName = "kelp-icon@1-8x.png" const kelpCcxtPath = "/ccxt" @@ -80,16 +79,10 @@ func init() { options.noElectron = serverCmd.Flags().Bool("no-electron", false, "open in browser instead of using electron") serverCmd.Run = func(ccmd *cobra.Command, args []string) { - basepath, e := kelpos.MakeOsPathBase() - if e != nil { - panic(errors.Wrap(e, "could not make ospath")) - } - log.Printf("ospath initialized with native path: %s", basepath.Native()) - log.Printf("ospath initialized with unix path: %s", basepath.Unix()) - isLocalMode := env == envDev isLocalDevMode := isLocalMode && *options.dev kos := kelpos.GetKelpOS() + var e error if isLocalMode { wd, e := os.Getwd() if e != nil { @@ -108,15 +101,15 @@ func init() { t := time.Now().Format("20060102T150405MST") logFilename := fmt.Sprintf("kelp-ui_%s.log", t) - logsDirPath := basepath.Join(kelpPrefsDirectory, logsDir) - log.Printf("calling mkdir on logsDirPath: %s ...", logsDirPath.AsString()) - e = kos.Mkdir(logsDirPath) + uiLogsDirPath := kos.GetDotKelpWorkingDir().Join(uiLogsDir) + log.Printf("calling mkdir on uiLogsDirPath: %s ...", uiLogsDirPath.AsString()) + e = kos.Mkdir(uiLogsDirPath) if e != nil { - panic(errors.Wrap(e, "could not mkdir on logsDirPath: "+logsDirPath.AsString())) + panic(errors.Wrap(e, "could not mkdir on uiLogsDirPath: "+uiLogsDirPath.AsString())) } // don't use explicit unix filepath here since it uses os.Open directly and won't work on windows - logFilepath = basepath.Join(kelpPrefsDirectory, logsDir, logFilename) + logFilepath = uiLogsDirPath.Join(logFilename) setLogFile(l, logFilepath.Native()) if *options.verbose { @@ -129,9 +122,11 @@ func init() { openBrowserWg.Add(1) if !isLocalDevMode { // don't use explicit unix filepath here since it uses os.Create directly and won't work on windows - trayIconPath := basepath.Join(kelpPrefsDirectory, kelpAssetsPath, trayIconName) + assetsDirPath := kos.GetDotKelpWorkingDir().Join(kelpAssetsPath) + log.Printf("assetsDirPath: %s", assetsDirPath.AsString()) + trayIconPath := assetsDirPath.Join(trayIconName) log.Printf("trayIconPath: %s", trayIconPath.AsString()) - e = writeTrayIcon(kos, trayIconPath, basepath) + e = writeTrayIcon(kos, trayIconPath, assetsDirPath) if e != nil { log.Fatal(errors.Wrap(e, "could not write tray icon")) } @@ -159,7 +154,7 @@ func init() { tailFilePort := startTailFileServer(tailFileCompiled) electronURL = fmt.Sprintf("http://localhost:%d", tailFilePort) } else { - tailFilepath := basepath.Join(kelpPrefsDirectory, "tail.html") + tailFilepath := kos.GetDotKelpWorkingDir().Join("tail.html") fileContents := []byte(tailFileCompiled) e := ioutil.WriteFile(tailFilepath.Native(), fileContents, 0644) if e != nil { @@ -244,7 +239,7 @@ func init() { ccxtGoos = "linux" } - ccxtDirPath := basepath.Join(kelpPrefsDirectory, kelpCcxtPath) + ccxtDirPath := kos.GetDotKelpWorkingDir().Join(kelpCcxtPath) ccxtFilenameNoExt := fmt.Sprintf("ccxt-rest_%s-x64", ccxtGoos) filenameWithExt := fmt.Sprintf("%s.zip", ccxtFilenameNoExt) @@ -256,7 +251,7 @@ func init() { } ccxtBinPath := ccxtDirPath.Join(ccxtFilenameNoExt, ccxtBinaryName) - unzipCcxtFile(kos, ccxtDirPath, ccxtBinPath, filenameWithExt, basepath) + unzipCcxtFile(kos, ccxtDirPath, ccxtBinPath, filenameWithExt) e = runCcxtBinary(kos, ccxtBinPath) if e != nil { @@ -265,9 +260,13 @@ func init() { } } + dataPath := kos.GetDotKelpWorkingDir().Join("bot_data") + botConfigsPath := dataPath.Join("configs") + botLogsPath := dataPath.Join("logs") s, e := backend.MakeAPIServer( kos, - basepath, + botConfigsPath, + botLogsPath, *options.horizonTestnetURI, apiTestNet, *options.horizonPubnetURI, @@ -280,7 +279,7 @@ func init() { panic(e) } - guiWebPath := basepath.Join("../gui/web") + guiWebPath := kos.GetBinDir().Join("../gui/web") if isLocalDevMode { // the frontend app checks the REACT_APP_API_PORT variable to be set when serving os.Setenv("REACT_APP_API_PORT", fmt.Sprintf("%d", *options.devAPIPort)) @@ -384,7 +383,6 @@ func unzipCcxtFile( ccxtDir *kelpos.OSPath, ccxtBinPath *kelpos.OSPath, filenameWithExt string, - originalDir *kelpos.OSPath, ) { if _, e := os.Stat(ccxtDir.Native()); !os.IsNotExist(e) { if _, e := os.Stat(ccxtBinPath.Native()); !os.IsNotExist(e) { @@ -393,7 +391,7 @@ func unzipCcxtFile( } log.Printf("unzipping file %s ... ", filenameWithExt) - zipCmd := fmt.Sprintf("cd %s && unzip %s && cd %s", ccxtDir.Unix(), filenameWithExt, originalDir.Unix()) + zipCmd := fmt.Sprintf("cd %s && unzip %s", ccxtDir.Unix(), filenameWithExt) _, e := kos.Blocking("zip", zipCmd) if e != nil { log.Fatal(errors.Wrap(e, fmt.Sprintf("unable to unzip file %s in directory %s", filenameWithExt, ccxtDir))) @@ -469,9 +467,7 @@ func generateStaticFiles(kos *kelpos.KelpOS, guiWebPath *kelpos.OSPath) { log.Println() } -func writeTrayIcon(kos *kelpos.KelpOS, trayIconPath *kelpos.OSPath, basepath *kelpos.OSPath) error { - assetsDirPath := basepath.Join(kelpPrefsDirectory, kelpAssetsPath) - log.Printf("assetsDirPath: %s", assetsDirPath.AsString()) +func writeTrayIcon(kos *kelpos.KelpOS, trayIconPath *kelpos.OSPath, assetsDirPath *kelpos.OSPath) error { if _, e := os.Stat(trayIconPath.Native()); !os.IsNotExist(e) { // file exists, don't write again return nil diff --git a/gui/backend/api_server.go b/gui/backend/api_server.go index dd757e1c0..df92c3290 100644 --- a/gui/backend/api_server.go +++ b/gui/backend/api_server.go @@ -15,10 +15,9 @@ import ( // APIServer is an instance of the API service type APIServer struct { - basepath *kelpos.OSPath - kelpBinName string - configsDir *kelpos.OSPath - logsDir *kelpos.OSPath + kelpBinPath *kelpos.OSPath + botConfigsPath *kelpos.OSPath + botLogsPath *kelpos.OSPath kos *kelpos.KelpOS horizonTestnetURI string horizonPubnetURI string @@ -34,7 +33,8 @@ type APIServer struct { // MakeAPIServer is a factory method func MakeAPIServer( kos *kelpos.KelpOS, - basepath *kelpos.OSPath, + botConfigsPath *kelpos.OSPath, + botLogsPath *kelpos.OSPath, horizonTestnetURI string, apiTestNet *horizonclient.Client, horizonPubnetURI string, @@ -43,9 +43,7 @@ func MakeAPIServer( noHeaders bool, quitFn func(), ) (*APIServer, error) { - kelpBinName := filepath.Base(os.Args[0]) - configsDir := basepath.Join("ops", "configs") - logsDir := basepath.Join("ops", "logs") + kelpBinPath := kos.GetBinDir().Join(filepath.Base(os.Args[0])) optionsMetadata, e := loadOptionsMetadata() if e != nil { @@ -53,10 +51,9 @@ func MakeAPIServer( } return &APIServer{ - basepath: basepath, - kelpBinName: kelpBinName, - configsDir: configsDir, - logsDir: logsDir, + kelpBinPath: kelpBinPath, + botConfigsPath: botConfigsPath, + botLogsPath: botLogsPath, kos: kos, horizonTestnetURI: horizonTestnetURI, horizonPubnetURI: horizonPubnetURI, @@ -125,7 +122,7 @@ func (s *APIServer) runKelpCommandBlocking(namespace string, cmd string) ([]byte // name of the folder in which the GUI version is unzipped. // To avoid these issues we only invoke with the binary name as opposed to the absolute path that contains the // directory name. see start_bot.go for some experimentation with absolute and relative paths - cmdString := fmt.Sprintf("./%s %s", s.kelpBinName, cmd) + cmdString := fmt.Sprintf("%s %s", s.kelpBinPath.Unix(), cmd) return s.kos.Blocking(namespace, cmdString) } @@ -135,19 +132,19 @@ func (s *APIServer) runKelpCommandBackground(namespace string, cmd string) (*kel // name of the folder in which the GUI version is unzipped. // To avoid these issues we only invoke with the binary name as opposed to the absolute path that contains the // directory name. see start_bot.go for some experimentation with absolute and relative paths - cmdString := fmt.Sprintf("./%s %s", s.kelpBinName, cmd) + cmdString := fmt.Sprintf("%s %s", s.kelpBinPath.Unix(), cmd) return s.kos.Background(namespace, cmdString) } func (s *APIServer) setupOpsDirectory() error { - e := s.kos.Mkdir(s.configsDir) + e := s.kos.Mkdir(s.botConfigsPath) if e != nil { - return fmt.Errorf("error setting up configs directory (%s): %s\n", s.configsDir, e) + return fmt.Errorf("error setting up configs directory (%s): %s\n", s.botConfigsPath, e) } - e = s.kos.Mkdir(s.logsDir) + e = s.kos.Mkdir(s.botLogsPath) if e != nil { - return fmt.Errorf("error setting up logs directory (%s): %s\n", s.logsDir, e) + return fmt.Errorf("error setting up logs directory (%s): %s\n", s.botLogsPath, e) } return nil diff --git a/gui/backend/autogenerate_bot.go b/gui/backend/autogenerate_bot.go index 45ed5d1f1..e14c6304a 100644 --- a/gui/backend/autogenerate_bot.go +++ b/gui/backend/autogenerate_bot.go @@ -49,7 +49,7 @@ func (s *APIServer) autogenerateBot(w http.ResponseWriter, r *http.Request) { filenamePair := bot.Filenames() sampleTrader := s.makeSampleTrader(kp.Seed()) - traderFilePath := s.configsDir.Join(filenamePair.Trader) + traderFilePath := s.botConfigsPath.Join(filenamePair.Trader) log.Printf("writing autogenerated bot config to file: %s\n", traderFilePath.AsString()) e = toml.WriteFile(traderFilePath.Native(), sampleTrader) if e != nil { @@ -58,7 +58,7 @@ func (s *APIServer) autogenerateBot(w http.ResponseWriter, r *http.Request) { } sampleBuysell := makeSampleBuysell() - strategyFilePath := s.configsDir.Join(filenamePair.Strategy) + strategyFilePath := s.botConfigsPath.Join(filenamePair.Strategy) log.Printf("writing autogenerated strategy config to file: %s\n", strategyFilePath.AsString()) e = toml.WriteFile(strategyFilePath.Native(), sampleBuysell) if e != nil { diff --git a/gui/backend/delete_bot.go b/gui/backend/delete_bot.go index 349588c91..26d72fa29 100644 --- a/gui/backend/delete_bot.go +++ b/gui/backend/delete_bot.go @@ -52,7 +52,7 @@ func (s *APIServer) deleteBot(w http.ResponseWriter, r *http.Request) { // delete configs botPrefix := model2.GetPrefix(botName) - botConfigPath := s.configsDir.Join(botPrefix) + botConfigPath := s.botConfigsPath.Join(botPrefix) _, e = s.kos.Blocking("rm", fmt.Sprintf("rm %s*", botConfigPath.Unix())) if e != nil { s.writeError(w, fmt.Sprintf("error running rm command for bot configs: %s\n", e)) diff --git a/gui/backend/generate_bot_name.go b/gui/backend/generate_bot_name.go index 375d06983..9afec29ac 100644 --- a/gui/backend/generate_bot_name.go +++ b/gui/backend/generate_bot_name.go @@ -57,7 +57,7 @@ func (s *APIServer) doGenerateBotName() (string, error) { } func (s *APIServer) prefixExists(prefix string) (bool, error) { - command := fmt.Sprintf("ls %s | grep %s", s.configsDir.Unix(), prefix) + command := fmt.Sprintf("ls %s | grep %s", s.botConfigsPath.Unix(), prefix) _, e := s.kos.Blocking("prefix", command) if e != nil { if strings.Contains(e.Error(), "exit status 1") { diff --git a/gui/backend/get_bot_config.go b/gui/backend/get_bot_config.go index ebedb2a8f..1f3d9a8ca 100644 --- a/gui/backend/get_bot_config.go +++ b/gui/backend/get_bot_config.go @@ -27,14 +27,14 @@ func (s *APIServer) getBotConfig(w http.ResponseWriter, r *http.Request) { } filenamePair := model2.GetBotFilenames(botName, "buysell") - traderFilePath := s.configsDir.Join(filenamePair.Trader) + traderFilePath := s.botConfigsPath.Join(filenamePair.Trader) var botConfig trader.BotConfig e = config.Read(traderFilePath.Native(), &botConfig) if e != nil { s.writeErrorJson(w, fmt.Sprintf("cannot read bot config at path '%s': %s\n", traderFilePath, e)) return } - strategyFilePath := s.configsDir.Join(filenamePair.Strategy) + strategyFilePath := s.botConfigsPath.Join(filenamePair.Strategy) var buysellConfig plugins.BuySellConfig e = config.Read(strategyFilePath.Native(), &buysellConfig) if e != nil { diff --git a/gui/backend/get_bot_info.go b/gui/backend/get_bot_info.go index 987a73a24..dbf9f7b4f 100644 --- a/gui/backend/get_bot_info.go +++ b/gui/backend/get_bot_info.go @@ -64,7 +64,7 @@ func (s *APIServer) runGetBotInfoDirect(w http.ResponseWriter, botName string) { } filenamePair := model2.GetBotFilenames(botName, buysell) - traderFilePath := s.configsDir.Join(filenamePair.Trader) + traderFilePath := s.botConfigsPath.Join(filenamePair.Trader) var botConfig trader.BotConfig e = config.Read(traderFilePath.Native(), &botConfig) if e != nil { diff --git a/gui/backend/list_bots.go b/gui/backend/list_bots.go index 6a15899d5..3f52aad1a 100644 --- a/gui/backend/list_bots.go +++ b/gui/backend/list_bots.go @@ -13,7 +13,7 @@ import ( func (s *APIServer) listBots(w http.ResponseWriter, r *http.Request) { log.Printf("listing bots\n") - resultBytes, e := s.kos.Blocking("ls", fmt.Sprintf("ls %s | sort", s.configsDir.Unix())) + resultBytes, e := s.kos.Blocking("ls", fmt.Sprintf("ls %s | sort", s.botConfigsPath.Unix())) if e != nil { s.writeErrorJson(w, fmt.Sprintf("error when listing bots: %s\n", e)) return diff --git a/gui/backend/start_bot.go b/gui/backend/start_bot.go index 69d54dbe2..22129ac87 100644 --- a/gui/backend/start_bot.go +++ b/gui/backend/start_bot.go @@ -61,17 +61,17 @@ func (s *APIServer) doStartBot(botName string, strategy string, iterations *uint // use relative paths, which is why it seems to work // Note that /mnt/c is unlikely to be valid in windows (but is valid in the linux subsystem) since it's usually prefixed by the // volume (C:\ etc.), which is why relative paths works so well here as it avoids this confusion. - traderRelativeConfigPath, e := s.configsDir.Join(filenamePair.Trader).RelFromPath(s.basepath) + traderRelativeConfigPath, e := s.botConfigsPath.Join(filenamePair.Trader).RelFromPath(s.kos.GetDotKelpWorkingDir()) if e != nil { return fmt.Errorf("unable to get relative path of trader config file from basepath: %s", e) } - stratRelativeConfigPath, e := s.configsDir.Join(filenamePair.Strategy).RelFromPath(s.basepath) + stratRelativeConfigPath, e := s.botConfigsPath.Join(filenamePair.Strategy).RelFromPath(s.kos.GetDotKelpWorkingDir()) if e != nil { return fmt.Errorf("unable to get relative path of strategy config file from basepath: %s", e) } - logRelativePrefixPath, e := s.logsDir.Join(logPrefix).RelFromPath(s.basepath) + logRelativePrefixPath, e := s.botLogsPath.Join(logPrefix).RelFromPath(s.kos.GetDotKelpWorkingDir()) if e != nil { return fmt.Errorf("unable to get relative path of log prefix path from basepath: %s", e) } diff --git a/gui/backend/upsert_bot_config.go b/gui/backend/upsert_bot_config.go index 78235bd78..a804ee131 100644 --- a/gui/backend/upsert_bot_config.go +++ b/gui/backend/upsert_bot_config.go @@ -87,7 +87,7 @@ func (s *APIServer) upsertBotConfig(w http.ResponseWriter, r *http.Request) { } filenamePair := model2.GetBotFilenames(req.Name, req.Strategy) - traderFilePath := s.configsDir.Join(filenamePair.Trader) + traderFilePath := s.botConfigsPath.Join(filenamePair.Trader) botConfig := req.TraderConfig log.Printf("upsert bot config to file: %s\n", traderFilePath.AsString()) e = toml.WriteFile(traderFilePath.Native(), &botConfig) @@ -96,7 +96,7 @@ func (s *APIServer) upsertBotConfig(w http.ResponseWriter, r *http.Request) { return } - strategyFilePath := s.configsDir.Join(filenamePair.Strategy) + strategyFilePath := s.botConfigsPath.Join(filenamePair.Strategy) strategyConfig := req.StrategyConfig log.Printf("upsert strategy config to file: %s\n", strategyFilePath.AsString()) e = toml.WriteFile(strategyFilePath.Native(), &strategyConfig) diff --git a/support/kelpos/kelpos.go b/support/kelpos/kelpos.go index cb70b1c15..d11a005f8 100644 --- a/support/kelpos/kelpos.go +++ b/support/kelpos/kelpos.go @@ -3,15 +3,21 @@ package kelpos import ( "fmt" "io" + "log" "os/exec" + "os/user" "sync" + "github.com/stellar/go/support/errors" "github.com/stellar/kelp/gui/model2" ) +const dotKelpDir = ".kelp" + // KelpOS is a struct that manages all subprocesses started by this Kelp process type KelpOS struct { - workingBinDir *OSPath + binDir *OSPath + dotKelpWorkingDir *OSPath processes map[string]Process processLock *sync.Mutex bots map[string]*BotInstance @@ -19,6 +25,16 @@ type KelpOS struct { silentRegistrations bool } +// GetBinDir accessor +func (kos *KelpOS) GetBinDir() *OSPath { + return kos.binDir +} + +// GetDotKelpWorkingDir accessor +func (kos *KelpOS) GetDotKelpWorkingDir() *OSPath { + return kos.dotKelpWorkingDir +} + // SetSilentRegistrations does not log every time we register and unregister commands func (kos *KelpOS) SetSilentRegistrations() { kos.silentRegistrations = true @@ -35,17 +51,36 @@ type Process struct { var singleton *KelpOS func init() { - path, e := MakeOsPathBase() + binDir, e := MakeOsPathBase() if e != nil { - panic(e) + panic(errors.Wrap(e, "could not make binDir")) } + log.Printf("binDir initialized: %s", binDir.AsString()) + + usr, e := user.Current() + if e != nil { + panic(errors.Wrap(e, "could not fetch current user (need to get home directory)")) + } + usrHomeDir, e := binDir.MakeFromNativePath(usr.HomeDir) + if e != nil { + panic(errors.Wrap(e, "could not make usrHomeDir from usr.HomeDir="+usr.HomeDir)) + } + log.Printf("Kelp is being run from user '%s' (Uid=%s, Name=%s, HomeDir=%s)", usr.Username, usr.Uid, usr.Name, usrHomeDir.AsString()) + + // file path for windows needs to be 260 characters (https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file) + // so we want to put it closer to the root volume in ~/.kelp (or C:\.kelp) so it does not throw an error + dotKelpWorkingDir := usrHomeDir.Join(dotKelpDir) + log.Printf("dotKelpWorkingDir initialized: %s", dotKelpWorkingDir.AsString()) + // using dotKelpWorkingDir as working directory since all our config files and log files are located in here and we want + // to have the shortest path lengths to accommodate for the 260 character file path limit in windows singleton = &KelpOS{ - workingBinDir: path, - processes: map[string]Process{}, - processLock: &sync.Mutex{}, - bots: map[string]*BotInstance{}, - botLock: &sync.Mutex{}, + binDir: binDir, + dotKelpWorkingDir: dotKelpWorkingDir, + processes: map[string]Process{}, + processLock: &sync.Mutex{}, + bots: map[string]*BotInstance{}, + botLock: &sync.Mutex{}, } } diff --git a/support/kelpos/ospath.go b/support/kelpos/ospath.go index aa737e9be..94aa63505 100644 --- a/support/kelpos/ospath.go +++ b/support/kelpos/ospath.go @@ -66,6 +66,24 @@ func MakeOsPathBase() (*OSPath, error) { return ospath, nil } +// MakeFromUnixPath returns a new OSPath at the passed in unix path string by using the existing OSPath +func (o *OSPath) MakeFromUnixPath(targetUnixPath string) (*OSPath, error) { + nativePath, e := convertUnixPathToNative(o.Unix(), targetUnixPath, o.Native()) + if e != nil { + return nil, fmt.Errorf("could not convert unix path (%s) to native: %s", targetUnixPath, e) + } + return makeOSPath(nativePath, targetUnixPath, false), nil +} + +// MakeFromNativePath returns a new OSPath at the passed in native path string by using the existing OSPath +func (o *OSPath) MakeFromNativePath(targetNativePath string) (*OSPath, error) { + unixPath, e := convertNativePathToUnix(o.Native(), targetNativePath, o.Unix()) + if e != nil { + return nil, fmt.Errorf("could not convert native path (%s) to unix: %s", targetNativePath, e) + } + return makeOSPath(targetNativePath, unixPath, false), nil +} + // Native returns the native representation of the path as a string func (o *OSPath) Native() string { return o.native @@ -108,16 +126,6 @@ func (o *OSPath) JoinRelPath(relPaths ...*OSPath) (*OSPath, error) { return o.Join(elems...), nil } -// RelFromBase returns a *OSPath that is relative to the basepath -func (o *OSPath) RelFromBase() (*OSPath, error) { - basepath, e := MakeOsPathBase() - if e != nil { - return nil, fmt.Errorf("unable to fetch ospathbase: %s", e) - } - - return o.RelFromPath(basepath) -} - // RelFromPath returns a *OSPath that is relative from the provided path func (o *OSPath) RelFromPath(basepath *OSPath) (*OSPath, error) { newRelNative, e := filepath.Rel(basepath.Native(), o.Native()) @@ -161,6 +169,15 @@ func convertNativePathToUnix(baseNative string, targetNative string, baseUnix st return toUnixFilepath(filepath.Join(baseUnix, toUnixFilepath(relBaseToTarget))), nil } +func convertUnixPathToNative(baseUnix string, targetUnix string, baseNative string) (string, error) { + relBaseToTarget, e := filepath.Rel(baseUnix, targetUnix) + if e != nil { + return "", fmt.Errorf("could not fetch relative path from baseUnix (%s) to targetUnix (%s): %s", baseUnix, targetUnix, e) + } + + return filepath.Join(baseNative, relBaseToTarget), nil +} + func getWorkingDirUnix() (string, error) { outputBytes, e := exec.Command("bash", "-c", "pwd").Output() if e != nil { diff --git a/support/kelpos/ospath_test.go b/support/kelpos/ospath_test.go index b862cc510..eedf60755 100644 --- a/support/kelpos/ospath_test.go +++ b/support/kelpos/ospath_test.go @@ -27,9 +27,7 @@ func TestOSPath(t *testing.T) { for _, k := range testCases { t.Run(k.basePathNative, func(t *testing.T) { // early exit if running on a disallowed platform to avoid false negatives - if !checkGoosAllowed(k.runGoos) { - return - } + skipIfGoosNotAllowed(t, k.runGoos) ospath1 := makeOSPath(k.basePathNative, "/mnt/c/testfolder", false) if !assert.Equal(t, false, ospath1.IsRelative()) { @@ -74,95 +72,153 @@ func TestOSPath(t *testing.T) { } } -func TestConvertNativePathToUnix(t *testing.T) { +func TestConvertBetweenNativeUnixPaths(t *testing.T) { testCases := []struct { - name string - runGoos []string - baseNative string - targetNative string - baseUnix string - wantTargetUnix string + name string + runGoos []string + baseNative string + targetNative string + baseUnix string + targetUnix string }{ { - name: "unix_forward", - runGoos: []string{"linux", "darwin"}, - baseNative: "/Users/a/test", - targetNative: "/Users/a/test/b", - baseUnix: "/Users/a/test", - wantTargetUnix: "/Users/a/test/b", - }, { - name: "unix_forward_trailslash", - runGoos: []string{"linux", "darwin"}, - baseNative: "/Users/a/test/", - targetNative: "/Users/a/test/b/", - baseUnix: "/Users/a/test/", - wantTargetUnix: "/Users/a/test/b", + name: "unix_forward", + runGoos: []string{"linux", "darwin"}, + baseNative: "/Users/a/test", + targetNative: "/Users/a/test/b", + baseUnix: "/Users/a/test", + targetUnix: "/Users/a/test/b", }, { - name: "unix_backward", - runGoos: []string{"linux", "darwin"}, - baseNative: "/Users/a/test", - targetNative: "/Users/a", - baseUnix: "/Users/a/test", - wantTargetUnix: "/Users/a", + name: "unix_backward", + runGoos: []string{"linux", "darwin"}, + baseNative: "/Users/a/test", + targetNative: "/Users/a", + baseUnix: "/Users/a/test", + targetUnix: "/Users/a", }, { - name: "unix_backward_trailslash", - runGoos: []string{"linux", "darwin"}, - baseNative: "/Users/a/test/", - targetNative: "/Users/a/", - baseUnix: "/Users/a/test/", - wantTargetUnix: "/Users/a", + name: "windows_forward", + runGoos: []string{"windows"}, + baseNative: "C:\\Users\\a\\test", + targetNative: "C:\\Users\\a\\test\\b", + baseUnix: "/Users/a/test", + targetUnix: "/Users/a/test/b", }, { - name: "windows_forward", - runGoos: []string{"windows"}, - baseNative: "C:\\Users\\a\\test", - targetNative: "C:\\Users\\a\\test\\b", - baseUnix: "/Users/a/test", - wantTargetUnix: "/Users/a/test/b", + name: "windows_backward", + runGoos: []string{"windows"}, + baseNative: "C:\\Users\\a\\test", + targetNative: "C:\\Users\\a", + baseUnix: "/Users/a/test", + targetUnix: "/Users/a", + }, + } + + for _, k := range testCases { + t.Run(k.name, func(t *testing.T) { + // early exit if running on a disallowed platform to avoid false negatives + skipIfGoosNotAllowed(t, k.runGoos) + + targetUnix, e := convertNativePathToUnix(k.baseNative, k.targetNative, k.baseUnix) + if !assert.NoError(t, e) { + return + } + if !assert.Equal(t, k.targetUnix, targetUnix) { + return + } + + targetNative, e := convertUnixPathToNative(k.baseUnix, k.targetUnix, k.baseNative) + if !assert.NoError(t, e) { + return + } + if !assert.Equal(t, k.targetNative, targetNative) { + return + } + }) + } +} + +func TestMakeBetweenNativeUnixPaths(t *testing.T) { + testCases := []struct { + name string + runGoos []string + basePathUnix string + targetPathUnix string + basePathNative string + targetPathNative string + }{ + { + name: "unix_forward", + runGoos: []string{"linux", "darwin"}, + basePathUnix: "/mnt/c/testfolder", + targetPathUnix: "/mnt/c/testfolder/a", + basePathNative: "/mnt/c/testfolder", + targetPathNative: "/mnt/c/testfolder/a", }, { - name: "windows_forward_trailslash", - runGoos: []string{"windows"}, - baseNative: "C:\\Users\\a\\test\\", - targetNative: "C:\\Users\\a\\test\\b\\", - baseUnix: "/Users/a/test/", - wantTargetUnix: "/Users/a/test/b", + name: "unix_backward", + runGoos: []string{"linux", "darwin"}, + basePathUnix: "/mnt/c/testfolder/subfolder", + targetPathUnix: "/mnt/c/a", + basePathNative: "/mnt/c/testfolder/subfolder", + targetPathNative: "/mnt/c/a", }, { - name: "windows_backward", - runGoos: []string{"windows"}, - baseNative: "C:\\Users\\a\\test", - targetNative: "C:\\Users\\a", - baseUnix: "/Users/a/test", - wantTargetUnix: "/Users/a", + name: "windows_forward", + runGoos: []string{"windows"}, + basePathUnix: "/mnt/c/testfolder", + targetPathUnix: "/mnt/c/testfolder/a", + basePathNative: "C:\\testfolder", + targetPathNative: "C:\\testfolder\\a", }, { - name: "windows_backward_trailslash", - runGoos: []string{"windows"}, - baseNative: "C:\\Users\\a\\test\\", - targetNative: "C:\\Users\\a\\", - baseUnix: "/Users/a/test/", - wantTargetUnix: "/Users/a", + name: "windows_backward", + runGoos: []string{"windows"}, + basePathUnix: "/mnt/c/testfolder/subfolder", + targetPathUnix: "/mnt/c/a", + basePathNative: "C:\\testfolder\\subfolder", + targetPathNative: "C:\\a", }, } for _, k := range testCases { t.Run(k.name, func(t *testing.T) { // early exit if running on a disallowed platform to avoid false negatives - if !checkGoosAllowed(k.runGoos) { + skipIfGoosNotAllowed(t, k.runGoos) + + basepath := makeOSPath(k.basePathNative, k.basePathUnix, false) + + nativePath, e := basepath.MakeFromUnixPath(k.targetPathUnix) + if !assert.NoError(t, e) { + return + } + if !assert.Equal(t, k.targetPathUnix, nativePath.Unix()) { + return + } + if !assert.Equal(t, k.targetPathNative, nativePath.Native()) { + return + } + if !assert.Equal(t, false, nativePath.IsRelative()) { return } - targetUnix, e := convertNativePathToUnix(k.baseNative, k.targetNative, k.baseUnix) + unixPath, e := basepath.MakeFromNativePath(k.targetPathNative) if !assert.NoError(t, e) { return } - assert.Equal(t, k.wantTargetUnix, targetUnix) + if !assert.Equal(t, k.targetPathUnix, unixPath.Unix()) { + return + } + if !assert.Equal(t, k.targetPathNative, unixPath.Native()) { + return + } + if !assert.Equal(t, false, unixPath.IsRelative()) { + return + } }) } } -func checkGoosAllowed(runGoos []string) bool { +func skipIfGoosNotAllowed(t *testing.T, runGoos []string) { for _, allowedGoos := range runGoos { if runtime.GOOS == allowedGoos { - return true + return } } - return false + t.Skipf("allowed GOOS values = %v but runtime.GOOS = %s", runGoos, runtime.GOOS) } diff --git a/support/kelpos/process.go b/support/kelpos/process.go index 062b3626f..640c064b0 100644 --- a/support/kelpos/process.go +++ b/support/kelpos/process.go @@ -97,7 +97,9 @@ func (kos *KelpOS) Blocking(namespace string, cmd string) ([]byte, error) { func (kos *KelpOS) Background(namespace string, cmd string) (*Process, error) { c := exec.Command("bash", "-c", cmd) // always execute commands from the working directory (specify as native since underlying OS handles it) - c.Dir = kos.workingBinDir.Native() + // using dotKelpWorkingDir as working directory since all our config files and log files are located in here and we want + // to have the shortest path lengths to accommodate for the 260 character file path limit in windows + c.Dir = kos.dotKelpWorkingDir.Native() log.Printf("process.Background is executing command: '%s' from directory '%s'", c.String(), c.Dir) stdinWriter, e := c.StdinPipe()