Kelp UI: refactor os paths used to accommodate for 260 character file…

… path limit in windows (#406)

* 1 - convertUnixPathToNative + test

* 2 - MakeFromUnixPath + test

* 3 - checkGoosAllowed -> skipIfGoosNotAllowed in tests

* 4 - remove unused RelFromBase

* 5 - rework folder structure to use ~/.kelp

1. ops now lives in ~/.kelp and is renamed to bot_data
2. /logs is now renamed to ui_logs
3. renamed path variables in APIServer to be botConfigsPath and botLogsPath

* 6 - manually expand ~ for kelp directory

* 7 - add comment about 260 character limit on windows

* 8 - MakeFromNativePath + test

* 9 - usr.HomeDir is in native form so make OSPath from native path

* 10 - fix MakeFromNativePath so it sets native and unix path correctly

* 11 - replace MakeFromUnixPath usage with a simple ospath.Join call

* 12 - set working directory to ~/.kelp and refactor paths to be saved and used via kos

this should give us the smallest path lengths on windows

* 13 - rename kelpBinName string to kelpBinPath *OSPath since we now run from the .kelp dir

* 14 - add comments on why we use ~/.kelp as the working directory instead of the binary directory as the wd

cmd/server_amd64.go

@@ -38,9 +38,8 @@ import (
 	"github.com/stellar/kelp/support/utils"
 )
 
-const kelpPrefsDirectory = ".kelp"
 const kelpAssetsPath = "/assets"
-const logsDir = "/logs"
+const uiLogsDir = "/ui_logs"
 const vendorDirectory = "/vendor"
 const trayIconName = "kelp-icon@1-8x.png"
 const kelpCcxtPath = "/ccxt"
@@ -80,16 +79,10 @@ func init() {
 	options.noElectron = serverCmd.Flags().Bool("no-electron", false, "open in browser instead of using electron")
 
 	serverCmd.Run = func(ccmd *cobra.Command, args []string) {
-		basepath, e := kelpos.MakeOsPathBase()
-		if e != nil {
-			panic(errors.Wrap(e, "could not make ospath"))
-		}
-		log.Printf("ospath initialized with native path: %s", basepath.Native())
-		log.Printf("ospath initialized with unix path: %s", basepath.Unix())
-
 		isLocalMode := env == envDev
 		isLocalDevMode := isLocalMode && *options.dev
 		kos := kelpos.GetKelpOS()
+		var e error
 		if isLocalMode {
 			wd, e := os.Getwd()
 			if e != nil {
@@ -108,15 +101,15 @@ func init() {
 			t := time.Now().Format("20060102T150405MST")
 			logFilename := fmt.Sprintf("kelp-ui_%s.log", t)
 
-			logsDirPath := basepath.Join(kelpPrefsDirectory, logsDir)
-			log.Printf("calling mkdir on logsDirPath: %s ...", logsDirPath.AsString())
-			e = kos.Mkdir(logsDirPath)
+			uiLogsDirPath := kos.GetDotKelpWorkingDir().Join(uiLogsDir)
+			log.Printf("calling mkdir on uiLogsDirPath: %s ...", uiLogsDirPath.AsString())
+			e = kos.Mkdir(uiLogsDirPath)
 			if e != nil {
-				panic(errors.Wrap(e, "could not mkdir on logsDirPath: "+logsDirPath.AsString()))
+				panic(errors.Wrap(e, "could not mkdir on uiLogsDirPath: "+uiLogsDirPath.AsString()))
 			}
 
 			// don't use explicit unix filepath here since it uses os.Open directly and won't work on windows
-			logFilepath = basepath.Join(kelpPrefsDirectory, logsDir, logFilename)
+			logFilepath = uiLogsDirPath.Join(logFilename)
 			setLogFile(l, logFilepath.Native())
 
 			if *options.verbose {
@@ -129,9 +122,11 @@ func init() {
 		openBrowserWg.Add(1)
 		if !isLocalDevMode {
 			// don't use explicit unix filepath here since it uses os.Create directly and won't work on windows
-			trayIconPath := basepath.Join(kelpPrefsDirectory, kelpAssetsPath, trayIconName)
+			assetsDirPath := kos.GetDotKelpWorkingDir().Join(kelpAssetsPath)
+			log.Printf("assetsDirPath: %s", assetsDirPath.AsString())
+			trayIconPath := assetsDirPath.Join(trayIconName)
 			log.Printf("trayIconPath: %s", trayIconPath.AsString())
-			e = writeTrayIcon(kos, trayIconPath, basepath)
+			e = writeTrayIcon(kos, trayIconPath, assetsDirPath)
 			if e != nil {
 				log.Fatal(errors.Wrap(e, "could not write tray icon"))
 			}
@@ -159,7 +154,7 @@ func init() {
 				tailFilePort := startTailFileServer(tailFileCompiled)
 				electronURL = fmt.Sprintf("http://localhost:%d", tailFilePort)
 			} else {
-				tailFilepath := basepath.Join(kelpPrefsDirectory, "tail.html")
+				tailFilepath := kos.GetDotKelpWorkingDir().Join("tail.html")
 				fileContents := []byte(tailFileCompiled)
 				e := ioutil.WriteFile(tailFilepath.Native(), fileContents, 0644)
 				if e != nil {
@@ -244,7 +239,7 @@ func init() {
 					ccxtGoos = "linux"
 				}
 
-				ccxtDirPath := basepath.Join(kelpPrefsDirectory, kelpCcxtPath)
+				ccxtDirPath := kos.GetDotKelpWorkingDir().Join(kelpCcxtPath)
 				ccxtFilenameNoExt := fmt.Sprintf("ccxt-rest_%s-x64", ccxtGoos)
 				filenameWithExt := fmt.Sprintf("%s.zip", ccxtFilenameNoExt)
 
@@ -256,7 +251,7 @@ func init() {
 				}
 
 				ccxtBinPath := ccxtDirPath.Join(ccxtFilenameNoExt, ccxtBinaryName)
-				unzipCcxtFile(kos, ccxtDirPath, ccxtBinPath, filenameWithExt, basepath)
+				unzipCcxtFile(kos, ccxtDirPath, ccxtBinPath, filenameWithExt)
 
 				e = runCcxtBinary(kos, ccxtBinPath)
 				if e != nil {
@@ -265,9 +260,13 @@ func init() {
 			}
 		}
 
+		dataPath := kos.GetDotKelpWorkingDir().Join("bot_data")
+		botConfigsPath := dataPath.Join("configs")
+		botLogsPath := dataPath.Join("logs")
 		s, e := backend.MakeAPIServer(
 			kos,
-			basepath,
+			botConfigsPath,
+			botLogsPath,
 			*options.horizonTestnetURI,
 			apiTestNet,
 			*options.horizonPubnetURI,
@@ -280,7 +279,7 @@ func init() {
 			panic(e)
 		}
 
-		guiWebPath := basepath.Join("../gui/web")
+		guiWebPath := kos.GetBinDir().Join("../gui/web")
 		if isLocalDevMode {
 			// the frontend app checks the REACT_APP_API_PORT variable to be set when serving
 			os.Setenv("REACT_APP_API_PORT", fmt.Sprintf("%d", *options.devAPIPort))
@@ -384,7 +383,6 @@ func unzipCcxtFile(
 	ccxtDir *kelpos.OSPath,
 	ccxtBinPath *kelpos.OSPath,
 	filenameWithExt string,
-	originalDir *kelpos.OSPath,
 ) {
 	if _, e := os.Stat(ccxtDir.Native()); !os.IsNotExist(e) {
 		if _, e := os.Stat(ccxtBinPath.Native()); !os.IsNotExist(e) {
@@ -393,7 +391,7 @@ func unzipCcxtFile(
 	}
 
 	log.Printf("unzipping file %s ... ", filenameWithExt)
-	zipCmd := fmt.Sprintf("cd %s && unzip %s && cd %s", ccxtDir.Unix(), filenameWithExt, originalDir.Unix())
+	zipCmd := fmt.Sprintf("cd %s && unzip %s", ccxtDir.Unix(), filenameWithExt)
 	_, e := kos.Blocking("zip", zipCmd)
 	if e != nil {
 		log.Fatal(errors.Wrap(e, fmt.Sprintf("unable to unzip file %s in directory %s", filenameWithExt, ccxtDir)))
@@ -469,9 +467,7 @@ func generateStaticFiles(kos *kelpos.KelpOS, guiWebPath *kelpos.OSPath) {
 	log.Println()
 }
 
-func writeTrayIcon(kos *kelpos.KelpOS, trayIconPath *kelpos.OSPath, basepath *kelpos.OSPath) error {
-	assetsDirPath := basepath.Join(kelpPrefsDirectory, kelpAssetsPath)
-	log.Printf("assetsDirPath: %s", assetsDirPath.AsString())
+func writeTrayIcon(kos *kelpos.KelpOS, trayIconPath *kelpos.OSPath, assetsDirPath *kelpos.OSPath) error {
 	if _, e := os.Stat(trayIconPath.Native()); !os.IsNotExist(e) {
 		// file exists, don't write again
 		return nil

gui/backend/api_server.go

@@ -15,10 +15,9 @@ import (
 
 // APIServer is an instance of the API service
 type APIServer struct {
-	basepath          *kelpos.OSPath
-	kelpBinName       string
-	configsDir        *kelpos.OSPath
-	logsDir           *kelpos.OSPath
+	kelpBinPath       *kelpos.OSPath
+	botConfigsPath    *kelpos.OSPath
+	botLogsPath       *kelpos.OSPath
 	kos               *kelpos.KelpOS
 	horizonTestnetURI string
 	horizonPubnetURI  string
@@ -34,7 +33,8 @@ type APIServer struct {
 // MakeAPIServer is a factory method
 func MakeAPIServer(
 	kos *kelpos.KelpOS,
-	basepath *kelpos.OSPath,
+	botConfigsPath *kelpos.OSPath,
+	botLogsPath *kelpos.OSPath,
 	horizonTestnetURI string,
 	apiTestNet *horizonclient.Client,
 	horizonPubnetURI string,
@@ -43,20 +43,17 @@ func MakeAPIServer(
 	noHeaders bool,
 	quitFn func(),
 ) (*APIServer, error) {
-	kelpBinName := filepath.Base(os.Args[0])
-	configsDir := basepath.Join("ops", "configs")
-	logsDir := basepath.Join("ops", "logs")
+	kelpBinPath := kos.GetBinDir().Join(filepath.Base(os.Args[0]))
 
 	optionsMetadata, e := loadOptionsMetadata()
 	if e != nil {
 		return nil, fmt.Errorf("error while loading options metadata when making APIServer: %s", e)
 	}
 
 	return &APIServer{
-		basepath:              basepath,
-		kelpBinName:           kelpBinName,
-		configsDir:            configsDir,
-		logsDir:               logsDir,
+		kelpBinPath:           kelpBinPath,
+		botConfigsPath:        botConfigsPath,
+		botLogsPath:           botLogsPath,
 		kos:                   kos,
 		horizonTestnetURI:     horizonTestnetURI,
 		horizonPubnetURI:      horizonPubnetURI,
@@ -125,7 +122,7 @@ func (s *APIServer) runKelpCommandBlocking(namespace string, cmd string) ([]byte
 	// name of the folder in which the GUI version is unzipped.
 	// To avoid these issues we only invoke with the binary name as opposed to the absolute path that contains the
 	// directory name. see start_bot.go for some experimentation with absolute and relative paths
-	cmdString := fmt.Sprintf("./%s %s", s.kelpBinName, cmd)
+	cmdString := fmt.Sprintf("%s %s", s.kelpBinPath.Unix(), cmd)
 	return s.kos.Blocking(namespace, cmdString)
 }
 
@@ -135,19 +132,19 @@ func (s *APIServer) runKelpCommandBackground(namespace string, cmd string) (*kel
 	// name of the folder in which the GUI version is unzipped.
 	// To avoid these issues we only invoke with the binary name as opposed to the absolute path that contains the
 	// directory name. see start_bot.go for some experimentation with absolute and relative paths
-	cmdString := fmt.Sprintf("./%s %s", s.kelpBinName, cmd)
+	cmdString := fmt.Sprintf("%s %s", s.kelpBinPath.Unix(), cmd)
 	return s.kos.Background(namespace, cmdString)
 }
 
 func (s *APIServer) setupOpsDirectory() error {
-	e := s.kos.Mkdir(s.configsDir)
+	e := s.kos.Mkdir(s.botConfigsPath)
 	if e != nil {
-		return fmt.Errorf("error setting up configs directory (%s): %s\n", s.configsDir, e)
+		return fmt.Errorf("error setting up configs directory (%s): %s\n", s.botConfigsPath, e)
 	}
 
-	e = s.kos.Mkdir(s.logsDir)
+	e = s.kos.Mkdir(s.botLogsPath)
 	if e != nil {
-		return fmt.Errorf("error setting up logs directory (%s): %s\n", s.logsDir, e)
+		return fmt.Errorf("error setting up logs directory (%s): %s\n", s.botLogsPath, e)
 	}
 
 	return nil

gui/backend/autogenerate_bot.go

@@ -49,7 +49,7 @@ func (s *APIServer) autogenerateBot(w http.ResponseWriter, r *http.Request) {
 
 	filenamePair := bot.Filenames()
 	sampleTrader := s.makeSampleTrader(kp.Seed())
-	traderFilePath := s.configsDir.Join(filenamePair.Trader)
+	traderFilePath := s.botConfigsPath.Join(filenamePair.Trader)
 	log.Printf("writing autogenerated bot config to file: %s\n", traderFilePath.AsString())
 	e = toml.WriteFile(traderFilePath.Native(), sampleTrader)
 	if e != nil {
@@ -58,7 +58,7 @@ func (s *APIServer) autogenerateBot(w http.ResponseWriter, r *http.Request) {
 	}
 
 	sampleBuysell := makeSampleBuysell()
-	strategyFilePath := s.configsDir.Join(filenamePair.Strategy)
+	strategyFilePath := s.botConfigsPath.Join(filenamePair.Strategy)
 	log.Printf("writing autogenerated strategy config to file: %s\n", strategyFilePath.AsString())
 	e = toml.WriteFile(strategyFilePath.Native(), sampleBuysell)
 	if e != nil {

gui/backend/delete_bot.go

@@ -52,7 +52,7 @@ func (s *APIServer) deleteBot(w http.ResponseWriter, r *http.Request) {
 
 	// delete configs
 	botPrefix := model2.GetPrefix(botName)
-	botConfigPath := s.configsDir.Join(botPrefix)
+	botConfigPath := s.botConfigsPath.Join(botPrefix)
 	_, e = s.kos.Blocking("rm", fmt.Sprintf("rm %s*", botConfigPath.Unix()))
 	if e != nil {
 		s.writeError(w, fmt.Sprintf("error running rm command for bot configs: %s\n", e))

gui/backend/generate_bot_name.go

@@ -57,7 +57,7 @@ func (s *APIServer) doGenerateBotName() (string, error) {
 }
 
 func (s *APIServer) prefixExists(prefix string) (bool, error) {
-	command := fmt.Sprintf("ls %s | grep %s", s.configsDir.Unix(), prefix)
+	command := fmt.Sprintf("ls %s | grep %s", s.botConfigsPath.Unix(), prefix)
 	_, e := s.kos.Blocking("prefix", command)
 	if e != nil {
 		if strings.Contains(e.Error(), "exit status 1") {

gui/backend/get_bot_config.go

@@ -27,14 +27,14 @@ func (s *APIServer) getBotConfig(w http.ResponseWriter, r *http.Request) {
 	}
 
 	filenamePair := model2.GetBotFilenames(botName, "buysell")
-	traderFilePath := s.configsDir.Join(filenamePair.Trader)
+	traderFilePath := s.botConfigsPath.Join(filenamePair.Trader)
 	var botConfig trader.BotConfig
 	e = config.Read(traderFilePath.Native(), &botConfig)
 	if e != nil {
 		s.writeErrorJson(w, fmt.Sprintf("cannot read bot config at path '%s': %s\n", traderFilePath, e))
 		return
 	}
-	strategyFilePath := s.configsDir.Join(filenamePair.Strategy)
+	strategyFilePath := s.botConfigsPath.Join(filenamePair.Strategy)
 	var buysellConfig plugins.BuySellConfig
 	e = config.Read(strategyFilePath.Native(), &buysellConfig)
 	if e != nil {

gui/backend/get_bot_info.go

@@ -64,7 +64,7 @@ func (s *APIServer) runGetBotInfoDirect(w http.ResponseWriter, botName string) {
 	}
 
 	filenamePair := model2.GetBotFilenames(botName, buysell)
-	traderFilePath := s.configsDir.Join(filenamePair.Trader)
+	traderFilePath := s.botConfigsPath.Join(filenamePair.Trader)
 	var botConfig trader.BotConfig
 	e = config.Read(traderFilePath.Native(), &botConfig)
 	if e != nil {

gui/backend/list_bots.go

@@ -13,7 +13,7 @@ import (
 
 func (s *APIServer) listBots(w http.ResponseWriter, r *http.Request) {
 	log.Printf("listing bots\n")
-	resultBytes, e := s.kos.Blocking("ls", fmt.Sprintf("ls %s | sort", s.configsDir.Unix()))
+	resultBytes, e := s.kos.Blocking("ls", fmt.Sprintf("ls %s | sort", s.botConfigsPath.Unix()))
 	if e != nil {
 		s.writeErrorJson(w, fmt.Sprintf("error when listing bots: %s\n", e))
 		return

gui/backend/start_bot.go

@@ -61,17 +61,17 @@ func (s *APIServer) doStartBot(botName string, strategy string, iterations *uint
 	// use relative paths, which is why it seems to work
 	// Note that /mnt/c is unlikely to be valid in windows (but is valid in the linux subsystem) since it's usually prefixed by the
 	// volume (C:\ etc.), which is why relative paths works so well here as it avoids this confusion.
-	traderRelativeConfigPath, e := s.configsDir.Join(filenamePair.Trader).RelFromPath(s.basepath)
+	traderRelativeConfigPath, e := s.botConfigsPath.Join(filenamePair.Trader).RelFromPath(s.kos.GetDotKelpWorkingDir())
 	if e != nil {
 		return fmt.Errorf("unable to get relative path of trader config file from basepath: %s", e)
 	}
 
-	stratRelativeConfigPath, e := s.configsDir.Join(filenamePair.Strategy).RelFromPath(s.basepath)
+	stratRelativeConfigPath, e := s.botConfigsPath.Join(filenamePair.Strategy).RelFromPath(s.kos.GetDotKelpWorkingDir())
 	if e != nil {
 		return fmt.Errorf("unable to get relative path of strategy config file from basepath: %s", e)
 	}
 
-	logRelativePrefixPath, e := s.logsDir.Join(logPrefix).RelFromPath(s.basepath)
+	logRelativePrefixPath, e := s.botLogsPath.Join(logPrefix).RelFromPath(s.kos.GetDotKelpWorkingDir())
 	if e != nil {
 		return fmt.Errorf("unable to get relative path of log prefix path from basepath: %s", e)
 	}

gui/backend/upsert_bot_config.go

@@ -87,7 +87,7 @@ func (s *APIServer) upsertBotConfig(w http.ResponseWriter, r *http.Request) {
 	}
 
 	filenamePair := model2.GetBotFilenames(req.Name, req.Strategy)
-	traderFilePath := s.configsDir.Join(filenamePair.Trader)
+	traderFilePath := s.botConfigsPath.Join(filenamePair.Trader)
 	botConfig := req.TraderConfig
 	log.Printf("upsert bot config to file: %s\n", traderFilePath.AsString())
 	e = toml.WriteFile(traderFilePath.Native(), &botConfig)
@@ -96,7 +96,7 @@ func (s *APIServer) upsertBotConfig(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	strategyFilePath := s.configsDir.Join(filenamePair.Strategy)
+	strategyFilePath := s.botConfigsPath.Join(filenamePair.Strategy)
 	strategyConfig := req.StrategyConfig
 	log.Printf("upsert strategy config to file: %s\n", strategyFilePath.AsString())
 	e = toml.WriteFile(strategyFilePath.Native(), &strategyConfig)