From a53fdeab0fbf2db8731486f086c04113f75c88e7 Mon Sep 17 00:00:00 2001 From: Thorsten de Buhr Date: Thu, 10 Aug 2023 11:30:24 +0200 Subject: [PATCH 1/5] [cpackget] add command shall download the pack and extract the license files to .Download #196 cpackget is already prepared to just extract the EULA and then quit, and to auto-accept EULA in a next step. Changed the program flow to skip showing the license / preparing the license UI if already the command line specifies the extract option. If the license file already exists it will be deleted (error if not possible). Added for encoded progress: * L: License file follows I: [L:F"C:\Users\user\AppData\Local\Arm\Packs\.Download\Keil.MDK-Middleware_Graphics.1.3.0.pack.license.rtf"] --- cmd/installer/pack.go | 16 +++++++++++++++- cmd/installer/root.go | 11 ++++++++++- cmd/ui/eula.go | 4 ++++ cmd/utils/encodedProgress.go | 1 + 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/cmd/installer/pack.go b/cmd/installer/pack.go index 7ad15d4..19efa49 100644 --- a/cmd/installer/pack.go +++ b/cmd/installer/pack.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/lu4p/cat" + "github.com/open-cmsis-pack/cpackget/cmd/errors" errs "github.com/open-cmsis-pack/cpackget/cmd/errors" "github.com/open-cmsis-pack/cpackget/cmd/ui" "github.com/open-cmsis-pack/cpackget/cmd/utils" @@ -460,7 +461,20 @@ func (p *PackType) extractEula(packPath string) error { eulaFileName := packPath + "." + path.Base(p.Pdsc.License) - log.Infof("Extracting embedded license to %v", eulaFileName) + if utils.GetEncodedProgress() { + log.Infof("[L:F\"%s\"]", eulaFileName) + } else { + log.Infof("Extracting embedded license to %v", eulaFileName) + } + + if utils.FileExists(eulaFileName) { + utils.UnsetReadOnly(eulaFileName) + os.Remove(eulaFileName) + } + if utils.FileExists(eulaFileName) { + log.Errorf("Cannot remove previous copy of license file: \"%s\"", eulaFileName) + return errors.ErrFailedCreatingFile + } return os.WriteFile(eulaFileName, eulaContents, utils.FileModeRO) } diff --git a/cmd/installer/root.go b/cmd/installer/root.go index 221838d..d04aec2 100644 --- a/cmd/installer/root.go +++ b/cmd/installer/root.go @@ -304,7 +304,9 @@ func DownloadPDSCFiles(skipInstalledPdscFiles bool, concurrency int, timeout int return nil } - log.Infof("[J%d:F\"%s\"]", numPdsc, Installation.PublicIndex) + if utils.GetEncodedProgress() { + log.Infof("[J%d:F\"%s\"]", numPdsc, Installation.PublicIndex) + } queue := concurrency for _, pdscTag := range pdscTags { @@ -1087,8 +1089,10 @@ func (p *PacksInstallationType) downloadPdscFile(pdscTag xml.PdscTag, skipInstal if skipInstalledPdscFiles { if utils.FileExists(pdscFilePath) { + log.Debugf("File already exists: \"%s\"", pdscFilePath) return nil } + log.Debugf("File does not exist and will be copied: \"%s\"", pdscFilePath) } pdscURL := pdscTag.URL @@ -1119,6 +1123,11 @@ func (p *PacksInstallationType) downloadPdscFile(pdscTag xml.PdscTag, skipInstal utils.UnsetReadOnly(pdscFilePath) err = utils.MoveFile(localFileName, pdscFilePath) utils.SetReadOnly(pdscFilePath) + + if !utils.FileExists(pdscFilePath) { + log.Errorf("File was not copied: \"%s\"", pdscFilePath) + } + return err } diff --git a/cmd/ui/eula.go b/cmd/ui/eula.go index 77f183b..2ede42b 100644 --- a/cmd/ui/eula.go +++ b/cmd/ui/eula.go @@ -40,6 +40,10 @@ type LicenseWindowType struct { // DisplayAndWaitForEULA prints out the license to the user through a UI // and waits for user confirmation. func DisplayAndWaitForEULA(licenseTitle, licenseContents string) (bool, error) { + if Extract { + return false, errs.ErrExtractEula + } + promptText := "License Agreement: [A]ccept [D]ecline [E]xtract" if !utils.IsTerminalInteractive() { diff --git a/cmd/utils/encodedProgress.go b/cmd/utils/encodedProgress.go index 752d450..3e121fd 100644 --- a/cmd/utils/encodedProgress.go +++ b/cmd/utils/encodedProgress.go @@ -46,6 +46,7 @@ func (p *EncodedProgress) Write(bs []byte) (int, error) { * P: Currently processed percentage * C: Currently processed bytes or numbers of files * J: Total number of files beeing processed + * L: License file follows */ func (p *EncodedProgress) Print() { newPercent := int(float64(p.current) / float64(p.total) * 100) From 8e864274771b7691ca94a1531de3f943d2d9eef0 Mon Sep 17 00:00:00 2001 From: Thorsten de Buhr Date: Thu, 10 Aug 2023 11:37:08 +0200 Subject: [PATCH 2/5] fixed pack import --- cmd/installer/pack.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/installer/pack.go b/cmd/installer/pack.go index 19efa49..d9070f0 100644 --- a/cmd/installer/pack.go +++ b/cmd/installer/pack.go @@ -14,7 +14,6 @@ import ( "strings" "github.com/lu4p/cat" - "github.com/open-cmsis-pack/cpackget/cmd/errors" errs "github.com/open-cmsis-pack/cpackget/cmd/errors" "github.com/open-cmsis-pack/cpackget/cmd/ui" "github.com/open-cmsis-pack/cpackget/cmd/utils" @@ -473,7 +472,7 @@ func (p *PackType) extractEula(packPath string) error { } if utils.FileExists(eulaFileName) { log.Errorf("Cannot remove previous copy of license file: \"%s\"", eulaFileName) - return errors.ErrFailedCreatingFile + return errs.ErrFailedCreatingFile } return os.WriteFile(eulaFileName, eulaContents, utils.FileModeRO) From b6fccb61cb5f77fd1515fbc3fbb9344b5b96bb91 Mon Sep 17 00:00:00 2001 From: Thorsten de Buhr Date: Thu, 10 Aug 2023 15:35:40 +0200 Subject: [PATCH 3/5] set concurrent downloads to 0 (disabled) because of issues with file system --- cmd/commands/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/commands/root.go b/cmd/commands/root.go index 693a5e6..ed47cbe 100644 --- a/cmd/commands/root.go +++ b/cmd/commands/root.go @@ -174,7 +174,7 @@ func NewCli() *cobra.Command { rootCmd.PersistentFlags().BoolP("quiet", "q", false, "Run cpackget silently, printing only error messages") rootCmd.PersistentFlags().BoolP("verbose", "v", false, "Sets verboseness level: None (Errors + Info + Warnings), -v (all + Debugging). Specify \"-q\" for no messages") rootCmd.PersistentFlags().StringP("pack-root", "R", defaultPackRoot, "Specifies pack root folder. Defaults to CMSIS_PACK_ROOT environment variable") - rootCmd.PersistentFlags().UintP("concurrent-downloads", "C", 5, "Number of concurrent batch downloads. Set to 0 to disable concurrency") + rootCmd.PersistentFlags().UintP("concurrent-downloads", "C", 0, "Number of concurrent batch downloads. Set to 0 to disable concurrency") rootCmd.PersistentFlags().UintP("timeout", "T", 0, "Set maximum duration (in seconds) of a download. Disabled by default") _ = viper.BindPFlag("concurrent-downloads", rootCmd.PersistentFlags().Lookup("concurrent-downloads")) _ = viper.BindPFlag("timeout", rootCmd.PersistentFlags().Lookup("timeout")) From 7e545c8ee4a109445f1dafc171e37ab0ea901d3e Mon Sep 17 00:00:00 2001 From: Thorsten de Buhr Date: Thu, 10 Aug 2023 16:01:49 +0200 Subject: [PATCH 4/5] added test: "test installing pack with license extracted but prev license exist" --- cmd/installer/root_pack_add_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cmd/installer/root_pack_add_test.go b/cmd/installer/root_pack_add_test.go index 256beb7..13a68f7 100644 --- a/cmd/installer/root_pack_add_test.go +++ b/cmd/installer/root_pack_add_test.go @@ -496,6 +496,32 @@ func TestAddPack(t *testing.T) { os.Remove(extractedLicensePath) }) + t.Run("test installing pack with license extracted but prev license exist", func(t *testing.T) { + localTestingDir := "test-add-pack-with-license-extracted-but-prev-license-exist" + assert.Nil(installer.SetPackRoot(localTestingDir, CreatePackRoot)) + installer.UnlockPackRoot() + defer removePackRoot(localTestingDir) + + packPath := packWithLicense + + extractedLicensePath := filepath.Join(installer.Installation.DownloadDir, filepath.Base(packPath)+".LICENSE.txt") + + ui.LicenseAgreed = nil + addPack(t, packPath, ConfigType{ + CheckEula: true, + ExtractEula: true, + }) + + addPack(t, packPath, ConfigType{ + CheckEula: true, + ExtractEula: true, + }) + + assert.True(utils.FileExists(extractedLicensePath)) + + os.Remove(extractedLicensePath) + }) + t.Run("test installing pack with missing license", func(t *testing.T) { // Missing license means it is specified in the PDSC file, but the actual license // file is not there From 48f8016217a5cc6c56e74b347c381cafdc2943bf Mon Sep 17 00:00:00 2001 From: Thorsten de Buhr Date: Fri, 11 Aug 2023 15:25:03 +0200 Subject: [PATCH 5/5] changed: - erronous files are deleted --- cmd/errors/errors.go | 3 +++ cmd/installer/root.go | 27 +++++++++++++++++++++++++++ cmd/installer/root_pack_add_test.go | 1 + 3 files changed, 31 insertions(+) diff --git a/cmd/errors/errors.go b/cmd/errors/errors.go index 1070124..ec4b318 100644 --- a/cmd/errors/errors.go +++ b/cmd/errors/errors.go @@ -94,4 +94,7 @@ var ( // Error/Flag to detect when a user has requested early termination ErrTerminatedByUser = errors.New("terminated by user request") + + ErrReadingPdsc = errors.New("reading PDSC failed") + ErrDownloadingPdsc = errors.New("download of some PDSC failed, please run 'cpackget update-index -a'") // add: -C 0 ) diff --git a/cmd/installer/root.go b/cmd/installer/root.go index d04aec2..aeb0a40 100644 --- a/cmd/installer/root.go +++ b/cmd/installer/root.go @@ -24,6 +24,8 @@ import ( const KeilDefaultPackRoot = "https://www.keil.com/pack/" +var stickyDownloadError bool = false + // GetDefaultCmsisPackRoot provides a default location // for the pack root if not provided. This is to enable // a "default mode", where the public index will be @@ -287,6 +289,7 @@ func RemovePdsc(pdscPath string) error { func massDownloadPdscFiles(pdscTag xml.PdscTag, skipInstalledPdscFiles bool, wg *sync.WaitGroup, timeout int) { if err := Installation.downloadPdscFile(pdscTag, skipInstalledPdscFiles, wg, timeout); err != nil { log.Error(err) + stickyDownloadError = true } } @@ -310,15 +313,18 @@ func DownloadPDSCFiles(skipInstalledPdscFiles bool, concurrency int, timeout int queue := concurrency for _, pdscTag := range pdscTags { + if concurrency == 0 || len(pdscTags) <= concurrency { if err := Installation.downloadPdscFile(pdscTag, skipInstalledPdscFiles, nil, timeout); err != nil { log.Error(err) + stickyDownloadError = true } } else { // Don't queue more downloads than specified if queue == 0 { if err := Installation.downloadPdscFile(pdscTag, skipInstalledPdscFiles, nil, timeout); err != nil { log.Error(err) + stickyDownloadError = true } wg.Add(concurrency) queue = concurrency @@ -329,6 +335,11 @@ func DownloadPDSCFiles(skipInstalledPdscFiles bool, concurrency int, timeout int } } } + + if stickyDownloadError { + return errs.ErrDownloadingPdsc + } + return nil } @@ -347,6 +358,8 @@ func UpdateInstalledPDSCFiles(pidxXML *xml.PidxXML, concurrency int, timeout int err := pdscXML.Read() if err != nil { log.Errorf("%s: %v", pdscFile, err) + utils.UnsetReadOnly(pdscFile) + os.Remove(pdscFile) continue } @@ -1121,13 +1134,27 @@ func (p *PacksInstallationType) downloadPdscFile(pdscTag xml.PdscTag, skipInstal } utils.UnsetReadOnly(pdscFilePath) + os.Remove(pdscFilePath) err = utils.MoveFile(localFileName, pdscFilePath) utils.SetReadOnly(pdscFilePath) + if err != nil { + return err + } + if !utils.FileExists(pdscFilePath) { log.Errorf("File was not copied: \"%s\"", pdscFilePath) } + pdscXML := xml.NewPdscXML(pdscFilePath) + err = pdscXML.Read() + if err != nil { + log.Errorf("XML File Read failed: %s : %v", pdscFilePath, err) + utils.UnsetReadOnly(pdscFilePath) + os.Remove(pdscFilePath) + return errs.ErrReadingPdsc + } + return err } diff --git a/cmd/installer/root_pack_add_test.go b/cmd/installer/root_pack_add_test.go index 13a68f7..eb4978b 100644 --- a/cmd/installer/root_pack_add_test.go +++ b/cmd/installer/root_pack_add_test.go @@ -511,6 +511,7 @@ func TestAddPack(t *testing.T) { CheckEula: true, ExtractEula: true, }) + assert.True(utils.FileExists(extractedLicensePath)) addPack(t, packPath, ConfigType{ CheckEula: true,