From 0952d6093cd17383e82fe9313a3eb9a68600295e Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Wed, 23 Aug 2023 22:06:06 +0200 Subject: [PATCH 01/14] Package install logged --- pkg/package_manager.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pkg/package_manager.go b/pkg/package_manager.go index 041d7d3b..15cc22b8 100644 --- a/pkg/package_manager.go +++ b/pkg/package_manager.go @@ -18,6 +18,7 @@ type PackageManager struct { instance *Instance SnapshotDeploySkipping bool + InstallLogged bool InstallRecursive bool SnapshotPatterns []string ToggledWorkflows []string @@ -188,6 +189,13 @@ func (pm *PackageManager) Upload(localPath string) (string, error) { } func (pm *PackageManager) Install(remotePath string) error { + if pm.InstallLogged { + return pm.installLogged(remotePath) + } + return pm.installRegular(remotePath) +} + +func (pm *PackageManager) installRegular(remotePath string) error { log.Infof("%s > installing package '%s'", pm.instance.ID(), remotePath) response, err := pm.instance.http.Request(). SetFormData(map[string]string{"cmd": "install", "recursive": fmt.Sprintf("%v", pm.InstallRecursive)}). @@ -208,6 +216,20 @@ func (pm *PackageManager) Install(remotePath string) error { return nil } +func (pm *PackageManager) installLogged(remotePath string) error { + log.Infof("%s > installing package '%s'", pm.instance.ID(), remotePath) + response, err := pm.instance.http.Request(). // TODO cmd as query param or not? + SetFormData(map[string]string{"cmd": "install", "recursive": fmt.Sprintf("%v", pm.InstallRecursive)}). + Post(ServiceHtmlPath + remotePath) + + // TODO parse HTML; process output line by line (do not buffer whole output) + // TODO log file per package; append for each deployment; separators with timestamps for each deployment + // TODO use logger? + + log.Infof("%s > installed package '%s'", pm.instance.ID(), remotePath) + return nil +} + func (pm *PackageManager) DeployWithChanged(localPath string) (bool, error) { if pm.instance.IsLocal() && pm.IsSnapshot(localPath) { // TODO remove local check; support remote as well return pm.deploySnapshot(localPath) From 5f71945826a110d0a4b2fe48c0f616b741d11e41 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Wed, 23 Aug 2023 22:17:01 +0200 Subject: [PATCH 02/14] Install logged, PoC --- pkg/package_manager.go | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/pkg/package_manager.go b/pkg/package_manager.go index 15cc22b8..418345a7 100644 --- a/pkg/package_manager.go +++ b/pkg/package_manager.go @@ -1,6 +1,7 @@ package pkg import ( + "bufio" "fmt" "github.com/samber/lo" log "github.com/sirupsen/logrus" @@ -207,7 +208,7 @@ func (pm *PackageManager) installRegular(remotePath string) error { } var status pkg.CommandResult if err = fmtx.UnmarshalJSON(response.RawBody(), &status); err != nil { - return fmt.Errorf("%s > cannot install package '%s'; cannot parse response: %w", pm.instance.ID(), remotePath, err) + return fmt.Errorf("%s > cannot install package '%s'; cannot parse JSON response: %w", pm.instance.ID(), remotePath, err) } if !status.Success { return fmt.Errorf("%s > cannot install package '%s'; unexpected status: %s", pm.instance.ID(), remotePath, status.Message) @@ -218,14 +219,25 @@ func (pm *PackageManager) installRegular(remotePath string) error { func (pm *PackageManager) installLogged(remotePath string) error { log.Infof("%s > installing package '%s'", pm.instance.ID(), remotePath) - response, err := pm.instance.http.Request(). // TODO cmd as query param or not? - SetFormData(map[string]string{"cmd": "install", "recursive": fmt.Sprintf("%v", pm.InstallRecursive)}). - Post(ServiceHtmlPath + remotePath) - - // TODO parse HTML; process output line by line (do not buffer whole output) - // TODO log file per package; append for each deployment; separators with timestamps for each deployment - // TODO use logger? - + // TODO cmd as query param or not? + response, err := pm.instance.http.Request(). + SetFormData(map[string]string{"cmd": "install", "recursive": fmt.Sprintf("%v", pm.InstallRecursive)}). + Post(ServiceHtmlPath + remotePath) + if err != nil { + return fmt.Errorf("%s > cannot install package '%s': %w", pm.instance.ID(), remotePath, err) + } else if response.IsError() { + return fmt.Errorf("%s > cannot install package '%s': '%s'", pm.instance.ID(), remotePath, response.Status()) + } + scanner := bufio.NewScanner(response.RawBody()) + for scanner.Scan() { + line := scanner.Text() + // TODO parse HTML; process output line by line (do not buffer whole output) + // TODO log file per package; append for each deployment; separators with timestamps for each deployment + // TODO use logger? + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("%s > cannot install package '%s': cannot parse HTML response: %w", pm.instance.ID(), remotePath, err) + } log.Infof("%s > installed package '%s'", pm.instance.ID(), remotePath) return nil } From 09e55b7911e5669acc210d789fee4e5212cf62f2 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Wed, 23 Aug 2023 22:21:02 +0200 Subject: [PATCH 03/14] Minor --- pkg/package_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/package_manager.go b/pkg/package_manager.go index 418345a7..f8cf4e92 100644 --- a/pkg/package_manager.go +++ b/pkg/package_manager.go @@ -233,7 +233,7 @@ func (pm *PackageManager) installLogged(remotePath string) error { line := scanner.Text() // TODO parse HTML; process output line by line (do not buffer whole output) // TODO log file per package; append for each deployment; separators with timestamps for each deployment - // TODO use logger? + // TODO use logger? aem/home/var/instance/author/aem-compose/var/log/package/install/filename.log } if err := scanner.Err(); err != nil { return fmt.Errorf("%s > cannot install package '%s': cannot parse HTML response: %w", pm.instance.ID(), remotePath, err) From e92c3781889ad3c9debe4db6c7f04b688d73d669 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 24 Aug 2023 09:28:37 +0200 Subject: [PATCH 04/14] Package install logged --- pkg/cfg/defaults.go | 6 ++++ pkg/instance.go | 1 - pkg/package_manager.go | 62 +++++++++++++++++++++++++++++++++++++----- pkg/pkg/constants.go | 6 ++++ 4 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 pkg/pkg/constants.go diff --git a/pkg/cfg/defaults.go b/pkg/cfg/defaults.go index 6a930f75..37888fda 100644 --- a/pkg/cfg/defaults.go +++ b/pkg/cfg/defaults.go @@ -81,6 +81,12 @@ func (c *Config) setDefaults() { v.SetDefault("instance.status.timeout", time.Millisecond*500) v.SetDefault("instance.package.install_recursive", true) + + v.SetDefault("instance.package.install.log.enabled", false) + v.SetDefault("instance.package.install.log.strict", true) + v.SetDefault("instance.package.install.log.console", false) + v.SetDefault("instance.package.install.log.dir", common.LogDir+"/package/install") + v.SetDefault("instance.package.snapshot_deploy_skipping", true) v.SetDefault("instance.package.snapshot_patterns", []string{"**/*-SNAPSHOT.zip"}) v.SetDefault("instance.package.toggled_workflows", []string{}) diff --git a/pkg/instance.go b/pkg/instance.go index 83abfd90..8760ec6c 100644 --- a/pkg/instance.go +++ b/pkg/instance.go @@ -316,5 +316,4 @@ func (i Instance) LockDir() string { } log.Panicf("%s > lock files for remote instances are not yet supported", i.ID()) return "" // TODO dir should reflect url or name? or configurable? for remote instances and features that are locked via local files like: pkg deploy skipping, SSL, ... - } diff --git a/pkg/package_manager.go b/pkg/package_manager.go index f8cf4e92..33e42fd1 100644 --- a/pkg/package_manager.go +++ b/pkg/package_manager.go @@ -11,7 +11,9 @@ import ( "github.com/wttech/aemc/pkg/common/pathx" "github.com/wttech/aemc/pkg/common/stringsx" "github.com/wttech/aemc/pkg/pkg" + "os" "path/filepath" + "strings" "time" ) @@ -19,8 +21,11 @@ type PackageManager struct { instance *Instance SnapshotDeploySkipping bool - InstallLogged bool InstallRecursive bool + InstallLogEnabled bool + InstallLogDir string + InstallLogConsole bool + InstallLogStrict bool SnapshotPatterns []string ToggledWorkflows []string } @@ -32,6 +37,10 @@ func NewPackageManager(res *Instance) *PackageManager { instance: res, SnapshotDeploySkipping: cv.GetBool("instance.package.snapshot_deploy_skipping"), + InstallLogEnabled: cv.GetBool("instance.package.install.log.enabled"), + InstallLogDir: cv.GetString("instance.package.install.log.dir"), + InstallLogConsole: cv.GetBool("instance.package.install.log.console"), + InstallLogStrict: cv.GetBool("instance.package.install.log.strict"), InstallRecursive: cv.GetBool("instance.package.install_recursive"), SnapshotPatterns: cv.GetStringSlice("instance.package.snapshot_patterns"), ToggledWorkflows: cv.GetStringSlice("instance.package.toggled_workflows"), @@ -190,7 +199,7 @@ func (pm *PackageManager) Upload(localPath string) (string, error) { } func (pm *PackageManager) Install(remotePath string) error { - if pm.InstallLogged { + if pm.InstallLogEnabled { return pm.installLogged(remotePath) } return pm.installRegular(remotePath) @@ -219,7 +228,7 @@ func (pm *PackageManager) installRegular(remotePath string) error { func (pm *PackageManager) installLogged(remotePath string) error { log.Infof("%s > installing package '%s'", pm.instance.ID(), remotePath) - // TODO cmd as query param or not? + response, err := pm.instance.http.Request(). SetFormData(map[string]string{"cmd": "install", "recursive": fmt.Sprintf("%v", pm.InstallRecursive)}). Post(ServiceHtmlPath + remotePath) @@ -228,16 +237,55 @@ func (pm *PackageManager) installLogged(remotePath string) error { } else if response.IsError() { return fmt.Errorf("%s > cannot install package '%s': '%s'", pm.instance.ID(), remotePath, response.Status()) } + + success := false + successWithErrors := false + + var installLog *log.Logger + if !pm.InstallLogConsole { + installLog = log.New() + installLogFilePath := fmt.Sprintf("%s/%s/%s.log", pm.InstallLogDir, pm.instance.ID(), filepath.Base(remotePath)) + if err := pathx.Ensure(filepath.Dir(installLogFilePath)); err != nil { + return err + } + installLogFile, err := os.OpenFile(installLogFilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + return fmt.Errorf("%s > cannot install package '%s': cannot open log file '%s'", pm.instance.ID(), remotePath, installLogFilePath) + } + defer installLogFile.Close() + installLog.SetOutput(installLogFile) + } + scanner := bufio.NewScanner(response.RawBody()) for scanner.Scan() { - line := scanner.Text() - // TODO parse HTML; process output line by line (do not buffer whole output) - // TODO log file per package; append for each deployment; separators with timestamps for each deployment - // TODO use logger? aem/home/var/instance/author/aem-compose/var/log/package/install/filename.log + htmlLine := scanner.Text() + if !success && strings.Contains(htmlLine, pkg.InstallSuccess) { + success = true + } + if !successWithErrors && strings.Contains(htmlLine, pkg.InstallSuccessWithErrors) { + successWithErrors = true + } + if !pm.InstallLogConsole { + installLog.Infof(htmlLine) + } else { + fmt.Println(htmlLine) + } } if err := scanner.Err(); err != nil { return fmt.Errorf("%s > cannot install package '%s': cannot parse HTML response: %w", pm.instance.ID(), remotePath, err) } + + failure := !success && !successWithErrors + if failure { + return fmt.Errorf("%s > cannot install package '%s': HTML response contains errors: %w", pm.instance.ID(), remotePath, err) + } else if successWithErrors { + if pm.InstallLogStrict { + return fmt.Errorf("%s > cannot install package '%s': HTML response contains errors: %w", pm.instance.ID(), remotePath, err) + } + log.Warnf("%s > installed package '%s': HTML response contains errors: %s", pm.instance.ID(), remotePath, err) + return nil + } + log.Infof("%s > installed package '%s'", pm.instance.ID(), remotePath) return nil } diff --git a/pkg/pkg/constants.go b/pkg/pkg/constants.go new file mode 100644 index 00000000..c8e8f2d2 --- /dev/null +++ b/pkg/pkg/constants.go @@ -0,0 +1,6 @@ +package pkg + +const ( + InstallSuccess = "" + InstallSuccessWithErrors = " Date: Thu, 24 Aug 2023 10:00:24 +0200 Subject: [PATCH 05/14] HTML reports --- pkg/package_manager.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/pkg/package_manager.go b/pkg/package_manager.go index 33e42fd1..09be39d3 100644 --- a/pkg/package_manager.go +++ b/pkg/package_manager.go @@ -10,6 +10,7 @@ import ( "github.com/wttech/aemc/pkg/common/osx" "github.com/wttech/aemc/pkg/common/pathx" "github.com/wttech/aemc/pkg/common/stringsx" + "github.com/wttech/aemc/pkg/common/timex" "github.com/wttech/aemc/pkg/pkg" "os" "path/filepath" @@ -241,19 +242,20 @@ func (pm *PackageManager) installLogged(remotePath string) error { success := false successWithErrors := false - var installLog *log.Logger + htmlFilePath := fmt.Sprintf("%s/%s/%s-%s.html", pm.InstallLogDir, pm.instance.ID(), filepath.Base(remotePath), timex.FileTimestampForNow()) + var htmlWriter *bufio.Writer + if !pm.InstallLogConsole { - installLog = log.New() - installLogFilePath := fmt.Sprintf("%s/%s/%s.log", pm.InstallLogDir, pm.instance.ID(), filepath.Base(remotePath)) - if err := pathx.Ensure(filepath.Dir(installLogFilePath)); err != nil { + if err := pathx.Ensure(filepath.Dir(htmlFilePath)); err != nil { return err } - installLogFile, err := os.OpenFile(installLogFilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + htmlFile, err := os.OpenFile(htmlFilePath, os.O_RDWR|os.O_CREATE, 0666) if err != nil { - return fmt.Errorf("%s > cannot install package '%s': cannot open log file '%s'", pm.instance.ID(), remotePath, installLogFilePath) + return fmt.Errorf("%s > cannot install package '%s': cannot open HTML log file '%s'", pm.instance.ID(), remotePath, htmlFilePath) } - defer installLogFile.Close() - installLog.SetOutput(installLogFile) + defer htmlFile.Close() + htmlWriter = bufio.NewWriter(htmlFile) + defer htmlWriter.Flush() } scanner := bufio.NewScanner(response.RawBody()) @@ -266,7 +268,10 @@ func (pm *PackageManager) installLogged(remotePath string) error { successWithErrors = true } if !pm.InstallLogConsole { - installLog.Infof(htmlLine) + _, err := htmlWriter.WriteString(htmlLine + osx.LineSep()) + if err != nil { + return fmt.Errorf("%s > cannot install package '%s': cannot write to HTML log file '%s'", pm.instance.ID(), remotePath, htmlFilePath) + } } else { fmt.Println(htmlLine) } @@ -276,16 +281,13 @@ func (pm *PackageManager) installLogged(remotePath string) error { } failure := !success && !successWithErrors - if failure { + if failure || (successWithErrors && pm.InstallLogStrict) { return fmt.Errorf("%s > cannot install package '%s': HTML response contains errors: %w", pm.instance.ID(), remotePath, err) - } else if successWithErrors { - if pm.InstallLogStrict { - return fmt.Errorf("%s > cannot install package '%s': HTML response contains errors: %w", pm.instance.ID(), remotePath, err) - } + } + if successWithErrors { log.Warnf("%s > installed package '%s': HTML response contains errors: %s", pm.instance.ID(), remotePath, err) return nil } - log.Infof("%s > installed package '%s'", pm.instance.ID(), remotePath) return nil } From 7dbbc36080fbf7277648784b57f9dd55550cba82 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 24 Aug 2023 10:16:35 +0200 Subject: [PATCH 06/14] Error reporting impr --- pkg/package_manager.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/package_manager.go b/pkg/package_manager.go index 09be39d3..3c9aa208 100644 --- a/pkg/package_manager.go +++ b/pkg/package_manager.go @@ -282,7 +282,10 @@ func (pm *PackageManager) installLogged(remotePath string) error { failure := !success && !successWithErrors if failure || (successWithErrors && pm.InstallLogStrict) { - return fmt.Errorf("%s > cannot install package '%s': HTML response contains errors: %w", pm.instance.ID(), remotePath, err) + if pm.InstallLogConsole { + return fmt.Errorf("%s > cannot install package '%s': HTML output contains errors", pm.instance.ID(), remotePath) + } + return fmt.Errorf("%s > cannot install package '%s': HTML report contains errors '%s'", pm.instance.ID(), remotePath, htmlFilePath) } if successWithErrors { log.Warnf("%s > installed package '%s': HTML response contains errors: %s", pm.instance.ID(), remotePath, err) From f56cf537c38a8e2f8d572a31d98e8adb208c07b5 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 24 Aug 2023 10:38:55 +0200 Subject: [PATCH 07/14] Docs --- README.md | 79 ++++++++++++++----- examples/docker/src/aem/default/etc/aem.yml | 11 +++ .../app_classic/aem/default/etc/aem.yml | 11 +++ pkg/project/app_cloud/aem/default/etc/aem.yml | 11 +++ pkg/project/instance/aem/default/etc/aem.yml | 11 +++ 5 files changed, 102 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index aae6fb45..b2b01f91 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,10 @@ AEMC is a versatile tool for managing Adobe Experience Manager (AEM) instances. * [Improving performance](#improving-performance) * [Increasing verbosity](#increasing-verbosity) * [Installing content packages](#installing-content-packages) + * [Installing packages with troubleshooting](#installing-packages-with-troubleshooting) * [Examples](#examples) - * ['SSL by Default' support](#ssl-by-default-support) - * [Global Trust Store management support](#global-trust-store-management-support) + * [SSL by Default](#ssl-by-default) + * [Global Trust Store](#global-trust-store) * [Contributing](#contributing) * [Authors](#authors) * [License](#license) @@ -366,6 +367,17 @@ instance: snapshot_deploy_skipping: true # Disable following workflow launchers for a package deployment time only toggled_workflows: [/libs/settings/workflow/launcher/config/asset_processing_on_sdk_*,/libs/settings/workflow/launcher/config/update_asset_*,/libs/settings/workflow/launcher/config/dam_*] + # Also sub-packages + install_recursive: true + # Use slower HTML endpoint for deployments but with better troubleshooting + install_log: + enabled: false + # Print HTML directly to console instead of writing to file + console: false + # Fail on case 'installed with errors' + strict: true + # Directory to place HTML report files + dir: aem/home/var/log/package/install # OSGi Framework osgi: @@ -503,11 +515,33 @@ To skip the instance health check for a single AEMC command, use the following s AEM_INSTANCE_CHECK_SKIP=true sh aemw package deploy --url my-package.zip ``` +### Installing packages with troubleshooting + +Starting from version 1.4.0 (see [#177](https://github.com/wttech/aemc/pull/177)), AEMC now supports AEM package installations using an HTML report serving endpoint, similar to CRX Package Manager's. While this method may result in slightly extended installation times, it provides a comprehensive HTML report detailing the package installation process. + +This new feature offers two distinct modes for leveraging its benefits: + +1. **Saving HTML Report to File:** + + To enable this mode and save the HTML report to a designated file, use the following shell command: + + ```shell + AEM_INSTANCE_PACKAGE_INSTALL_LOG_ENABLED=true sh aemw package deploy --url my-package.zip + ``` + +2. **Direct Console Output of HTML Report:** + + To directly print the HTML report to the console, execute this shell command: + + ```shell + AEM_INSTANCE_PACKAGE_INSTALL_LOG_ENABLED=true AEM_INSTANCE_PACKAGE_INSTALL_LOG_CONSOLE=true sh aemw package deploy --url my-package.zip + ``` + # Examples -## 'SSL by Default' support +## SSL by Default -AEM Compose supports 'SSL by Default' feature of AEM. +AEM Compose supports *SSL by Default* feature of AEM. This feature requires: - certificate file in PEM format @@ -517,7 +551,7 @@ This feature requires: - hostname for HTTPS connector (used by AEM to check if the setup was successful; has to be reachable by AEM) - port for HTTPS connector -To set up 'SSL by Default', run: +To set up *SSL by Default*, run: ```shell sh aemw ssl setup \ @@ -545,36 +579,39 @@ See the reference documentation: [AEM 6.5 > Administering Guide > SSL by Default For local environment remember to set different port numbers for author and publish instances. -## Global Trust Store management support +## Global Trust Store AEM Compose supports managing the trust store of AEM instances. This feature supports: + - creation of the general trust store if it does not exist -```shell -sh aemw gts create --password PASSWORD_HERE -A -``` + ```shell + sh aemw gts create --password PASSWORD_HERE -A + ``` + - getting the general trust store status -```shell -sh aemw gts status -A -``` + ```shell + sh aemw gts status -A + ``` - adding a certificate to the general trust store -```shell -sh aemw gts certificate add --path -A -``` + ```shell + sh aemw gts certificate add --path -A + ``` + This command will add a certificate to the general trust store only if not exists in trust store and will return the alias of the certificate. Command `certificate add` supports certificates in PEM and DER formats. - reading a certificate from the general trust store (by alias) -```shell -sh aemw gts certificate read --alias -A -``` + ```shell + sh aemw gts certificate read --alias -A + ``` - removing a certificate from the general trust store (by alias) -```shell -sh aemw gts certificate remove --alias -A -``` + ```shell + sh aemw gts certificate remove --alias -A + ``` # Contributing diff --git a/examples/docker/src/aem/default/etc/aem.yml b/examples/docker/src/aem/default/etc/aem.yml index 69e1cc9f..ed5347b2 100755 --- a/examples/docker/src/aem/default/etc/aem.yml +++ b/examples/docker/src/aem/default/etc/aem.yml @@ -139,6 +139,17 @@ instance: snapshot_deploy_skipping: true # Disable following workflow launchers for a package deployment time only toggled_workflows: [/libs/settings/workflow/launcher/config/asset_processing_on_sdk_*,/libs/settings/workflow/launcher/config/update_asset_*,/libs/settings/workflow/launcher/config/dam_*] + # Also sub-packages + install_recursive: true + # Use slower HTML endpoint for deployments but with better troubleshooting + install_log: + enabled: false + # Print HTML directly to console instead of writing to file + console: false + # Fail on case 'installed with errors' + strict: true + # Directory to place HTML report files + dir: aem/home/var/log/package/install # OSGi Framework osgi: diff --git a/pkg/project/app_classic/aem/default/etc/aem.yml b/pkg/project/app_classic/aem/default/etc/aem.yml index 99d2edf8..f691290d 100755 --- a/pkg/project/app_classic/aem/default/etc/aem.yml +++ b/pkg/project/app_classic/aem/default/etc/aem.yml @@ -138,6 +138,17 @@ instance: snapshot_deploy_skipping: true # Disable following workflow launchers for a package deployment time only toggled_workflows: [/libs/settings/workflow/launcher/config/update_asset_*,/libs/settings/workflow/launcher/config/dam_*] + # Also sub-packages + install_recursive: true + # Use slower HTML endpoint for deployments but with better troubleshooting + install_log: + enabled: false + # Print HTML directly to console instead of writing to file + console: false + # Fail on case 'installed with errors' + strict: true + # Directory to place HTML report files + dir: aem/home/var/log/package/install # OSGi Framework osgi: diff --git a/pkg/project/app_cloud/aem/default/etc/aem.yml b/pkg/project/app_cloud/aem/default/etc/aem.yml index b4297ef1..4a6b0b96 100755 --- a/pkg/project/app_cloud/aem/default/etc/aem.yml +++ b/pkg/project/app_cloud/aem/default/etc/aem.yml @@ -138,6 +138,17 @@ instance: snapshot_deploy_skipping: true # Disable following workflow launchers for a package deployment time only toggled_workflows: [/libs/settings/workflow/launcher/config/asset_processing_on_sdk_*,/libs/settings/workflow/launcher/config/dam_*] + # Also sub-packages + install_recursive: true + # Use slower HTML endpoint for deployments but with better troubleshooting + install_log: + enabled: false + # Print HTML directly to console instead of writing to file + console: false + # Fail on case 'installed with errors' + strict: true + # Directory to place HTML report files + dir: aem/home/var/log/package/install # OSGi Framework osgi: diff --git a/pkg/project/instance/aem/default/etc/aem.yml b/pkg/project/instance/aem/default/etc/aem.yml index 69e1cc9f..ed5347b2 100755 --- a/pkg/project/instance/aem/default/etc/aem.yml +++ b/pkg/project/instance/aem/default/etc/aem.yml @@ -139,6 +139,17 @@ instance: snapshot_deploy_skipping: true # Disable following workflow launchers for a package deployment time only toggled_workflows: [/libs/settings/workflow/launcher/config/asset_processing_on_sdk_*,/libs/settings/workflow/launcher/config/update_asset_*,/libs/settings/workflow/launcher/config/dam_*] + # Also sub-packages + install_recursive: true + # Use slower HTML endpoint for deployments but with better troubleshooting + install_log: + enabled: false + # Print HTML directly to console instead of writing to file + console: false + # Fail on case 'installed with errors' + strict: true + # Directory to place HTML report files + dir: aem/home/var/log/package/install # OSGi Framework osgi: From b67c44cca00bc004bb9b39aaa19e72adacf60c35 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 24 Aug 2023 10:39:42 +0200 Subject: [PATCH 08/14] Minor YAML --- pkg/cfg/defaults.go | 8 ++++---- pkg/package_manager.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/cfg/defaults.go b/pkg/cfg/defaults.go index 37888fda..b179763c 100644 --- a/pkg/cfg/defaults.go +++ b/pkg/cfg/defaults.go @@ -82,10 +82,10 @@ func (c *Config) setDefaults() { v.SetDefault("instance.package.install_recursive", true) - v.SetDefault("instance.package.install.log.enabled", false) - v.SetDefault("instance.package.install.log.strict", true) - v.SetDefault("instance.package.install.log.console", false) - v.SetDefault("instance.package.install.log.dir", common.LogDir+"/package/install") + v.SetDefault("instance.package.install_log.enabled", false) + v.SetDefault("instance.package.install_log.strict", true) + v.SetDefault("instance.package.install_log.console", false) + v.SetDefault("instance.package.install_log.dir", common.LogDir+"/package/install") v.SetDefault("instance.package.snapshot_deploy_skipping", true) v.SetDefault("instance.package.snapshot_patterns", []string{"**/*-SNAPSHOT.zip"}) diff --git a/pkg/package_manager.go b/pkg/package_manager.go index 3c9aa208..df1bf273 100644 --- a/pkg/package_manager.go +++ b/pkg/package_manager.go @@ -38,10 +38,10 @@ func NewPackageManager(res *Instance) *PackageManager { instance: res, SnapshotDeploySkipping: cv.GetBool("instance.package.snapshot_deploy_skipping"), - InstallLogEnabled: cv.GetBool("instance.package.install.log.enabled"), - InstallLogDir: cv.GetString("instance.package.install.log.dir"), - InstallLogConsole: cv.GetBool("instance.package.install.log.console"), - InstallLogStrict: cv.GetBool("instance.package.install.log.strict"), + InstallLogEnabled: cv.GetBool("instance.package.install_log.enabled"), + InstallLogDir: cv.GetString("instance.package.install_log.dir"), + InstallLogConsole: cv.GetBool("instance.package.install_log.console"), + InstallLogStrict: cv.GetBool("instance.package.install_log.strict"), InstallRecursive: cv.GetBool("instance.package.install_recursive"), SnapshotPatterns: cv.GetStringSlice("instance.package.snapshot_patterns"), ToggledWorkflows: cv.GetStringSlice("instance.package.toggled_workflows"), From df9b9fec2817ecc1ce4a1705d02995d0b4ea820b Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 24 Aug 2023 10:47:13 +0200 Subject: [PATCH 09/14] Install log => HTML --- README.md | 6 ++-- examples/docker/src/aem/default/etc/aem.yml | 2 +- pkg/cfg/defaults.go | 8 +++--- pkg/package_manager.go | 28 +++++++++---------- .../app_classic/aem/default/etc/aem.yml | 2 +- pkg/project/app_cloud/aem/default/etc/aem.yml | 2 +- pkg/project/instance/aem/default/etc/aem.yml | 2 +- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index b2b01f91..d4b543e9 100644 --- a/README.md +++ b/README.md @@ -370,7 +370,7 @@ instance: # Also sub-packages install_recursive: true # Use slower HTML endpoint for deployments but with better troubleshooting - install_log: + install_html: enabled: false # Print HTML directly to console instead of writing to file console: false @@ -526,7 +526,7 @@ This new feature offers two distinct modes for leveraging its benefits: To enable this mode and save the HTML report to a designated file, use the following shell command: ```shell - AEM_INSTANCE_PACKAGE_INSTALL_LOG_ENABLED=true sh aemw package deploy --url my-package.zip + AEM_INSTANCE_PACKAGE_INSTALL_HTML_ENABLED=true sh aemw package deploy --url my-package.zip ``` 2. **Direct Console Output of HTML Report:** @@ -534,7 +534,7 @@ This new feature offers two distinct modes for leveraging its benefits: To directly print the HTML report to the console, execute this shell command: ```shell - AEM_INSTANCE_PACKAGE_INSTALL_LOG_ENABLED=true AEM_INSTANCE_PACKAGE_INSTALL_LOG_CONSOLE=true sh aemw package deploy --url my-package.zip + AEM_INSTANCE_PACKAGE_INSTALL_HTML_ENABLED=true AEM_INSTANCE_PACKAGE_INSTALL_HTML_CONSOLE=true sh aemw package deploy --url my-package.zip ``` # Examples diff --git a/examples/docker/src/aem/default/etc/aem.yml b/examples/docker/src/aem/default/etc/aem.yml index ed5347b2..1a7a3936 100755 --- a/examples/docker/src/aem/default/etc/aem.yml +++ b/examples/docker/src/aem/default/etc/aem.yml @@ -142,7 +142,7 @@ instance: # Also sub-packages install_recursive: true # Use slower HTML endpoint for deployments but with better troubleshooting - install_log: + install_html: enabled: false # Print HTML directly to console instead of writing to file console: false diff --git a/pkg/cfg/defaults.go b/pkg/cfg/defaults.go index b179763c..3492ff09 100644 --- a/pkg/cfg/defaults.go +++ b/pkg/cfg/defaults.go @@ -82,10 +82,10 @@ func (c *Config) setDefaults() { v.SetDefault("instance.package.install_recursive", true) - v.SetDefault("instance.package.install_log.enabled", false) - v.SetDefault("instance.package.install_log.strict", true) - v.SetDefault("instance.package.install_log.console", false) - v.SetDefault("instance.package.install_log.dir", common.LogDir+"/package/install") + v.SetDefault("instance.package.install_html.enabled", false) + v.SetDefault("instance.package.install_html.strict", true) + v.SetDefault("instance.package.install_html.console", false) + v.SetDefault("instance.package.install_html.dir", common.LogDir+"/package/install") v.SetDefault("instance.package.snapshot_deploy_skipping", true) v.SetDefault("instance.package.snapshot_patterns", []string{"**/*-SNAPSHOT.zip"}) diff --git a/pkg/package_manager.go b/pkg/package_manager.go index df1bf273..0b7f8175 100644 --- a/pkg/package_manager.go +++ b/pkg/package_manager.go @@ -23,10 +23,10 @@ type PackageManager struct { SnapshotDeploySkipping bool InstallRecursive bool - InstallLogEnabled bool - InstallLogDir string - InstallLogConsole bool - InstallLogStrict bool + InstallHTMLEnabled bool + InstallHTMLDir string + InstallHTMLConsole bool + InstallHTMLStrict bool SnapshotPatterns []string ToggledWorkflows []string } @@ -38,10 +38,10 @@ func NewPackageManager(res *Instance) *PackageManager { instance: res, SnapshotDeploySkipping: cv.GetBool("instance.package.snapshot_deploy_skipping"), - InstallLogEnabled: cv.GetBool("instance.package.install_log.enabled"), - InstallLogDir: cv.GetString("instance.package.install_log.dir"), - InstallLogConsole: cv.GetBool("instance.package.install_log.console"), - InstallLogStrict: cv.GetBool("instance.package.install_log.strict"), + InstallHTMLEnabled: cv.GetBool("instance.package.install_html.enabled"), + InstallHTMLDir: cv.GetString("instance.package.install_html.dir"), + InstallHTMLConsole: cv.GetBool("instance.package.install_html.console"), + InstallHTMLStrict: cv.GetBool("instance.package.install_html.strict"), InstallRecursive: cv.GetBool("instance.package.install_recursive"), SnapshotPatterns: cv.GetStringSlice("instance.package.snapshot_patterns"), ToggledWorkflows: cv.GetStringSlice("instance.package.toggled_workflows"), @@ -200,7 +200,7 @@ func (pm *PackageManager) Upload(localPath string) (string, error) { } func (pm *PackageManager) Install(remotePath string) error { - if pm.InstallLogEnabled { + if pm.InstallHTMLEnabled { return pm.installLogged(remotePath) } return pm.installRegular(remotePath) @@ -242,10 +242,10 @@ func (pm *PackageManager) installLogged(remotePath string) error { success := false successWithErrors := false - htmlFilePath := fmt.Sprintf("%s/%s/%s-%s.html", pm.InstallLogDir, pm.instance.ID(), filepath.Base(remotePath), timex.FileTimestampForNow()) + htmlFilePath := fmt.Sprintf("%s/%s/%s-%s.html", pm.InstallHTMLDir, pm.instance.ID(), filepath.Base(remotePath), timex.FileTimestampForNow()) var htmlWriter *bufio.Writer - if !pm.InstallLogConsole { + if !pm.InstallHTMLConsole { if err := pathx.Ensure(filepath.Dir(htmlFilePath)); err != nil { return err } @@ -267,7 +267,7 @@ func (pm *PackageManager) installLogged(remotePath string) error { if !successWithErrors && strings.Contains(htmlLine, pkg.InstallSuccessWithErrors) { successWithErrors = true } - if !pm.InstallLogConsole { + if !pm.InstallHTMLConsole { _, err := htmlWriter.WriteString(htmlLine + osx.LineSep()) if err != nil { return fmt.Errorf("%s > cannot install package '%s': cannot write to HTML log file '%s'", pm.instance.ID(), remotePath, htmlFilePath) @@ -281,8 +281,8 @@ func (pm *PackageManager) installLogged(remotePath string) error { } failure := !success && !successWithErrors - if failure || (successWithErrors && pm.InstallLogStrict) { - if pm.InstallLogConsole { + if failure || (successWithErrors && pm.InstallHTMLStrict) { + if pm.InstallHTMLConsole { return fmt.Errorf("%s > cannot install package '%s': HTML output contains errors", pm.instance.ID(), remotePath) } return fmt.Errorf("%s > cannot install package '%s': HTML report contains errors '%s'", pm.instance.ID(), remotePath, htmlFilePath) diff --git a/pkg/project/app_classic/aem/default/etc/aem.yml b/pkg/project/app_classic/aem/default/etc/aem.yml index f691290d..86e39740 100755 --- a/pkg/project/app_classic/aem/default/etc/aem.yml +++ b/pkg/project/app_classic/aem/default/etc/aem.yml @@ -141,7 +141,7 @@ instance: # Also sub-packages install_recursive: true # Use slower HTML endpoint for deployments but with better troubleshooting - install_log: + install_html: enabled: false # Print HTML directly to console instead of writing to file console: false diff --git a/pkg/project/app_cloud/aem/default/etc/aem.yml b/pkg/project/app_cloud/aem/default/etc/aem.yml index 4a6b0b96..58c902c9 100755 --- a/pkg/project/app_cloud/aem/default/etc/aem.yml +++ b/pkg/project/app_cloud/aem/default/etc/aem.yml @@ -141,7 +141,7 @@ instance: # Also sub-packages install_recursive: true # Use slower HTML endpoint for deployments but with better troubleshooting - install_log: + install_html: enabled: false # Print HTML directly to console instead of writing to file console: false diff --git a/pkg/project/instance/aem/default/etc/aem.yml b/pkg/project/instance/aem/default/etc/aem.yml index ed5347b2..1a7a3936 100755 --- a/pkg/project/instance/aem/default/etc/aem.yml +++ b/pkg/project/instance/aem/default/etc/aem.yml @@ -142,7 +142,7 @@ instance: # Also sub-packages install_recursive: true # Use slower HTML endpoint for deployments but with better troubleshooting - install_log: + install_html: enabled: false # Print HTML directly to console instead of writing to file console: false From 5bd9bc7932b9c7588800f7de0ea171cc5d9b51a4 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 24 Aug 2023 10:49:21 +0200 Subject: [PATCH 10/14] Refactorings --- pkg/package_manager.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/package_manager.go b/pkg/package_manager.go index 0b7f8175..51e25001 100644 --- a/pkg/package_manager.go +++ b/pkg/package_manager.go @@ -201,12 +201,12 @@ func (pm *PackageManager) Upload(localPath string) (string, error) { func (pm *PackageManager) Install(remotePath string) error { if pm.InstallHTMLEnabled { - return pm.installLogged(remotePath) + return pm.installHTML(remotePath) } - return pm.installRegular(remotePath) + return pm.installJSON(remotePath) } -func (pm *PackageManager) installRegular(remotePath string) error { +func (pm *PackageManager) installJSON(remotePath string) error { log.Infof("%s > installing package '%s'", pm.instance.ID(), remotePath) response, err := pm.instance.http.Request(). SetFormData(map[string]string{"cmd": "install", "recursive": fmt.Sprintf("%v", pm.InstallRecursive)}). @@ -227,7 +227,7 @@ func (pm *PackageManager) installRegular(remotePath string) error { return nil } -func (pm *PackageManager) installLogged(remotePath string) error { +func (pm *PackageManager) installHTML(remotePath string) error { log.Infof("%s > installing package '%s'", pm.instance.ID(), remotePath) response, err := pm.instance.http.Request(). From b0721f89edaec903733492b095a0d5f1e70736e2 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 24 Aug 2023 10:51:31 +0200 Subject: [PATCH 11/14] Naming --- pkg/package_manager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/package_manager.go b/pkg/package_manager.go index 51e25001..7385c913 100644 --- a/pkg/package_manager.go +++ b/pkg/package_manager.go @@ -251,7 +251,7 @@ func (pm *PackageManager) installHTML(remotePath string) error { } htmlFile, err := os.OpenFile(htmlFilePath, os.O_RDWR|os.O_CREATE, 0666) if err != nil { - return fmt.Errorf("%s > cannot install package '%s': cannot open HTML log file '%s'", pm.instance.ID(), remotePath, htmlFilePath) + return fmt.Errorf("%s > cannot install package '%s': cannot open HTML report file '%s'", pm.instance.ID(), remotePath, htmlFilePath) } defer htmlFile.Close() htmlWriter = bufio.NewWriter(htmlFile) @@ -270,7 +270,7 @@ func (pm *PackageManager) installHTML(remotePath string) error { if !pm.InstallHTMLConsole { _, err := htmlWriter.WriteString(htmlLine + osx.LineSep()) if err != nil { - return fmt.Errorf("%s > cannot install package '%s': cannot write to HTML log file '%s'", pm.instance.ID(), remotePath, htmlFilePath) + return fmt.Errorf("%s > cannot install package '%s': cannot write to HTML report file '%s'", pm.instance.ID(), remotePath, htmlFilePath) } } else { fmt.Println(htmlLine) From 597ef712cfcb652e645b72a81bbcd1d5b9d2f92d Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 24 Aug 2023 10:55:44 +0200 Subject: [PATCH 12/14] Doc --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d4b543e9..10904a70 100644 --- a/README.md +++ b/README.md @@ -502,7 +502,7 @@ By default, the timeout is set to `10m`, but you have the option to increase it To set the timeout for a single AEMC command, use the following syntax: ```shell -AEM_INSTANCE_HTTP_TIMEOUT=0 sh aemw package deploy --url my-package.zip +AEM_INSTANCE_HTTP_TIMEOUT=0 sh aemw package deploy --file my-package.zip ``` It's important to be aware that AEMaaCS also has its own timeout for requests made to the Package Manager UI. For detailed information, please refer to the [documentation](https://experienceleague.adobe.com/docs/experience-manager-cloud-service/content/implementing/developer-tools/package-manager.html?lang=en#aemaacs-packages). @@ -512,7 +512,7 @@ To skip the instance health check after running a command that would normally tr To skip the instance health check for a single AEMC command, use the following syntax: ```shell -AEM_INSTANCE_CHECK_SKIP=true sh aemw package deploy --url my-package.zip +AEM_INSTANCE_CHECK_SKIP=true sh aemw package deploy --file my-package.zip ``` ### Installing packages with troubleshooting @@ -526,7 +526,7 @@ This new feature offers two distinct modes for leveraging its benefits: To enable this mode and save the HTML report to a designated file, use the following shell command: ```shell - AEM_INSTANCE_PACKAGE_INSTALL_HTML_ENABLED=true sh aemw package deploy --url my-package.zip + AEM_INSTANCE_PACKAGE_INSTALL_HTML_ENABLED=true sh aemw package deploy --file my-package.zip ``` 2. **Direct Console Output of HTML Report:** @@ -534,7 +534,7 @@ This new feature offers two distinct modes for leveraging its benefits: To directly print the HTML report to the console, execute this shell command: ```shell - AEM_INSTANCE_PACKAGE_INSTALL_HTML_ENABLED=true AEM_INSTANCE_PACKAGE_INSTALL_HTML_CONSOLE=true sh aemw package deploy --url my-package.zip + AEM_INSTANCE_PACKAGE_INSTALL_HTML_ENABLED=true AEM_INSTANCE_PACKAGE_INSTALL_HTML_CONSOLE=true sh aemw package deploy --file my-package.zip ``` # Examples From 173db8dbf20b5d2fe5bf3c209d23f776feb2fef7 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 24 Aug 2023 10:59:43 +0200 Subject: [PATCH 13/14] Doc --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 10904a70..0de87308 100644 --- a/README.md +++ b/README.md @@ -521,17 +521,13 @@ Starting from version 1.4.0 (see [#177](https://github.com/wttech/aemc/pull/177) This new feature offers two distinct modes for leveraging its benefits: -1. **Saving HTML Report to File:** - - To enable this mode and save the HTML report to a designated file, use the following shell command: +1. Saving HTML report to file: ```shell AEM_INSTANCE_PACKAGE_INSTALL_HTML_ENABLED=true sh aemw package deploy --file my-package.zip ``` -2. **Direct Console Output of HTML Report:** - - To directly print the HTML report to the console, execute this shell command: +2. Direct console output of HTML report: ```shell AEM_INSTANCE_PACKAGE_INSTALL_HTML_ENABLED=true AEM_INSTANCE_PACKAGE_INSTALL_HTML_CONSOLE=true sh aemw package deploy --file my-package.zip From 2b8cfe279535feffbf0bc18e1991979da8806832 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 24 Aug 2023 11:05:44 +0200 Subject: [PATCH 14/14] Docs --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index 0de87308..92a1e23b 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ AEMC is a versatile tool for managing Adobe Experience Manager (AEM) instances. * [Installing content packages](#installing-content-packages) * [Installing packages with troubleshooting](#installing-packages-with-troubleshooting) * [Examples](#examples) + * [Replication agents](#replication-agents) * [SSL by Default](#ssl-by-default) * [Global Trust Store](#global-trust-store) * [Contributing](#contributing) @@ -535,6 +536,37 @@ This new feature offers two distinct modes for leveraging its benefits: # Examples +## Replication agents + +1. Configuring publish agent on AEM author: + + ```shell + PROPS=" + enabled: true + transportUri: http://localhost:4503/bin/receive?sling:authRequestLogin=1 + transportUser: admin + transportPassword: admin + userId: admin + " + echo "$PROPS" | sh aemw repl agent setup -A --location "author" --name "publish" + ``` + +2. Configuring flush agent on AEM publish: + + ```shell + PROPS=" + enabled: true + transportUri: http://localhost/dispatcher/invalidate.cache + protocolHTTPHeaders: + - 'CQ-Action: {action}' + - 'CQ-Handle: {path}' + - 'CQ-Path: {path}' + - 'Host: flush' + " + echo "$PROPS" | sh aemw repl agent setup -P --location "publish" --name "flush" + ``` + If needed, update `localhost` to the value on which AEM dispatcher is available, e.g.`localhost:8080`. + ## SSL by Default AEM Compose supports *SSL by Default* feature of AEM.