From faf1d16692ead1115d34fdf7b84c256c94a321da Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Fri, 21 Aug 2020 18:53:25 -0400 Subject: [PATCH] [Xamarin.Android.Build.Tests] Import regression tests from job 1 (#5030) Context: https://github.com/xamarin/QualityAssurance/pull/3639 This is an initial effort to migrate non duplicated and relevant tests from QualityAssurance into xamarin-android. The majority of test categories that ran in the first regression test job have been audited, and a few tests have been imported while the rest have been removed. The remaining regression test categories that were ran as part of the first and second regression test jobs have been merged, and the third job has been removed. Device tests that searched for `.__override__` directory content after a Release build have been updated to have a stricter assert. These tests will also now set `android:debuggable="true"` to allow usage of the run-as command, which is required when attempting to list the content of the potential `/data/data/*` location. The DeviceTest base class has been updated to ensure logcat output is captured on failure by escaping the destination path, and a duplicate `AddTestAttachment` call has been removed. --- build-tools/automation/azure-pipelines.yaml | 46 +---- .../run-integrated-regression-tests.yaml | 47 +---- .../Test/Mono.Android-Test.Shared.projitems | 1 + .../Test/Mono.Android-Tests.csproj | 1 + .../Test/System.Text/EncodingTests.cs | 22 +++ .../PackagingTest.cs | 29 ++- .../Tasks/LinkerTests.cs | 47 +++++ .../Utilities/DeviceTest.cs | 7 +- .../Tests/InstallTests.cs | 176 +++++++++++------- .../Mono.Android-TestsAppBundle.csproj | 1 + .../Mono.Android-TestsMultiDex.csproj | 1 + 11 files changed, 213 insertions(+), 165 deletions(-) create mode 100644 src/Mono.Android/Test/System.Text/EncodingTests.cs diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index e76e9243d47..e817d25fc12 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -1103,7 +1103,7 @@ stages: condition: and(eq(dependencies.mac_build.result, 'Succeeded'), eq(variables['RunAllTests'], true)) jobs: - # Check - "Xamarin.Android (Test Integrated Regression - macOS)" + # Check - "Xamarin.Android (Regression Tests Mac-1)" - job: integrated_regression_mac_1 displayName: Mac-1 pool: @@ -1119,7 +1119,7 @@ stages: parameters: node_id: 1 - # Check - "Xamarin.Android (Test Integrated Regression - macOS)" + # Check - "Xamarin.Android (Regression Tests Mac-2)" - job: integrated_regression_mac_2 displayName: Mac-2 pool: @@ -1135,23 +1135,7 @@ stages: parameters: node_id: 2 - # Check - "Xamarin.Android (Test Integrated Regression - macOS)" - - job: integrated_regression_mac_3 - displayName: Mac-3 - pool: - name: VSEng-Xamarin-Mac-Devices - demands: - - android - timeoutInMinutes: 240 - cancelTimeoutInMinutes: 5 - workspace: - clean: all - steps: - - template: yaml-templates/run-integrated-regression-tests.yaml - parameters: - node_id: 3 - - # Check - "Xamarin.Android (Test Integrated Regression - Windows)" + # Check - "Xamarin.Android (Regression Tests Windows-1)" - job: integrated_regression_Win_1 displayName: Windows-1 pool: @@ -1173,7 +1157,7 @@ stages: - template: remove-visualstudio.yml@yaml - # Check - "Xamarin.Android (Test Integrated Regression - Windows)" + # Check - "Xamarin.Android (Regression Tests Windows-2)" - job: integrated_regression_Win_2 displayName: Windows-2 pool: @@ -1195,28 +1179,6 @@ stages: - template: remove-visualstudio.yml@yaml - # Check - "Xamarin.Android (Test Integrated Regression - Windows)" - - job: integrated_regression_Win_3 - displayName: Windows-3 - pool: - name: VSEng-Xamarin-Win-XMA - demands: - - android - timeoutInMinutes: 240 - cancelTimeoutInMinutes: 5 - workspace: - clean: all - variables: - XQA_VISUALSTUDIO_LOCATION: '%ProgramFiles(x86)%\Microsoft Visual Studio\2019\Enterprise' - steps: - - template: remove-visualstudio.yml@yaml - - - template: yaml-templates\run-integrated-regression-tests.yaml - parameters: - node_id: 3 - - - template: remove-visualstudio.yml@yaml - # TimeZoneInfo tests - template: yaml-templates\run-timezoneinfo-tests.yaml parameters: diff --git a/build-tools/automation/yaml-templates/run-integrated-regression-tests.yaml b/build-tools/automation/yaml-templates/run-integrated-regression-tests.yaml index b9b46ff7b86..e997250bf9f 100644 --- a/build-tools/automation/yaml-templates/run-integrated-regression-tests.yaml +++ b/build-tools/automation/yaml-templates/run-integrated-regression-tests.yaml @@ -90,51 +90,6 @@ steps: testRunTitle: Smoke Tests on Device nunitConsoleExtraArgs: --where "cat == RegressionDeviceTests" - - template: run-nunit-tests.yaml - parameters: - nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe - testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll - testRunTitle: Smoke Tests - nunitConsoleExtraArgs: --where "cat == RegressionTests" - - - template: run-nunit-tests.yaml - parameters: - nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe - testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll - testRunTitle: Installer Tests - nunitConsoleExtraArgs: --where "cat == InstallationTests" - - - template: run-nunit-tests.yaml - parameters: - nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe - testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll - testRunTitle: Fast Deploy Tests - nunitConsoleExtraArgs: --where "cat == FastDevTests" - condition: and(succeeded(), eq(variables['XA.Commercial.Build'], 'true')) - - - template: run-nunit-tests.yaml - parameters: - nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe - testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll - testRunTitle: Debug Tests - nunitConsoleExtraArgs: --where "cat == DebuggerTests" - condition: and(succeeded(), eq(variables['XA.Commercial.Build'], 'true')) - - - template: run-nunit-tests.yaml - parameters: - nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe - testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll - testRunTitle: Linker Tests - nunitConsoleExtraArgs: --where "cat == LinkerTests" - - - template: run-nunit-tests.yaml - parameters: - nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe - testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll - testRunTitle: Wearable Tests - nunitConsoleExtraArgs: --where "cat == Wearable" - -- ${{ if eq(parameters.node_id, 2) }}: - template: run-nunit-tests.yaml parameters: nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe @@ -149,7 +104,7 @@ steps: testRunTitle: AOT Tests nunitConsoleExtraArgs: --where "cat == AotSupport" -- ${{ if eq(parameters.node_id, 3) }}: +- ${{ if eq(parameters.node_id, 2) }}: - template: run-nunit-tests.yaml parameters: nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe diff --git a/src/Mono.Android/Test/Mono.Android-Test.Shared.projitems b/src/Mono.Android/Test/Mono.Android-Test.Shared.projitems index 9ee41241229..8904936743c 100644 --- a/src/Mono.Android/Test/Mono.Android-Test.Shared.projitems +++ b/src/Mono.Android/Test/Mono.Android-Test.Shared.projitems @@ -42,6 +42,7 @@ + diff --git a/src/Mono.Android/Test/Mono.Android-Tests.csproj b/src/Mono.Android/Test/Mono.Android-Tests.csproj index fa55152cccc..c507c72a00c 100644 --- a/src/Mono.Android/Test/Mono.Android-Tests.csproj +++ b/src/Mono.Android/Test/Mono.Android-Tests.csproj @@ -18,6 +18,7 @@ Properties\AndroidManifest.xml armeabi-v7a;x86 False + All true ..\..\..\product.snk v11.0 diff --git a/src/Mono.Android/Test/System.Text/EncodingTests.cs b/src/Mono.Android/Test/System.Text/EncodingTests.cs new file mode 100644 index 00000000000..fd3da6aaea8 --- /dev/null +++ b/src/Mono.Android/Test/System.Text/EncodingTests.cs @@ -0,0 +1,22 @@ +using System; +using System.Text; + +using NUnit.Framework; + +namespace Xamarin.Android.RuntimeTests +{ + [TestFixture] + public class EncodingTests + { + EncodingInfo[] EncodingTestData = Encoding.GetEncodings (); + + [Test, TestCaseSource (nameof (EncodingTestData))] + public void GetAllAvailableEncodings (EncodingInfo info) + { + // Requires All or can throw: + // System.NotSupportedException : Encoding 37 data could not be found. Make sure you have correct international codeset assembly installed and enabled. + Encoding enc = info.GetEncoding (); + Assert.IsNotNull (enc.EncodingName, $"Failed to get Encoding from '{info.DisplayName}'."); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index 1a4eede245b..448a4009306 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -399,7 +399,11 @@ public void CheckSignApk ([Values(true, false)] bool useApkSigner, [Values(true, proj.SetProperty (proj.ReleaseProperties, "AndroidSigningKeyPass", Uri.EscapeDataString (pass)); proj.SetProperty (proj.ReleaseProperties, "AndroidSigningStorePass", Uri.EscapeDataString (pass)); proj.SetProperty (proj.ReleaseProperties, KnownProperties.AndroidCreatePackagePerAbi, perAbiApk); - proj.SetAndroidSupportedAbis ("armeabi-v7a", "x86"); + if (perAbiApk) { + proj.SetAndroidSupportedAbis ("armeabi-v7a", "x86", "arm64-v8a", "x86_64"); + } else { + proj.SetAndroidSupportedAbis ("armeabi-v7a", "x86"); + } using (var b = CreateApkBuilder (Path.Combine ("temp", TestContext.CurrentContext.Test.Name))) { var bin = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath); Assert.IsTrue (b.Build (proj), "First build failed"); @@ -417,6 +421,17 @@ public void CheckSignApk ([Values(true, false)] bool useApkSigner, [Values(true, } } + // Make sure the APKs have unique version codes + if (perAbiApk) { + int armManifestCode = GetVersionCodeFromIntermediateManifest (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android", "armeabi-v7a", "AndroidManifest.xml")); + int x86ManifestCode = GetVersionCodeFromIntermediateManifest (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android", "x86", "AndroidManifest.xml")); + int arm64ManifestCode = GetVersionCodeFromIntermediateManifest (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android", "arm64-v8a", "AndroidManifest.xml")); + int x86_64ManifestCode = GetVersionCodeFromIntermediateManifest (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android", "x86_64", "AndroidManifest.xml")); + var versionList = new List { armManifestCode, x86ManifestCode, arm64ManifestCode, x86_64ManifestCode }; + Assert.True (versionList.Distinct ().Count () == versionList.Count, + $"APK version codes were not unique - armeabi-v7a: {armManifestCode}, x86: {x86ManifestCode}, arm64-v8a: {arm64ManifestCode}, x86_64: {x86_64ManifestCode}"); + } + var item = proj.AndroidResources.First (x => x.Include () == "Resources\\values\\Strings.xml"); item.TextContent = () => proj.StringsXml.Replace ("${PROJECT_NAME}", "Foo"); item.Timestamp = null; @@ -435,6 +450,18 @@ public void CheckSignApk ([Values(true, false)] bool useApkSigner, [Values(true, } } } + + int GetVersionCodeFromIntermediateManifest (string manifestFilePath) + { + var doc = XDocument.Load (manifestFilePath); + var versionCode = doc.Descendants () + .Where (e => e.Name == "manifest") + .Select (m => m.Attribute ("{http://schemas.android.com/apk/res/android}versionCode")).FirstOrDefault (); + + if (!int.TryParse (versionCode?.Value, out int parsedCode)) + Assert.Fail ($"Unable to parse 'versionCode' value from manifest content: {File.ReadAllText (manifestFilePath)}."); + return parsedCode; + } } [Test] diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index 0ee9968fc1a..80489d73508 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -241,5 +241,52 @@ public void LinkDescription () } } } + + [Test] + public void LinkWithNullAttribute () + { + var proj = new XamarinAndroidApplicationProject { + IsRelease = true, + OtherBuildItems = { + new BuildItem ("Compile", "NullAttribute.cs") { TextContent = () => @" +using System; +using Android.Content; +using Android.Widget; +namespace UnnamedProject { + public class MyAttribute : Attribute + { + Type[] types; + + public Type[] Types { + get { return types; } + } + + public MyAttribute (Type[] ta) + { + types = ta; + } + } + + [MyAttribute (null)] + public class AttributedButtonStub : Button + { + public AttributedButtonStub (Context context) : base (context) + { + } + } +}" + }, + } + }; + + proj.MainActivity = proj.DefaultMainActivity.Replace ("//${AFTER_ONCREATE}", +$@" var myButton = new AttributedButtonStub (this); + myButton.Text = ""Bug #35710""; +"); + + using (var b = CreateApkBuilder ()) { + Assert.IsTrue (b.Build (proj), "Building a project with a null attribute value should have succeded."); + } + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs index 4564f0cd93a..5dda6a31e8a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs @@ -29,19 +29,14 @@ protected override void CleanupTest () string deviceLog = Path.Combine (outputDir, "logcat-failed.log"); string remote = "/data/local/tmp/screenshot.png"; RunAdbCommand ($"shell screencap {remote}"); + RunAdbCommand ($"logcat -d > \"{deviceLog}\""); RunAdbCommand ($"pull {remote} \"{local}\""); RunAdbCommand ($"shell rm {remote}"); - RunAdbCommand ($"logcat -d > {deviceLog}"); if (File.Exists (local)) { TestContext.AddTestAttachment (local); } else { TestContext.WriteLine ($"{local} did not exist!"); } - if (File.Exists (deviceLog)) { - TestContext.AddTestAttachment (deviceLog); - } else { - TestContext.WriteLine ($"{deviceLog} did not exist!"); - } } base.CleanupTest (); diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs index d085eb9681e..1422122e655 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Xamarin.ProjectTools; using System.IO; @@ -25,6 +25,31 @@ static byte [] GetKeystore () } } + string[] GetOverrideDirectoryPaths (string packageName) + { + return new string [] { + $"/data/data/{packageName}/files/.__override__", + $"/storage/emulated/0/Android/data/{packageName}/files/.__override__", + $"/mnt/shell/emulated/0/Android/data/{packageName}/files/.__override__", + $"/storage/sdcard/Android/data/{packageName}/files/.__override__", + }; + } + + string GetContentFromAllOverrideDirectories (string packageName, bool useRunAsCommand = true) + { + var adbShellArgs = $"shell ls"; + if (useRunAsCommand) + adbShellArgs = $"shell run-as {packageName} ls"; + + var directorylist = string.Empty; + foreach (var dir in GetOverrideDirectoryPaths (packageName)) { + var listing = RunAdbCommand ($"{adbShellArgs} {dir}"); + if (!listing.Contains ("No such file or directory")) + directorylist += $"{listing} "; + } + return directorylist; + } + [Test] public void ReInstallIfUserUninstalled ([Values (false, true)] bool isRelease) { @@ -64,6 +89,8 @@ public void InstallAndUnInstall ([Values (false, true)] bool isRelease) IsRelease = isRelease, }; if (isRelease) { + // Set debuggable=true to allow run-as command usage with a release build + proj.AndroidManifest = proj.AndroidManifest.Replace (" InlineData.ResxWithContents ("Cancel") + }, + new BuildItem ("EmbeddedResource", "Foo.es.resx") { + TextContent = () => InlineData.ResxWithContents ("Cancelar") + } + } + }; + + using (var builder = CreateApkBuilder ()) { + Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); + var projectOutputPath = Path.Combine (Root, builder.ProjectDirectory, proj.OutputPath); + var resourceFilesFromDisk = Directory.EnumerateFiles (projectOutputPath, "*.resources.dll", SearchOption.AllDirectories) + .Select (r => r = r.Replace (projectOutputPath, string.Empty).Replace ("\\", "/")); + + var overrideContents = string.Empty; + foreach (var dir in GetOverrideDirectoryPaths (proj.PackageName)) { + overrideContents += RunAdbCommand ($"shell find {dir}"); + } + builder.Uninstall (proj); + Assert.IsTrue (resourceFilesFromDisk.Any (), $"Unable to find any localized assemblies in {resourceFilesFromDisk}"); + foreach (var res in resourceFilesFromDisk) { + StringAssert.Contains (res, overrideContents, $"{res} did not exist in the .__override__ directory.\nFound:{overrideContents}"); + } + + } + } } } diff --git a/tests/Runtime-AppBundle/Mono.Android-TestsAppBundle.csproj b/tests/Runtime-AppBundle/Mono.Android-TestsAppBundle.csproj index 97e60b7325e..8bf77175a1c 100644 --- a/tests/Runtime-AppBundle/Mono.Android-TestsAppBundle.csproj +++ b/tests/Runtime-AppBundle/Mono.Android-TestsAppBundle.csproj @@ -18,6 +18,7 @@ Properties\AndroidManifest.xml armeabi-v7a;x86 False + All d8 aab <_MonoAndroidTest>..\..\src\Mono.Android\Test\ diff --git a/tests/Runtime-MultiDex/Mono.Android-TestsMultiDex.csproj b/tests/Runtime-MultiDex/Mono.Android-TestsMultiDex.csproj index 69b981d2a58..75a4c09266f 100644 --- a/tests/Runtime-MultiDex/Mono.Android-TestsMultiDex.csproj +++ b/tests/Runtime-MultiDex/Mono.Android-TestsMultiDex.csproj @@ -18,6 +18,7 @@ Properties\AndroidManifest.xml armeabi-v7a;x86 False + All true d8 true