Reorganize dependencies and infrastructure for ACS (#2278)

* Reorganize dependencies

* Doesn't compile yet

* Add some methods that are missing when compiling with the full library

However, this now includes a lot of code that's very likely not really used.
Need to find a way around this.

* Remove unused step

This feature isn't used anywhere

* Separate debug and release also for the samples

So that compiling them works

* Now release builds again

* Extend workflow options

* Update runtime version

* Map file contains a summary

* Make sure the test assemblies are linked

For some reason, the linux build fails now due to these
apparently missing.

* Fix build issue

* Using a reference now fails on Windows

* Undo this again

* New attempt at solving that strange issue

* Tool packs in release mode

Problem: Would pack with the previous release

* Use current version of full package

* Ensure tool is really built

* Undo general exclusion of launchsettings.json

Can still commit some, if useful

* Added comments about experimental state

* Return to 1.0.x version

Better keep the low version as long as this
is experimental

* Minor review findings

* Use Arcade on tools and devices directories to fix ACS package not staying as preview

* Fix build issue due to trying to generate symbols for tool package

* Also print the runtime version

* bin path has changed

* Also update settings

---------

Co-authored-by: Jose Perez Rodriguez <joperezr@microsoft.com>

.github/workflows/arduino.yml

@@ -17,12 +17,16 @@ jobs:
   build:
     # The type of runner that the job will run on
     runs-on: windows-latest
+    strategy:
+        matrix:
+            configuration: [Release]
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       - name: Checkout
         uses: actions/checkout@master
 
       - run: |
-          eng/ArduinoCsCI.cmd %USERPROFILE% Debug
+          eng/ArduinoCsCI.cmd %USERPROFILE% ${{ matrix.configuration }}
+
 

build.proj

@@ -36,6 +36,7 @@
       <_ProjectsToPackage Include="$(MSBuildThisFileDirectory)src\Iot.Device.Bindings\Iot.Device.Bindings.csproj" />
       <_ProjectsToPackage Include="$(MSBuildThisFileDirectory)src\System.Device.Gpio\System.Device.Gpio.csproj" />
       <_ProjectsToPackage Include="$(MSBuildThisFileDirectory)src\Iot.Device.Bindings.SkiaSharpAdapter\Iot.Device.Bindings.SkiaSharpAdapter.csproj" />
+      <_ProjectsToPackage Include="$(MSBuildThisFileDirectory)tools\ArduinoCsCompiler\Frontend\Frontend.csproj"/>
 
       <ProjectReference Condition="'$(BuildPackages)' == 'true'" Include="@(_ProjectsToPackage)" Targets="Pack" />
   </ItemGroup>

eng/ArduinoCsCI.cmd

@@ -5,7 +5,7 @@ REM Second argument is either "Debug" or "Release"
 if %1!==! goto :usage
 
 REM Defines the revision to check out in the ExtendedConfigurableFirmata repo
-set FIRMATA_SIMULATOR_CHECKOUT_REVISION=4a3b895c062c8e48685b9018d642d2c5ea84c354
+set FIRMATA_SIMULATOR_CHECKOUT_REVISION=66b1863f3a4b0999f2c1cb7133332cbd25730252
 set RUN_COMPILER_TESTS=FALSE
 
 choco install -y --no-progress arduino-cli
@@ -20,7 +20,7 @@ REM directly execute PS, we can ignore any test errors.
 powershell -ExecutionPolicy ByPass -command "%~dp0common\Build.ps1" -restore -build -ci -configuration %2 -preparemachine
 
 set ArduinoRootDir=%1\Documents\Arduino
-set acspath=%~dp0\..\tools\ArduinoCsCompiler\Frontend\bin\%2\net6.0\acs.exe
+set acspath=%~dp0\..\artifacts\bin\Frontend\%2\net6.0\acs.exe
 
 git clone https://github.com/firmata/ConfigurableFirmata %ArduinoRootDir%\libraries\ConfigurableFirmata
 git clone https://github.com/pgrawehr/ExtendedConfigurableFirmata %ArduinoRootDir%\ExtendedConfigurableFirmata
@@ -35,6 +35,8 @@ if %errorlevel%==0 set RUN_COMPILER_TESTS=TRUE
 
 dir %ArduinoRootDir%
 
+if NOT exist %acspath% goto error
+
 %acspath% --help
 
 rem Write runtime data to ExtendedConfigurableFirmata directory, before building
@@ -70,14 +72,18 @@ REM information about all tests being executed (as this test run can take 30 min
 echo Starting basic arduino tests
 dotnet test -c %2 --no-build --no-restore --filter feature=firmata -l "console;verbosity=normal" -maxcpucount:1
 
-echo on
-if %RUN_COMPILER_TESTS%==TRUE (
-echo Starting extended Arduino compiler tests
-dotnet test -c %2 --no-build --no-restore --filter feature=firmata-compiler -l "console;verbosity=normal" -maxcpucount:1
-)
+if errorlevel 1 goto error
 
+echo on
+echo Run full compiler against simple example
+REM The current directory is tools/ArduinoCsCompiler
+%acspath% compile --run %~dp0\..\artifacts\bin\BlinkingLed\%2\net6.0\BlinkingLed.exe --network localhost --mapfile BlinkingLed-tokenmap.txt
 if errorlevel 1 goto error
 
+REM copy token map to output (so we have something to compare the history of sizes, if something changes unexpectedly)
+type BlinkingLed-tokenmap.txt
+
+
 popd
 taskkill /im ExtendedConfigurableFirmataSim.exe /f
 

eng/Versions.external.props

@@ -4,6 +4,10 @@
     <PackageReference Include="UnitsNet" Version="5.31.0" />
   </ItemGroup>
 
+  <PropertyGroup>
+    <MicrosoftNETTestSdkVersion>17.8.0</MicrosoftNETTestSdkVersion>
+  </PropertyGroup>
+
   <!-- Automatically include these assemblies in all test projects -->
   <ItemGroup Condition="$(MSBuildProjectName.EndsWith('Tests'))">
      <PackageReference Include="Moq" Version="4.16.1" />
@@ -13,7 +17,6 @@
        <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="xunit.runner.utility" Version="2.6.3" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
     <PackageReference Include="Shouldly" Version="4.2.1" />
   </ItemGroup>
 </Project>

src/devices/Arduino/tests/Arduino.Tests.csproj

@@ -11,7 +11,6 @@
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\Arduino.csproj" />
-    <ProjectReference Include="..\..\..\..\tools\ArduinoCsCompiler\Frontend\Frontend.csproj" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingPackageVersion)" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsolePackageVersion)" />
     <PackageReference Include="System.IO.Ports" Version="$(SystemIOPortsPackageVersion)" />

src/devices/Camera/tests/TestCamera/TestCamera.csproj

@@ -10,7 +10,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
     <PackageReference Include="xunit" Version="2.6.2" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.5.3">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

src/devices/Directory.Build.props

@@ -1,8 +1,6 @@
 <Project>
-  <Import Project="../../eng/Compilers.props" />
-  <Import Project="../../eng/Versions.props" />
-  <Import Project="../../eng/Analyzers.props" />
-  <Import Project="../../eng/Versions.external.props" />
+  <Import Project="$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))" />
+
   <PropertyGroup>
     <!-- Most projects should use this value to build for all possible target frameworks -->
     <DefaultBindingTfms>net6.0;netstandard2.0;</DefaultBindingTfms>

src/devices/Display/Large4Digit7SegmentDisplay.cs

@@ -254,7 +254,7 @@ public void Write(string value, Alignment alignment = Alignment.Left)
                     break;
                 case 5 when value[2] != ':':
 #pragma warning disable CA2208 // False positive because the message uses the parameter name "value"
-                    throw new ArgumentException("value[2] must be a ':'", nameof(value));
+                    throw new ArgumentException($"{nameof(value)}[2] must be a ':'", nameof(value));
 #pragma warning restore CA2208
                 case 5:
                     Dots |= Dot.CenterColon;

tools/ArduinoCsCompiler/ArduinoCsCompiler.csproj

@@ -25,9 +25,15 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.8.0" />
+  </ItemGroup>
+  <ItemGroup Condition="$(Configuration)=='Debug'">
     <ProjectReference Include="..\..\src\devices\Arduino\Arduino.csproj" />
     <ProjectReference Include="..\..\src\System.Device.Gpio\System.Device.Gpio.csproj" />
   </ItemGroup>
+  <ItemGroup Condition="$(Configuration)!='Debug'">
+    <ProjectReference Include="..\..\src\System.Device.Gpio\System.Device.Gpio.csproj"/>
+    <ProjectReference Include="..\..\src\Iot.Device.Bindings\Iot.Device.Bindings.csproj"/>
+  </ItemGroup>
 
   <ItemGroup>
     <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">

tools/ArduinoCsCompiler/ArduinoCsCompiler.sln

@@ -39,6 +39,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Button", "..\..\src\devices
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BlinkingLed", "samples\BlinkingLed\BlinkingLed.csproj", "{072A16C3-61D5-429D-85BC-09EF4A8CEE8D}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Iot.Device.Bindings", "..\..\src\Iot.Device.Bindings\Iot.Device.Bindings.csproj", "{AD276307-C242-45FE-AAC4-E6713E8C9AC3}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -105,6 +107,10 @@ Global
 		{072A16C3-61D5-429D-85BC-09EF4A8CEE8D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{072A16C3-61D5-429D-85BC-09EF4A8CEE8D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{072A16C3-61D5-429D-85BC-09EF4A8CEE8D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AD276307-C242-45FE-AAC4-E6713E8C9AC3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AD276307-C242-45FE-AAC4-E6713E8C9AC3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AD276307-C242-45FE-AAC4-E6713E8C9AC3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AD276307-C242-45FE-AAC4-E6713E8C9AC3}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

tools/ArduinoCsCompiler/CompilerCommandHandler.cs

@@ -30,7 +30,21 @@ public CompilerCommandHandler(MicroCompiler compiler)
             _logger = this.GetCurrentClassLogger();
         }
 
-        public IlCapabilities? IlCapabilities => _ilCapabilities;
+        /// <summary>
+        /// Query the hardware capabilities.
+        /// Use the setter to set the value to null, to force an update
+        /// </summary>
+        public IlCapabilities? IlCapabilities
+        {
+            get
+            {
+                return _ilCapabilities;
+            }
+            set
+            {
+                _ilCapabilities = value;
+            }
+        }
 
         protected override void OnErrorMessage(string message, Exception? exception)
         {

tools/ArduinoCsCompiler/CompilerSettings.cs

@@ -94,8 +94,16 @@ public bool ForceFlashWrite
             set;
         }
 
+        /// <summary>
+        /// True to support language preview features
+        /// </summary>
         public bool UsePreviewFeatures { get; set; }
 
+        /// <summary>
+        /// The name of the process, if applicable
+        /// </summary>
+        public string? ProcessName { get; set; }
+
         object ICloneable.Clone()
         {
             return MemberwiseClone();

tools/ArduinoCsCompiler/Directory.build.props

@@ -1,8 +1,7 @@
 <Project>
-  <Import Project="../../eng/Analyzers.props" />
-  <Import Project="../../eng/Versions.props" />
-  <Import Project="../../eng/Versions.external.props" />
+  <Import Project="$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))" />
+
   <PropertyGroup>
-	<Nullable>enable</Nullable>
+	  <Nullable>enable</Nullable>
   </PropertyGroup>
 </Project>

tools/ArduinoCsCompiler/ExecutionSet.cs

@@ -1504,10 +1504,20 @@ public void SuppressNamespace(string namespacePrefix, bool includingSubNamespace
             }
         }
 
-        public void WriteMapFile(string tokenMapFile)
+        public void WriteMapFile(string tokenMapFile, IlCapabilities ilCapabilities)
         {
             using StreamWriter w = new StreamWriter(tokenMapFile);
 
+            w.WriteLine("Program data summary");
+            w.WriteLine("====================");
+            w.WriteLine($"Program Name: {CompilerSettings.ProcessName ?? "N/A"}");
+            w.WriteLine(ilCapabilities);
+            w.WriteLine($"Total number of classes: {_classes.Count}");
+            w.WriteLine($"Total number of methods: {_methods.Count}");
+            w.WriteLine($"Total number of static fields: {_staticFieldsUsed.Count}");
+            w.WriteLine($"Total number of string constants: {_strings.Count}");
+            w.WriteLine();
+
             w.WriteLine("Token map file, ordered by token number");
             w.WriteLine("=======================================");
             w.WriteLine();

tools/ArduinoCsCompiler/Frontend/CompilerRun.cs

@@ -57,7 +57,7 @@ public override bool RunCommand()
 
                 _compiler = new MicroCompiler(board, true);
 
-                if (!_compiler.QueryBoardCapabilities(out var caps))
+                if (!_compiler.QueryBoardCapabilities(true, out var caps))
                 {
                     Logger.LogError("Couldn't query board capabilities. Possibly incompatible firmware");
                     return false;
@@ -144,6 +144,7 @@ private void RunCompiler(FileInfo inputInfo)
                 UseFlashForProgram = true,
                 UsePreviewFeatures = CommandLineOptions.UsePreviewFeatures,
                 AdditionalSuppressions = CommandLineOptions.Suppressions,
+                ProcessName = inputInfo.Name,
             };
 
             Logger.LogInformation("Collecting method information and metadata...");
@@ -158,17 +159,15 @@ private void RunCompiler(FileInfo inputInfo)
                 Logger.LogDebug($"Class {stat.Key.FullName}: {stat.Value.TotalBytes} Bytes");
             }
 
-            if (!string.IsNullOrEmpty(CommandLineOptions.TokenMapFile))
-            {
-                set.WriteMapFile(CommandLineOptions.TokenMapFile);
-            }
-
             Logger.LogInformation($"Compile successful. {ErrorManager.NumErrors} Errors, {ErrorManager.NumWarnings} Warnings");
 
             if (!CommandLineOptions.CompileOnly)
             {
                 set.Load(false);
 
+                // Call this after load, so we have an updated flash usage value available.
+                WriteTokenMap(set);
+
                 if (CommandLineOptions.Run == false)
                 {
                     Logger.LogInformation("Program uploaded successfully. Execution not started. Reset the board to start execution.");
@@ -290,6 +289,28 @@ private void RunCompiler(FileInfo inputInfo)
                     }
                 }
             }
+            else
+            {
+                WriteTokenMap(set); // If we don't load, do this anyway
+            }
+        }
+
+        private void WriteTokenMap(ExecutionSet set)
+        {
+            if (_compiler == null)
+            {
+                return;
+            }
+
+            if (!string.IsNullOrEmpty(CommandLineOptions.TokenMapFile))
+            {
+                if (!_compiler.QueryBoardCapabilities(true, out var caps))
+                {
+                    caps = new IlCapabilities(); // Illegal, but that should be obvious to anyone
+                }
+
+                set.WriteMapFile(CommandLineOptions.TokenMapFile, caps);
+            }
         }
 
         private void PrintStateFromDebuggerData((DebuggerDataKind Kind, byte[] Data) x)

tools/ArduinoCsCompiler/Frontend/ExecRun.cs

@@ -25,7 +25,7 @@ public override bool RunCommand()
 
         var compiler = new MicroCompiler(_board, true);
 
-        if (!compiler.QueryBoardCapabilities(out var caps))
+        if (!compiler.QueryBoardCapabilities(true, out var caps))
         {
             Logger.LogError("Couldn't query board capabilities. Possibly incompatible firmware");
             return false;

tools/ArduinoCsCompiler/Frontend/Frontend.csproj

@@ -9,20 +9,53 @@
     <AssemblyName>acs</AssemblyName>
     <RootNamespace>ArduinoCsCompiler</RootNamespace>
     <Product>Arduino CS Compiler</Product>
-    <Description>A translation compiler that retargets C# applications so they can run on a microcontroller using the unnamed firmware.</Description>
+    <Description>A translation compiler that retargets C# applications so they can run on a microcontroller using a special firmware. See Readme for further information.</Description>
     <Authors>The .NET Foundation community</Authors>
     <NeutralLanguage>en</NeutralLanguage>
+    <IsPackable>True</IsPackable>
+    <!-- This is a tool package so we disable generating a symbol package. -->
+    <IncludeSymbols>false</IncludeSymbols>
+    <LangVersion>latest</LangVersion>
+    <PackAsTool>True</PackAsTool>
+    <ToolCommandName>dotnet-acs</ToolCommandName>
+    <PackageOutputPath>..\..\..\artifacts\packages\$(Configuration)\Shipping</PackageOutputPath>
+    <MajorVersion>1</MajorVersion>
+    <MinorVersion>0</MinorVersion>
+    <PatchVersion>2</PatchVersion>
+    <!-- Stay in beta mode -->
+    <SuppressFinalPackageVersion>true</SuppressFinalPackageVersion>
   </PropertyGroup>
 
   <ItemGroup>
     <Compile Include="*.cs" />
+
+    <!-- Required for packaging -->
+    <None Include="$(PackageIconFullPath)" Pack="true" PackagePath="$(PackageIcon)" Visible="false" />
+    <None Include="$(RepoRoot)/src/devices/README-nuget.md" Pack="true" Visible="false" PackagePath="\README.md" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="..\..\..\src\devices\Arduino\samples\ApiChecker\RgbLedTest.cs" Link="RgbLedTest.cs" />
+    <Compile Include="..\..\..\src\devices\Arduino\samples\ApiChecker\TestCases.cs" Link="TestCases.cs" />
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="CommandLineParser" Version="2.9.1" />
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\..\..\src\devices\Arduino\Arduino.csproj" />
-    <ProjectReference Include="..\..\..\src\devices\Arduino\samples\ApiChecker\Arduino.sample.csproj" />
     <ProjectReference Include="..\ArduinoCsCompiler.csproj" />
   </ItemGroup>
+  <!-- In debug mode, we take the dependencies from the current build, in release mode we link to the officially released packages.
+  This implies that any application the user want's to translate do the same. If they're not part of the same repository,
+  they will always link to the packages, not the projects.
+  -->
+  <ItemGroup Condition="$(Configuration)=='Debug'">
+    <ProjectReference Include="..\..\..\src\devices\Mcp3xxx\Mcp3xxx.csproj" />
+    <ProjectReference Include="..\..\..\src\devices\Bmxx80\Bmxx80.csproj" />
+    <ProjectReference Include="..\..\..\src\devices\Arduino\Arduino.csproj" />
+    <ProjectReference Include="..\..\..\src\System.Device.Gpio\System.Device.Gpio.csproj" />
+  </ItemGroup>
+  <ItemGroup Condition="$(Configuration)!='Debug'">
+    <ProjectReference Include="..\..\..\src\System.Device.Gpio\System.Device.Gpio.csproj" />
+    <ProjectReference Include="..\..\..\src\Iot.Device.Bindings\Iot.Device.Bindings.csproj" />
+  </ItemGroup>
 </Project>

tools/ArduinoCsCompiler/Frontend/Program.cs

@@ -15,9 +15,7 @@
 using System.Threading;
 using CommandLine;
 using Iot.Device.Arduino;
-using Iot.Device.Common;
 using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
 using UnitsNet;
 
 namespace ArduinoCsCompiler
@@ -34,6 +32,8 @@ private static int Main(string[] args)
             }
 
             Console.WriteLine($"ArduinoCsCompiler - Version {version.Version}");
+            Console.WriteLine("This tool is experimental - expect many missing features and that the behavior will change.");
+            Console.WriteLine($"Active runtime version {RuntimeInformation.FrameworkDescription}");
             bool runResult = false;
 
             var parser = new Parser(x =>