Automatically generate greenline images for tests and provide for tes…

…t failure. (#1438)

CoreGraphics.Drawing.UnitTests can now be invoked in one of two modes, mediated by command line flags:

* `--compare[=/path/to/reference/folder]` (_default mode_) compare all rendered images against their baselines, failing the test if any images differ by even a single pixel.
* `--generate [--out=/path/to/output/directory]` generate output images only (do not run comparisons.)

A failing test will emit a brightly-colored diff image to `Greenlines.TestImage.CASE.TEST.png` in the _output directory_, and attach attributes to the gtest output XML data specifying where to find the images. A compatible test runner can load these images and display them side-by-side.

**Open Questions/Concerns**
* I do not like the design of `rgba/bgraPixels`.
* I do not take stride into account; if the image is width-padded, the tests will ikely fail.
* The reference platform loads images in RGBA, and we load them in BGRA. This is not an incompatibility, but it does make manual buffer manipulation dicey.
  * The fix here is to render the image into a NEW context, but that presents a few issues of its own:
    * We must trust our DrawImage function
    * We must further validate bitmap context, and trust our bitmap context as an integral part of test execution.
* All tests will be disabled as of the merge of #1434. We can re-enable them and reintroduce reference images after build validation.

Closes #1271.

build/Tests/UnitTests/CoreGraphics.Drawing/CoreGraphics.Drawing.UnitTests.vcxproj

@@ -238,7 +238,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="$(StarboardBasePath)\tests\unittests\Framework\Framework.cpp" />
-    <ClCompile Include="$(StarboardBasePath)\tests\unittests\CoreGraphics.Drawing\EntryPoint.cpp" />
+    <ClangCompile Include="$(StarboardBasePath)\tests\unittests\CoreGraphics.Drawing\EntryPoint.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClangCompile Include="$(StarboardBasePath)\tests\unittests\CoreGraphics.Drawing\CGContextBlendModeTests.cpp" />
@@ -247,12 +247,16 @@
     <ClangCompile Include="$(StarboardBasePath)\tests\unittests\CoreGraphics.Drawing\CGContextDrawing_ShadowTests.cpp" />
     <ClangCompile Include="$(StarboardBasePath)\tests\unittests\CoreGraphics.Drawing\CGPathDrawingTests.cpp" />
     <ClangCompile Include="$(StarboardBasePath)\tests\unittests\CoreGraphics.Drawing\DrawingTest.cpp" />
+    <ClangCompile Include="$(StarboardBasePath)\tests\unittests\CoreGraphics.drawing\ImageComparison.cpp" />
+    <ClangCompile Include="$(StarboardBasePath)\tests\unittests\CoreGraphics.drawing\ImageHelpers.cpp" />
   </ItemGroup>
   <Target Name="CopyTestResourcesToOutput" AfterTargets="AfterBuild">
     <ItemGroup>
-      <TestResourceFile Include="..\..\..\..\tests\unittests\CoreGraphics.drawing\data\*" />
+      <TestResourceFile Include="$(MSBuildThisFileDirectory)..\..\..\..\tests\unittests\CoreGraphics.Drawing\data\*" />
+      <ReferenceImageFile Include="$(MSBuildThisFileDirectory)..\..\..\..\tests\unittests\CoreGraphics.Drawing\data\reference\*" />
     </ItemGroup>
-    <Copy SourceFiles="@(TestResourceFile)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" />
+    <Copy SourceFiles="@(TestResourceFile)" DestinationFolder="$(OutDir)\data" SkipUnchangedFiles="True" />
+    <Copy SourceFiles="@(ReferenceImageFile)" DestinationFolder="$(OutDir)\data\reference" SkipUnchangedFiles="True" />
   </Target>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

tests/UnitTests/CoreGraphics.drawing/DrawingTest.cpp

@@ -15,9 +15,9 @@
 //******************************************************************************
 
 #include "DrawingTest.h"
-
-#include <CoreFoundation/CoreFoundation.h>
-#include <ImageIO/ImageIO.h>
+#include "DrawingTestConfig.h"
+#include "ImageComparison.h"
+#include "ImageHelpers.h"
 
 #include <Starboard/SmartTypes.h>
 #include <memory>
@@ -70,47 +70,68 @@ CFStringRef testing::DrawTest::CreateOutputFilename() {
 void testing::DrawTest::TearDown() {
     CGContextRef context = GetDrawingContext();
 
+    // Generate image from context.
     woc::unique_cf<CGImageRef> image{ CGBitmapContextCreateImage(context) };
     ASSERT_NE(nullptr, image);
 
-    woc::unique_cf<CFMutableDataRef> imageData{ CFDataCreateMutable(nullptr, 1048576) };
-    woc::unique_cf<CGImageDestinationRef> imageDest{ CGImageDestinationCreateWithData(imageData.get(), CFSTR("public.png"), 1, nullptr) };
-    CGImageDestinationAddImage(imageDest.get(), image.get(), nullptr);
-    CGImageDestinationFinalize(imageDest.get());
-
+    // Generate output filename (generally, TestCase.TestName.png).
     woc::unique_cf<CFStringRef> originalFilename{ CreateOutputFilename() };
 
     woc::unique_cf<CFMutableStringRef> filename{ CFStringCreateMutableCopy(nullptr, 0, originalFilename.get()) };
 
     CFStringFindAndReplace(filename.get(), CFSTR("DISABLED_"), CFSTR(""), CFRange{ 0, CFStringGetLength(filename.get()) }, 0);
     CFStringFindAndReplace(filename.get(), CFSTR("/"), CFSTR("_"), CFRange{ 0, CFStringGetLength(filename.get()) }, 0);
 
-    // This is only populated if CFStringGetCStringPtr fails.
-    std::unique_ptr<char[]> owningFilenamePtr;
-
-    char* rawFilename = const_cast<char*>(CFStringGetCStringPtr(filename.get(), kCFStringEncodingUTF8));
-    size_t len = 0;
-
-    if (!rawFilename) {
-        CFRange filenameRange{ 0, CFStringGetLength(filename.get()) };
-        CFIndex requiredBufferLength = 0;
-        CFStringGetBytes(filename.get(), filenameRange, kCFStringEncodingUTF8, 0, FALSE, nullptr, 0, &requiredBufferLength);
-        owningFilenamePtr.reset(new char[requiredBufferLength]);
-        rawFilename = owningFilenamePtr.get();
-        CFStringGetBytes(filename.get(),
-                         filenameRange,
-                         kCFStringEncodingUTF8,
-                         0,
-                         FALSE,
-                         (UInt8*)rawFilename,
-                         requiredBufferLength,
-                         &requiredBufferLength);
-        len = requiredBufferLength;
-    } else {
-        len = strlen(rawFilename);
+    auto drawingConfig = DrawingTestConfig::Get();
+
+    // Write the context image to the output file.
+    woc::unique_cf<CFDataRef> actualImageData{ _CFDataCreatePNGFromCGImage(image.get()) };
+    ASSERT_NE(nullptr, actualImageData);
+
+    woc::unique_cf<CFStringRef> outputPath{
+        CFStringCreateWithFormat(nullptr, nullptr, CFSTR("%s/%@"), drawingConfig->GetOutputPath().c_str(), filename.get())
+    };
+    ASSERT_NE(nullptr, outputPath);
+    outputPath.reset(_CFStringCreateAbsolutePath(outputPath.get()));
+
+    ASSERT_TRUE(_WriteCFDataToFile(actualImageData.get(), outputPath.get()));
+
+    // For comparisons ...
+    if (drawingConfig->GetMode() == DrawingTestMode::Compare) {
+        // ... load the reference image (same name, different directory as the output image)
+        woc::unique_cf<CFStringRef> referenceFilename{
+            CFStringCreateWithFormat(nullptr, nullptr, CFSTR("%s/%@"), drawingConfig->GetComparisonPath().c_str(), filename.get())
+        };
+        referenceFilename.reset(_CFStringCreateAbsolutePath(referenceFilename.get()));
+
+        woc::unique_cf<CGImageRef> referenceImage{ _CGImageCreateFromPNGFile(referenceFilename.get()) };
+        ASSERT_NE(nullptr, referenceImage);
+
+        // And fire off a comparator.
+        PixelByPixelImageComparator comparator;
+        auto delta = comparator.CompareImages(referenceImage.get(), image.get());
+
+        if (delta.result != ImageComparisonResult::Same) {
+            if (delta.result == ImageComparisonResult::Incomparable) {
+                ADD_FAILURE() << "images are incomparable due to a mismatch in dimensions, presence, or byte length";
+            } else {
+                ADD_FAILURE() << "images differ nontrivially";
+            }
+
+            woc::unique_cf<CFStringRef> deltaFilename{
+                CFStringCreateWithFormat(nullptr, nullptr, CFSTR("%s/Greenline.%@"), drawingConfig->GetOutputPath().c_str(), filename.get())
+            };
+            deltaFilename.reset(_CFStringCreateAbsolutePath(deltaFilename.get()));
+
+            woc::unique_cf<CFDataRef> encodedDeltaImageData{ _CFDataCreatePNGFromCGImage(delta.deltaImage.get()) };
+
+            _WriteCFDataToFile(encodedDeltaImageData.get(), deltaFilename.get());
+
+            RecordProperty("expectedImage", CFStringGetCStringPtr(referenceFilename.get(), kCFStringEncodingUTF8));
+            RecordProperty("actualImage", CFStringGetCStringPtr(outputPath.get(), kCFStringEncodingUTF8));
+            RecordProperty("deltaImage", CFStringGetCStringPtr(deltaFilename.get(), kCFStringEncodingUTF8));
+        }
     }
-    woc::unique_cf<CFURLRef> url{ CFURLCreateFromFileSystemRepresentation(nullptr, (UInt8*)rawFilename, strlen(rawFilename), FALSE) };
-    ASSERT_TRUE(CFURLWriteDataAndPropertiesToResource(url.get(), imageData.get(), nullptr, nullptr));
 }
 
 void testing::DrawTest::SetUpContext() {

tests/UnitTests/CoreGraphics.drawing/EntryPoint.cpp

@@ -16,9 +16,83 @@
 
 #include <windows.h>
 #include <TestFramework.h>
+#include <memory>
+
+#include <CoreFoundation/CoreFoundation.h>
+
+#include "DrawingTestConfig.h"
+
+template <size_t N>
+static int strconstprefix(const char* left, const char (&right)[N]) {
+    // The null terminator would fail a prefix match; don't count it.
+    return strncmp(left, right, N-1);
+}
+
+static std::string __ModuleDirectory() {
+    static std::string modulePath = []() {
+        std::string modulePath(MAX_PATH, '\0');
+        GetModuleFileNameA(nullptr, &modulePath[0], modulePath.capacity());
+        auto pos = modulePath.rfind('\\');
+        if (pos == std::string::npos) {
+            pos = modulePath.rfind('/');
+        }
+
+        if (pos != std::string::npos) {
+            modulePath.erase(pos);
+        }
+
+        return modulePath;
+    }();
+    return modulePath;
+}
+
+class CommandLineDrawingTestConfigImpl : public DrawingTestConfigImpl {
+private:
+    DrawingTestMode _mode;
+    std::string _outputPath;
+    std::string _comparisonPath;
+
+public:
+    CommandLineDrawingTestConfigImpl(int argc, char** argv)
+        : _mode(DrawingTestMode::Compare), _outputPath(__ModuleDirectory()), _comparisonPath(__ModuleDirectory() + "/data/reference") {
+        for (int i = 1; i < argc; ++i) {
+            char* arg = argv[i];
+            if (!arg) {
+                break;
+            }
+
+            if (strcmp(arg, "--generate") == 0) {
+                _mode = DrawingTestMode::Generate;
+            } else if (strconstprefix(arg, "--out=") == 0) {
+                _outputPath = std::move(std::string(arg + 6));
+            } else if (strconstprefix(arg, "--compare=") == 0) {
+                _comparisonPath = std::move(std::string(arg + 10));
+            }
+        }
+    }
+
+    virtual DrawingTestMode GetMode() override {
+        return _mode;
+    }
+
+    virtual std::string GetOutputPath() override {
+        return _outputPath;
+    }
+
+    virtual std::string GetComparisonPath() override {
+        return _comparisonPath;
+    }
+};
+
+std::shared_ptr<DrawingTestConfigImpl> _configImpl;
+/* static */ DrawingTestConfigImpl* DrawingTestConfig::Get() {
+    return _configImpl.get();
+}
 
 #ifdef WIN32
+#include <COMIncludes.h>
 #include <wrl\wrappers\corewrappers.h>
+#include <COMIncludes_end.h>
 using namespace Microsoft::WRL::Wrappers;
 #endif
 
@@ -31,6 +105,9 @@ int main(int argc, char** argv) {
     }
 #endif
     testing::InitGoogleTest(&argc, argv);
+
+    _configImpl = std::move(std::make_shared<CommandLineDrawingTestConfigImpl>(argc, argv));
+
     auto result = RUN_ALL_TESTS();
     return result;
 }

tests/UnitTests/CoreGraphics.drawing/Makefile

@@ -2,85 +2,10 @@
 # Copyright (c) Microsoft. All rights reserved.
 #
 
-all: CoreGraphics.Drawing.UnitTests.exe
-.PHONY: all
+# CoreGraphics.Drawing.UnitTests.exe
+UT_NAME = CoreGraphics.Drawing
+UT_FILES := $(wildcard *.cpp)
+UT_RESOURCES := data
+UT_FRAMEWORKS = CoreGraphics ImageIO
 
-include $(CLEAR_VARS)
-
-CLANG := clang++
-OSX_VERSION_MIN := 10.10
-
-SRCDIR := .
-SRCFILES := $(wildcard $(SRCDIR)/*.m*) $(wildcard $(SRCDIR)/*.cpp)
-
-OUTDIR := $(SRCDIR)/../../../build/Tests/UnitTests/CoreGraphics.Drawing/OSX
-
-OUT_OBJ_DIR := $(OUTDIR)/Objects
-
-OUTTARGETS := $(addsuffix .o, $(addprefix $(OUT_OBJ_DIR)/, $(basename $(notdir $(SRCFILES)))))
-
-RESOURCE_DIR := $(SRCDIR)/data
-RESOURCE_FILES := $(addprefix data/, $(notdir $(wildcard $(RESOURCE_DIR)/*)))
-COPY_FILES := $(addprefix $(OUTDIR)/, $(RESOURCE_FILES))
-
-INC := $(SRCDIR)/../../frameworks/include $(SRCDIR)/../../frameworks/gtest $(SRCDIR)/../../frameworks/gtest/include 
-INC += $(SRCDIR)/../../frameworks/OSXShims/include $(SRCDIR)/../../../include/xplat
-INC_PARAMS := $(INC:%=-I %)
-
-CFLAGS := -std=c++14
-CFLAGS += $(INC_PARAMS)
-CFLAGS += -DTARGET_OS_MAC
-CFLAGS += -DWINOBJCRT_IMPEXP=
-CFLAGS += -DLOGGING_IMPEXP=
-CFLAGS += -mmacosx-version-min=$(OSX_VERSION_MIN)
-
-FRAMEWORKS := CoreFoundation CoreGraphics ImageIO Foundation
-LDFLAGS := -mmacosx-version-min=$(OSX_VERSION_MIN)
-LDFLAGS += $(foreach o,$(FRAMEWORKS),-framework $(o))
-
-SHIMS := $(OUT_OBJ_DIR)/windows.o $(OUT_OBJ_DIR)/IwMalloc.o $(OUT_OBJ_DIR)/LoggingNative.o
-
-$(OUT_OBJ_DIR)/IwMalloc.o : $(SRCDIR)/../../../Frameworks/WinObjCRT/MemoryManagement.cpp
-	mkdir -p $(OUTDIR)/Objects
-	$(CLANG) -c -o $@ $(CFLAGS) $<
-
-$(OUT_OBJ_DIR)/windows.o : $(SRCDIR)/../../frameworks/OSXShims/src/windows.cpp
-	mkdir -p $(OUTDIR)/Objects
-	$(CLANG) -c -o $@ $(CFLAGS) $<
-
-$(OUT_OBJ_DIR)/LoggingNative.o : $(SRCDIR)/../../frameworks/OSXShims/src/LoggingNative.mm
-	mkdir -p $(OUTDIR)/Objects
-	$(CLANG) -c -o $@ $(CFLAGS) $<
-
-$(OUT_OBJ_DIR)/%.o : $(SRCDIR)/%.m
-	mkdir -p $(OUTDIR)/Objects
-	$(CLANG) -c -o $@ $(CFLAGS) -x objective-c $^
-
-$(OUT_OBJ_DIR)/%.o : $(SRCDIR)/%.mm
-	mkdir -p $(OUTDIR)/Objects
-	$(CLANG) -c -o $@ $(CFLAGS) -x objective-c++ $^
-
-$(OUT_OBJ_DIR)/%.o : $(SRCDIR)/%.cpp
-	mkdir -p $(OUTDIR)/Objects
-	$(CLANG) -c -o $@ $(CFLAGS) -x c++ $^
-
-.PHONY: clean
-
-clean:
-	- $(RM) -r $(OUT_OBJ_DIR) $(OUTDIR)
-
-$(OUTDIR)/data/% :
-	mkdir -p $(OUTDIR)/data
-	cp -f $(RESOURCE_DIR)/$* $@
-
-$(OUTDIR)/% :
-	cp -f $(RF_RESOURCE_DIR)/$* $@
-
-$(OUTDIR)/CoreGraphics.Drawing.UnitTests.exe : $(SHIMS) $(OUTTARGETS) $(COPY_FILES)
-	$(CLANG) -o $@ -std=c++14 $(INC_PARAMS) $(SRCDIR)/../../frameworks/gtest/src/gtest-all.cc \
-	$(LDFLAGS) $(SRCDIR)/../../../build/Tests/UnitTests/CoreGraphics.Drawing/OSX/Objects/*.o
-	@echo "\nDone, please run from $(shell CDPATH="" cd "$(dir $@)"; pwd)/$(notdir $@)"
-
-.PHONY: CoreGraphics.Drawing.UnitTests.exe
-
-CoreGraphics.Drawing.UnitTests.exe : $(OUTDIR)/CoreGraphics.Drawing.UnitTests.exe
+include ../UT.common.mk

tests/frameworks/OSXShims/include/windows.h

@@ -25,6 +25,10 @@
 #define _MAX_PATH PATH_MAX
 #endif
 
+#ifndef MAX_PATH
+#define MAX_PATH PATH_MAX
+#endif
+
 #ifndef _MAX_FNAME
 #define _MAX_FNAME NAME_MAX
 #endif

tests/unittests/CoreGraphics.drawing/CGContextBlendModeTests.cpp

@@ -83,9 +83,9 @@ INSTANTIATE_TEST_CASE_P(DISABLED_CompositionModes,
                         ::testing::Combine(::testing::ValuesIn(alphas), ::testing::ValuesIn(compositionModes)));
 
 INSTANTIATE_TEST_CASE_P(DISABLED_PorterDuffModes,
-                                 CGContextBlendMode,
-                                 ::testing::Combine(::testing::ValuesIn(alphas), ::testing::ValuesIn(porterDuffBlendModes)));
+                        CGContextBlendMode,
+                        ::testing::Combine(::testing::ValuesIn(alphas), ::testing::ValuesIn(porterDuffBlendModes)));
 
 INSTANTIATE_TEST_CASE_P(DISABLED_OperatorBlendModes,
-                                 CGContextBlendMode,
-                                 ::testing::Combine(::testing::ValuesIn(alphas), ::testing::ValuesIn(blendOperators)));
+                        CGContextBlendMode,
+                        ::testing::Combine(::testing::ValuesIn(alphas), ::testing::ValuesIn(blendOperators)));

tests/unittests/CoreGraphics.drawing/DrawingTestConfig.h

@@ -0,0 +1,31 @@
+//******************************************************************************
+//
+// Copyright (c) Microsoft. All rights reserved.
+//
+// This code is licensed under the MIT License (MIT).
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+//
+//******************************************************************************
+
+#pragma once
+
+enum class DrawingTestMode : int { Generate, Compare };
+
+class DrawingTestConfigImpl {
+public:
+    virtual DrawingTestMode GetMode() = 0;
+    virtual std::string GetOutputPath() = 0;
+    virtual std::string GetComparisonPath() = 0;
+};
+
+class DrawingTestConfig {
+public:
+    static DrawingTestConfigImpl* Get();
+};