Automapping: Added per-input-layer properties for ignoring flip flags

This can greatly reduce the number of layers necessary to describe all
matching input tiles.

Closes #3803

NEWS.md

@@ -18,6 +18,7 @@
 * tmxrasterizer: Fixed --hide/show-layer to work on group layers (#3899)
 * tmxviewer: Added support for viewing JSON maps (#3866)
 * AutoMapping: Ignore empty outputs per-rule (#3523)
+* Automapping: Added per-input-layer properties for ignoring flip flags (#3803)
 * AutoMapping: Always apply output sets with empty index
 * Windows: Fixed the support for WebP images (updated to Qt 6.6.1, #3661)
 * Fixed possible crash after assigning to tiled.activeAsset

docs/manual/automapping.md

@@ -113,6 +113,7 @@ Everything after the first underscore is the **name**, which determines which la
 
 The **index** is optional, and is not related to the input indices. Instead, output indices are used to randomize the output: every time the rule finds a match, a random output index is chosen and only the output layers with that index will have their contents placed into the working map.
 
+{bdg-primary}`New in Tiled 1.10.3`
 For convenience, Tiled 1.10.3 introduced two changes to the behavior related to indexes. If an output index is completely empty for a given rule, it will never be chosen for that rule. This is useful when some rules have more random options than others. Also, when no index is specified, that part of the rule's output will always apply when the rule matches. This can be used to combine an unconditional part of a rule's output with a random part.
 
 #### Random Output Example
@@ -216,6 +217,18 @@ AutoEmpty (alias: StrictEmpty)
 
   Normally, empty tiles are simply ignored. When **AutoEmpty** is `true`, empty tiles within the input region match empty tiles in the target layer. This can only happen when you have multiple input/inputnot layers and some of the tiles that are part of the same rule are empty while others are not. Usually, using the [Empty]{.tile .empty} [special tile](#specialtiles) is the best way to specify an empty tile, but this property is useful when you have multiple input layers, some of which need to match many empty tiles. Note that the input region is defined by *all* input layers, regardless of index.
 
+IgnoreHorizontalFlip {bdg-primary}`New in Tiled 1.10.3`
+: This boolean layer property can be added to `input` and `inputnot` layers to also match horizontally flipped versions of the input tile.
+
+IgnoreVerticalFlip
+: This boolean layer property can be added to `input` and `inputnot` layers to also match vertically flipped versions of the input tile.
+
+IgnoreDiagonalFlip
+: This boolean layer property can be added to `input` and `inputnot` layers to also match anti-diagonally flipped versions of the input tile. This kind of flip is used for 90-degree rotation of tiles.
+
+IgnoreHexRotate120
+: This boolean layer property can be added to `input` and `inputnot` layers to also match 120-degree rotated tiles on hexagonal maps. However, note that Automapping currently does not really work for hexagonal maps since it does not take into account the staggered axis.
+
 (outputProbability)=
 Probability {bdg-primary}`New in Tiled 1.10`
 : This float layer property can be added to `output` layers to control the probability that a given output index will be chosen. The probabilities for each output index are relative to one another, and default to 1.0. For example, if you have outputA with probability 2 and outputB with probability 0.5, A will be chosen four times as often as B. If multiple output layers with the same index have their **Probability** set, the last (top-most) layer's probability will be used.

src/libtiled/tilelayer.h

@@ -73,7 +73,14 @@ class TILEDSHARED_EXPORT Cell
     Q_PROPERTY(bool rotatedHexagonal120 READ rotatedHexagonal120 WRITE setRotatedHexagonal120)
 
 public:
-    static Cell empty;
+    enum Flags {
+        FlippedHorizontally     = 0x01,
+        FlippedVertically       = 0x02,
+        FlippedAntiDiagonally   = 0x04,
+        RotatedHexagonal120     = 0x08,
+        Checked                 = 0x10,
+        VisualFlags             = FlippedHorizontally | FlippedVertically | FlippedAntiDiagonally | RotatedHexagonal120
+    };
 
     Cell() = default;
 
@@ -93,7 +100,7 @@ class TILEDSHARED_EXPORT Cell
     {
         return _tileset == other._tileset
                 && _tileId == other._tileId
-                && (_flags & VisualFlags) == (other._flags & VisualFlags);
+                && flags() == other.flags();
     }
 
     bool operator != (const Cell &other) const
@@ -125,16 +132,9 @@ class TILEDSHARED_EXPORT Cell
     void setTile(Tile *tile);
     bool refersTile(const Tile *tile) const;
 
-private:
-    enum Flags {
-        FlippedHorizontally     = 0x01,
-        FlippedVertically       = 0x02,
-        FlippedAntiDiagonally   = 0x04,
-        RotatedHexagonal120     = 0x08,
-        Checked                 = 0x10,
-        VisualFlags             = FlippedHorizontally | FlippedVertically | FlippedAntiDiagonally | RotatedHexagonal120
-    };
+    static Cell empty;
 
+private:
     Tileset *_tileset = nullptr;
     int _tileId = -1;
     int _flags = 0;

src/tiled/automapper.cpp

@@ -116,9 +116,9 @@ static MatchType matchType(const Tile *tile)
  */
 struct CompileContext
 {
-    QVector<Cell> anyOf;
-    QVector<Cell> noneOf;
-    QVector<Cell> inputCells;
+    QVector<MatchCell> anyOf;
+    QVector<MatchCell> noneOf;
+    QVector<MatchCell> inputCells;
 };
 
 struct ApplyContext
@@ -277,21 +277,34 @@ void AutoMapper::setupRuleMapProperties()
 
 void AutoMapper::setupInputLayerProperties(InputLayer &inputLayer)
 {
-    inputLayer.strictEmpty = false;
-
     QMapIterator<QString, QVariant> it(inputLayer.tileLayer->properties());
     while (it.hasNext()) {
         it.next();
 
         const QString &name = it.key();
         const QVariant &value = it.value();
 
-        if (name.compare(QLatin1String("strictempty"), Qt::CaseInsensitive) == 0 ||
-                name.compare(QLatin1String("autoempty"), Qt::CaseInsensitive) == 0) {
-            if (value.canConvert(QMetaType::Bool)) {
-                inputLayer.strictEmpty = value.toBool();
-                continue;
-            }
+        if (checkOption(name, value, QLatin1String("StrictEmpty"), inputLayer.strictEmpty))
+            continue;
+        if (checkOption(name, value, QLatin1String("AutoEmpty"), inputLayer.strictEmpty))
+            continue;
+
+        bool ignoreFlip;
+        if (checkOption(name, value, QLatin1String("IgnoreHorizontalFlip"), ignoreFlip) && ignoreFlip) {
+            inputLayer.flagsMask &= ~Cell::FlippedHorizontally;
+            continue;
+        }
+        if (checkOption(name, value, QLatin1String("IgnoreVerticalFlip"), ignoreFlip) && ignoreFlip) {
+            inputLayer.flagsMask &= ~Cell::FlippedVertically;
+            continue;
+        }
+        if (checkOption(name, value, QLatin1String("IgnoreDiagonalFlip"), ignoreFlip) && ignoreFlip) {
+            inputLayer.flagsMask &= ~Cell::FlippedAntiDiagonally;
+            continue;
+        }
+        if (checkOption(name, value, QLatin1String("IgnoreHexRotate120"), ignoreFlip) && ignoreFlip) {
+            inputLayer.flagsMask &= ~Cell::RotatedHexagonal120;
+            continue;
         }
 
         addWarning(tr("Ignoring unknown property '%2' = '%3' on layer '%4' (rule map '%1')")
@@ -468,8 +481,7 @@ bool AutoMapper::setupRuleMapLayers()
 
             setup.mInputLayerNames.insert(layerName);
 
-            InputLayer inputLayer;
-            inputLayer.tileLayer = tileLayer;
+            InputLayer inputLayer { tileLayer };
             setupInputLayerProperties(inputLayer);
 
             auto &inputSet = find_or_emplace<InputSet>(setup.mInputSets, [&setName] (const InputSet &set) {
@@ -740,17 +752,17 @@ static void forEachPointInRegion(const QRegion &region, Callback callback)
  */
 static void collectCellsInRegion(const QVector<InputLayer> &list,
                                  const QRegion &r,
-                                 QVector<Cell> &cells)
+                                 QVector<MatchCell> &cells)
 {
     for (const InputLayer &inputLayer : list) {
         forEachPointInRegion(r, [&] (int x, int y) {
             const Cell &cell = inputLayer.tileLayer->cellAt(x, y);
             switch (matchType(cell.tile())) {
             case MatchType::Tile:
-                appendUnique(cells, cell);
+                appendUnique(cells, { cell, inputLayer.flagsMask });
                 break;
             case MatchType::Empty:
-                appendUnique(cells, Cell());
+                appendUnique(cells, MatchCell());
                 break;
             default:
                 break;
@@ -795,14 +807,16 @@ bool AutoMapper::compileRule(QVector<RuleInputSet> &inputSets,
  * Returns whether this combination can match at all. A match is not possible,
  * when \a anyOf is non-empty, but all cells in \a anyOf are also in \a noneOf.
  */
-static bool optimizeAnyNoneOf(QVector<Cell> &anyOf, QVector<Cell> &noneOf)
+static bool optimizeAnyNoneOf(QVector<MatchCell> &anyOf, QVector<MatchCell> &noneOf)
 {
-    auto compareCell = [] (const Cell &a, const Cell &b) {
+    auto compareCell = [] (const MatchCell &a, const MatchCell &b) {
         if (a.tileset() != b.tileset())
             return a.tileset() < b.tileset();
         if (a.tileId() != b.tileId())
             return a.tileId() < b.tileId();
-        return a.flags() < b.flags();
+        if (a.flags() != b.flags())
+            return a.flags() < b.flags();
+        return a.flagsMask < b.flagsMask;
     };
 
     // First sort and erase duplicates
@@ -851,9 +865,9 @@ bool AutoMapper::compileInputSet(RuleInputSet &index,
 {
     const QPoint topLeft = inputRegion.boundingRect().topLeft();
 
-    QVector<Cell> &anyOf = compileContext.anyOf;
-    QVector<Cell> &noneOf = compileContext.noneOf;
-    QVector<Cell> &inputCells = compileContext.inputCells;
+    QVector<MatchCell> &anyOf = compileContext.anyOf;
+    QVector<MatchCell> &noneOf = compileContext.noneOf;
+    QVector<MatchCell> &inputCells = compileContext.inputCells;
 
     for (const InputConditions &conditions : inputSet.layers) {
         inputCells.clear();
@@ -874,16 +888,16 @@ bool AutoMapper::compileInputSet(RuleInputSet &index,
                 switch (matchType(cell.tile())) {
                 case MatchType::Unknown:
                     if (inputLayer.strictEmpty)
-                        anyOf.append(cell);
+                        anyOf.append({ cell, inputLayer.flagsMask });
                     break;
                 case MatchType::Tile:
-                    anyOf.append(cell);
+                    anyOf.append({ cell, inputLayer.flagsMask });
                     break;
                 case MatchType::Empty:
-                    anyOf.append(Cell());
+                    anyOf.append(MatchCell());
                     break;
                 case MatchType::NonEmpty:
-                    noneOf.append(Cell());
+                    noneOf.append(MatchCell());
                     break;
                 case MatchType::Other:
                     // The "any other tile" case is implemented as "none of the
@@ -906,16 +920,16 @@ bool AutoMapper::compileInputSet(RuleInputSet &index,
                 switch (matchType(cell.tile())) {
                 case MatchType::Unknown:
                     if (inputLayer.strictEmpty)
-                        noneOf.append(cell);
+                        noneOf.append({ cell, inputLayer.flagsMask });
                     break;
                 case MatchType::Tile:
-                    noneOf.append(cell);
+                    noneOf.append({ cell, inputLayer.flagsMask });
                     break;
                 case MatchType::Empty:
-                    noneOf.append(Cell());
+                    noneOf.append(MatchCell());
                     break;
                 case MatchType::NonEmpty:
-                    anyOf.append(Cell());
+                    anyOf.append(MatchCell());
                     break;
                 case MatchType::Other:
                     // This is the "not any other tile" case, which is
@@ -941,7 +955,7 @@ bool AutoMapper::compileInputSet(RuleInputSet &index,
                     if (inputCells.isEmpty())
                         collectCellsInRegion(conditions.listYes, inputRegion, inputCells);
                     noneOf.append(inputCells);
-                    noneOf.append(Cell());
+                    noneOf.append(MatchCell());
                 }
             }
 
@@ -960,16 +974,16 @@ bool AutoMapper::compileInputSet(RuleInputSet &index,
                 const bool emptyAllowed = (anyOf.isEmpty() ||
                                            std::any_of(anyOf.cbegin(),
                                                        anyOf.cend(),
-                                                       [] (const Cell &cell) { return cell.isEmpty(); }))
+                                                       [] (const MatchCell &cell) { return cell.isEmpty(); }))
                         && std::none_of(noneOf.cbegin(),
                                         noneOf.cend(),
-                                        [] (const Cell &cell) { return cell.isEmpty(); });
+                                        [] (const MatchCell &cell) { return cell.isEmpty(); });
 
                 if (!emptyAllowed)
                     canMatch = false;
             }
 
-            if (anyOf.size() > 0 || noneOf.size() > 0) {
+            if (!anyOf.empty() || !noneOf.empty()) {
                 index.cells.append(anyOf);
                 index.cells.append(noneOf);
 
@@ -1134,6 +1148,14 @@ void AutoMapper::autoMap(const QRegion &where,
     }
 }
 
+static bool cellMatches(const MatchCell &matchCell, const Cell &cell)
+{
+    const auto flagsMask = matchCell.flagsMask;
+    return matchCell.tileset() == cell.tileset()
+            && matchCell.tileId() == cell.tileId()
+            && (matchCell.flags() & flagsMask) == (cell.flags() & flagsMask);
+}
+
 /**
  * Checks whether the given \a inputSet matches at the given \a offset.
  */
@@ -1152,8 +1174,8 @@ static bool matchInputIndex(const RuleInputSet &inputSet, QPoint offset, AutoMap
             bool anyMatch = !pos.anyCount;
 
             for (auto c = std::exchange(nextCell, nextCell + pos.anyCount); c < nextCell; ++c) {
-                const Cell &desired = inputSet.cells[c];
-                if (desired.isEmpty() ? cell.isEmpty() : desired == cell) {
+                const MatchCell &desired = inputSet.cells[c];
+                if (desired.isEmpty() ? cell.isEmpty() : cellMatches(desired, cell)) {
                     anyMatch = true;
                     break;
                 }
@@ -1164,8 +1186,8 @@ static bool matchInputIndex(const RuleInputSet &inputSet, QPoint offset, AutoMap
 
             // Match fails as soon as any of the "none" tiles is seen
             for (auto c = std::exchange(nextCell, nextCell + pos.noneCount); c < nextCell; ++c) {
-                const Cell &undesired = inputSet.cells[c];
-                if (undesired.isEmpty() ? cell.isEmpty() : undesired == cell)
+                const MatchCell &undesired = inputSet.cells[c];
+                if (undesired.isEmpty() ? cell.isEmpty() : cellMatches(undesired, cell))
                     return false;
             }
         }

src/tiled/automapper.h

@@ -36,7 +36,6 @@
 #include <QVector>
 
 #include <memory>
-#include <type_traits>
 #include <unordered_map>
 #include <vector>
 
@@ -53,13 +52,20 @@ class MapDocument;
 
 struct InputLayer
 {
+    explicit InputLayer(const TileLayer *tileLayer = nullptr)
+        : tileLayer(tileLayer)
+    {}
+
     const TileLayer *tileLayer;
-    bool strictEmpty;
+    bool strictEmpty = false;
+    int flagsMask = Cell::VisualFlags;
 };
 
 struct InputConditions
 {
-    InputConditions(const QString &layerName) : layerName(layerName) {}
+    explicit InputConditions(const QString &layerName)
+        : layerName(layerName)
+    {}
 
     QString layerName;
     QVector<InputLayer> listYes;        // "input"
@@ -175,6 +181,16 @@ struct RuleInputLayerPos
     int noneCount;                  // none of these cells
 };
 
+struct MatchCell : Cell
+{
+    int flagsMask = Cell::VisualFlags;
+
+    bool operator == (const MatchCell &other) const
+    {
+        return Cell::operator==(other) && flagsMask == other.flagsMask;
+    }
+};
+
 /**
  * An efficient structure for matching purposes. Each data structure has a
  * single container, which keeps things packed together in memory.
@@ -183,7 +199,7 @@ struct RuleInputSet
 {
     QVector<RuleInputLayer> layers;
     QVector<RuleInputLayerPos> positions;
-    QVector<Cell> cells;
+    QVector<MatchCell> cells;
 };
 
 struct RuleOutputTileLayer
@@ -305,7 +321,7 @@ class TILED_EDITOR_EXPORT AutoMapper : public QObject
         int autoMappingRadius = 0;
     };
 
-    using GetCell = std::add_pointer_t<const Cell &(int x, int y, const TileLayer &tileLayer)>;
+    using GetCell = const Cell &(*)(int x, int y, const TileLayer &tileLayer);
 
     /**
      * Constructs an AutoMapper.

src/tiled/propertybrowser.cpp

@@ -458,6 +458,10 @@ static void addAutomappingProperties(Properties &properties, const Object *objec
         if (layer->name().startsWith(QLatin1String("input"), Qt::CaseInsensitive)) {
             mergeProperties(properties, QVariantMap {
                 { QStringLiteral("AutoEmpty"), false },
+                { QStringLiteral("IgnoreHorizontalFlip"), false },
+                { QStringLiteral("IgnoreVerticalFlip"), false },
+                { QStringLiteral("IgnoreDiagonalFlip"), false },
+                // { QStringLiteral("IgnoreHexRotate120"), false },
             });
         } else if (layer->name().startsWith(QLatin1String("output"), Qt::CaseInsensitive)) {
             mergeProperties(properties, QVariantMap {

tests/automapping/ignore-flip/map-result.tmx

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<map version="1.10" tiledversion="1.10.2" orientation="orthogonal" renderorder="right-down" width="4" height="2" tilewidth="16" tileheight="16" infinite="0" nextlayerid="6" nextobjectid="1">
+ <tileset firstgid="1" source="../spr_test_tileset.tsx"/>
+ <layer id="1" name="set" width="4" height="2">
+  <data encoding="csv">
+12,11,10,9,
+8,7,6,5
+</data>
+ </layer>
+</map>