internal: improve tryGenerateInstance (#858)

## Summary

Replace the meta-type instantiation plus subsequent test for whether
the resulting type is still generic with directly testing for whether
the to-be-instantiated type contains unbound type variables.

Apart from being more efficient, this also allows for removing the now-
unused `prepareMetatypeForSigmatch` procedure, which is a step
towards removing the "meta-types allowed" mode from `semtypinst`.

## Details

Compared to `containsGenericType`, the new `containsUnresolvedTypeVar`
looks for unbound type variables instead of for generic types. This
slightly changes the meaning of `tryGenerateInstance` from "produces
non-meta, concrete type on success" to "produces concrete type on
success". However, since the single usage-site of `tryGenerateInstance`
(`procParamTypeRel`) already tests for meta types being returned, no
additional adjustment is required.

compiler/sem/semtypinst.nim

@@ -984,46 +984,72 @@ proc generateTypeInstance*(p: PContext, pt: TIdTable, info: TLineInfo,
     var position = 0
     recomputeFieldPositions(objType, objType.n, position)
 
-proc prepareMetatypeForSigmatch*(p: PContext, pt: TIdTable, info: TLineInfo,
-                                 t: PType): PType =
-  var typeMap = initLayeredTypeMap(pt)
-  var cl = initTypeVars(p, typeMap, info, nil)
-  cl.allowMetaTypes = true
-  pushInfoContext(p.config, info)
-  result = replaceTypeVarsT(cl, t)
-  popInfoContext(p.config)
-
 template generateTypeInstance*(p: PContext, pt: TIdTable, arg: PNode,
                                t: PType): untyped =
   generateTypeInstance(p, pt, arg.info, t)
 
+proc containsUnboundTypeVar(pt: TIdTable, t: PType): bool =
+  ## Given the type `t`, answers the question of whether `t` either is or uses
+  ## an unbound type variable somewhere. `pt` provides the bindings.
+  proc nodeContainsUnbound(pt: TIdTable, n: PNode): bool =
+    case n.kind
+    of nkSym:
+      result = isUnresolvedSym(n.sym) and idTableGet(pt, n.sym.typ) == nil
+    of nkWithoutSons - {nkSym}:
+      result = false
+    of nkWithSons:
+      for it in n.items:
+        if nodeContainsUnbound(pt, it):
+          return true
+
+  case t.kind
+  of tyStatic, tyGenericParam, tyTypeDesc, tyTypeClasses:
+    # a type variable; it's an unbound one if it's not in the lookup table
+    result = idTableGet(pt, t) == nil
+  of tyGenericBody:
+    # this can only be a composite type-class that wasn't lifted
+    result = true
+  of tyGenericInvocation:
+    # we only look at the arguments and ignore the body. If the body is a
+    # meta-type, that's fine; we don't guarantee that the resulting type is
+    # not a meta-type
+    result = false
+    for i in 1..<t.len:
+      if containsUnboundTypeVar(pt, t[i]):
+        result = true
+        break
+
+  of tyDistinct, tyArray, tySet, tySink, tyTuple, tyVar, tyLent, tyPtr, tyRef,
+     tySequence, tyOpenArray, tyVarargs, tyUncheckedArray,
+     tyProc, tyOrdinal, tyAlias, tyObject:
+    # can contain type variables
+    for it in t.sons.items:
+      # the element can be nil for proc types
+      if it != nil and containsUnboundTypeVar(pt, it):
+        result = true
+        break
+
+  of tyRange:
+    result = containsUnboundTypeVar(pt, t) or nodeContainsUnbound(pt, t.n)
+  of tyFromExpr:
+    result = nodeContainsUnbound(pt, t.n)
+  of tyGenericInst, tyInferred:
+    result = containsUnboundTypeVar(pt, t[^1])
+  of tyNone, tyEmpty, tyError, IntegralTypes, tyNil, tyUntyped, tyTyped,
+     tyPointer, tyString, tyCstring, tyForward, tyVoid:
+    # neither a type variables nor can it contain one
+    result = false
+
 proc tryGenerateInstance*(c: PContext, pt: TIdTable, info: TLineInfo, t: PType): PType =
-  ## Tries to resolve the generic type `t` to a concrete type. Returns `nil` on
-  ## failure, and the resolved type on success.
+  ## Tries to resolve the generic type `t` to a non-generic one, using the type
+  ## bindings provided by `pt`. Returns `nil` on failure, and the resolved type
+  ## on success. Do note that the returned type can still be a meta type.
   ##
   ## XXX: this procedure is only a workaround. ``replaceTypeVarsT`` should
   ##      properly support the case where it's not certain whether all
   ##      referenced type parameters are resolved already
   assert containsGenericType(t)
-  result = prepareMetatypeForSigmatch(c, pt, info, t)
-
-  proc containsGenericType2(t: PType): bool =
-    if t.kind == tyGenericInst:
-      # check the parameters:
-      for i in 1..<t.len-1:
-        if t[i].isMetaType:
-          return true
-
-    result = containsGenericType(t)
-
-  # ``handleGenericInvocation`` doesn't properly propagate the ``tfHasMeta``
-  # flag, so we don't rely on it here. We also need to use a special-purpose
-  # ``containsGenericType`` -- otherwise generic phantom types wouldn't be
-  # detected as being generic
-  if containsGenericType2(result):
+  if containsUnboundTypeVar(pt, t):
     result = nil
   else:
-    # ``prepareMetatypeForSigmatch`` doesn't produce proper instances, but
-    # since we now know that all referenced type parameters can be resolved,
-    # we can produce a proper one
     result = generateTypeInstance(c, pt, info, t)