From 25f87a30faadee9d55beaa44b2c8e7cf73c16b11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fallas=20Avenda=C3=B1o?= Date: Wed, 22 Nov 2023 10:12:00 -0600 Subject: [PATCH] Add decorations processing (#99) * First commit for decorations * Add support for `inlining` --- cli/cli.js | 8 +- cli/morphir-elm-gen.js | 1 + cli/src/Morphir/Elm/Target.elm | 2 +- src/Morphir/Scala/AST.elm | 2 +- src/Morphir/Scala/PrettyPrinter.elm | 18 ++++ src/Morphir/Snowpark/AccessElementMapping.elm | 14 ++- src/Morphir/Snowpark/Backend.elm | 91 +++++++++++++---- src/Morphir/Snowpark/Customization.elm | 99 +++++++++++++++++++ .../FunctionMappingsForPlainScala.elm | 15 ++- .../MapExpressionsToDataFrameOperations.elm | 14 +-- src/Morphir/Snowpark/MapFunctionsMapping.elm | 35 ++++++- src/Morphir/Snowpark/MappingContext.elm | 44 ++++++++- .../Snowpark/RecordWrapperGenerator.elm | 25 ++++- src/Morphir/Snowpark/TypeRefMapping.elm | 14 ++- .../Snowpark/UserDefinedFunctionMapping.elm | 2 +- tests/Morphir/Snowpark/AccessElementTests.elm | 4 +- .../Snowpark/FunctionGenerationTests.elm | 5 +- .../Morphir/Snowpark/MappingContextTests.elm | 4 +- tests/Morphir/Snowpark/PatternMatchTests.elm | 4 +- .../Snowpark/RecordWrapperGenerationTests.elm | 6 +- 20 files changed, 349 insertions(+), 58 deletions(-) create mode 100644 src/Morphir/Snowpark/Customization.elm diff --git a/cli/cli.js b/cli/cli.js index 31703c7dd..703b91233 100644 --- a/cli/cli.js +++ b/cli/cli.js @@ -84,8 +84,14 @@ async function gen(input, outputPath, options) { opts.limitToModules = options.modulesToInclude ? options.modulesToInclude.split(",") : null opts.includeCodecs = options.includeCodecs ? true : false opts.filename = options.filename == '' ? '' : options.filename - const fileMap = await generate(opts, JSON.parse(morphirIrJson.toString())) + if (options.decorations) { + if (await fileExist(path.resolve(options.decorations))) { + options.decorationsObj = JSON.parse(await readFile(path.resolve(options.decorations))) + } + } + + const fileMap = await generate(opts, JSON.parse(morphirIrJson.toString())) const writePromises = fileMap.map(async ([ [dirPath, fileName], content diff --git a/cli/morphir-elm-gen.js b/cli/morphir-elm-gen.js index 45433c374..946307456 100644 --- a/cli/morphir-elm-gen.js +++ b/cli/morphir-elm-gen.js @@ -23,6 +23,7 @@ program .option('-s, --include-codecs', 'Generate JSON codecs', false) .option('-f, --filename ', 'Filename of the generated JSON Schema.', '') .option('-ls, --include ', 'Limit what will be included.', '') + .option('-dec, --decorations ', 'JSON file with decorations') .parse(process.argv) cli.gen(program.opts().input, path.resolve(program.opts().output), program.opts()) diff --git a/cli/src/Morphir/Elm/Target.elm b/cli/src/Morphir/Elm/Target.elm index a1721c3b4..19ba2c92d 100644 --- a/cli/src/Morphir/Elm/Target.elm +++ b/cli/src/Morphir/Elm/Target.elm @@ -57,7 +57,7 @@ decodeOptions gen = Decode.map SparkOptions (Decode.succeed Morphir.Scala.Spark.Backend.Options) Ok "Snowpark" -> - Decode.map SnowparkOptions (Decode.succeed Morphir.Snowpark.Backend.Options) + Decode.map SnowparkOptions Morphir.Snowpark.Backend.decodeOptions Ok "JsonSchema" -> Decode.map JsonSchemaOptions Morphir.JsonSchema.Backend.Codec.decodeOptions diff --git a/src/Morphir/Scala/AST.elm b/src/Morphir/Scala/AST.elm index ca00d5d1b..0dc10409c 100644 --- a/src/Morphir/Scala/AST.elm +++ b/src/Morphir/Scala/AST.elm @@ -17,7 +17,7 @@ module Morphir.Scala.AST exposing ( Name, Path, Documented, Annotated, withAnnotation, withoutAnnotation, CompilationUnit, PackageDecl - , ImportDecl, ImportName, Mod(..), TypeDecl(..), ArgDecl, ArgValue(..), MemberDecl(..) + , ImportDecl, ImportName(..), Mod(..), TypeDecl(..), ArgDecl, ArgValue(..), MemberDecl(..) , Type(..), Value(..), Pattern(..), Lit(..), Generator(..) , nameOfTypeDecl ) diff --git a/src/Morphir/Scala/PrettyPrinter.elm b/src/Morphir/Scala/PrettyPrinter.elm index 1c50fda75..4a00b5363 100644 --- a/src/Morphir/Scala/PrettyPrinter.elm +++ b/src/Morphir/Scala/PrettyPrinter.elm @@ -72,11 +72,29 @@ mapCompilationUnit opt cu = concat [ concat [ "package ", dotSep (prefixKeywords cu.packageDecl), newLine ] , newLine + , mapImports cu.imports , cu.typeDecls |> List.map (mapDocumented (mapAnnotated (mapTypeDecl opt))) |> String.join (newLine ++ newLine) ] +mapImports : List ImportDecl -> Doc +mapImports imports = + case imports of + [] -> + "" + importsList -> + (importsList |> List.map mapImport) ++ [ newLine ] + |> concat + + +mapImport : ImportDecl -> Doc +mapImport importDecl = + concat + [ "import " + , String.join "." (importDecl.packagePrefix) + , newLine + ] mapTypeDecl : Options -> TypeDecl -> Doc mapTypeDecl opt typeDecl = diff --git a/src/Morphir/Snowpark/AccessElementMapping.elm b/src/Morphir/Snowpark/AccessElementMapping.elm index 865dbc75f..aaad7db46 100644 --- a/src/Morphir/Snowpark/AccessElementMapping.elm +++ b/src/Morphir/Snowpark/AccessElementMapping.elm @@ -17,7 +17,6 @@ import Morphir.IR.Literal exposing (Literal(..)) import Morphir.Snowpark.MappingContext exposing (ValueMappingContext) import Morphir.Snowpark.MappingContext exposing (isUnionTypeWithoutParams , getReplacementForIdentifier) -import Html.Attributes exposing (name) import Morphir.IR.FQName as FQName import Morphir.IR.Value exposing (Value(..)) import Morphir.Snowpark.ReferenceUtils exposing (isValueReferenceToSimpleTypesRecord @@ -30,6 +29,7 @@ import Morphir.Snowpark.MappingContext exposing (isUnionTypeWithParams) import Morphir.IR.Value as Value import Morphir.Snowpark.Constants exposing (MapValueType) import Morphir.IR.Type as Type +import Morphir.Snowpark.Utils exposing (tryAlternatives) checkForDataFrameVariableReference : Value ta (IrType.Type ()) -> ValueMappingContext -> Maybe String @@ -100,9 +100,15 @@ mapConstructorAccess tpe name ctx = _ -> Scala.Literal (Scala.StringLit "Constructor access not converted") -mapReferenceAccess : (IrType.Type ()) -> FQName.FQName -> ValueMappingContext -> Scala.Value -mapReferenceAccess tpe name ctx = - if MappingContext.isDataFrameFriendlyType tpe ctx.typesContextInfo || + +mapReferenceAccess : (IrType.Type ()) -> FQName.FQName -> MapValueType () -> ValueMappingContext -> Scala.Value +mapReferenceAccess tpe name mapValue ctx = + if Dict.member name ctx.globalValuesToInline then + ctx.globalValuesToInline + |> Dict.get name + |> Maybe.map (\definition -> mapValue definition.body ctx) + |> Maybe.withDefault Scala.Wildcard + else if MappingContext.isDataFrameFriendlyType tpe ctx.typesContextInfo || MappingContext.isListOfDataFrameFriendlyType tpe ctx.typesContextInfo then let nsName = scalaPathToModule name diff --git a/src/Morphir/Snowpark/Backend.elm b/src/Morphir/Snowpark/Backend.elm index c0ad109d4..ccf930feb 100644 --- a/src/Morphir/Snowpark/Backend.elm +++ b/src/Morphir/Snowpark/Backend.elm @@ -1,4 +1,4 @@ -module Morphir.Snowpark.Backend exposing (mapDistribution, Options, mapFunctionDefinition) +module Morphir.Snowpark.Backend exposing (mapDistribution, Options, mapFunctionDefinition, decodeOptions) import Dict import List @@ -32,26 +32,47 @@ import Morphir.Snowpark.MappingContext as MappingContext exposing ( import Morphir.Snowpark.MappingContext exposing (FunctionClassification(..)) import Morphir.Snowpark.FunctionMappingsForPlainScala as FunctionMappingsForPlainScala import Morphir.Snowpark.MapExpressionsToDataFrameOperations exposing (mapValue) +import Json.Decode as Decode exposing (Decoder) +import Morphir.IR.Decoration.Codec exposing (decodeNodeIDByValuePairs) +import Morphir.SDK.Dict as SDKDict +import Morphir.IR.NodeId exposing (NodeID) +import Morphir.Snowpark.Customization exposing ( loadCustomizationOptions + , CustomizationOptions + , tryToApplyPostConversionCustomization + , emptyCustomizationOptions ) +import Morphir.Snowpark.Constants exposing (typeRefForSnowparkType) type alias Options = - {} + { decorations : Maybe (SDKDict.Dict NodeID Decode.Value) } + +decodeOptions : Decoder Options +decodeOptions = + Decode.map Options + (Decode.maybe + (Decode.field "decorationsObj" decodeNodeIDByValuePairs)) mapDistribution : Options -> Distribution -> FileMap -mapDistribution _ distro = +mapDistribution opts distro = + let + loadedOpts : CustomizationOptions + loadedOpts = opts.decorations + |> Maybe.map loadCustomizationOptions + |> Maybe.withDefault emptyCustomizationOptions + in case distro of Distribution.Library packageName _ packageDef -> - mapPackageDefinition distro packageName packageDef + mapPackageDefinition distro packageName packageDef loadedOpts -mapPackageDefinition : Distribution -> Package.PackageName -> Package.Definition () (Type ()) -> FileMap -mapPackageDefinition _ packagePath packageDef = +mapPackageDefinition : Distribution -> Package.PackageName -> Package.Definition () (Type ()) -> CustomizationOptions -> FileMap +mapPackageDefinition _ packagePath packageDef customizationOptions = let - contextInfo = MappingContext.processDistributionModules packagePath packageDef + contextInfo = MappingContext.processDistributionModules packagePath packageDef customizationOptions generatedScala = packageDef.modules |> Dict.toList |> List.concatMap (\( modulePath, moduleImpl ) -> - mapModuleDefinition packagePath modulePath moduleImpl contextInfo + mapModuleDefinition packagePath modulePath moduleImpl contextInfo customizationOptions ) in generatedScala @@ -67,8 +88,8 @@ mapPackageDefinition _ packagePath packageDef = |> Dict.fromList -mapModuleDefinition : Package.PackageName -> Path -> AccessControlled (Module.Definition () (Type ())) -> GlobalDefinitionInformation () -> List Scala.CompilationUnit -mapModuleDefinition currentPackagePath currentModulePath accessControlledModuleDef ctxInfo = +mapModuleDefinition : Package.PackageName -> Path -> AccessControlled (Module.Definition () (Type ())) -> GlobalDefinitionInformation () -> CustomizationOptions -> List Scala.CompilationUnit +mapModuleDefinition currentPackagePath currentModulePath accessControlledModuleDef ctxInfo customizationOptions = let ( scalaPackagePath, moduleName ) = case currentModulePath |> List.reverse of @@ -88,22 +109,24 @@ mapModuleDefinition currentPackagePath currentModulePath accessControlledModuleD |> RecordWrapperGenerator.generateRecordWrappers currentPackagePath currentModulePath ctxInfo |> List.map (\doc -> { annotations = doc.value.annotations, value = Scala.MemberTypeDecl (doc.value.value) } ) - functionMembers : List (Scala.Annotated Scala.MemberDecl) - functionMembers = + (functionMembers, generatedImports) = accessControlledModuleDef.value.values |> Dict.toList - |> List.concatMap + |> List.map (\( valueName, accessControlledValueDef ) -> - [ mapFunctionDefinition valueName accessControlledValueDef currentPackagePath currentModulePath ctxInfo - ] + (processFunctionMember valueName accessControlledValueDef currentPackagePath currentModulePath ctxInfo customizationOptions) ) - |> List.map Scala.withoutAnnotation + |> List.unzip + |> \(members, imports) -> + ((members |> List.concat |> List.map Scala.withoutAnnotation) + , imports |> List.concat) + moduleUnit : Scala.CompilationUnit moduleUnit = { dirPath = scalaPackagePath , fileName = (moduleName |> Name.toTitleCase) ++ ".scala" , packageDecl = scalaPackagePath - , imports = [] + , imports = generatedImports , typeDecls = [( Scala.Documented (Just (String.join "" [ "Generated based on ", currentModulePath |> Path.toString Name.toTitleCase "." ])) (Scala.Annotated [] (Scala.Object @@ -130,8 +153,18 @@ mapModuleDefinition currentPackagePath currentModulePath accessControlledModuleD [ moduleUnit ] +processFunctionMember : Name.Name -> AccessControlled (Documented (Value.Definition () (Type ()))) -> Package.PackageName -> Path -> GlobalDefinitionInformation () -> CustomizationOptions -> (List Scala.MemberDecl , List Scala.ImportDecl) +processFunctionMember valueName accessControlledValueDef currentPackagePath currentModulePath ctxInfo customizationOptions = + let + fullFunctionName = FQName.fQName currentPackagePath currentModulePath valueName + mappedFunction = mapFunctionDefinition valueName accessControlledValueDef currentPackagePath currentModulePath ctxInfo + in + Maybe.withDefault + ([ mappedFunction ], [] ) + (tryToApplyPostConversionCustomization fullFunctionName mappedFunction customizationOptions) + mapFunctionDefinition : Name.Name -> AccessControlled (Documented (Value.Definition () (Type ()))) -> Path -> Path -> GlobalDefinitionInformation () -> Scala.MemberDecl -mapFunctionDefinition functionName body currentPackagePath modulePath (typeContextInfo, functionsInfo) = +mapFunctionDefinition functionName body currentPackagePath modulePath (typeContextInfo, functionsInfo, inlineInfo) = let fullFunctionName = FQName.fQName currentPackagePath modulePath functionName functionClassification = getFunctionClassification fullFunctionName functionsInfo @@ -141,7 +174,8 @@ mapFunctionDefinition functionName body currentPackagePath modulePath (typeConte , parameters = parameterNames , functionClassificationInfo = functionsInfo , currentFunctionClassification = functionClassification - , packagePath = currentPackagePath} + , packagePath = currentPackagePath + , globalValuesToInline = inlineInfo } localDeclarations = body.value.value.inputTypes |> List.filterMap (checkForDataFrameColumndsDeclaration typeContextInfo) @@ -152,7 +186,7 @@ mapFunctionDefinition functionName body currentPackagePath modulePath (typeConte { modifiers = [] , name = mapValueName functionName , typeArgs = [] - , args = parameters + , args = addImplicitSession parameters , returnType = Just returnTypeToGenerate , body = @@ -161,6 +195,21 @@ mapFunctionDefinition functionName body currentPackagePath modulePath (typeConte (declarations, Just bodyToUse) -> Just (Scala.Block declarations bodyToUse) (_, _) -> Nothing } +addImplicitSession : List (List Scala.ArgDecl) -> List (List Scala.ArgDecl) +addImplicitSession args = + case args of + [] -> + args + _ -> + let + implicitArg = + { modifiers = [ Scala.Implicit ] + , tpe = typeRefForSnowparkType "Session" + , name = "sfSession" + , defaultValue = Nothing + } + in + args ++ [ [ implicitArg ] ] includeDataFrameInfo : List (Scala.MemberDecl, (String, FQName.FQName)) -> ValueMappingContext -> ValueMappingContext includeDataFrameInfo declInfos ctx = @@ -225,7 +274,7 @@ processParameters inputTypes currentFunctionClassification ctx = inputTypes |> List.map (generateArgumentDeclarationForFunction ctx currentFunctionClassification) -mapFunctionBody : Value.Definition ta (Type ()) -> ValueMappingContext -> Maybe Scala.Value +mapFunctionBody : Value.Definition () (Type ()) -> ValueMappingContext -> Maybe Scala.Value mapFunctionBody value ctx = let functionToMap = diff --git a/src/Morphir/Snowpark/Customization.elm b/src/Morphir/Snowpark/Customization.elm new file mode 100644 index 000000000..2fa97e359 --- /dev/null +++ b/src/Morphir/Snowpark/Customization.elm @@ -0,0 +1,99 @@ +module Morphir.Snowpark.Customization exposing (CustomizationOptions, loadCustomizationOptions, emptyCustomizationOptions, generateCacheCode, tryToApplyPostConversionCustomization) + +import Morphir.IR.FQName exposing (FQName) +import Set exposing (Set) +import Morphir.Scala.AST as Scala exposing (ImportName(..)) +import Json.Decode as Decode exposing (Decoder) +import Morphir.IR.Decoration.Codec exposing (decodeNodeIDByValuePairs) +import Morphir.SDK.Dict as SDKDict +import Morphir.IR.NodeId exposing (NodeID) +import Morphir.IR.NodeId exposing (NodeID(..)) +import Json.Decode exposing (decodeString) + + +type alias CustomizationOptions = + { functionsToInline: Set FQName + , functionsToCache: Set FQName + } + +emptyCustomizationOptions : CustomizationOptions +emptyCustomizationOptions = CustomizationOptions Set.empty Set.empty + +loadCustomizationOptions : SDKDict.Dict NodeID Decode.Value -> CustomizationOptions +loadCustomizationOptions optionsDict = + let + decodeFirstElementAsString = + Decode.decodeValue (Decode.index 0 Decode.string) + optsList = + optionsDict + |> SDKDict.toList + functions = + optsList + |> List.filterMap (\(id, value) -> + case id of + ValueID fullName _ -> Just (fullName, value) + _ -> Nothing) + (functionsToCache, valuesToInline) = + functions + |> List.foldr + (\(fullName, valueDec) ((toCache, toInline) as current) -> + case decodeFirstElementAsString valueDec of + Ok "cacheResult" -> + (Set.insert fullName toCache, toInline) + Ok "inlineElement" -> + (toCache, Set.insert fullName toInline) + _ -> + current) + (Set.empty, Set.empty) + in + { functionsToInline = valuesToInline + , functionsToCache = functionsToCache + } + +generateCacheCode : String -> Scala.MemberDecl -> Maybe Scala.Type -> Scala.MemberDecl +generateCacheCode cacheName decl returnType = + case decl of + Scala.FunctionDecl funcInfo -> + let + funcArgs = + funcInfo.args |> List.concatMap (\t -> t |> List.map (\t2 -> Scala.Variable t2.name)) + body = + Maybe.withDefault (Scala.Variable "_") funcInfo.body + expressionToCache = + case returnType of + Just (Scala.TypeRef _ "DataFrame") -> + Scala.Select body "cacheResult" + _ -> + body + getOrElseUpdateCall = + Scala.Apply (Scala.Ref [cacheName] "getOrElseUpdate") + [ Scala.ArgValue Nothing (Scala.Tuple funcArgs) + , Scala.ArgValue Nothing expressionToCache + ] + in + Scala.FunctionDecl { funcInfo | body = Just getOrElseUpdateCall } + _ -> + decl + +tryToApplyPostConversionCustomization : FQName -> Scala.MemberDecl -> CustomizationOptions -> Maybe (List Scala.MemberDecl , List Scala.ImportDecl) +tryToApplyPostConversionCustomization fullFunctionName mappedFunction customizationOptions = + case (Set.member fullFunctionName customizationOptions.functionsToCache , mappedFunction) of + (True, Scala.FunctionDecl funcInfo) -> + Just ( + [ generateCacheCode (funcInfo.name ++ "Cache") mappedFunction funcInfo.returnType, + Scala.ValueDecl + { modifiers = [] + , pattern = Scala.NamedMatch (funcInfo.name ++ "Cache") + , valueType = Just (Scala.TypeApply + (Scala.TypeRef [ "scala", "collection", "concurrent" ] "Map") + [ Scala.TupleType (funcInfo.args |> List.concatMap (\t -> t |> List.map (\t2 -> t2.tpe) )) , Maybe.withDefault (Scala.TypeVar "_") funcInfo.returnType ] ) + , value = + Scala.Select (Scala.New [ "java", "util", "concurrent" ] "ConcurrentHashMap" []) "asScala" + } + ], + [ Scala.ImportDecl False [ "scala", "collection", "JavaConverters", "_"] [] ] ) + _ -> + if Set.member fullFunctionName customizationOptions.functionsToInline then + Just ([], []) + else + Nothing \ No newline at end of file diff --git a/src/Morphir/Snowpark/FunctionMappingsForPlainScala.elm b/src/Morphir/Snowpark/FunctionMappingsForPlainScala.elm index b16fcf95a..cc445abf8 100644 --- a/src/Morphir/Snowpark/FunctionMappingsForPlainScala.elm +++ b/src/Morphir/Snowpark/FunctionMappingsForPlainScala.elm @@ -19,8 +19,11 @@ import Morphir.Snowpark.ReferenceUtils exposing (mapLiteralToPlainLiteral) import Morphir.Snowpark.LetMapping exposing (mapLetDefinition) import Morphir.Snowpark.MapExpressionsToDataFrameOperations as MapDfOperations import Morphir.Snowpark.PatternMatchMapping exposing (PatternMatchValues) +import Morphir.Snowpark.AccessElementMapping exposing (mapReferenceAccess) +import Morphir.Snowpark.MappingContext exposing (isUnionTypeWithoutParams) +import Morphir.IR.FQName as FQName -mapValueForPlainScala : IrValueType a -> ValueMappingContext -> Scala.Value +mapValueForPlainScala : IrValueType () -> ValueMappingContext -> Scala.Value mapValueForPlainScala value ctx = case value of Apply _ _ _ -> @@ -29,6 +32,8 @@ mapValueForPlainScala value ctx = mapLiteralToPlainLiteral tpe literal LetDefinition _ name definition body -> mapLetDefinition name definition body mapValueForPlainScala ctx + Reference tpe name -> + mapReferenceAccess tpe name mapValueForPlainScala ctx PatternMatch tpe expr cases -> mapPatternMatch (tpe, expr, cases) mapValueForPlainScala ctx _ -> @@ -54,12 +59,17 @@ mapCaseOfCase mapValue ctx (sourcePattern, sourceExpr) = (Scala.UnapplyMatch [] "Some" [ (Scala.NamedMatch (Name.toCamelCase varName)) ], convertedExpr) (ConstructorPattern _ ([["morphir"],["s","d","k"]],[["maybe"]],["nothing"]) []) -> (Scala.UnapplyMatch [] "None" [], convertedExpr) + (ConstructorPattern (TypeIR.Reference _ fullTypeName _) fullName []) -> + if isUnionTypeWithoutParams fullTypeName ctx.typesContextInfo then + (Scala.LiteralMatch (Scala.StringLit (Name.toTitleCase (FQName.getLocalName fullName))), convertedExpr) + else + (Scala.NamedMatch "CONSTRUCTOR_PATTERN_NOT_CONVERTED", convertedExpr) _ -> (Scala.NamedMatch "PATTERN_NOT_CONVERTED", convertedExpr) -mapFunctionCall : ValueIR.Value ta (TypeIR.Type ()) -> Constants.MapValueType ta -> ValueMappingContext -> Scala.Value +mapFunctionCall : ValueIR.Value () (TypeIR.Type ()) -> Constants.MapValueType () -> ValueMappingContext -> Scala.Value mapFunctionCall value mapValue ctx = case value of ValueIR.Apply _ func arg -> @@ -67,7 +77,6 @@ mapFunctionCall value mapValue ctx = _ -> Scala.Literal (Scala.StringLit "invalid function call") - mergeMappingDictionaries : FunctionMappingTable ta -> FunctionMappingTable ta -> FunctionMappingTable ta mergeMappingDictionaries first second = Dict.union first second diff --git a/src/Morphir/Snowpark/MapExpressionsToDataFrameOperations.elm b/src/Morphir/Snowpark/MapExpressionsToDataFrameOperations.elm index 459c65a71..bb0e6b7dd 100644 --- a/src/Morphir/Snowpark/MapExpressionsToDataFrameOperations.elm +++ b/src/Morphir/Snowpark/MapExpressionsToDataFrameOperations.elm @@ -32,7 +32,7 @@ import Morphir.Snowpark.ReferenceUtils exposing (getListTypeParameter) import Morphir.Snowpark.MappingContext exposing (FunctionClassification(..)) import Morphir.Snowpark.LetMapping exposing (mapLetDefinition) -mapValue : Value ta (Type ()) -> ValueMappingContext -> Scala.Value +mapValue : Value () (Type ()) -> ValueMappingContext -> Scala.Value mapValue value ctx = case value of Literal tpe literal -> @@ -46,7 +46,7 @@ mapValue value ctx = List listType values -> mapListCreation listType values ctx Reference tpe name -> - mapReferenceAccess tpe name ctx + mapReferenceAccess tpe name mapValue ctx Apply _ _ _ -> MapFunctionsMapping.mapFunctionsMapping value mapValue ctx PatternMatch tpe expr cases -> @@ -65,7 +65,7 @@ mapValue value ctx = Scala.Literal (Scala.StringLit ("Unsupported element")) -mapRecordCreation : Type () -> Dict.Dict (Name.Name) (Value ta (Type ())) -> ValueMappingContext -> Scala.Value +mapRecordCreation : Type () -> Dict.Dict (Name.Name) (Value () (Type ())) -> ValueMappingContext -> Scala.Value mapRecordCreation tpe fields ctx = if isTypeRefToRecordWithComplexTypes tpe ctx.typesContextInfo then mapRecordCreationToCaseClassCreation tpe fields ctx @@ -86,7 +86,7 @@ mapRecordCreation tpe fields ctx = Scala.Literal (Scala.StringLit ("Record creation not converted2")) -mapRecordCreationToCaseClassCreation : Type () -> Dict.Dict (Name.Name) (Value ta (Type ())) -> ValueMappingContext -> Scala.Value +mapRecordCreationToCaseClassCreation : Type () -> Dict.Dict (Name.Name) (Value () (Type ())) -> ValueMappingContext -> Scala.Value mapRecordCreationToCaseClassCreation tpe fields ctx = case tpe of Type.Reference _ fullName [] -> @@ -108,7 +108,7 @@ mapRecordCreationToCaseClassCreation tpe fields ctx = _ -> Scala.Literal (Scala.StringLit ("Record creation not converted")) -mapListCreation : (Type ()) -> List (Value ta (Type ())) -> ValueMappingContext -> Scala.Value +mapListCreation : (Type ()) -> List (Value () (Type ())) -> ValueMappingContext -> Scala.Value mapListCreation tpe values ctx = let listOfRecordWithSimpleTypes = typeRefIsListOf tpe (\innerTpe -> isTypeRefToRecordWithSimpleTypes innerTpe ctx.typesContextInfo) @@ -120,13 +120,13 @@ mapListCreation tpe values ctx = else case (getListTypeParameter tpe |> Maybe.map (\t -> isTypeReferenceToSimpleTypesRecord t ctx.typesContextInfo) |> Maybe.withDefault Nothing, values) of (Just (path, name), []) -> - Scala.Apply (Scala.Select (Scala.Ref path (Name.toTitleCase name)) "createEmptyDataFrame") [ Scala.ArgValue Nothing (Scala.Literal (Scala.NullLit)) ] + Scala.Apply (Scala.Select (Scala.Ref path (Name.toTitleCase name)) "createEmptyDataFrame") [ Scala.ArgValue Nothing (Scala.Variable "sfSession") ] _ -> Scala.Apply (Scala.Variable "Seq") (values |> List.map (\v -> Scala.ArgValue Nothing (mapValue v ctx))) -mapIfThenElse : Value ta (Type ()) -> Value ta (Type ()) -> Value ta (Type ()) -> ValueMappingContext -> Scala.Value +mapIfThenElse : Value () (Type ()) -> Value () (Type ()) -> Value () (Type ()) -> ValueMappingContext -> Scala.Value mapIfThenElse condition thenExpr elseExpr ctx = let whenCall = diff --git a/src/Morphir/Snowpark/MapFunctionsMapping.elm b/src/Morphir/Snowpark/MapFunctionsMapping.elm index 0726e97a9..97d60642c 100644 --- a/src/Morphir/Snowpark/MapFunctionsMapping.elm +++ b/src/Morphir/Snowpark/MapFunctionsMapping.elm @@ -88,7 +88,7 @@ dataFrameMappings = ] |> Dict.fromList -mapFunctionsMapping : ValueIR.Value ta (TypeIR.Type ()) -> Constants.MapValueType ta -> ValueMappingContext -> Scala.Value +mapFunctionsMapping : ValueIR.Value () (TypeIR.Type ()) -> Constants.MapValueType () -> ValueMappingContext -> Scala.Value mapFunctionsMapping value mapValue ctx = case value of ValueIR.Apply _ function arg -> @@ -106,20 +106,45 @@ getFullNameIfReferencedElement value = _ -> Nothing -mapUncurriedFunctionCall : (IrValueType ta, List (IrValueType ta)) -> Constants.MapValueType ta -> FunctionMappingTable ta -> ValueMappingContext -> Scala.Value +mapUncurriedFunctionCall : (IrValueType (), List (IrValueType ())) -> Constants.MapValueType () -> FunctionMappingTable () -> ValueMappingContext -> Scala.Value mapUncurriedFunctionCall (func, args) mapValue mappings ctx = let + funcNameIfAvailable = + getFullNameIfReferencedElement func + inlineFunctionIfAvailable = + funcNameIfAvailable + |> Maybe.map (\fullName -> Dict.get fullName mappings) + |> Maybe.withDefault Nothing builtinMappingFunction = getFullNameIfReferencedElement func - |> Maybe.map (\fullName -> Dict.get fullName mappings) + |> Maybe.map (getInliningFunctionIfRequired ctx) |> Maybe.withDefault Nothing in - case builtinMappingFunction of - Just mappingFunc -> + case (inlineFunctionIfAvailable, builtinMappingFunction) of + (Just inliningFunction, _) -> + inliningFunction (func, args) mapValue ctx + (_, Just mappingFunc) -> mappingFunc (func, args) mapValue ctx _ -> tryToConvertUserFunctionCall (func, args) mapValue ctx +getInliningFunctionIfRequired : ValueMappingContext -> FQName.FQName -> Maybe (MappingFunctionType ()) +getInliningFunctionIfRequired ctx name = + Dict.get name ctx.globalValuesToInline + |> Maybe.map (\definition -> + \(_, args) mapValue innerCtx -> + let + convertedArgs = + List.map2 (\arg (paramName, _, _) -> (paramName, mapValue arg ctx)) args definition.inputTypes + + newCtx = + convertedArgs + |> List.foldr (\(paramName, value) currentCtx -> + addReplacementForIdentifier paramName value currentCtx) innerCtx + in + mapValue definition.body newCtx) + + mapListMemberFunction : (IrValueType ta, List (IrValueType ta)) -> Constants.MapValueType ta -> ValueMappingContext -> Scala.Value mapListMemberFunction ( _, args ) mapValue ctx = case args of diff --git a/src/Morphir/Snowpark/MappingContext.elm b/src/Morphir/Snowpark/MappingContext.elm index e077ceb99..2f9af5240 100644 --- a/src/Morphir/Snowpark/MappingContext.elm +++ b/src/Morphir/Snowpark/MappingContext.elm @@ -58,6 +58,8 @@ import Morphir.IR.Name exposing (Name) import Morphir.IR.Path as Path import Morphir.IR.Value as Value import Morphir.Snowpark.Utils exposing (tryAlternatives) +import Morphir.Snowpark.Customization exposing ( CustomizationOptions ) +import Morphir.IR.FQName as FQName type TypeDefinitionClassification a = RecordWithSimpleTypes (List (Name, Type a)) @@ -80,8 +82,11 @@ type alias FunctionClassificationInformation = type alias MappingContextInfo a = Dict FQName (TypeClassificationState a) +type alias InlineValuesCollection = + Dict FQName (Value.Definition () (Type ())) + type alias GlobalDefinitionInformation a = - (MappingContextInfo a, FunctionClassificationInformation) + (MappingContextInfo a, FunctionClassificationInformation, InlineValuesCollection) type alias ValueMappingContext = { parameters: List Name @@ -92,6 +97,7 @@ type alias ValueMappingContext = , dataFrameColumnsObjects: Dict FQName String , functionClassificationInfo: FunctionClassificationInformation , currentFunctionClassification : FunctionClassification + , globalValuesToInline : Dict FQName.FQName (Value.Definition () (Type.Type ())) } emptyValueMappingContext : ValueMappingContext @@ -103,6 +109,7 @@ emptyValueMappingContext = { parameters = [] , dataFrameColumnsObjects = Dict.empty , functionClassificationInfo = Dict.empty , currentFunctionClassification = Unknown + , globalValuesToInline = Dict.empty } isLocalVariableDefinition : Name -> ValueMappingContext -> Bool @@ -496,8 +503,8 @@ processSecondPassOnType name typeClassification ctx = _ -> ctx _ -> ctx -processDistributionModules : Package.PackageName -> Package.Definition () (Type ()) -> GlobalDefinitionInformation () -processDistributionModules packagePath package = +processDistributionModules : Package.PackageName -> Package.Definition () (Type ()) -> CustomizationOptions -> GlobalDefinitionInformation () +processDistributionModules packagePath package customizationOptions = let moduleList = package.modules @@ -524,4 +531,33 @@ processDistributionModules packagePath package = in Dict.insert fullFunctionName classfication current) Dict.empty - in (secondPass, functionsClassifed) + valuesToInline = + collectValuesToInline customizationOptions.functionsToInline package.modules + + in (secondPass, functionsClassifed, valuesToInline) + + +collectValuesToInline : Set FQName -> Dict ModuleName (AccessControlled (Module.Definition () (Type ()))) -> Dict FQName (Value.Definition () (Type ())) +collectValuesToInline namesToInline modules = + collectingValuesToInline (Set.toList namesToInline) modules Dict.empty + +collectingValuesToInline : List FQName -> Dict ModuleName (AccessControlled (Module.Definition () (Type ()))) -> Dict FQName (Value.Definition () (Type ())) -> Dict FQName (Value.Definition () (Type ())) +collectingValuesToInline namesToInline modules current = + case namesToInline of + (first::rest) -> + let + newDict = + case Dict.get (FQName.getModulePath first) modules of + Just mod -> + case Dict.get (FQName.getLocalName first) mod.value.values of + Just { value } -> + Dict.insert first value.value current + _ -> + current + _ -> + current + in + collectingValuesToInline rest modules newDict + + _ -> + current \ No newline at end of file diff --git a/src/Morphir/Snowpark/RecordWrapperGenerator.elm b/src/Morphir/Snowpark/RecordWrapperGenerator.elm index ad6c298b2..f5c276441 100644 --- a/src/Morphir/Snowpark/RecordWrapperGenerator.elm +++ b/src/Morphir/Snowpark/RecordWrapperGenerator.elm @@ -30,7 +30,7 @@ For union types without parameters we are going to generate an object definition |-} generateRecordWrappers : Package.PackageName -> ModuleName -> GlobalDefinitionInformation () -> Dict Name (AccessControlled (Documented (Type.Definition ()))) -> List (Scala.Documented (Scala.Annotated Scala.TypeDecl)) -generateRecordWrappers packageName moduleName (ctx, _) typesInModule = +generateRecordWrappers packageName moduleName (ctx, _, _) typesInModule = typesInModule |> Dict.toList |> List.concatMap (processTypeDeclaration packageName moduleName ctx) @@ -187,6 +187,20 @@ classForRecordWrapper name fields = ) )) +emptyDataFrameField : Scala.Annotated (Scala.MemberDecl) +emptyDataFrameField = + Scala.Annotated + [] + (Scala.ValueDecl + { modifiers = [] + , pattern = Scala.NamedMatch ("emptyDataFrameCache") + , valueType = Just (Scala.TypeApply + (Scala.TypeRef ["scala","collection","mutable"] "HashMap") + [ Scala.TypeVar "Boolean" ,typeRefForSnowparkType "DataFrame"] ) + , value = + Scala.New ["scala","collection","mutable"] "HashMap" [] + }) + emptyDataFrameMethod : Scala.Annotated (Scala.MemberDecl) emptyDataFrameMethod = let @@ -199,6 +213,11 @@ emptyDataFrameMethod = , name = "session" , defaultValue = Nothing } + getOrElseUpdateCall = + Scala.Apply (Scala.Ref ["emptyDataFrameCache"] "getOrElseUpdate") + [ Scala.ArgValue Nothing (Scala.Literal (Scala.BooleanLit True)) + , Scala.ArgValue Nothing (Scala.Apply createDataFrameMethod createDataFrameArgs) + ] in Scala.Annotated [] @@ -208,7 +227,7 @@ emptyDataFrameMethod = , typeArgs = [] , args = [ [ arg ] ] , returnType = Just <| typeRefForSnowparkType "DataFrame" - , body = Just <| Scala.Apply createDataFrameMethod createDataFrameArgs + , body = Just getOrElseUpdateCall -- <| Scala.Apply createDataFrameMethod createDataFrameArgs }) objectForRecordWrapper : Name -> (List (Field ())) -> MappingContextInfo () -> (Scala.Documented (Scala.Annotated Scala.TypeDecl)) @@ -226,7 +245,7 @@ objectForRecordWrapper name fields ctx = , name = nameToUse , members = - (members ++ [ schema, emptyDataFrameMethod ]) + (members ++ [ schema, emptyDataFrameMethod, emptyDataFrameField ]) , extends = [ Scala.TypeRef [] nameToUse ] , body = diff --git a/src/Morphir/Snowpark/TypeRefMapping.elm b/src/Morphir/Snowpark/TypeRefMapping.elm index cc2faae85..74d350cb9 100644 --- a/src/Morphir/Snowpark/TypeRefMapping.elm +++ b/src/Morphir/Snowpark/TypeRefMapping.elm @@ -135,9 +135,21 @@ mapToScalaTypes : Type () -> MappingContextInfo () -> Scala.Type mapToScalaTypes typeReference ctx = tryAlternatives [ (\_ -> checkDataFrameCase typeReference ctx) , (\_ -> checkForBasicTypeToScala typeReference ctx) - , (\_ -> checkComplexRecordCase typeReference ctx) ] + , (\_ -> checkComplexRecordCase typeReference ctx) + , (\_ -> checkUnionTypeForPlainScala typeReference ctx) ] |> Maybe.withDefault (Scala.TypeVar "TypeNotConverted") +checkUnionTypeForPlainScala : Type () -> MappingContextInfo () -> Maybe Scala.Type +checkUnionTypeForPlainScala tpe ctx = + case tpe of + Type.Reference _ fullTypeName _ -> + if isUnionTypeWithoutParams fullTypeName ctx then + Just <| Scala.TypeVar "String" + else + Nothing + _ -> + Nothing + mapTypeReferenceForColumnOperations : Type () -> MappingContextInfo () -> Scala.Type mapTypeReferenceForColumnOperations typeReference ctx = tryAlternatives [ (\_ -> checkDataFrameCaseToArray typeReference ctx) diff --git a/src/Morphir/Snowpark/UserDefinedFunctionMapping.elm b/src/Morphir/Snowpark/UserDefinedFunctionMapping.elm index 5584e05f3..bd9859c39 100644 --- a/src/Morphir/Snowpark/UserDefinedFunctionMapping.elm +++ b/src/Morphir/Snowpark/UserDefinedFunctionMapping.elm @@ -18,7 +18,7 @@ import Morphir.Snowpark.MappingContext exposing (isLocalVariableDefinition) import Morphir.Snowpark.MappingContext exposing (isFunctionReceivingDataFrameExpressions) import Morphir.Snowpark.ReferenceUtils exposing (getFunctionInputTypes) -tryToConvertUserFunctionCall : ((Value a (Type ())), List (Value a (Type ()))) -> Constants.MapValueType a -> ValueMappingContext -> Scala.Value +tryToConvertUserFunctionCall : ((Value () (Type ())), List (Value () (Type ()))) -> Constants.MapValueType () -> ValueMappingContext -> Scala.Value tryToConvertUserFunctionCall (func, args) mapValue ctx = case func of ValueIR.Reference functionType functionName -> diff --git a/tests/Morphir/Snowpark/AccessElementTests.elm b/tests/Morphir/Snowpark/AccessElementTests.elm index 7d45b7f67..5a200b7ff 100644 --- a/tests/Morphir/Snowpark/AccessElementTests.elm +++ b/tests/Morphir/Snowpark/AccessElementTests.elm @@ -3,6 +3,7 @@ module Morphir.Snowpark.AccessElementTests exposing (mapFieldAccessTests) import Expect import Test exposing (Test, describe, test) import Dict exposing (Dict(..)) +import Set import Morphir.Snowpark.MapExpressionsToDataFrameOperations exposing (mapValue) import Morphir.Scala.AST as Scala import Morphir.IR.Value as Value @@ -40,7 +41,8 @@ constructorReference = mapFieldAccessTests: Test mapFieldAccessTests = let - (calculatedContext, _) = MappingContext.processDistributionModules testDistributionName testDistributionPackage + customizationOptions = {functionsToInline = Set.empty, functionsToCache = Set.empty} + (calculatedContext, _, _) = MappingContext.processDistributionModules testDistributionName testDistributionPackage customizationOptions valueMapContext = { emptyValueMappingContext | typesContextInfo = calculatedContext } assertMapFieldAccess = test ("Convert record field reference") <| diff --git a/tests/Morphir/Snowpark/FunctionGenerationTests.elm b/tests/Morphir/Snowpark/FunctionGenerationTests.elm index 2e6225ca6..6765f35bc 100644 --- a/tests/Morphir/Snowpark/FunctionGenerationTests.elm +++ b/tests/Morphir/Snowpark/FunctionGenerationTests.elm @@ -19,11 +19,13 @@ import Morphir.Snowpark.CommonTestUtils exposing (stringTypeInstance , testDistributionPackage) import Morphir.Snowpark.Constants exposing (typeRefForSnowparkType) import Morphir.IR.Path as Path +import Set functionGenTests: Test functionGenTests = let - calculatedContext = MappingContext.processDistributionModules testDistributionName testDistributionPackage + customizationOptions = {functionsToInline = Set.empty, functionsToCache = Set.empty} + calculatedContext = MappingContext.processDistributionModules testDistributionName testDistributionPackage customizationOptions typeOfRecord = Type.Reference () (FQName.fromString "UTest:MyMod:Emp" ":") [] functionDefinition = public { doc = "" @@ -45,6 +47,7 @@ functionGenTests = , typeArgs = [] , args = [[Scala.ArgDecl [] (Scala.TypeRef ["uTest", "MyMod"] "Emp") "a" Nothing] , [Scala.ArgDecl [] (typeRefForSnowparkType "Column") "b" Nothing] + , [Scala.ArgDecl [ Scala.Implicit ] (typeRefForSnowparkType "Session") "sfSession" Nothing] ] , returnType = Just <| typeRefForSnowparkType "Column" , body = Just expectedFunctionBody diff --git a/tests/Morphir/Snowpark/MappingContextTests.elm b/tests/Morphir/Snowpark/MappingContextTests.elm index 719f4bd0e..20c9f8d65 100644 --- a/tests/Morphir/Snowpark/MappingContextTests.elm +++ b/tests/Morphir/Snowpark/MappingContextTests.elm @@ -1,6 +1,7 @@ module Morphir.Snowpark.MappingContextTests exposing (typeClassificationTests) import Dict +import Set import Test exposing (Test, describe, test) import Expect import Morphir.IR.Path as Path @@ -67,7 +68,8 @@ testDistributionPackage = typeClassificationTests : Test typeClassificationTests = let - (calculatedContext, _) = MappingContext.processDistributionModules testDistributionName testDistributionPackage + customizationOptions = {functionsToInline = Set.empty, functionsToCache = Set.empty} + (calculatedContext, _, _) = MappingContext.processDistributionModules testDistributionName testDistributionPackage customizationOptions assertCount = test ("Types in context") <| \_ -> diff --git a/tests/Morphir/Snowpark/PatternMatchTests.elm b/tests/Morphir/Snowpark/PatternMatchTests.elm index 0ae3e4fb1..8a8fd2bc8 100644 --- a/tests/Morphir/Snowpark/PatternMatchTests.elm +++ b/tests/Morphir/Snowpark/PatternMatchTests.elm @@ -3,6 +3,7 @@ module Morphir.Snowpark.PatternMatchTests exposing (caseOfGenTests) import Expect import Test exposing (Test, describe, test) import Dict exposing (Dict(..)) +import Set import Morphir.Scala.AST as Scala import Morphir.IR.Value as Value import Morphir.IR.Type as Type @@ -27,7 +28,8 @@ a2Lit = (Literal.stringLiteral "a") caseOfGenTests: Test caseOfGenTests = let - (calculatedContext, _) = MappingContext.processDistributionModules testDistributionName testDistributionPackage + customizationOptions = {functionsToInline = Set.empty, functionsToCache = Set.empty} + (calculatedContext, _ , _) = MappingContext.processDistributionModules testDistributionName testDistributionPackage customizationOptions cases = [ (Value.LiteralPattern str aLit, Value.Literal str a2Lit) , (Value.WildcardPattern str, Value.Literal str (Literal.stringLiteral "D"))] inputCase = Value.PatternMatch stringTypeInstance (Value.Literal str (Literal.stringLiteral "X")) cases diff --git a/tests/Morphir/Snowpark/RecordWrapperGenerationTests.elm b/tests/Morphir/Snowpark/RecordWrapperGenerationTests.elm index f99f4271b..01641693c 100644 --- a/tests/Morphir/Snowpark/RecordWrapperGenerationTests.elm +++ b/tests/Morphir/Snowpark/RecordWrapperGenerationTests.elm @@ -2,6 +2,7 @@ module Morphir.Snowpark.RecordWrapperGenerationTests exposing (..) import Dict +import Set import Test exposing (Test, describe, test) import Expect import Morphir.IR.Path as Path @@ -40,7 +41,8 @@ testDistributionPackage = typeClassificationTests : Test typeClassificationTests = let - calculatedContext = MappingContext.processDistributionModules testDistributionName testDistributionPackage + customizationOptions = {functionsToInline = Set.empty, functionsToCache = Set.empty} + calculatedContext = MappingContext.processDistributionModules testDistributionName testDistributionPackage customizationOptions firstModule = Dict.get [(Name.fromString "MyMod")] testDistributionPackage.modules |> Maybe.map (\access -> access.value) assertItCreatedWrapper = test ("Wrapper creation") <| @@ -53,7 +55,7 @@ typeClassificationTests = |> Maybe.withDefault [] in - Expect.equal ["Trait:Emp1:2", "Object:Emp1:4", "Class:Emp1Wrapper:2"] generationResult + Expect.equal ["Trait:Emp1:2", "Object:Emp1:5", "Class:Emp1Wrapper:2"] generationResult in describe "resolveTNam"