Skip to content

Commit

Permalink
Add decorations processing (finos#99)
Browse files Browse the repository at this point in the history
* First commit for decorations

* Add support for `inlining`
  • Loading branch information
sfc-gh-lfallasavendano committed Dec 13, 2023
1 parent e6a525b commit 9e576bd
Show file tree
Hide file tree
Showing 20 changed files with 349 additions and 58 deletions.
8 changes: 7 additions & 1 deletion cli/cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,14 @@ async function gen(input, outputPath, options) {
opts.limitToModules = options.modulesToInclude ? options.modulesToInclude.split(",") : null
opts.includeCodecs = options.includeCodecs ? true : false
opts.filename = options.filename == '' ? '' : options.filename
const fileMap = await generate(opts, JSON.parse(morphirIrJson.toString()))

if (options.decorations) {
if (await fileExist(path.resolve(options.decorations))) {
options.decorationsObj = JSON.parse(await readFile(path.resolve(options.decorations)))
}
}

const fileMap = await generate(opts, JSON.parse(morphirIrJson.toString()))
const writePromises =
fileMap.map(async ([
[dirPath, fileName], content
Expand Down
1 change: 1 addition & 0 deletions cli/morphir-elm-gen.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ program
.option('-s, --include-codecs', 'Generate JSON codecs', false)
.option('-f, --filename <filename>', 'Filename of the generated JSON Schema.', '')
.option('-ls, --include <comma.separated,list.of,strings>', 'Limit what will be included.', '')
.option('-dec, --decorations <filename>', 'JSON file with decorations')
.parse(process.argv)

cli.gen(program.opts().input, path.resolve(program.opts().output), program.opts())
Expand Down
2 changes: 1 addition & 1 deletion cli/src/Morphir/Elm/Target.elm
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ decodeOptions gen =
Decode.map SparkOptions (Decode.succeed Morphir.Scala.Spark.Backend.Options)

Ok "Snowpark" ->
Decode.map SnowparkOptions (Decode.succeed Morphir.Snowpark.Backend.Options)
Decode.map SnowparkOptions Morphir.Snowpark.Backend.decodeOptions

Ok "JsonSchema" ->
Decode.map JsonSchemaOptions Morphir.JsonSchema.Backend.Codec.decodeOptions
Expand Down
2 changes: 1 addition & 1 deletion src/Morphir/Scala/AST.elm
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

module Morphir.Scala.AST exposing
( Name, Path, Documented, Annotated, withAnnotation, withoutAnnotation, CompilationUnit, PackageDecl
, ImportDecl, ImportName, Mod(..), TypeDecl(..), ArgDecl, ArgValue(..), MemberDecl(..)
, ImportDecl, ImportName(..), Mod(..), TypeDecl(..), ArgDecl, ArgValue(..), MemberDecl(..)
, Type(..), Value(..), Pattern(..), Lit(..), Generator(..)
, nameOfTypeDecl
)
Expand Down
18 changes: 18 additions & 0 deletions src/Morphir/Scala/PrettyPrinter.elm
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,29 @@ mapCompilationUnit opt cu =
concat
[ concat [ "package ", dotSep (prefixKeywords cu.packageDecl), newLine ]
, newLine
, mapImports cu.imports
, cu.typeDecls
|> List.map (mapDocumented (mapAnnotated (mapTypeDecl opt)))
|> String.join (newLine ++ newLine)
]

mapImports : List ImportDecl -> Doc
mapImports imports =
case imports of
[] ->
""
importsList ->
(importsList |> List.map mapImport) ++ [ newLine ]
|> concat


mapImport : ImportDecl -> Doc
mapImport importDecl =
concat
[ "import "
, String.join "." (importDecl.packagePrefix)
, newLine
]

mapTypeDecl : Options -> TypeDecl -> Doc
mapTypeDecl opt typeDecl =
Expand Down
14 changes: 10 additions & 4 deletions src/Morphir/Snowpark/AccessElementMapping.elm
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import Morphir.IR.Literal exposing (Literal(..))
import Morphir.Snowpark.MappingContext exposing (ValueMappingContext)
import Morphir.Snowpark.MappingContext exposing (isUnionTypeWithoutParams
, getReplacementForIdentifier)
import Html.Attributes exposing (name)
import Morphir.IR.FQName as FQName
import Morphir.IR.Value exposing (Value(..))
import Morphir.Snowpark.ReferenceUtils exposing (isValueReferenceToSimpleTypesRecord
Expand All @@ -30,6 +29,7 @@ import Morphir.Snowpark.MappingContext exposing (isUnionTypeWithParams)
import Morphir.IR.Value as Value
import Morphir.Snowpark.Constants exposing (MapValueType)
import Morphir.IR.Type as Type
import Morphir.Snowpark.Utils exposing (tryAlternatives)


checkForDataFrameVariableReference : Value ta (IrType.Type ()) -> ValueMappingContext -> Maybe String
Expand Down Expand Up @@ -100,9 +100,15 @@ mapConstructorAccess tpe name ctx =
_ ->
Scala.Literal (Scala.StringLit "Constructor access not converted")

mapReferenceAccess : (IrType.Type ()) -> FQName.FQName -> ValueMappingContext -> Scala.Value
mapReferenceAccess tpe name ctx =
if MappingContext.isDataFrameFriendlyType tpe ctx.typesContextInfo ||

mapReferenceAccess : (IrType.Type ()) -> FQName.FQName -> MapValueType () -> ValueMappingContext -> Scala.Value
mapReferenceAccess tpe name mapValue ctx =
if Dict.member name ctx.globalValuesToInline then
ctx.globalValuesToInline
|> Dict.get name
|> Maybe.map (\definition -> mapValue definition.body ctx)
|> Maybe.withDefault Scala.Wildcard
else if MappingContext.isDataFrameFriendlyType tpe ctx.typesContextInfo ||
MappingContext.isListOfDataFrameFriendlyType tpe ctx.typesContextInfo then
let
nsName = scalaPathToModule name
Expand Down
91 changes: 70 additions & 21 deletions src/Morphir/Snowpark/Backend.elm
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
module Morphir.Snowpark.Backend exposing (mapDistribution, Options, mapFunctionDefinition)
module Morphir.Snowpark.Backend exposing (mapDistribution, Options, mapFunctionDefinition, decodeOptions)

import Dict
import List
Expand Down Expand Up @@ -32,26 +32,47 @@ import Morphir.Snowpark.MappingContext as MappingContext exposing (
import Morphir.Snowpark.MappingContext exposing (FunctionClassification(..))
import Morphir.Snowpark.FunctionMappingsForPlainScala as FunctionMappingsForPlainScala
import Morphir.Snowpark.MapExpressionsToDataFrameOperations exposing (mapValue)
import Json.Decode as Decode exposing (Decoder)
import Morphir.IR.Decoration.Codec exposing (decodeNodeIDByValuePairs)
import Morphir.SDK.Dict as SDKDict
import Morphir.IR.NodeId exposing (NodeID)
import Morphir.Snowpark.Customization exposing ( loadCustomizationOptions
, CustomizationOptions
, tryToApplyPostConversionCustomization
, emptyCustomizationOptions )
import Morphir.Snowpark.Constants exposing (typeRefForSnowparkType)

type alias Options =
{}
{ decorations : Maybe (SDKDict.Dict NodeID Decode.Value) }

decodeOptions : Decoder Options
decodeOptions =
Decode.map Options
(Decode.maybe
(Decode.field "decorationsObj" decodeNodeIDByValuePairs))

mapDistribution : Options -> Distribution -> FileMap
mapDistribution _ distro =
mapDistribution opts distro =
let
loadedOpts : CustomizationOptions
loadedOpts = opts.decorations
|> Maybe.map loadCustomizationOptions
|> Maybe.withDefault emptyCustomizationOptions
in
case distro of
Distribution.Library packageName _ packageDef ->
mapPackageDefinition distro packageName packageDef
mapPackageDefinition distro packageName packageDef loadedOpts

mapPackageDefinition : Distribution -> Package.PackageName -> Package.Definition () (Type ()) -> FileMap
mapPackageDefinition _ packagePath packageDef =
mapPackageDefinition : Distribution -> Package.PackageName -> Package.Definition () (Type ()) -> CustomizationOptions -> FileMap
mapPackageDefinition _ packagePath packageDef customizationOptions =
let
contextInfo = MappingContext.processDistributionModules packagePath packageDef
contextInfo = MappingContext.processDistributionModules packagePath packageDef customizationOptions
generatedScala =
packageDef.modules
|> Dict.toList
|> List.concatMap
(\( modulePath, moduleImpl ) ->
mapModuleDefinition packagePath modulePath moduleImpl contextInfo
mapModuleDefinition packagePath modulePath moduleImpl contextInfo customizationOptions
)
in
generatedScala
Expand All @@ -67,8 +88,8 @@ mapPackageDefinition _ packagePath packageDef =
|> Dict.fromList


mapModuleDefinition : Package.PackageName -> Path -> AccessControlled (Module.Definition () (Type ())) -> GlobalDefinitionInformation () -> List Scala.CompilationUnit
mapModuleDefinition currentPackagePath currentModulePath accessControlledModuleDef ctxInfo =
mapModuleDefinition : Package.PackageName -> Path -> AccessControlled (Module.Definition () (Type ())) -> GlobalDefinitionInformation () -> CustomizationOptions -> List Scala.CompilationUnit
mapModuleDefinition currentPackagePath currentModulePath accessControlledModuleDef ctxInfo customizationOptions =
let
( scalaPackagePath, moduleName ) =
case currentModulePath |> List.reverse of
Expand All @@ -88,22 +109,24 @@ mapModuleDefinition currentPackagePath currentModulePath accessControlledModuleD
|> RecordWrapperGenerator.generateRecordWrappers currentPackagePath currentModulePath ctxInfo
|> List.map (\doc -> { annotations = doc.value.annotations, value = Scala.MemberTypeDecl (doc.value.value) } )

functionMembers : List (Scala.Annotated Scala.MemberDecl)
functionMembers =
(functionMembers, generatedImports) =
accessControlledModuleDef.value.values
|> Dict.toList
|> List.concatMap
|> List.map
(\( valueName, accessControlledValueDef ) ->
[ mapFunctionDefinition valueName accessControlledValueDef currentPackagePath currentModulePath ctxInfo
]
(processFunctionMember valueName accessControlledValueDef currentPackagePath currentModulePath ctxInfo customizationOptions)
)
|> List.map Scala.withoutAnnotation
|> List.unzip
|> \(members, imports) ->
((members |> List.concat |> List.map Scala.withoutAnnotation)
, imports |> List.concat)

moduleUnit : Scala.CompilationUnit
moduleUnit =
{ dirPath = scalaPackagePath
, fileName = (moduleName |> Name.toTitleCase) ++ ".scala"
, packageDecl = scalaPackagePath
, imports = []
, imports = generatedImports
, typeDecls = [( Scala.Documented (Just (String.join "" [ "Generated based on ", currentModulePath |> Path.toString Name.toTitleCase "." ]))
(Scala.Annotated []
(Scala.Object
Expand All @@ -130,8 +153,18 @@ mapModuleDefinition currentPackagePath currentModulePath accessControlledModuleD
[ moduleUnit ]


processFunctionMember : Name.Name -> AccessControlled (Documented (Value.Definition () (Type ()))) -> Package.PackageName -> Path -> GlobalDefinitionInformation () -> CustomizationOptions -> (List Scala.MemberDecl , List Scala.ImportDecl)
processFunctionMember valueName accessControlledValueDef currentPackagePath currentModulePath ctxInfo customizationOptions =
let
fullFunctionName = FQName.fQName currentPackagePath currentModulePath valueName
mappedFunction = mapFunctionDefinition valueName accessControlledValueDef currentPackagePath currentModulePath ctxInfo
in
Maybe.withDefault
([ mappedFunction ], [] )
(tryToApplyPostConversionCustomization fullFunctionName mappedFunction customizationOptions)

mapFunctionDefinition : Name.Name -> AccessControlled (Documented (Value.Definition () (Type ()))) -> Path -> Path -> GlobalDefinitionInformation () -> Scala.MemberDecl
mapFunctionDefinition functionName body currentPackagePath modulePath (typeContextInfo, functionsInfo) =
mapFunctionDefinition functionName body currentPackagePath modulePath (typeContextInfo, functionsInfo, inlineInfo) =
let
fullFunctionName = FQName.fQName currentPackagePath modulePath functionName
functionClassification = getFunctionClassification fullFunctionName functionsInfo
Expand All @@ -141,7 +174,8 @@ mapFunctionDefinition functionName body currentPackagePath modulePath (typeConte
, parameters = parameterNames
, functionClassificationInfo = functionsInfo
, currentFunctionClassification = functionClassification
, packagePath = currentPackagePath}
, packagePath = currentPackagePath
, globalValuesToInline = inlineInfo }
localDeclarations =
body.value.value.inputTypes
|> List.filterMap (checkForDataFrameColumndsDeclaration typeContextInfo)
Expand All @@ -152,7 +186,7 @@ mapFunctionDefinition functionName body currentPackagePath modulePath (typeConte
{ modifiers = []
, name = mapValueName functionName
, typeArgs = []
, args = parameters
, args = addImplicitSession parameters
, returnType =
Just returnTypeToGenerate
, body =
Expand All @@ -161,6 +195,21 @@ mapFunctionDefinition functionName body currentPackagePath modulePath (typeConte
(declarations, Just bodyToUse) -> Just (Scala.Block declarations bodyToUse)
(_, _) -> Nothing
}
addImplicitSession : List (List Scala.ArgDecl) -> List (List Scala.ArgDecl)
addImplicitSession args =
case args of
[] ->
args
_ ->
let
implicitArg =
{ modifiers = [ Scala.Implicit ]
, tpe = typeRefForSnowparkType "Session"
, name = "sfSession"
, defaultValue = Nothing
}
in
args ++ [ [ implicitArg ] ]

includeDataFrameInfo : List (Scala.MemberDecl, (String, FQName.FQName)) -> ValueMappingContext -> ValueMappingContext
includeDataFrameInfo declInfos ctx =
Expand Down Expand Up @@ -225,7 +274,7 @@ processParameters inputTypes currentFunctionClassification ctx =
inputTypes |> List.map (generateArgumentDeclarationForFunction ctx currentFunctionClassification)


mapFunctionBody : Value.Definition ta (Type ()) -> ValueMappingContext -> Maybe Scala.Value
mapFunctionBody : Value.Definition () (Type ()) -> ValueMappingContext -> Maybe Scala.Value
mapFunctionBody value ctx =
let
functionToMap =
Expand Down
99 changes: 99 additions & 0 deletions src/Morphir/Snowpark/Customization.elm
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
module Morphir.Snowpark.Customization exposing (CustomizationOptions, loadCustomizationOptions, emptyCustomizationOptions, generateCacheCode, tryToApplyPostConversionCustomization)

import Morphir.IR.FQName exposing (FQName)
import Set exposing (Set)
import Morphir.Scala.AST as Scala exposing (ImportName(..))
import Json.Decode as Decode exposing (Decoder)
import Morphir.IR.Decoration.Codec exposing (decodeNodeIDByValuePairs)
import Morphir.SDK.Dict as SDKDict
import Morphir.IR.NodeId exposing (NodeID)
import Morphir.IR.NodeId exposing (NodeID(..))
import Json.Decode exposing (decodeString)


type alias CustomizationOptions =
{ functionsToInline: Set FQName
, functionsToCache: Set FQName
}

emptyCustomizationOptions : CustomizationOptions
emptyCustomizationOptions = CustomizationOptions Set.empty Set.empty

loadCustomizationOptions : SDKDict.Dict NodeID Decode.Value -> CustomizationOptions
loadCustomizationOptions optionsDict =
let
decodeFirstElementAsString =
Decode.decodeValue (Decode.index 0 Decode.string)
optsList =
optionsDict
|> SDKDict.toList
functions =
optsList
|> List.filterMap (\(id, value) ->
case id of
ValueID fullName _ -> Just (fullName, value)
_ -> Nothing)
(functionsToCache, valuesToInline) =
functions
|> List.foldr
(\(fullName, valueDec) ((toCache, toInline) as current) ->
case decodeFirstElementAsString valueDec of
Ok "cacheResult" ->
(Set.insert fullName toCache, toInline)
Ok "inlineElement" ->
(toCache, Set.insert fullName toInline)
_ ->
current)
(Set.empty, Set.empty)
in
{ functionsToInline = valuesToInline
, functionsToCache = functionsToCache
}

generateCacheCode : String -> Scala.MemberDecl -> Maybe Scala.Type -> Scala.MemberDecl
generateCacheCode cacheName decl returnType =
case decl of
Scala.FunctionDecl funcInfo ->
let
funcArgs =
funcInfo.args |> List.concatMap (\t -> t |> List.map (\t2 -> Scala.Variable t2.name))
body =
Maybe.withDefault (Scala.Variable "_") funcInfo.body
expressionToCache =
case returnType of
Just (Scala.TypeRef _ "DataFrame") ->
Scala.Select body "cacheResult"
_ ->
body
getOrElseUpdateCall =
Scala.Apply (Scala.Ref [cacheName] "getOrElseUpdate")
[ Scala.ArgValue Nothing (Scala.Tuple funcArgs)
, Scala.ArgValue Nothing expressionToCache
]
in
Scala.FunctionDecl { funcInfo | body = Just getOrElseUpdateCall }
_ ->
decl

tryToApplyPostConversionCustomization : FQName -> Scala.MemberDecl -> CustomizationOptions -> Maybe (List Scala.MemberDecl , List Scala.ImportDecl)
tryToApplyPostConversionCustomization fullFunctionName mappedFunction customizationOptions =
case (Set.member fullFunctionName customizationOptions.functionsToCache , mappedFunction) of
(True, Scala.FunctionDecl funcInfo) ->
Just (
[ generateCacheCode (funcInfo.name ++ "Cache") mappedFunction funcInfo.returnType,
Scala.ValueDecl
{ modifiers = []
, pattern = Scala.NamedMatch (funcInfo.name ++ "Cache")
, valueType = Just (Scala.TypeApply
(Scala.TypeRef [ "scala", "collection", "concurrent" ] "Map")
[ Scala.TupleType (funcInfo.args |> List.concatMap (\t -> t |> List.map (\t2 -> t2.tpe) )) , Maybe.withDefault (Scala.TypeVar "_") funcInfo.returnType ] )
, value =
Scala.Select (Scala.New [ "java", "util", "concurrent" ] "ConcurrentHashMap" []) "asScala"
}
],
[ Scala.ImportDecl False [ "scala", "collection", "JavaConverters", "_"] [] ] )
_ ->
if Set.member fullFunctionName customizationOptions.functionsToInline then
Just ([], [])
else
Nothing
Loading

0 comments on commit 9e576bd

Please sign in to comment.