(puppetlabsGH-141) Update Sidecar to extract Function V4 API information

Previously the Sidecar could only emit information in function V3 API format.
Now that the Sidecar Protocol has been changed to emit V4 information, this
commit updates the Sidecar to extact the V4 information.

This commit:
* Updates the sidecar_protocol_extensions for the Puppet Function object to
  adapt the V3 function information into the V4 version
* Updates the tests for the sidecar_protocol_extensions for the new parameters
  for the V4 API functions
* Updates the Puppet Strings Helper to extract the required information from
  YARN in order to populate the V4 API function metadata.  In particular the
  concepts of function type and arity no longer apply, and have been replaced
  using Function Signatures.

lib/puppet-languageserver-sidecar/puppet_strings_helper.rb

@@ -155,8 +155,8 @@ def populate_functions_from_yard_registry!
         obj.calling_source   = obj.source
         obj.line             = item[:line]
         obj.doc              = item[:docstring][:text]
-        obj.arity            = -1 # We don't care about arity
-        obj.function_version = item[:type] == 'ruby4x' ? 4 : 3
+        # 'ruby3x' functions are version 3.  'ruby4x' and 'puppet' are version 4
+        obj.function_version = item[:type] == 'ruby3x' ? 3 : 4
 
         # Try and determine the function call site from the source file
         char = item[:source].index(":#{func_name}")
@@ -165,20 +165,36 @@ def populate_functions_from_yard_registry!
           obj.length = func_name.length + 1
         end
 
-        case item[:type]
-        when 'ruby3x'
-          obj.function_version = 3
-          # This is a bit hacky but it works (probably).  Puppet-Strings doesn't rip this information out, but you do have the
-          # the source to query
-          obj.type = item[:source].match(/:rvalue/) ? :rvalue : :statement
-        when 'ruby4x'
-          obj.function_version = 4
-          # All ruby functions are statements
-          obj.type = :statement
-        else
-          PuppetLanguageServerSidecar.log_message(:error, "[#{self.class}] Unknown function type #{item[:type]}")
+        # Note that puppet strings doesn't populate the method signatures for V3 functions
+        # Also, we don't have access to the arity of V3 functions so we can't reverse engineer the signature
+        item[:signatures].each do |signature|
+          sig = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionSignature.new
+
+          sig.key = signature[:signature]
+          sig.doc = signature[:docstring][:text]
+          signature[:docstring][:tags].each do |tag|
+            case tag[:tag_name]
+            when 'param'
+              sig.parameters << {
+                'name'  => tag[:name],
+                'types' => tag[:types],
+                'doc'   => tag[:text]
+              }
+            when 'return'
+              sig.return_types = tag[:types]
+            end
+          end
+          obj.signatures << sig
         end
 
+        # Extract other common information
+        # TODO: Other common tags include `example`, `overload`
+        pre_docs = ''
+        pre_docs += "This uses the legacy Ruby function API\n" if item[:type] == 'ruby3x'
+        since_tag = item[:docstring][:tags].find { |tag| tag[:tag_name] == 'since' }
+        pre_docs += "Since #{since_tag[:text]}\n" unless since_tag.nil?
+        obj.doc = pre_docs + "\n" + obj.doc unless pre_docs.empty?
+
         @cache[source_path].functions << obj
       end
     end

lib/puppet-languageserver-sidecar/sidecar_protocol_extensions.rb

@@ -43,10 +43,52 @@ def self.from_puppet(name, item)
         obj.source         = item[:source_location][:source]
         obj.calling_source = obj.source
         obj.line           = item[:source_location][:line]
-        # TODO: name ?
-        obj.doc   = item[:doc]
-        obj.arity = item[:arity]
-        obj.type  = item[:type]
+        # This method is only called for V3 API functions. Therefore we need to transform this V3 function into V4 metadata
+        obj.doc = 'This uses the legacy Ruby function API'
+        obj.doc += "\n\n" + item[:doc] unless item[:doc].nil?
+        obj.function_version = 3
+
+        # V3 functions don't explicitly define signatures.  We can craft an approximation using arity
+        # From - https://github.com/puppetlabs/puppet/blob/904023ffc59bc6771c0c7abf2a1d2c4acf941b09/lib/puppet/parser/functions.rb#L159-L168
+        #
+        # @option options [Integer] :arity (-1) the
+        #   [arity](https://en.wikipedia.org/wiki/Arity) of the function.  When
+        #   specified as a positive integer the function is expected to receive
+        #   _exactly_ the specified number of arguments.  When specified as a
+        #   negative number, the function is expected to receive _at least_ the
+        #   absolute value of the specified number of arguments incremented by one.
+        #   For example, a function with an arity of `-4` is expected to receive at
+        #   minimum 3 arguments.  A function with the default arity of `-1` accepts
+        #   zero or more arguments.  A function with an arity of 2 must be provided
+        #   with exactly two arguments, no more and no less.  Added in Puppet 3.1.0.
+        #
+        # Find the number of required parameters
+        sig_params = []
+        param_text = []
+        required_params = item[:arity] < 0 ? -item[:arity] - 1 : item[:arity]
+        (1..required_params).each do |index|
+          sig_params << {
+            'name'  => "param#{index}",
+            'types' => ['Any']
+          }
+          param_text << "Any $param#{index}"
+        end
+        # Add optional parameters if needed
+        if item[:arity] < 0
+          sig_params << {
+            'name'  => '*args',
+            'types' => ['Optional[Any]']
+          }
+          param_text << 'Optional[Any] *args'
+        end
+
+        # V3 functions only have a single signature
+        obj.signatures << PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionSignature.new.from_h!(
+          'key'          => "#{name}(#{param_text.join(', ')})",
+          'doc'          => item[:doc],
+          'parameters'   => sig_params,
+          'return_types' => item[:type] == :rvalue ? ['Any'] : ['Undef'] # :rvalue type functions return something. :statement type functions don't return anything so Undef
+        )
         obj
       end
     end

spec/languageserver-sidecar/integration/puppet-languageserver-sidecar/featureflag_puppetstrings/featureflag_puppetstrings_spec.rb

@@ -274,7 +274,7 @@ def expect_same_array_content(a, b)
 
         # Make sure the function has the right properties
         func = child_with_key(deserial, :fixture_function)
-        expect(func.doc).to eq('doc_fixture_function')
+        expect(func.doc).to match(/doc_fixture_function/)
         expect(func.source).to match(/valid_module_workspace/)
 
         # Make sure the function has the right properties
@@ -380,7 +380,7 @@ def expect_same_array_content(a, b)
 
         # Make sure the function has the right properties
         func = child_with_key(deserial, :env_function)
-        expect(func.doc).to eq('doc_env_function')
+        expect(func.doc).to match(/doc_env_function/)
         expect(func.source).to match(/valid_environment_workspace/)
 
         # Make sure the function has the right properties

spec/languageserver-sidecar/integration/puppet-languageserver-sidecar/puppet-languageserver-sidecar_spec.rb

@@ -156,7 +156,7 @@ def with_temporary_file(content)
 
         # Make sure the function has the right properties
         func = child_with_key(deserial, :fixture_function)
-        expect(func.doc).to eq('doc_fixture_function')
+        expect(func.doc).to match(/doc_fixture_function/)
         expect(func.source).to match(/valid_module_workspace/)
       end
     end
@@ -247,7 +247,7 @@ def with_temporary_file(content)
 
         # Make sure the function has the right properties
         func = child_with_key(deserial, :env_function)
-        expect(func.doc).to eq('doc_env_function')
+        expect(func.doc).to match(/doc_env_function/)
         expect(func.source).to match(/valid_environment_workspace/)
       end
     end

spec/languageserver-sidecar/unit/puppet-languageserver-sidecar/sidecar_protocol_extensions_spec.rb

@@ -84,15 +84,19 @@
     describe '.from_puppet' do
       it 'should populate from a Puppet function object' do
         result = subject_klass.from_puppet(puppet_funcname, puppet_func)
-
-        expect(result.doc).to eq(puppet_func[:doc])
-        expect(result.arity).to eq(puppet_func[:arity])
-        expect(result.type).to eq(puppet_func[:type])
+        # This method is only called for V3 API functions.
+        # Therefore we need to assert the V3 function into V4 metadata
+
+        expect(result.doc).to match(puppet_func[:doc])
+        expect(result.function_version).to eq(3)
+        expect(result.signatures.count).to eq(1)
+        expect(result.signatures[0].doc).to match(puppet_func[:doc])
+        expect(result.signatures[0].key).to eq("#{puppet_funcname}()")
       end
     end
 
     describe '#from_json' do
-      [:doc, :arity, :type].each do |testcase|
+      [:doc, :function_version, :signatures].each do |testcase|
         it "should deserialize a serialized #{testcase} value" do
           serial = subject_klass.from_puppet(puppet_funcname, puppet_func)
           deserial = subject_klass.new.from_json!(serial.to_json)