Counter utf-16 characters

[mattn]

Fixes #282

[mattn]

Benchmark

function! s:count_utf16_code_units(str) abort
  let l:len = strchars(a:str)
  let l:i = 0
  let l:cnt = 0

  while l:i < l:len
    let l:chr = strcharpart(a:str, l:i, 1)
    if char2nr(l:chr) > 0x10000
      let l:cnt = l:cnt + 2
    else
      let l:cnt = l:cnt + 1
    endif

    let l:i = l:i + 1
  endwhile

  return l:cnt
endfunction

function! s:my_count_utf16_code_units(str) abort
  let l:rs = split(a:str, '\zs')
  let l:len = len(l:rs)
  return l:len + count(l:rs, 'char2nr(v:val)>0x10000')
endfunction

function! s:benchmark()
  let s = 'a𐐀b'
  let n = 100000

  let start = reltime()
  for i in range(n)
    call strlen(s)
  endfor
  echo printf('strlen: %f', reltimefloat(reltime(start)))

  let start = reltime()
  for i in range(n)
    call s:count_utf16_code_units(s)
  endfor
  echo printf('s:count_utf16_code_units: %f', reltimefloat(reltime(start)))

  let start = reltime()
  for i in range(n)
    call s:my_count_utf16_code_units(s)
  endfor
  echo printf('s:my_count_utf16_code_units: %f', reltimefloat(reltime(start)))
endfunction

call s:benchmark()

strlen: 0.229879
s:count_utf16_code_units: 4.050134
s:my_count_utf16_code_units: 1.107014

[natebosch]

When I encountered this I also needed to replace all occurrence of str[start:end] with strcharpart(). Was that not necessary here?

[mattn]

It is no needed that replace all of strlen to this function. This function should be used for only calculate offset.

[mattn]

function! s:count_utf16_code_units(str) abort
  let l:len = strchars(a:str)
  let l:i = 0
  let l:cnt = 0

  while l:i < l:len
    let l:chr = strcharpart(a:str, l:i, 1)
    if char2nr(l:chr) > 0x10000
      let l:cnt = l:cnt + 2
    else
      let l:cnt = l:cnt + 1
    endif

    let l:i = l:i + 1
  endwhile

  return l:cnt
endfunction

function! s:my_count_utf16_code_units(str) abort
  let l:rs = split(a:str, '\zs')
  return len(l:rs) + count(l:rs, 'char2nr(v:val)=>0x10000')
endfunction

function! s:benchmark(name, n, case)
  let start = reltime()
  for i in range(a:n)
    call strlen(a:case)
  endfor
  echo printf('%s: strlen: %f', a:name, reltimefloat(reltime(start)))

  let start = reltime()
  for i in range(a:n)
    call s:count_utf16_code_units(a:case)
  endfor
  echo printf('%s: s:count_utf16_code_units: %f', a:name, reltimefloat(reltime(start)))

  let start = reltime()
  for i in range(a:n)
    call s:my_count_utf16_code_units(a:case)
  endfor
  echo printf('%s: s:my_count_utf16_code_units: %f', a:name, reltimefloat(reltime(start)))
endfunction

call s:benchmark('short string', 100000, 'a𐐀b')
call s:benchmark('long string', 10000, repeat('a𐐀b', 200))

short string: strlen: 0.249764
short string: s:count_utf16_code_units: 4.046245
short string: s:my_count_utf16_code_units: 1.054177
long string: strlen: 0.025720
long string: s:count_utf16_code_units: 60.834453
long string: s:my_count_utf16_code_units: 4.178993

[prabirshrestha]

There are some interesting approach and comments going on currently at microsoft/language-server-protocol#376 (comment)

[Avi-D-coder]

Is vim-lsp currently using UTF-8/bytes, codepoints or grapheme clusters?

[prabirshrestha]

There is now webworkers in ducktape vim. Perf heavy things could go there :) bobpepin/vim#1

[prabirshrestha]

@mattn Any updates on this? One option I can think of is having this under a feature flag turned off by default.

[mattn]

Sorry delay, I'll fix soon.

[hrsh7th]

I met strange vim's behavior at trying to use this branch.

The behavior is col(a:m) is not match lsp#utils#strlen(getline(a:m)[:col(a:m)]) in insert-mode.

In my environment, those unmatching were fixable below code.

function! lsp#utils#col(m) abort
    if g:lsp_use_utf16
        let col = lsp#utils#strlen(getline(a:m)[:col(a:m) - 1])

        if mode() ==# 'i' && col(a:m) == col('$')
            let col = col + 1
        endif

        return col
    endif
    return col(a:m)
endfunction

(This branch is very great work for me. Thanks mattn)

[clason]

@mattn Unfortunately, this PR doesn't help with the simple example from #425 (not even with @hrsh7th change). Am I missing something?

[hrsh7th]

@clason I missunderstood. My environemnt had other patch. It solves utf8 character problems maybe (This patch is vim-lsc's diff logic). But still remaining utf16 character problem after applying this patch.
I'm sorry, I should had learning character encoding and checking my environment before post comment...

[mattn]

If someone already found the problem of this patch, please point it. Or if you already have another patch, please send new PR.

[clason]

@hrsh7th Could you share your "other patch"? If it works for UTF-8, that'd already be progress.

[hrsh7th]

I opend PR (#447).

It includes other patch (it is ported vim-lsc's diff logic).

@clason Please test #447.

Sorry for my confusing movements...

[mattn]

Fixed by #447