Merge pull request #115 from oist/vc/deprecations

Get MXNet.jl ready for v0.5

.travis.yml

@@ -6,6 +6,7 @@ os:
   - osx
 julia:
   - 0.4
+  - 0.5
   - nightly
 
 # dependent apt packages

deps/build.jl

@@ -1,3 +1,4 @@
+using Compat
 ################################################################################
 # First try to detect and load existing libmxnet
 ################################################################################
@@ -21,7 +22,7 @@ if !libmxnet_detected
   ################################################################################
   # If not found, try to build automatically using BinDeps
   ################################################################################
-  @windows_only begin
+  if is_windows()
     info("Please follow the libmxnet documentation on how to build manually")
     info("or to install pre-build packages:")
     info("http://mxnet.readthedocs.io/en/latest/how_to/build.html#building-on-windows")
@@ -60,7 +61,9 @@ if !libmxnet_detected
         FileRule(joinpath(_libdir, "libmxnet.so"), @build_steps begin
           ChangeDirectory("$_mxdir")
           `cp make/config.mk config.mk`
-          @osx_only `cp make/osx.mk config.mk`
+          if is_apple()
+            `cp make/osx.mk config.mk`
+          end
           `sed -i -s 's/USE_OPENCV = 1/USE_OPENCV = 0/' config.mk`
           `sed -i -s "s/MSHADOW_CFLAGS = \(.*\)/MSHADOW_CFLAGS = \1 $ilp64/" mshadow/make/mshadow.mk`
           `cp ../../cblas.h include/cblas.h`

src/base.jl

@@ -43,7 +43,7 @@ function mx_get_last_error()
   if msg == C_NULL
     throw(MXError("Failed to get last error message"))
   end
-  return @compat String(msg)
+  return unsafe_string(msg)
 end
 
 "Utility macro to call MXNet API functions"

src/callback.jl

@@ -51,7 +51,7 @@ For example, the :func:`speedometer` callback is defined as
 function every_n_batch(callback :: Function, n :: Int; call_on_0 :: Bool = false)
   BatchCallback(n, call_on_0, callback)
 end
-function Base.call(cb :: BatchCallback, state :: OptimizationState)
+@compat function (cb :: BatchCallback)(state :: OptimizationState)
   if state.curr_batch == 0
     if cb.call_on_0
       cb.callback(state)
@@ -107,7 +107,7 @@ A convenient function to construct a callback that runs every ``n`` full data-pa
 function every_n_epoch(callback :: Function, n :: Int; call_on_0 :: Bool = false)
   EpochCallback(n, call_on_0, callback)
 end
-function Base.call{T<:Real}(cb :: EpochCallback, model :: Any, state :: OptimizationState, metric :: Vector{Tuple{Base.Symbol, T}})
+@compat function (cb :: EpochCallback){T<:Real}(model :: Any, state :: OptimizationState, metric :: Vector{Tuple{Base.Symbol, T}})
   if state.curr_epoch == 0
     if cb.call_on_0
       cb.callback(model, state, metric)

src/executor.jl

@@ -23,7 +23,7 @@ function Executor(hdr :: MX_ExecutorHandle, symbol :: SymbolicNode,
   ref_hdrs = Ref{Ptr{MX_handle}}(0)
   @mxcall(:MXExecutorOutputs, (MX_handle, Ref{MX_uint}, Ref{Ptr{MX_handle}}),
           hdr, ref_size, ref_hdrs)
-  out_hdrs = pointer_to_array(ref_hdrs[], ref_size[])
+  out_hdrs = unsafe_wrap(Array, ref_hdrs[], ref_size[])
   out_arrays = [NDArray(MX_NDArrayHandle(x)) for x in out_hdrs]
 
   arg_names = list_arguments(symbol)
@@ -217,5 +217,5 @@ Can be used to get an estimated about the memory cost.
 function debug_str(self :: Executor)
   s_ref = Ref{Cstring}()
   @mxcall(:MXExecutorPrint, (MX_handle, Ptr{Cstring}), self.handle, s_ref)
-  @compat String(s_ref[])
+  unsafe_string(s_ref[])
 end

src/io.jl

@@ -559,15 +559,15 @@ function _define_data_iter_creator(hdr :: MX_handle; gen_docs::Bool=false)
           (MX_handle, Ref{char_p}, Ref{char_p}, Ref{MX_uint}, Ref{char_pp}, Ref{char_pp}, Ref{char_pp}),
           hdr, ref_name, ref_desc, ref_narg, ref_arg_names, ref_arg_types, ref_arg_descs)
 
-  iter_name = Symbol(String(ref_name[]))
+  iter_name = Symbol(unsafe_wrap(String, ref_name[]))
 
   if gen_docs
     if endswith(string(iter_name), "Iter")
       f_desc = "Can also be called with the alias ``$(string(iter_name)[1:end-4] * "Provider")``.\n"
     else
       f_desc = ""
     end
-    f_desc *= String(ref_desc[]) * "\n\n"
+    f_desc *= unsafe_string(ref_desc[]) * "\n\n"
     f_desc *= ":param Base.Symbol data_name: keyword argument, default ``:data``. The name of the data.\n"
     f_desc *= ":param Base.Symbol label_name: keyword argument, default ``:softmax_label``. " *
               "The name of the label. Could be ``nothing`` if no label is presented in this dataset.\n\n"
@@ -578,8 +578,8 @@ function _define_data_iter_creator(hdr :: MX_handle; gen_docs::Bool=false)
 
   defun = quote
     function $iter_name(; kwargs...)
-      arg_keys = AbstractString[string(k) for (k,v) in kwargs]
-      arg_vals = AbstractString[dump_mx_param(v) for (k,v) in kwargs]
+      arg_keys = String[string(k) for (k,v) in kwargs]
+      arg_vals = String[dump_mx_param(v) for (k,v) in kwargs]
       ref_hdr  = Ref{MX_handle}(0)
 
       @mxcall(:MXDataIterCreateIter, (MX_handle, MX_uint, char_pp, char_pp, Ref{MX_handle}),
@@ -603,7 +603,7 @@ function _import_io_iterators(;gen_docs::Bool=false)
   @mxcall(:MXListDataIters, (Ref{MX_uint}, Ref{Ptr{MX_handle}}), n_ref, h_ref)
 
   n_creators = n_ref[]
-  h_creators = pointer_to_array(h_ref[], n_creators)
+  h_creators = unsafe_wrap(Array, h_ref[], n_creators)
 
   if gen_docs
     docs = Dict{Base.Symbol, AbstractString}()

src/kvstore.jl

@@ -87,7 +87,7 @@ end
 function get_type(self :: KVStore)
   type_ref = Ref{char_p}(0)
   @mxcall(:MXKVStoreGetType, (MX_handle, Ref{char_p}), self, type_ref)
-  return Symbol(@compat String(type_ref[]))
+  return Symbol(unsafe_wrap(String, type_ref[]))
 end
 
 function get_num_workers(self :: KVStore)

src/model.jl

@@ -385,8 +385,8 @@ function fit(self :: FeedForward, optimizer :: AbstractOptimizer, data :: Abstra
 
   train_execs = Array(Executor, num_dev)
   for i = 1:num_dev
-    data_shapes = [k => tuple(v[1:end-1]...,length(slices[i])) for (k,v) in provide_data(data)]
-    label_shapes = [k => tuple(v[1:end-1]...,length(slices[i])) for (k,v) in provide_label(data)]
+    data_shapes = Dict([k => tuple(v[1:end-1]...,length(slices[i])) for (k,v) in provide_data(data)])
+    label_shapes = Dict([k => tuple(v[1:end-1]...,length(slices[i])) for (k,v) in provide_label(data)])
     train_execs[i] = simple_bind(self.arch, self.ctx[i]; grad_req=grad_req, data_shapes..., label_shapes...)
     dbg_str = mx.debug_str(train_execs[i])
     info(string("TempSpace: ", split(dbg_str, ['\n'])[end-2]..., " on ", self.ctx[i]))

src/ndarray.jl

@@ -251,7 +251,7 @@ function size(arr :: NDArray)
   ref_shape = Ref{Ptr{MX_uint}}(0)
   @mxcall(:MXNDArrayGetShape, (MX_handle, Ref{MX_uint}, Ref{Ptr{MX_uint}}),
           arr, ref_ndim, ref_shape)
-  tuple(map(Int, flipdim(pointer_to_array(ref_shape[], ref_ndim[]),1))...)
+  tuple(map(Int, flipdim(unsafe_wrap(Array, ref_shape[], ref_ndim[]),1))...)
 end
 function size(arr :: NDArray, dim :: Int)
   size(arr)[dim]
@@ -824,8 +824,7 @@ end
 function try_get_shared(arr :: NDArray)
   if context(arr).device_type == CPU
     # try to do data sharing
-    vec = pointer_to_array(pointer(arr), length(arr))
-    return reshape(vec, size(arr))
+    return unsafe_wrap(Array, pointer(arr), size(arr))
   else
     # impossible to share, just copying
     return copy(arr)
@@ -876,11 +875,11 @@ function load(filename::AbstractString, ::Type{NDArray})
   out_name_size = out_name_size[]
   out_size      = out_size[]
   if out_name_size == 0
-    return [NDArray(MX_NDArrayHandle(hdr)) for hdr in pointer_to_array(out_hdrs[], out_size)]
+    return [NDArray(MX_NDArrayHandle(hdr)) for hdr in unsafe_wrap(Array, out_hdrs[], out_size)]
   else
     @assert out_size == out_name_size
-    return Dict([(Symbol(@compat String(k)), NDArray(MX_NDArrayHandle(hdr))) for (k,hdr) in
-                 zip(pointer_to_array(out_names[], out_size), pointer_to_array(out_hdrs[], out_size))])
+    return Dict([(Symbol(unsafe_wrap(String, k)), NDArray(MX_NDArrayHandle(hdr))) for (k,hdr) in
+                 zip(unsafe_wrap(Array, out_names[], out_size), unsafe_wrap(Array, out_hdrs[], out_size))])
   end
 end
 
@@ -998,8 +997,10 @@ function _get_function_description(handle :: MX_handle)
           ref_arg_types, ref_arg_descs, ref_ret_type)
 
   name = Symbol(unsafe_wrap(String, ref_name[]))
-
-  desc = unsafe_wrap(String, ref_desc[]) * "\n\n"
+  signature = _format_signature(Int(ref_narg[]), ref_arg_names)
+  desc = "    " * string(name) * "(" * signature * ")\n\n"
+  desc *= unsafe_wrap(String, ref_desc[]) * "\n\n"
+  desc *= "# Arguments\n"
   desc *= _format_docstring(Int(ref_narg[]), ref_arg_names, ref_arg_types, ref_arg_descs)
   return name, desc
 end
@@ -1081,9 +1082,17 @@ macro _import_ndarray_functions()
     name, desc = _get_function_description(handle)
     exprs = _get_function_expressions(handle, name)
 
-    expr = quote
-      $(exprs...)
-      @doc $desc $name
+    # TODO(vchuravy): Fix this in a more elegant way once we only support
+    # v0.5
+    if isdefined(Base, name) || isdefined(name)
+      expr = quote
+        $(exprs...)
+      end
+    else
+      expr = quote
+        $(exprs...)
+        @doc $desc $name
+      end
     end
 
     push!(func_exprs, expr)

src/symbolic-node.jl

@@ -32,18 +32,19 @@ function Base.copy(self :: SymbolicNode)
   Base.deepcopy(self)
 end
 
+# TODO(vchuravy) How to add documentation to the v0.5 style call overloading
 """
     call(self :: SymbolicNode, args :: SymbolicNode...)
     call(self :: SymbolicNode; kwargs...)
 
 Make a new node by composing ``self`` with ``args``. Or the arguments
 can be specified using keyword arguments.
 """
-function Base.call(self :: SymbolicNode, args :: SymbolicNode...)
+@compat function (self::SymbolicNode)(args :: SymbolicNode...)
   s = deepcopy(self)
   _compose!(s, args...)
 end
-function Base.call(self :: SymbolicNode; kwargs...)
+@compat function (self::SymbolicNode)(;kwargs...)
   s = deepcopy(self)
   _compose!(s; kwargs...)
 end
@@ -55,8 +56,8 @@ macro _list_symbol_info(self, func_name)
     @mxcall($func_name, (MX_handle, Ref{MX_uint}, Ref{char_pp}),
             $self, ref_sz, ref_names)
     narg = ref_sz[]
-    names = pointer_to_array(ref_names[], narg)
-    names = [Symbol(@compat String(x)) for x in names]
+    names = unsafe_wrap(Array, ref_names[], narg)
+    names = [Symbol(unsafe_wrap(String, x)) for x in names]
     return names
   end
 end
@@ -123,13 +124,13 @@ Get attribute attached to this :class:`SymbolicNode` belonging to key.
 :return: The value belonging to key as a :class:`Nullable`.
 """
 function get_attr(self :: SymbolicNode, key :: Symbol)
-  key_s = @compat String(string(key))
+  key_s = string(key)
   ref_out = Ref{Cstring}()
   ref_success = Ref{Cint}(-1)
   @mxcall(:MXSymbolGetAttr, (MX_handle, Cstring, Ref{Cstring}, Ref{Cint}),
           self, key_s, ref_out, ref_success)
   if ref_success[] == 1
-    return Nullable{String}(@compat String(ref_out[]))
+    return Nullable{String}(unsafe_string(ref_out[]))
   else
     return Nullable{String}()
   end
@@ -147,11 +148,11 @@ function list_attr(self :: SymbolicNode)
   @mxcall(:MXSymbolListAttrShallow, (MX_handle, Ref{MX_uint}, Ref{char_pp}),
             self, ref_sz, ref_strings)
   narg = 2*ref_sz[]
-  strings = pointer_to_array(ref_strings[], narg)
+  strings = unsafe_wrap(Array, ref_strings[], narg)
   out = Dict{Symbol, String}()
   for i in 1:2:narg
-    key = Symbol(@compat String(strings[i]))
-    value = @compat String(strings[i+1])
+    key = Symbol(unsafe_wrap(String, strings[i]))
+    value = unsafe_string(strings[i+1]) # Creates a copy of string
     out[key] = value
   end
   return out
@@ -169,11 +170,11 @@ function list_all_attr(self :: SymbolicNode)
   @mxcall(:MXSymbolListAttr, (MX_handle, Ref{MX_uint}, Ref{char_pp}),
             self, ref_sz, ref_strings)
   narg = 2*ref_sz[]
-  strings = pointer_to_array(ref_strings[], narg)
+  strings = unsafe_wrap(Array, ref_strings[], narg)
   out = Dict{Symbol, String}()
   for i in 1:2:narg
-    key = Symbol(@compat String(strings[i]))
-    value = @compat String(strings[i+1])
+    key = Symbol(unsafe_wrap(String, strings[i]))
+    value = unsafe_string(strings[i+1])
     out[key] = value
   end
   return out
@@ -191,8 +192,8 @@ the attributes of a :class:`SymbolicNode` that is already been used somewhere el
 cause unexpected behavior and inconsistency.
 """
 function set_attr(self :: SymbolicNode, key :: Symbol, value :: AbstractString)
-  key_s = @compat String(string(key))
-  value_s = @compat String(value)
+  key_s = string(key)
+  value_s = String(value)
 
   @mxcall(:MXSymbolSetAttr, (MX_handle, Cstring, Cstring), self, key_s, value_s)
 end
@@ -231,10 +232,10 @@ function Group(nodes :: SymbolicNode...)
 end
 
 function _build_shapes(shape_size::MX_uint, shape_ndim::Ptr{MX_uint}, shape_data::Ptr{Ptr{MX_uint}})
-  shape_ndim = pointer_to_array(shape_ndim, shape_size)
-  shape_data = pointer_to_array(shape_data, shape_size)
+  shape_ndim = unsafe_wrap(Array, shape_ndim, shape_size)
+  shape_data = unsafe_wrap(Array, shape_data, shape_size)
   shapes = map(1:shape_size) do i
-    my_shape = pointer_to_array(shape_data[i], shape_ndim[i])
+    my_shape = unsafe_wrap(Array, shape_data[i], shape_ndim[i])
     tuple(flipdim(Int[my_shape...],1)...)
   end
   convert(Vector{Tuple}, shapes)
@@ -332,9 +333,9 @@ function _infer_type(self, keys, arg_type_data)
   if ref_complete[] == 0
     return (nothing, nothing, nothing)
   else
-    in_type = pointer_to_array(ref_in_type_data[], ref_in_type_size[])
-    out_type = pointer_to_array(ref_out_type_data[], ref_out_type_size[])
-    aux_type = pointer_to_array(ref_aux_type_data[], ref_aux_type_size[])
+    in_type = unsafe_wrap(Array, ref_in_type_data[], ref_in_type_size[])
+    out_type = unsafe_wrap(Array, ref_out_type_data[], ref_out_type_size[])
+    aux_type = unsafe_wrap(Array, ref_aux_type_data[], ref_aux_type_size[])
     return ([fromTypeFlag(TypeFlag(t)) for t in in_type],
             [fromTypeFlag(TypeFlag(t)) for t in out_type],
             [fromTypeFlag(TypeFlag(t)) for t in aux_type])
@@ -528,7 +529,7 @@ Convert a :class:`SymbolicNode` into a JSON string.
 function to_json(self :: SymbolicNode)
   ref_json = Ref{char_p}(0)
   @mxcall(:MXSymbolSaveToJSON, (MX_handle, Ref{char_p}), self, ref_json)
-  return @compat String(ref_json[])
+  return unsafe_string(ref_json[])
 end
 
 """
@@ -586,14 +587,17 @@ function _define_atomic_symbol_creator(hdr :: MX_handle)
   kv_nargs_s  = unsafe_wrap(String, ref_kv_nargs[])
   kv_nargs    = Symbol(kv_nargs_s)
 
+  signature = _format_signature(Int(ref_nargs[]), ref_arg_names)
+  f_desc = "    " * func_name_s * "(" * signature * ")\n\n"
   f_desc = unsafe_wrap(String, ref_desc[]) * "\n\n"
   if !isempty(kv_nargs_s)
-    f_desc *= "This function support variable length positional :class:`SymbolicNode` inputs.\n\n"
+    f_desc *= "This function support variable length positional `SymbolicNode` inputs.\n\n"
   end
+  f_desc *= "# Arguments\n"
   f_desc *= _format_docstring(Int(ref_nargs[]), ref_arg_names, ref_arg_types, ref_arg_descs)
-  f_desc *= ":param Symbol name: The name of the :class:`SymbolicNode`. (e.g. `:my_symbol`), optional.\n"
-  f_desc *= ":param Dict{Symbol, AbstractString} attrs: The attributes associated with this :class:`SymbolicNode`.\n\n"
-  f_desc *= ":return: $(_format_typestring(unsafe_wrap(String, ref_ret_type[]))).\n\n"
+  f_desc *= "* `name::Symbol`: The name of the `SymbolicNode`. (e.g. `:my_symbol`), optional.\n"
+  f_desc *= "* `attrs::Dict{Symbol, AbstractString}`: The attributes associated with this `SymbolicNode`.\n\n"
+  f_desc *= "Returns `$(_format_typestring(unsafe_wrap(String, ref_ret_type[])))`."
 
   # function $func_name(args...; kwargs...)
   func_head = Expr(:call, func_name, Expr(:parameters, Expr(:..., :kwargs)), Expr(:..., :args))
@@ -675,9 +679,14 @@ function _define_atomic_symbol_creator(hdr :: MX_handle)
   end
 
   func_def = Expr(:function, func_head, Expr(:block, func_body))
-  quote
-    $func_def
-    @doc $f_desc $func_name
+  # TODO(vchuravy) find a more elegant solution fro v0.5
+  if isdefined(Base, func_name) || isdefined(func_name)
+    return func_def
+  else
+    return quote
+      $func_def
+      @doc $f_desc $func_name
+    end
   end
 end