From 692e87c1888b31af439d5fc42e4b5f8197fc5dc9 Mon Sep 17 00:00:00 2001 From: Koichi Akabe Date: Sat, 4 Aug 2018 08:35:13 +0900 Subject: [PATCH] Implement contrib functions in Python --- primitiv-core | 2 +- primitiv/_device.pxd | 2 +- primitiv/_function.pxd | 18 +- primitiv/_function.pyx | 189 +++++++++++--------- primitiv/_graph.pxd | 4 +- primitiv/_initializer.pxd | 2 +- primitiv/_model.pxd | 2 +- primitiv/_optimizer.pxd | 2 +- primitiv/_parameter.pxd | 2 +- primitiv/_shape.pxd | 2 +- primitiv/_tensor.pxd | 2 +- primitiv/devices/_cuda_device.pxd | 2 +- primitiv/devices/_eigen_device.pxd | 2 +- primitiv/devices/_naive_device.pxd | 2 +- primitiv/devices/_opencl_device.pxd | 2 +- primitiv/initializers/_initializer_impl.pxd | 2 +- primitiv/optimizers/_optimizer_impl.pxd | 2 +- primitiv/py_optimizer.h | 3 +- 18 files changed, 127 insertions(+), 115 deletions(-) diff --git a/primitiv-core b/primitiv-core index c4e6ec2..cb10da9 160000 --- a/primitiv-core +++ b/primitiv-core @@ -1 +1 @@ -Subproject commit c4e6ec2be9732afdb21c6c272f8adbc2d0621765 +Subproject commit cb10da975b0b63f4f816917500dd353755fc0424 diff --git a/primitiv/_device.pxd b/primitiv/_device.pxd index b7526e3..fa3740f 100644 --- a/primitiv/_device.pxd +++ b/primitiv/_device.pxd @@ -1,4 +1,4 @@ -cdef extern from "primitiv/device.h": +cdef extern from "primitiv/core/device.h": cdef cppclass CppDevice "primitiv::Device": void dump_description() except + diff --git a/primitiv/_function.pxd b/primitiv/_function.pxd index bd0ec59..a97ef21 100644 --- a/primitiv/_function.pxd +++ b/primitiv/_function.pxd @@ -9,7 +9,7 @@ from primitiv._shape cimport CppShape from primitiv._parameter cimport CppParameter -cdef extern from "primitiv/functions.h": +cdef extern from "primitiv/core/basic_functions.h": CppTensor func_input_tensor "primitiv::functions::input_tensor" (const CppShape &shape, const vector[float] &data, CppDevice *dev) except + CppNode func_input_node "primitiv::functions::input_node" (const CppShape &shape, const vector[float] &data, CppDevice *dev, CppGraph *g) except + CppTensor func_parameter_tensor "primitiv::functions::parameter_tensor" (CppParameter ¶m) except + @@ -40,12 +40,7 @@ cdef extern from "primitiv/functions.h": Var func_prelu "primitiv::functions::prelu" [Var](const Var &x, float a) except + Var func_elu "primitiv::functions::elu" [Var](const Var &x, float a) except + Var func_selu "primitiv::functions::selu" [Var](const Var &x, float a, float s) except + - CppNode func_sum "primitiv::functions::sum" (const vector[CppNode] &xs) except + - CppTensor func_sum "primitiv::functions::sum" (const vector[CppTensor] &xs) except + Var func_sum "primitiv::functions::sum" [Var](const Var &x, unsigned dim) except + - CppNode func_mean "primitiv::functions::mean" (const vector[CppNode] &xs) except + - CppTensor func_mean "primitiv::functions::mean" (const vector[CppTensor] &xs) except + - Var func_mean "primitiv::functions::mean" [Var](const Var &x, unsigned dim) except + Var func_broadcast "primitiv::functions::broadcast" [Var](const Var &x, unsigned dim, unsigned size) except + Var func_logsumexp "primitiv::functions::logsumexp" [Var](const Var &x, unsigned dim) except + Var func_log_softmax "primitiv::functions::log_softmax" [Var](const Var &x, unsigned dim) except + @@ -58,13 +53,8 @@ cdef extern from "primitiv/functions.h": CppTensor func_constant_tensor "primitiv::functions::constant_tensor" (const CppShape &shape, float k, CppDevice *dev) except + CppNode func_constant_node "primitiv::functions::constant_node" (const CppShape &shape, float k, CppDevice *dev, CppGraph *g) except + - CppTensor func_zeros_tensor "primitiv::functions::zeros_tensor" (const CppShape &shape, CppDevice *dev) except + - CppNode func_zeros_node "primitiv::functions::zeros_node" (const CppShape &shape, CppDevice *dev, CppGraph *g) except + - CppTensor func_ones_tensor "primitiv::functions::ones_tensor" (const CppShape &shape, CppDevice *dev) except + - CppNode func_ones_node "primitiv::functions::ones_node" (const CppShape &shape, CppDevice *dev, CppGraph *g) except + CppTensor func_identity_tensor "primitiv::functions::identity_tensor" (unsigned size, CppDevice *dev) except + CppNode func_identity_node "primitiv::functions::identity_node" (unsigned size, CppDevice *dev, CppGraph *g) except + - Var func_dropout "primitiv::functions::dropout" [Var](const Var &x, float rate, bool enabled) except + Var func_positive "primitiv::functions::positive" [Var](const Var &x) except + Var func_negative "primitiv::functions::negative" [Var](const Var &x) except + @@ -82,13 +72,11 @@ cdef extern from "primitiv/functions.h": Var func_divide "primitiv::functions::divide" [Var](const Var &a, const Var &b) except + -cdef extern from "primitiv/functions.h": +cdef extern from "primitiv/core/basic_functions.h": Var func_batch_sum "primitiv::functions::batch::sum" [Var](const Var &x) except + - Var func_batch_mean "primitiv::functions::batch::mean" [Var](const Var &x) except + - Var func_batch_normalize "primitiv::functions::batch::normalize" [Var](const Var &x) except + -cdef extern from "primitiv/functions.h": +cdef extern from "primitiv/core/basic_functions.h": CppNode func_random_bernoulli_node "primitiv::functions::random::bernoulli_node" (const CppShape &shape, float p, CppDevice *dev, CppGraph *g) except + CppTensor func_random_bernoulli_tensor "primitiv::functions::random::bernoulli_tensor" (const CppShape &shape, float p, CppDevice *dev) except + diff --git a/primitiv/_function.pyx b/primitiv/_function.pyx index e0fe574..76358d9 100644 --- a/primitiv/_function.pyx +++ b/primitiv/_function.pyx @@ -158,32 +158,13 @@ class functions: def elu(Node x, float a): return wrapNode(func_elu(x.wrapped, a)) - @staticmethod - def selu(Node x, float a, float s): - return wrapNode(func_selu(x.wrapped, a, s)) - @staticmethod def sum(x, dim = None): - cdef vector[CppNode] xs - cdef Node node if isinstance(x, list): - for node in x: - xs.push_back(node.wrapped) - return wrapNode(func_sum(xs)) + return functions.sum_list(x) else: return wrapNode(func_sum(( x).wrapped, dim)) - @staticmethod - def mean(x, dim = None): - cdef vector[CppNode] xs - cdef Node node - if isinstance(x, list): - for node in x: - xs.push_back(node.wrapped) - return wrapNode(func_mean(xs)) - else: - return wrapNode(func_mean(( x).wrapped, dim)) - @staticmethod def broadcast(Node x, unsigned dim, unsigned size): return wrapNode(func_broadcast(x.wrapped, dim, size)) @@ -243,43 +224,76 @@ class functions: get_cpp_device(device), get_cpp_graph(graph))) @staticmethod - def zeros(shape, Device device = None, Graph graph = None): + def identity(unsigned size, Device device = None, Graph graph = None): if device is None: device = Device.get_default() if graph is None: graph = Graph.get_default() - return wrapNode(func_zeros_node(normShape(shape).wrapped, - get_cpp_device(device), get_cpp_graph(graph))) + return wrapNode(func_identity_node(size, get_cpp_device(device), get_cpp_graph(graph))) + + # contrib functions @staticmethod - def ones(shape, Device device = None, Graph graph = None): - if device is None: - device = Device.get_default() - if graph is None: - graph = Graph.get_default() - return wrapNode(func_ones_node(normShape(shape).wrapped, - get_cpp_device(device), get_cpp_graph(graph))) + def selu(Node x, float a=1.6732632423543772848170429916717, float s=1.0507009873554804934193349852946): + return s * functions.elu(x, a); @staticmethod - def identity(unsigned size, Device device = None, Graph graph = None): - if device is None: - device = Device.get_default() - if graph is None: - graph = Graph.get_default() - return wrapNode(func_identity_node(size, get_cpp_device(device), get_cpp_graph(graph))) + def sum_list(list xs): + if not xs: + raise TypeError("No nodes to sum.") + ret = xs[0] + for x in xs[1:]: + ret = ret + x + return ret + + @staticmethod + def mean(x, dim = None): + if isinstance(x, list): + return functions.sum_list(x) / len(x) + else: + return functions.sum(x, dim) / x.shape()[dim] + + @staticmethod + def zeros(shape, Device dev = None, Graph g = None): + return functions.constant(shape, 0.0, dev, g) + + @staticmethod + def ones(shape, Device dev = None, Graph g = None): + return functions.constant(shape, 1.0, dev, g) + + @staticmethod + def dropout(Node x, float rate, bool enabled): + if not enabled: + return x + if rate == 1.0: + return 0.0 * x + p = 1.0 - rate + return (1.0 / p) * x * functions.random.bernoulli(x.shape(), p, x.device()) + + # end contrib functions class batch: @staticmethod def sum(Node x): return wrapNode(func_batch_sum[CppNode](x.wrapped)) + # contrib functions + @staticmethod def mean(Node x): - return wrapNode(func_batch_mean[CppNode](x.wrapped)) + return functions.batch.sum(x) / x.shape().batch() @staticmethod def normalize(Node x): - return wrapNode(func_batch_normalize[CppNode](x.wrapped)) + if not x.shape().has_batch(): + return x + b = x.shape().batch() + scale = b / (b - 1) + m = functions.batch.mean(x) + v = scale * (functions.batch.mean(x * x) - m * m) + return (x - m) / functions.sqrt(v + 1e-8) + + # end contrib functions class random: @staticmethod @@ -327,10 +341,6 @@ class functions: return wrapNode(func_random_gumbel_node(normShape(shape).wrapped, mu, beta, get_cpp_device(device), get_cpp_graph(graph))) - @staticmethod - def dropout(Node x, float rate, bool enabled): - return wrapNode(func_dropout(x.wrapped, rate, enabled)) - class tensor_functions: @@ -472,32 +482,13 @@ class tensor_functions: def elu(Tensor x, float a): return Tensor.get_wrapper_with_new(new CppTensor(func_elu(x.wrapped[0], a))) - @staticmethod - def selu(Tensor x, float a, float s): - return Tensor.get_wrapper_with_new(new CppTensor(func_selu(x.wrapped[0], a, s))) - @staticmethod def sum(x, dim = None): - cdef vector[CppTensor] xs - cdef Tensor t if isinstance(x, list): - for t in x: - xs.push_back(t.wrapped[0]) - return Tensor.get_wrapper_with_new(new CppTensor(func_sum(xs))) + return tensor_functions.sum_list(x) else: return Tensor.get_wrapper_with_new(new CppTensor(func_sum(( x).wrapped[0], dim))) - @staticmethod - def mean(x, dim = None): - cdef vector[CppTensor] xs - cdef Tensor t - if isinstance(x, list): - for t in x: - xs.push_back(t.wrapped[0]) - return Tensor.get_wrapper_with_new(new CppTensor(func_mean(xs))) - else: - return Tensor.get_wrapper_with_new(new CppTensor(func_mean(( x).wrapped[0], dim))) - @staticmethod def broadcast(Tensor x, unsigned dim, unsigned size): return Tensor.get_wrapper_with_new(new CppTensor(func_broadcast(x.wrapped[0], dim, size))) @@ -554,38 +545,76 @@ class tensor_functions: return Tensor.get_wrapper_with_new(new CppTensor(func_constant_tensor(normShape(shape).wrapped, k, get_cpp_device(device)))) + @staticmethod - def zeros(shape, Device device = None): + def identity(unsigned size, Device device = None): if device is None: device = Device.get_default() - return Tensor.get_wrapper_with_new(new CppTensor(func_zeros_tensor(normShape(shape).wrapped, - get_cpp_device(device)))) + return Tensor.get_wrapper_with_new(new CppTensor(func_identity_tensor(size, get_cpp_device(device)))) + + # contrib functions @staticmethod - def ones(shape, Device device = None): - if device is None: - device = Device.get_default() - return Tensor.get_wrapper_with_new(new CppTensor(func_ones_tensor(normShape(shape).wrapped, - get_cpp_device(device)))) + def selu(Node x, float a=1.6732632423543772848170429916717, float s=1.0507009873554804934193349852946): + return s * tensor_functions.elu(x, a); @staticmethod - def identity(unsigned size, Device device = None): - if device is None: - device = Device.get_default() - return Tensor.get_wrapper_with_new(new CppTensor(func_identity_tensor(size, get_cpp_device(device)))) + def sum_list(list xs): + if not xs: + raise TypeError("No nodes to sum.") + ret = xs[0] + for x in xs[1:]: + ret = ret + x + return ret + + @staticmethod + def mean(x, dim = None): + if isinstance(x, list): + return tensor_functions.sum_list(x) / len(x) + else: + return tensor_functions.sum(x, dim) / x.shape()[dim] + + @staticmethod + def zeros(shape, Device dev = None): + return tensor_functions.constant(shape, 0.0, dev) + + @staticmethod + def ones(shape, Device dev = None): + return tensor_functions.constant(shape, 1.0, dev) + + @staticmethod + def dropout(Node x, float rate, bool enabled): + if not enabled: + return x + if rate == 1.0: + return 0.0 * x + p = 1.0 - rate + return (1.0 / p) * x * tensor_functions.random.bernoulli(x.shape(), p, x.device()) + + # end contrib functions class batch: @staticmethod def sum(Tensor x): return Tensor.get_wrapper_with_new(new CppTensor(func_batch_sum[CppTensor](x.wrapped[0]))) + # contrib functions + @staticmethod - def mean(Tensor x): - return Tensor.get_wrapper_with_new(new CppTensor(func_batch_mean[CppTensor](x.wrapped[0]))) + def mean(Node x): + return tensor_functions.batch.sum(x) / x.shape().batch() @staticmethod - def normalize(Tensor x): - return Tensor.get_wrapper_with_new(new CppTensor(func_batch_normalize[CppTensor](x.wrapped[0]))) + def normalize(Node x): + if not x.shape().has_batch(): + return x + b = x.shape().batch() + scale = b / (b - 1) + m = tensor_functions.batch.mean(x) + v = scale * (tensor_functions.batch.mean(x * x) - m * m) + return (x - m) / tensor_functions.sqrt(v + 1e-8) + + # end contrib functions class random: @staticmethod @@ -622,7 +651,3 @@ class tensor_functions: device = Device.get_default() return Tensor.get_wrapper_with_new(new CppTensor(func_random_gumbel_tensor(normShape(shape).wrapped, mu, beta, get_cpp_device(device)))) - - @staticmethod - def dropout(Tensor x, float rate, bool enabled): - return Tensor.get_wrapper_with_new(new CppTensor(func_dropout(x.wrapped[0], rate, enabled))) diff --git a/primitiv/_graph.pxd b/primitiv/_graph.pxd index 6c68614..3c60c83 100644 --- a/primitiv/_graph.pxd +++ b/primitiv/_graph.pxd @@ -6,7 +6,7 @@ from primitiv._shape cimport CppShape from primitiv._tensor cimport CppTensor -cdef extern from "primitiv/graph.h" nogil: +cdef extern from "primitiv/core/graph.h" nogil: cdef cppclass CppNode "primitiv::Node": CppNode(CppNode &&src) except + CppNode() except + @@ -23,7 +23,7 @@ cdef extern from "primitiv/graph.h" nogil: void backward() except + -cdef extern from "primitiv/graph.h" nogil: +cdef extern from "primitiv/core/graph.h" nogil: cdef cppclass CppGraph "primitiv::Graph": CppGraph() except + void clear() except + diff --git a/primitiv/_initializer.pxd b/primitiv/_initializer.pxd index 86c233b..a352794 100644 --- a/primitiv/_initializer.pxd +++ b/primitiv/_initializer.pxd @@ -1,7 +1,7 @@ from primitiv._tensor cimport CppTensor -cdef extern from "primitiv/initializer.h": +cdef extern from "primitiv/core/initializer.h": cdef cppclass CppInitializer "primitiv::Initializer": CppInitializer() except + void apply(CppTensor &x) except + diff --git a/primitiv/_model.pxd b/primitiv/_model.pxd index a7796a2..39c4aac 100644 --- a/primitiv/_model.pxd +++ b/primitiv/_model.pxd @@ -7,7 +7,7 @@ from primitiv._device cimport CppDevice from primitiv._parameter cimport CppParameter -cdef extern from "primitiv/model.h": +cdef extern from "primitiv/core/model.h": cdef cppclass CppModel "primitiv::Model": CppModel() except + void load(string &path, bool with_stats, CppDevice *device) except + diff --git a/primitiv/_optimizer.pxd b/primitiv/_optimizer.pxd index 23d9e37..ac141fb 100644 --- a/primitiv/_optimizer.pxd +++ b/primitiv/_optimizer.pxd @@ -10,7 +10,7 @@ from primitiv._parameter cimport CppParameter, Parameter from primitiv._shape cimport CppShape -cdef extern from "primitiv/optimizer.h": +cdef extern from "primitiv/core/optimizer.h": cdef cppclass CppOptimizer "primitiv::Optimizer": CppOptimizer(CppOptimizer &&) except + CppOptimizer() except + diff --git a/primitiv/_parameter.pxd b/primitiv/_parameter.pxd index 02f155c..4d240d2 100644 --- a/primitiv/_parameter.pxd +++ b/primitiv/_parameter.pxd @@ -9,7 +9,7 @@ from primitiv._device cimport CppDevice from primitiv._initializer cimport CppInitializer, Initializer -cdef extern from "primitiv/parameter.h": +cdef extern from "primitiv/core/parameter.h": cdef cppclass CppParameter "primitiv::Parameter": CppParameter() except + CppParameter(const CppShape &shape, const vector[float] &value, CppDevice *device) except + diff --git a/primitiv/_shape.pxd b/primitiv/_shape.pxd index b4beadc..e5df1d7 100644 --- a/primitiv/_shape.pxd +++ b/primitiv/_shape.pxd @@ -3,7 +3,7 @@ from libcpp.string cimport string from libcpp cimport bool -cdef extern from "primitiv/shape.h": +cdef extern from "primitiv/core/shape.h": cdef cppclass CppShape "primitiv::Shape": CppShape() except + CppShape(vector[unsigned] &dims, unsigned batch) except + diff --git a/primitiv/_tensor.pxd b/primitiv/_tensor.pxd index e9040cf..fdd91e3 100644 --- a/primitiv/_tensor.pxd +++ b/primitiv/_tensor.pxd @@ -5,7 +5,7 @@ from primitiv._device cimport CppDevice from primitiv._shape cimport CppShape -cdef extern from "primitiv/tensor.h" nogil: +cdef extern from "primitiv/core/tensor.h" nogil: cdef cppclass CppTensor "primitiv::Tensor": CppTensor(CppTensor &&src) except + CppTensor() except + diff --git a/primitiv/devices/_cuda_device.pxd b/primitiv/devices/_cuda_device.pxd index 8d43185..19c147b 100644 --- a/primitiv/devices/_cuda_device.pxd +++ b/primitiv/devices/_cuda_device.pxd @@ -1,7 +1,7 @@ from primitiv._device cimport CppDevice, Device -cdef extern from "primitiv/cuda_device.h": +cdef extern from "primitiv/devices/cuda/device.h": cdef cppclass CppCUDA "primitiv::devices::CUDA" (CppDevice): CppCUDA(unsigned device_id) except + CppCUDA(unsigned device_id, unsigned rng_seed) except + diff --git a/primitiv/devices/_eigen_device.pxd b/primitiv/devices/_eigen_device.pxd index 8d59ea7..84a2c4d 100644 --- a/primitiv/devices/_eigen_device.pxd +++ b/primitiv/devices/_eigen_device.pxd @@ -1,7 +1,7 @@ from primitiv._device cimport CppDevice, Device -cdef extern from "primitiv/eigen_device.h": +cdef extern from "primitiv/devices/eigen/device.h": cdef cppclass CppEigen "primitiv::devices::Eigen" (CppDevice): CppEigen() except + CppEigen(unsigned rng_seed) except + diff --git a/primitiv/devices/_naive_device.pxd b/primitiv/devices/_naive_device.pxd index 9184f68..a247e3d 100644 --- a/primitiv/devices/_naive_device.pxd +++ b/primitiv/devices/_naive_device.pxd @@ -1,7 +1,7 @@ from primitiv._device cimport CppDevice, Device -cdef extern from "primitiv/naive_device.h": +cdef extern from "primitiv/devices/naive/device.h": cdef cppclass CppNaive "primitiv::devices::Naive" (CppDevice): CppNaive() except + CppNaive(unsigned rng_seed) except + diff --git a/primitiv/devices/_opencl_device.pxd b/primitiv/devices/_opencl_device.pxd index f4c7bc3..f7788cf 100644 --- a/primitiv/devices/_opencl_device.pxd +++ b/primitiv/devices/_opencl_device.pxd @@ -1,7 +1,7 @@ from primitiv._device cimport CppDevice, Device -cdef extern from "primitiv/opencl_device.h": +cdef extern from "primitiv/devices/opencl/device.h": cdef cppclass CppOpenCL "primitiv::devices::OpenCL" (CppDevice): CppOpenCL(unsigned platform_id, unsigned device_id) except + CppOpenCL(unsigned platform_id, unsigned device_id, unsigned rng_seed) except + diff --git a/primitiv/initializers/_initializer_impl.pxd b/primitiv/initializers/_initializer_impl.pxd index 7cd41bb..433b1a5 100644 --- a/primitiv/initializers/_initializer_impl.pxd +++ b/primitiv/initializers/_initializer_impl.pxd @@ -1,7 +1,7 @@ from primitiv._initializer cimport CppInitializer, Initializer -cdef extern from "primitiv/initializer_impl.h": +cdef extern from "primitiv/core/initializer_impl.h": cdef cppclass CppConstant "primitiv::initializers::Constant" (CppInitializer): CppConstant(float k) diff --git a/primitiv/optimizers/_optimizer_impl.pxd b/primitiv/optimizers/_optimizer_impl.pxd index 5a349a4..2654281 100644 --- a/primitiv/optimizers/_optimizer_impl.pxd +++ b/primitiv/optimizers/_optimizer_impl.pxd @@ -5,7 +5,7 @@ from primitiv._device cimport CppDevice from primitiv._optimizer cimport CppOptimizer, Optimizer -cdef extern from "primitiv/optimizer_impl.h": +cdef extern from "primitiv/core/optimizer_impl.h": cdef cppclass CppSGD "primitiv::optimizers::SGD" (CppOptimizer): CppSGD(float eta) float eta() diff --git a/primitiv/py_optimizer.h b/primitiv/py_optimizer.h index 961cf7a..4602603 100644 --- a/primitiv/py_optimizer.h +++ b/primitiv/py_optimizer.h @@ -1,8 +1,7 @@ #ifndef PRIMITIV_PYTHON_PY_OPTIMIZER_H_ #define PRIMITIV_PYTHON_PY_OPTIMIZER_H_ -#include -#include +#include __PYX_EXTERN_C int primitiv_python_optimizer_get_configs( PyObject *obj,