From a6e238b314f6b8c72e20e4c293a5985359c57f87 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sun, 10 Dec 2023 21:10:10 -0800 Subject: [PATCH 01/95] switching machines --- include/ClientPool.h | 56 +++++++++++++++++++++++++++++++++ include/Export.h | 9 ++++-- src/ClientPool.cpp | 73 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 include/ClientPool.h create mode 100644 src/ClientPool.cpp diff --git a/include/ClientPool.h b/include/ClientPool.h new file mode 100644 index 00000000..4c4a3c48 --- /dev/null +++ b/include/ClientPool.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include "Client.h" + +namespace douban { +namespace mc { + +template +void duplicate_strings(const char* const * strs, const size_t n, + std::vector >& out, std::vector& refs) { + out.resize(n); + refs.resize(n); + for (int i = 0; i < n; i++) { + if (strs == NULL || strs[i] == NULL) { + out[i][0] = '\0'; + refs[i] = NULL; + continue; + } + std::snprintf(out[i].data(), N, "%s", strs[i]); + refs[i] = &out[i]; + } +} + +class ClientPool { +public: + ClientPool(); + ~ClientPool(); + void config(config_options_t opt, int val); + int init(const char* const * hosts, const uint32_t* ports, const size_t n, + const char* const * aliases = NULL); + int updateServers(const char* const * hosts, const uint32_t* ports, const size_t n, + const char* const * aliases = NULL); + Client* acquire(); + void release(Client* ref); + +protected: + int growPool(int by); + + bool m_opt_changed[CLIENT_CONFIG_OPTION_COUNT]; + int m_opt_value[CLIENT_CONFIG_OPTION_COUNT]; + std::vector m_clients; + int m_initial_clients; + int m_max_clients; + int m_max_growth; + + std::vector> m_hosts_data; + std::vector> m_aliases_data; + std::vector m_ports; + + std::vector m_hosts; + std::vector m_aliases; +}; + +} // namespace mc +} // namespace douban \ No newline at end of file diff --git a/include/Export.h b/include/Export.h index 97a026c0..daa3fc72 100644 --- a/include/Export.h +++ b/include/Export.h @@ -9,12 +9,17 @@ typedef enum { - CFG_POLL_TIMEOUT, + // Client config options + CFG_POLL_TIMEOUT = 0, CFG_CONNECT_TIMEOUT, CFG_RETRY_TIMEOUT, CFG_HASH_FUNCTION, - CFG_MAX_RETRIES + CFG_MAX_RETRIES, + + // ClientPool config options + CFG_MAX_CLIENTS } config_options_t; +#define CLIENT_CONFIG_OPTION_COUNT 5 typedef enum { diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp new file mode 100644 index 00000000..4217809b --- /dev/null +++ b/src/ClientPool.cpp @@ -0,0 +1,73 @@ +#include + +#include "ClientPool.h" + +namespace douban { +namespace mc { + +// default max of 4 clients to match memcached's default of 4 worker threads +ClientPool::ClientPool(): m_initial_clients(1), m_max_clients(4), m_max_growth(4) { + memset(m_opt_changed, false, sizeof m_opt_changed); +} + + +ClientPool::ClientPool() { +} + + +void ClientPool::config(config_options_t opt, int val) { + switch (val) { + case CFG_MAX_CLIENTS: + m_max_clients = val; + break; + default: + if (opt < CLIENT_CONFIG_OPTION_COUNT) { + m_opt_changed[opt] = true; + m_opt_value[opt] = val; + for (auto client : m_clients) { + client->config(opt, val); + } + } + break; + } +} + + +int ClientPool::init(const char* const * hosts, const uint32_t* ports, const size_t n, + const char* const * aliases) { + updateServers(hosts, ports, n, aliases); + growPool(m_initial_clients); +} + + +int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, const size_t n, + const char* const* aliases) { + duplicate_strings(hosts, n, m_hosts_data, m_hosts); + duplicate_strings(aliases, n, m_aliases_data, m_aliases); + + m_ports.resize(n); + std::copy(ports, ports + n, m_ports.begin()); + + for (auto client : m_clients) { + client->updateServers(hosts, ports, n, aliases); + } +} + + +int ClientPool::growPool(int by) { + // TODO +} + + +Client* acquire() { + // TODO +} + + +void release(Client* ref) { + // TODO +} + + +} // namespace mc +} // namespace douban \ No newline at end of file From 22b1e47026a54e50e28ae1c1b74449a3cae0a37b Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 11 Dec 2023 14:28:32 -0800 Subject: [PATCH 02/95] switch back... --- include/ClientPool.h | 15 +++++------ include/Export.h | 8 ++++-- src/ClientPool.cpp | 59 +++++++++++++++++++++++++++++++++----------- 3 files changed, 58 insertions(+), 24 deletions(-) diff --git a/include/ClientPool.h b/include/ClientPool.h index 4c4a3c48..4840cf02 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -35,17 +35,18 @@ class ClientPool { void release(Client* ref); protected: - int growPool(int by); + int growPool(size_t by); + int setup(Client* c); bool m_opt_changed[CLIENT_CONFIG_OPTION_COUNT]; int m_opt_value[CLIENT_CONFIG_OPTION_COUNT]; - std::vector m_clients; - int m_initial_clients; - int m_max_clients; - int m_max_growth; + std::vector m_clients; + size_t m_initial_clients; + size_t m_max_clients; + size_t m_max_growth; - std::vector> m_hosts_data; - std::vector> m_aliases_data; + std::vector > m_hosts_data; + std::vector > m_aliases_data; std::vector m_ports; std::vector m_hosts; diff --git a/include/Export.h b/include/Export.h index daa3fc72..1bf20f9d 100644 --- a/include/Export.h +++ b/include/Export.h @@ -16,10 +16,14 @@ typedef enum { CFG_HASH_FUNCTION, CFG_MAX_RETRIES, + // type separator to track number of Client config options to save + CLIENT_CONFIG_OPTION_COUNT, + // ClientPool config options - CFG_MAX_CLIENTS + CFG_INITIAL_CLIENTS, + CFG_MAX_CLIENTS, + CFG_MAX_GROWTH } config_options_t; -#define CLIENT_CONFIG_OPTION_COUNT 5 typedef enum { diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 4217809b..a8eee5a5 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include "ClientPool.h" @@ -10,64 +12,91 @@ ClientPool::ClientPool(): m_initial_clients(1), m_max_clients(4), m_max_growth(4 memset(m_opt_changed, false, sizeof m_opt_changed); } - ClientPool::ClientPool() { } - void ClientPool::config(config_options_t opt, int val) { switch (val) { + case CFG_INITIAL_CLIENTS: + m_initial_clients = val; + break; case CFG_MAX_CLIENTS: m_max_clients = val; break; + case CFG_MAX_GROWTH: + m_max_growth = val; + break; default: if (opt < CLIENT_CONFIG_OPTION_COUNT) { m_opt_changed[opt] = true; m_opt_value[opt] = val; for (auto client : m_clients) { - client->config(opt, val); + client.config(opt, val); } } break; } } - int ClientPool::init(const char* const * hosts, const uint32_t* ports, const size_t n, const char* const * aliases) { updateServers(hosts, ports, n, aliases); - growPool(m_initial_clients); + return growPool(m_initial_clients); } - int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, const size_t n, const char* const* aliases) { + // TODO: acquire lock; one update at a time duplicate_strings(hosts, n, m_hosts_data, m_hosts); duplicate_strings(aliases, n, m_aliases_data, m_aliases); m_ports.resize(n); std::copy(ports, ports + n, m_ports.begin()); - for (auto client : m_clients) { - client->updateServers(hosts, ports, n, aliases); - } + std::atomic rv = 0; + std::for_each(std::execution::par, m_clients.begin(), m_clients.end(), + [this, &rv](Client& c) { + const int idx = &c - &m_clients[0]; + // TODO: acquire lock idx + const int err = c.updateServers(m_hosts.data(), m_ports.data(), + m_hosts.size(), m_aliases.data()); + if (err != 0) { + rv.store(err, std::memory_order_relaxed); + } + }); + return rv; } - -int ClientPool::growPool(int by) { - // TODO +int ClientPool::setup(Client* c) { + for (int i = 0; i < CLIENT_CONFIG_OPTION_COUNT; i++) { + if (m_opt_changed) { + c->config(static_cast(i), m_opt_value[i]); + } + } + c->init(m_hosts.data(), m_ports.data(), m_hosts.size(), m_aliases.data()); } +int ClientPool::growPool(size_t by) { + // TODO: acquire same lock as updateServers + // TODO: client locks start busy + assert(by > 0); + size_t from = m_clients.size(); + m_clients.resize(from + by); + auto start = m_clients.data() + from; + std::for_each(std::execution::par, start, start + by, [this](Client& c) { + setup(&c); + const int idx = &c - &m_clients[0]; + // TODO: release lock idx + }); +} Client* acquire() { // TODO } - void release(Client* ref) { // TODO } - } // namespace mc -} // namespace douban \ No newline at end of file +} // namespace douban From 9fe14b3faebcbf245a6d9a4ef6ad0ace5463488a Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 12 Dec 2023 16:37:44 -0800 Subject: [PATCH 03/95] LockPool --- include/ClientPool.h | 6 ++- include/Common.h | 2 +- include/LockPool.h | 90 ++++++++++++++++++++++++++++++++++++++++++++ src/ClientPool.cpp | 34 ++++++++++------- src/Common.cpp | 4 +- 5 files changed, 118 insertions(+), 18 deletions(-) create mode 100644 include/LockPool.h diff --git a/include/ClientPool.h b/include/ClientPool.h index 4840cf02..152bfc23 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -1,7 +1,9 @@ #pragma once #include +#include #include "Client.h" +#include "LockPool.h" namespace douban { namespace mc { @@ -22,7 +24,7 @@ void duplicate_strings(const char* const * strs, const size_t n, } } -class ClientPool { +class ClientPool : LockPool { public: ClientPool(); ~ClientPool(); @@ -51,6 +53,8 @@ class ClientPool { std::vector m_hosts; std::vector m_aliases; + + std::mutex m_pool_lock; }; } // namespace mc diff --git a/include/Common.h b/include/Common.h index f7a22de8..9295465a 100644 --- a/include/Common.h +++ b/include/Common.h @@ -226,7 +226,7 @@ typedef enum { } op_code_t; const char* errCodeToString(err_code_t err); -bool isLocalSocket(const char* host); +bool isUnixSocket(const char* host); server_string_split_t splitServerString(char* input); } // namespace mc diff --git a/include/LockPool.h b/include/LockPool.h new file mode 100644 index 00000000..1c807273 --- /dev/null +++ b/include/LockPool.h @@ -0,0 +1,90 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace douban { +namespace mc { + +class OrderedLock { + std::queue m_fifo_locks; + std::mutex m_fifo_access; +public: + OrderedLock() {}; + void lock() { + std::unique_lock acquire(m_fifo_access); + std::condition_variable signal; + m_fifo_locks.emplace(&signal); + signal.wait(acquire); + } + + bool unlock() { + std::unique_lock acquire(m_fifo_access); + if (m_fifo_locks.empty()) { + return true; + } + m_fifo_locks.front()->notify_one(); + m_fifo_locks.pop(); + return m_fifo_locks.empty(); + } +}; + +class LockPool : OrderedLock { + std::deque m_available; +protected: + std::mutex m_available_lock; + std::vector m_thread_workers; + std::atomic m_waiting; +public: + + LockPool() : m_waiting(false) {} + ~LockPool() { + std::lock_guard freeing(m_available_lock); + for (auto worker : m_thread_workers) { + std::lock_guard freeing_worker(*worker); + delete worker; + } + } + + void addWorkers(size_t n) { + for (int i = n; i > 0; i--) { + m_thread_workers.emplace_back(new std::mutex); + } + std::lock_guard queueing_growth(m_available_lock); + // static_cast needed for some versions of C++ + std::transform( + m_thread_workers.end() - n, m_thread_workers.end(), std::back_inserter(m_available), + static_cast(std::addressof)); + m_waiting = true; + for (int i = n; i-- > 0; !unlock()) {} + } + + std::mutex** acquireWorker() { + if (!m_waiting) { + lock(); + } + std::lock_guard acquiring(m_available_lock); + const auto res = m_available.front(); + m_available.pop_front(); + m_waiting = m_available.empty(); + return res; + } + + void releaseWorker(std::mutex** worker) { + std::lock_guard acquiring(m_available_lock); + m_available.push_front(worker); + m_waiting = unlock(); + } + + size_t workerIndex(std::mutex** worker) { + return worker - m_thread_workers.data(); + } +}; + +} // namespace mc +} // namespace douban diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index a8eee5a5..4403ba38 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -1,4 +1,3 @@ -#include #include #include @@ -16,6 +15,7 @@ ClientPool::ClientPool() { } void ClientPool::config(config_options_t opt, int val) { + std::lock_guard config_pool(m_pool_lock); switch (val) { case CFG_INITIAL_CLIENTS: m_initial_clients = val; @@ -46,7 +46,7 @@ int ClientPool::init(const char* const * hosts, const uint32_t* ports, const siz int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, const size_t n, const char* const* aliases) { - // TODO: acquire lock; one update at a time + std::lock_guard updating_clients(m_pool_lock); duplicate_strings(hosts, n, m_hosts_data, m_hosts); duplicate_strings(aliases, n, m_aliases_data, m_aliases); @@ -54,10 +54,11 @@ int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, c std::copy(ports, ports + n, m_ports.begin()); std::atomic rv = 0; - std::for_each(std::execution::par, m_clients.begin(), m_clients.end(), + std::lock_guard updating(m_available_lock); + std::for_each(std::execution::par_unseq, m_clients.begin(), m_clients.end(), [this, &rv](Client& c) { - const int idx = &c - &m_clients[0]; - // TODO: acquire lock idx + const int idx = &c - m_clients.data(); + std::lock_guard updating_worker(*m_thread_workers[idx]); const int err = c.updateServers(m_hosts.data(), m_ports.data(), m_hosts.size(), m_aliases.data()); if (err != 0) { @@ -77,25 +78,30 @@ int ClientPool::setup(Client* c) { } int ClientPool::growPool(size_t by) { - // TODO: acquire same lock as updateServers - // TODO: client locks start busy + std::lock_guard growing_pool(m_pool_lock); assert(by > 0); size_t from = m_clients.size(); m_clients.resize(from + by); auto start = m_clients.data() + from; - std::for_each(std::execution::par, start, start + by, [this](Client& c) { + std::for_each(std::execution::par_unseq, start, start + by, + [this](Client& c) { setup(&c); - const int idx = &c - &m_clients[0]; - // TODO: release lock idx }); + addWorkers(by); } -Client* acquire() { - // TODO +Client* ClientPool::acquire() { + // TODO: grow pool if necessary + std::mutex** mux = acquireWorker(); + (**mux).lock(); + return &m_clients[workerIndex(mux)]; } -void release(Client* ref) { - // TODO +void ClientPool::release(Client* ref) { + const int idx = ref - m_clients.data(); + std::mutex** mux = &m_thread_workers[idx]; + (**mux).unlock(); + releaseWorker(mux); } } // namespace mc diff --git a/src/Common.cpp b/src/Common.cpp index 2e75ffca..148bdd92 100644 --- a/src/Common.cpp +++ b/src/Common.cpp @@ -50,9 +50,9 @@ const char* errCodeToString(err_code_t err) { } } -bool isLocalSocket(const char* host) { +bool isUnixSocket(const char* host) { // errors on the side of false negatives, allowing syntax expansion; - // starting slash to denote socket paths is from pylibmc + // starting slash syntax is from libmemcached return host[0] == '/'; } From 02c941d044891fda1a2f693345aff63ec6460fc3 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 13 Dec 2023 09:08:34 -0800 Subject: [PATCH 04/95] ClientPool --- include/ClientPool.h | 17 +++++++++-------- include/LockPool.h | 6 ++++-- src/ClientPool.cpp | 37 +++++++++++++++++++++++++++++++------ 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/include/ClientPool.h b/include/ClientPool.h index 152bfc23..49caef44 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -1,7 +1,5 @@ #pragma once -#include -#include #include "Client.h" #include "LockPool.h" @@ -29,16 +27,18 @@ class ClientPool : LockPool { ClientPool(); ~ClientPool(); void config(config_options_t opt, int val); - int init(const char* const * hosts, const uint32_t* ports, const size_t n, - const char* const * aliases = NULL); - int updateServers(const char* const * hosts, const uint32_t* ports, const size_t n, - const char* const * aliases = NULL); + int init(const char* const * hosts, const uint32_t* ports, + const size_t n, const char* const * aliases = NULL); + int updateServers(const char* const * hosts, const uint32_t* ports, + const size_t n, const char* const * aliases = NULL); Client* acquire(); void release(Client* ref); -protected: +private: int growPool(size_t by); int setup(Client* c); + bool shouldGrowUnsafe(); + int autoGrowUnsafe(); bool m_opt_changed[CLIENT_CONFIG_OPTION_COUNT]; int m_opt_value[CLIENT_CONFIG_OPTION_COUNT]; @@ -55,7 +55,8 @@ class ClientPool : LockPool { std::vector m_aliases; std::mutex m_pool_lock; + mutable std::shared_mutex m_acquiring_growth; }; } // namespace mc -} // namespace douban \ No newline at end of file +} // namespace douban diff --git a/include/LockPool.h b/include/LockPool.h index 1c807273..f3b3ba73 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -10,10 +10,11 @@ namespace douban { namespace mc { - + class OrderedLock { std::queue m_fifo_locks; std::mutex m_fifo_access; + public: OrderedLock() {}; void lock() { @@ -36,12 +37,13 @@ class OrderedLock { class LockPool : OrderedLock { std::deque m_available; + protected: std::mutex m_available_lock; std::vector m_thread_workers; std::atomic m_waiting; -public: +public: LockPool() : m_waiting(false) {} ~LockPool() { std::lock_guard freeing(m_available_lock); diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 4403ba38..95878b6b 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -1,4 +1,3 @@ -#include #include #include "ClientPool.h" @@ -41,6 +40,7 @@ void ClientPool::config(config_options_t opt, int val) { int ClientPool::init(const char* const * hosts, const uint32_t* ports, const size_t n, const char* const * aliases) { updateServers(hosts, ports, n, aliases); + std::shared_lock initializing(m_acquiring_growth); return growPool(m_initial_clients); } @@ -77,21 +77,46 @@ int ClientPool::setup(Client* c) { c->init(m_hosts.data(), m_ports.data(), m_hosts.size(), m_aliases.data()); } +// if called outside acquire, needs to own m_acquiring_growth int ClientPool::growPool(size_t by) { - std::lock_guard growing_pool(m_pool_lock); assert(by > 0); + std::lock_guard growing_pool(m_pool_lock); size_t from = m_clients.size(); m_clients.resize(from + by); - auto start = m_clients.data() + from; + const auto start = m_clients.data() + from; + std::atomic rv = 0; std::for_each(std::execution::par_unseq, start, start + by, - [this](Client& c) { - setup(&c); + [this, &rv](Client& c) { + const int err = setup(&c); + if (err != 0) { + rv.store(err, std::memory_order_relaxed); + } }); addWorkers(by); + return rv; +} + +bool ClientPool::shouldGrowUnsafe() { + return m_clients.size() < m_max_clients && !m_waiting; +} + +int ClientPool::autoGrowUnsafe() { + return growPool(MIN(m_max_clients - m_clients.size(), + MIN(m_max_growth, m_clients.size()))); } Client* ClientPool::acquire() { - // TODO: grow pool if necessary + m_acquiring_growth.lock_shared(); + const auto growing = shouldGrowUnsafe(); + m_acquiring_growth.unlock_shared(); + if (growing) { + m_acquiring_growth.lock(); + if (shouldGrowUnsafe()) { + autoGrowUnsafe(); + } + m_acquiring_growth.unlock(); + } + std::mutex** mux = acquireWorker(); (**mux).lock(); return &m_clients[workerIndex(mux)]; From 3fb5dd7f9a983184a29cd44d8b934f46e5c33a50 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 13 Dec 2023 12:49:17 -0800 Subject: [PATCH 05/95] missed renamed symbol --- src/ClientPool.cpp | 10 +++++----- src/Connection.cpp | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 95878b6b..b376dbd4 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -37,15 +37,15 @@ void ClientPool::config(config_options_t opt, int val) { } } -int ClientPool::init(const char* const * hosts, const uint32_t* ports, const size_t n, - const char* const * aliases) { +int ClientPool::init(const char* const * hosts, const uint32_t* ports, + const size_t n, const char* const * aliases) { updateServers(hosts, ports, n, aliases); - std::shared_lock initializing(m_acquiring_growth); + std::unique_lock initializing(m_acquiring_growth); return growPool(m_initial_clients); } -int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, const size_t n, - const char* const* aliases) { +int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, + const size_t n, const char* const* aliases) { std::lock_guard updating_clients(m_pool_lock); duplicate_strings(hosts, n, m_hosts_data, m_hosts); duplicate_strings(aliases, n, m_aliases_data, m_aliases); diff --git a/src/Connection.cpp b/src/Connection.cpp index 0b95c8b3..b44c4963 100644 --- a/src/Connection.cpp +++ b/src/Connection.cpp @@ -47,7 +47,7 @@ Connection::~Connection() { int Connection::init(const char* host, uint32_t port, const char* alias) { snprintf(m_host, sizeof m_host, "%s", host); m_port = port; - m_unixSocket = isLocalSocket(m_host); + m_unixSocket = isUnixSocket(m_host); if (alias == NULL) { m_hasAlias = false; if (m_unixSocket) { From b7305cab9898992c769c0dd5e8056f7f948438c7 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 13 Dec 2023 14:58:28 -0800 Subject: [PATCH 06/95] cppcheck fixes --- include/ClientPool.h | 2 +- include/LockPool.h | 2 +- src/ClientPool.cpp | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/include/ClientPool.h b/include/ClientPool.h index 49caef44..4005b1ad 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -32,7 +32,7 @@ class ClientPool : LockPool { int updateServers(const char* const * hosts, const uint32_t* ports, const size_t n, const char* const * aliases = NULL); Client* acquire(); - void release(Client* ref); + void release(const Client* ref); private: int growPool(size_t by); diff --git a/include/LockPool.h b/include/LockPool.h index f3b3ba73..2ed54dc5 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -83,7 +83,7 @@ class LockPool : OrderedLock { m_waiting = unlock(); } - size_t workerIndex(std::mutex** worker) { + size_t workerIndex(const std::mutex** worker) { return worker - m_thread_workers.data(); } }; diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index b376dbd4..71ea5d4d 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -8,6 +8,7 @@ namespace mc { // default max of 4 clients to match memcached's default of 4 worker threads ClientPool::ClientPool(): m_initial_clients(1), m_max_clients(4), m_max_growth(4) { memset(m_opt_changed, false, sizeof m_opt_changed); + memset(m_opt_value, 0, sizeof m_opt_value); } ClientPool::ClientPool() { @@ -70,7 +71,7 @@ int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, int ClientPool::setup(Client* c) { for (int i = 0; i < CLIENT_CONFIG_OPTION_COUNT; i++) { - if (m_opt_changed) { + if (m_opt_changed[i]) { c->config(static_cast(i), m_opt_value[i]); } } @@ -122,7 +123,7 @@ Client* ClientPool::acquire() { return &m_clients[workerIndex(mux)]; } -void ClientPool::release(Client* ref) { +void ClientPool::release(const Client* ref) { const int idx = ref - m_clients.data(); std::mutex** mux = &m_thread_workers[idx]; (**mux).unlock(); From c936fdf0e9b1169854c1c7113e4148e8d3bc9fa7 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 13 Dec 2023 15:43:55 -0800 Subject: [PATCH 07/95] minor build fixes --- include/ClientPool.h | 4 ++-- include/LockPool.h | 2 +- src/ClientPool.cpp | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/include/ClientPool.h b/include/ClientPool.h index 4005b1ad..ee7a6e56 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -11,14 +11,14 @@ void duplicate_strings(const char* const * strs, const size_t n, std::vector >& out, std::vector& refs) { out.resize(n); refs.resize(n); - for (int i = 0; i < n; i++) { + for (size_t i = 0; i < n; i++) { if (strs == NULL || strs[i] == NULL) { out[i][0] = '\0'; refs[i] = NULL; continue; } std::snprintf(out[i].data(), N, "%s", strs[i]); - refs[i] = &out[i]; + refs[i] = out[i].data(); } } diff --git a/include/LockPool.h b/include/LockPool.h index 2ed54dc5..f3b3ba73 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -83,7 +83,7 @@ class LockPool : OrderedLock { m_waiting = unlock(); } - size_t workerIndex(const std::mutex** worker) { + size_t workerIndex(std::mutex** worker) { return worker - m_thread_workers.data(); } }; diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 71ea5d4d..57d835f8 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -11,7 +11,7 @@ ClientPool::ClientPool(): m_initial_clients(1), m_max_clients(4), m_max_growth(4 memset(m_opt_value, 0, sizeof m_opt_value); } -ClientPool::ClientPool() { +ClientPool::~ClientPool() { } void ClientPool::config(config_options_t opt, int val) { @@ -75,7 +75,7 @@ int ClientPool::setup(Client* c) { c->config(static_cast(i), m_opt_value[i]); } } - c->init(m_hosts.data(), m_ports.data(), m_hosts.size(), m_aliases.data()); + return c->init(m_hosts.data(), m_ports.data(), m_hosts.size(), m_aliases.data()); } // if called outside acquire, needs to own m_acquiring_growth @@ -93,6 +93,8 @@ int ClientPool::growPool(size_t by) { rv.store(err, std::memory_order_relaxed); } }); + // adds workers with non-zero return values + // if changed, acquire should probably raise rather than hang addWorkers(by); return rv; } From a4662c295ef8fca026e1bc266c028340ccd9b43e Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Thu, 14 Dec 2023 11:42:36 -0800 Subject: [PATCH 08/95] std::execution::par[_unseq] requires -fexceptions --- CMakeLists.txt | 2 +- include/ClientPool.h | 4 ++-- src/ClientPool.cpp | 20 +++++++++++--------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 04a4799a..8ebd9f84 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: Debug Release." FORCE) endif (NOT CMAKE_BUILD_TYPE) -set(CMAKE_CXX_FLAGS_COMMON "-Wall -fno-rtti -fno-exceptions") +set(CMAKE_CXX_FLAGS_COMMON "-Wall -fno-rtti") set(CMAKE_CXX_FLAGS_DEBUG "-DDEBUG -g2 ${CMAKE_CXX_FLAGS_COMMON}" CACHE STRING "CXX DEBUG FLAGS" FORCE) set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3 ${CMAKE_CXX_FLAGS_COMMON}" CACHE STRING "CXX RELEASE FLAGS" FORCE) set(CMAKE_INSTALL_INCLUDE include CACHE PATH "Output directory for header files") diff --git a/include/ClientPool.h b/include/ClientPool.h index ee7a6e56..81c317fb 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -37,8 +37,8 @@ class ClientPool : LockPool { private: int growPool(size_t by); int setup(Client* c); - bool shouldGrowUnsafe(); - int autoGrowUnsafe(); + inline bool shouldGrowUnsafe(); + int autoGrow(); bool m_opt_changed[CLIENT_CONFIG_OPTION_COUNT]; int m_opt_value[CLIENT_CONFIG_OPTION_COUNT]; diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 57d835f8..fc7c74bf 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -1,4 +1,5 @@ #include +#include #include "ClientPool.h" @@ -15,6 +16,7 @@ ClientPool::~ClientPool() { } void ClientPool::config(config_options_t opt, int val) { + assert(m_clients.size() == 0); std::lock_guard config_pool(m_pool_lock); switch (val) { case CFG_INITIAL_CLIENTS: @@ -99,13 +101,17 @@ int ClientPool::growPool(size_t by) { return rv; } -bool ClientPool::shouldGrowUnsafe() { +inline bool ClientPool::shouldGrowUnsafe() { return m_clients.size() < m_max_clients && !m_waiting; } -int ClientPool::autoGrowUnsafe() { - return growPool(MIN(m_max_clients - m_clients.size(), - MIN(m_max_growth, m_clients.size()))); +int ClientPool::autoGrow() { + std::unique_lock growing(m_acquiring_growth); + if (shouldGrowUnsafe()) { + return growPool(MIN(m_max_clients - m_clients.size(), + MIN(m_max_growth, m_clients.size()))); + } + return 0; } Client* ClientPool::acquire() { @@ -113,11 +119,7 @@ Client* ClientPool::acquire() { const auto growing = shouldGrowUnsafe(); m_acquiring_growth.unlock_shared(); if (growing) { - m_acquiring_growth.lock(); - if (shouldGrowUnsafe()) { - autoGrowUnsafe(); - } - m_acquiring_growth.unlock(); + std::thread acquire_overflow(&ClientPool::autoGrow, this); } std::mutex** mux = acquireWorker(); From f5eca1c46a56cbd56e3185064f583b24a97931c9 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Thu, 14 Dec 2023 15:35:49 -0800 Subject: [PATCH 09/95] act support --- .github/workflows/golang.yml | 1 + .github/workflows/manual.yml | 89 ++++++++++++++++++++++++++++++++++++ .github/workflows/python.yml | 2 + include/ClientPool.h | 1 + src/ClientPool.cpp | 1 - 5 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/manual.yml diff --git a/.github/workflows/golang.yml b/.github/workflows/golang.yml index c542e54e..56e04387 100644 --- a/.github/workflows/golang.yml +++ b/.github/workflows/golang.yml @@ -25,6 +25,7 @@ jobs: uses: actions/setup-go@v2 with: go-version: ${{ matrix.gover }} + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Start memcached servers run: ./misc/memcached_server start - name: Run gotest diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml new file mode 100644 index 00000000..49c6e32c --- /dev/null +++ b/.github/workflows/manual.yml @@ -0,0 +1,89 @@ +name: manual + +on: + workflow_dispatch: + +jobs: + cpptest: + runs-on: ubuntu-latest + strategy: + matrix: + compiler: ["gcc"] + build_type: ["Debug"] + + steps: + - uses: actions/checkout@v2 + - name: Setup system dependencies + run: | + sudo apt-get update + sudo apt-get -y install memcached g++ + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools future pytest greenify gevent numpy + - name: Run cpptest + run: | + if [[ ${{ matrix.compiler }} = "gcc" ]]; then export CC=gcc CXX=g++; fi + if [[ ${{ matrix.compiler }} = "clang" ]]; then export CC=clang CXX=clang++; fi + ./misc/travis/cpptest.sh + + pylatest: + runs-on: ubuntu-latest + strategy: + matrix: + pyver: ["3.12"] + compiler: ["gcc"] + build_type: ["Debug"] + + steps: + - uses: actions/checkout@v2 + - name: Setup system dependencies + run: | + sudo apt-get update + sudo apt-get -y install valgrind memcached g++ + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.pyver }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools future pytest greenify gevent numpy + - name: Start memcached servers + run: ./misc/memcached_server startall + - name: Run unittest + run: | + if [[ ${{ matrix.compiler }} = "gcc" ]]; then export CC=gcc CXX=g++; fi + if [[ ${{ matrix.compiler }} = "clang" ]]; then export CC=clang CXX=clang++; fi + ./misc/travis/unittest.sh + - name: Stop memcached servers + run: ./misc/memcached_server stopall + + goclang: + runs-on: ubuntu-latest + strategy: + matrix: + gover: ["1.15"] + compiler: ["clang"] + + steps: + - uses: actions/checkout@v2 + - name: Setup system dependencies + run: | + sudo apt-get update + sudo apt-get -y install memcached + - name: Set up Golang + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.gover }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Start memcached servers + run: ./misc/memcached_server start + - name: Run gotest + run: | + export CC=clang CXX=clang++ + clang++ -v + ./misc/travis/gotest.sh + - name: Stop memcached servers + run: ./misc/memcached_server stop diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index f364f304..be6837ce 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -25,6 +25,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.pyver }} + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install python dependencies run: | python -m pip install --upgrade pip @@ -55,6 +56,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.pyver }} + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install python dependencies run: | python -m pip install --upgrade pip diff --git a/include/ClientPool.h b/include/ClientPool.h index 81c317fb..b42193e0 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -1,5 +1,6 @@ #pragma once +#include #include "Client.h" #include "LockPool.h" diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index fc7c74bf..ac54e749 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -1,6 +1,5 @@ #include #include - #include "ClientPool.h" namespace douban { From 448458bd5fd21c2d7f2c85c284c36ec5df22bc81 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 15 Dec 2023 15:28:36 -0800 Subject: [PATCH 10/95] python bindings --- include/ClientPool.h | 30 ++++++++++-- include/Export.h | 1 + include/LockPool.h | 33 ++++++------- libmc/__init__.py | 12 +++-- libmc/_client.pyx | 109 ++++++++++++++++++++++++++++++++++++------- src/Client.cpp | 7 +++ src/ClientPool.cpp | 60 +++++++++++++++--------- 7 files changed, 190 insertions(+), 62 deletions(-) diff --git a/include/ClientPool.h b/include/ClientPool.h index b42193e0..42cd756b 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -9,7 +9,7 @@ namespace mc { template void duplicate_strings(const char* const * strs, const size_t n, - std::vector >& out, std::vector& refs) { + std::deque >& out, std::vector& refs) { out.resize(n); refs.resize(n); for (size_t i = 0; i < n; i++) { @@ -23,6 +23,26 @@ void duplicate_strings(const char* const * strs, const size_t n, } } +class irange { +public: + irange(int n) : i(0), _end(n) {} + + bool operator!=(const irange& other) const { return i != _end; } + irange& operator++() { ++i; return *this; } + int operator*() const { return i; } + irange begin() const { return *this; } + irange end() const { return *this; } + +private: + int i; + int _end; +}; + +typedef struct { + Client c; + int index; +} IndexedClient; + class ClientPool : LockPool { public: ClientPool(); @@ -32,6 +52,8 @@ class ClientPool : LockPool { const size_t n, const char* const * aliases = NULL); int updateServers(const char* const * hosts, const uint32_t* ports, const size_t n, const char* const * aliases = NULL); + IndexedClient* _acquire(); + void _release(const IndexedClient* idx); Client* acquire(); void release(const Client* ref); @@ -43,13 +65,13 @@ class ClientPool : LockPool { bool m_opt_changed[CLIENT_CONFIG_OPTION_COUNT]; int m_opt_value[CLIENT_CONFIG_OPTION_COUNT]; - std::vector m_clients; + std::deque m_clients; size_t m_initial_clients; size_t m_max_clients; size_t m_max_growth; - std::vector > m_hosts_data; - std::vector > m_aliases_data; + std::deque > m_hosts_data; + std::deque > m_aliases_data; std::vector m_ports; std::vector m_hosts; diff --git a/include/Export.h b/include/Export.h index 1bf20f9d..e7db3e58 100644 --- a/include/Export.h +++ b/include/Export.h @@ -15,6 +15,7 @@ typedef enum { CFG_RETRY_TIMEOUT, CFG_HASH_FUNCTION, CFG_MAX_RETRIES, + CFG_SET_FAILOVER, // type separator to track number of Client config options to save CLIENT_CONFIG_OPTION_COUNT, diff --git a/include/LockPool.h b/include/LockPool.h index f3b3ba73..acff22ad 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -15,7 +15,7 @@ class OrderedLock { std::queue m_fifo_locks; std::mutex m_fifo_access; -public: +protected: OrderedLock() {}; void lock() { std::unique_lock acquire(m_fifo_access); @@ -36,37 +36,42 @@ class OrderedLock { }; class LockPool : OrderedLock { - std::deque m_available; + std::deque m_available; + std::list m_mux_mallocs; protected: std::mutex m_available_lock; - std::vector m_thread_workers; + std::deque m_thread_workers; std::atomic m_waiting; -public: LockPool() : m_waiting(false) {} ~LockPool() { std::lock_guard freeing(m_available_lock); for (auto worker : m_thread_workers) { std::lock_guard freeing_worker(*worker); - delete worker; + } + for (auto mem : m_mux_mallocs) { + delete[] mem; } } void addWorkers(size_t n) { - for (int i = n; i > 0; i--) { - m_thread_workers.emplace_back(new std::mutex); - } std::lock_guard queueing_growth(m_available_lock); + const auto from = m_thread_workers.size(); + for (int i = from + n; i > from; i--) { + m_available.push_back(i); + } + const auto muxes = new std::mutex[n]; + m_mux_mallocs.push_back(&muxes); // static_cast needed for some versions of C++ std::transform( - m_thread_workers.end() - n, m_thread_workers.end(), std::back_inserter(m_available), - static_cast(std::addressof)); + muxes, muxes + n, std::back_inserter(m_thread_workers), + static_cast(std::addressof)); m_waiting = true; for (int i = n; i-- > 0; !unlock()) {} } - std::mutex** acquireWorker() { + int acquireWorker() { if (!m_waiting) { lock(); } @@ -77,15 +82,11 @@ class LockPool : OrderedLock { return res; } - void releaseWorker(std::mutex** worker) { + void releaseWorker(int worker) { std::lock_guard acquiring(m_available_lock); m_available.push_front(worker); m_waiting = unlock(); } - - size_t workerIndex(std::mutex** worker) { - return worker - m_thread_workers.data(); - } }; } // namespace mc diff --git a/libmc/__init__.py b/libmc/__init__.py index a4395a6b..ade3209f 100644 --- a/libmc/__init__.py +++ b/libmc/__init__.py @@ -1,6 +1,6 @@ import os from ._client import ( - PyClient, ThreadUnsafe, + PyClient, PyClientPool as ClientPool, ThreadUnsafe, encode_value, decode_value, @@ -8,6 +8,10 @@ MC_POLL_TIMEOUT, MC_CONNECT_TIMEOUT, MC_RETRY_TIMEOUT, + MC_SET_FAILOVER, + MC_INITIAL_CLIENTS, + MC_MAX_CLIENTS, + MC_MAX_GROWTH, MC_HASH_MD5, MC_HASH_FNV1_32, @@ -42,10 +46,12 @@ class Client(PyClient): __all__ = [ - 'Client', 'ThreadUnsafe', '__VERSION__', 'encode_value', 'decode_value', + 'Client', 'ClientPool', 'ThreadUnsafe', '__VERSION__', + 'encode_value', 'decode_value', 'MC_DEFAULT_EXPTIME', 'MC_POLL_TIMEOUT', 'MC_CONNECT_TIMEOUT', - 'MC_RETRY_TIMEOUT', + 'MC_RETRY_TIMEOUT', 'MC_SET_FAILOVER', 'MC_INITIAL_CLIENTS', + 'MC_MAX_CLIENTS', 'MC_MAX_GROWTH', 'MC_HASH_MD5', 'MC_HASH_FNV1_32', 'MC_HASH_FNV1A_32', 'MC_HASH_CRC_32', diff --git a/libmc/_client.pyx b/libmc/_client.pyx index 44719d0a..f95f3257 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -55,6 +55,11 @@ cdef extern from "Export.h": CFG_RETRY_TIMEOUT CFG_HASH_FUNCTION CFG_MAX_RETRIES + CFG_SET_FAILOVER + + CFG_INITIAL_CLIENTS + CFG_MAX_CLIENTS + CFG_MAX_GROWTH ctypedef enum hash_function_options_t: OPT_HASH_MD5 @@ -217,6 +222,23 @@ cdef extern from "Client.h" namespace "douban::mc": const char* errCodeToString(err_code_t err) nogil + +cdef extern from "ClientPool.h" namespace "douban::mc": + ctypedef struct IndexedClient: + Client c + int index + + cdef cppclass ClientPool: + ClientPool() + void config(config_options_t opt, int val) nogil + int init(const char* const * hosts, const uint32_t* ports, size_t n, + const char* const * aliases) nogil + int updateServers(const char* const * hosts, const uint32_t* ports, size_t n, + const char* const * aliases) nogil + IndexedClient* _acquire() nogil + void _release(const IndexedClient* ref) nogil + + cdef uint32_t MC_DEFAULT_PORT = 11211 cdef flags_t _FLAG_EMPTY = 0 cdef flags_t _FLAG_PICKLE = 1 << 0 @@ -237,6 +259,10 @@ MC_POLL_TIMEOUT = PyInt_FromLong(CFG_POLL_TIMEOUT) MC_CONNECT_TIMEOUT = PyInt_FromLong(CFG_CONNECT_TIMEOUT) MC_RETRY_TIMEOUT = PyInt_FromLong(CFG_RETRY_TIMEOUT) MC_MAX_RETRIES = PyInt_FromLong(CFG_MAX_RETRIES) +MC_SET_FAILOVER = PyInt_FromLong(CFG_SET_FAILOVER) +MC_INITIAL_CLIENTS = PyInt_FromLong(CFG_INITIAL_CLIENTS) +MC_MAX_CLIENTS = PyInt_FromLong(CFG_MAX_CLIENTS) +MC_MAX_GROWTH = PyInt_FromLong(CFG_MAX_GROWTH) MC_HASH_MD5 = PyInt_FromLong(OPT_HASH_MD5) @@ -335,32 +361,20 @@ class ThreadUnsafe(Exception): pass -cdef class PyClient: +cdef class PyClientSettings: cdef readonly list servers cdef readonly int comp_threshold - cdef Client* _imp cdef bool_t do_split cdef bool_t noreply cdef bytes prefix cdef hash_function_options_t hash_fn cdef bool_t failover cdef basestring encoding - cdef err_code_t last_error - cdef object _thread_ident - cdef object _created_stack def __cinit__(self, list servers, bool_t do_split=True, int comp_threshold=0, noreply=False, basestring prefix=None, hash_function_options_t hash_fn=OPT_HASH_MD5, failover=False, encoding='utf8'): self.servers = servers - self._imp = new Client() - self._imp.config(CFG_HASH_FUNCTION, hash_fn) - rv = self._update_servers(servers, True) - if failover: - self._imp.enableConsistentFailover() - else: - self._imp.disableConsistentFailover() - self.do_split = do_split self.comp_threshold = comp_threshold self.noreply = noreply @@ -372,10 +386,38 @@ cdef class PyClient: else: self.prefix = None + self.init() + + def init(): + pass + + cpdef _args(): + return (self.servers, self.do_split, self.comp_threshold, self.noreply, + self.prefix, self.hash_fn, self.failover, self.encoding) + + def __reduce__(self): + return (self.__class__, self._args()) + + +cdef class PyClient(PyClientSettings): + cdef Client* _imp + cdef err_code_t last_error + cdef object _thread_ident + cdef object _created_stack + + def init(): self.last_error = RET_OK self._thread_ident = None self._created_stack = traceback.extract_stack() + self._imp = new Client() + self._imp.config(CFG_HASH_FUNCTION, self.hash_fn) + rv = self._update_servers(self.servers, True) + if self.failover: + self._imp.enableConsistentFailover() + else: + self._imp.disableConsistentFailover() + cdef _update_servers(self, list servers, bool_t init): cdef int rv = 0 cdef size_t n = len(servers) @@ -417,9 +459,6 @@ cdef class PyClient: def __dealloc__(self): del self._imp - def __reduce__(self): - return (PyClient, (self.servers, self.do_split, self.comp_threshold, self.noreply, self.prefix, self.hash_fn, self.failover, self.encoding)) - def config(self, int opt, int val): self._imp.config(opt, val) @@ -1091,3 +1130,41 @@ cdef class PyClient: def get_last_strerror(self): return errCodeToString(self.last_error) + + +class PyPoolClient(PyClient): + cdef IndexedClient* _indexed + + def init(): + self.last_error = RET_OK + self._thread_ident = None + self._created_stack = traceback.extract_stack() + + + +class PyClientPool(PyClientSettings): + worker = PyPoolClient + cdef list clients + + def init(): + self._imp = new ClientPool() + self._imp.config(CFG_HASH_FUNCTION, self.hash_fn) + self.clients = [] + + def setup(self, IndexedClientClient* imp): + worker = __class__.worker(*self._args()) + worker._indexed = imp + worker._imp = imp.c + return worker + + def acquire(self): + worker = self._imp._acquire() + if worker.index >= len(self.clients): + clients += [None] * (worker.index - len(self.clients)) + clients.append(setup(worker)) + elif self.clients[worker.index] == None: + self.clients[i] = setup(worker); + return self.clients[i] + + def release(self, PyPoolClient worker): + self._imp._release(worker._indexed) diff --git a/src/Client.cpp b/src/Client.cpp index 889c9800..7db3ad40 100644 --- a/src/Client.cpp +++ b/src/Client.cpp @@ -33,6 +33,13 @@ void Client::config(config_options_t opt, int val) { case CFG_MAX_RETRIES: setMaxRetries(val); break; + case CFG_SET_FAILOVER: + assert(val == 0 || val == 1); + if (val == 0) { + disableConsistentFailover(); + } else { + enableConsistentFailover(); + } default: break; } diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index ac54e749..2ddd5eca 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "ClientPool.h" namespace douban { @@ -15,11 +16,22 @@ ClientPool::~ClientPool() { } void ClientPool::config(config_options_t opt, int val) { - assert(m_clients.size() == 0); std::lock_guard config_pool(m_pool_lock); + if (opt < CLIENT_CONFIG_OPTION_COUNT) { + m_opt_changed[opt] = true; + m_opt_value[opt] = val; + for (auto &client : m_clients) { + client.c.config(opt, val); + } + return; + } + std::unique_lock initializing(m_acquiring_growth); switch (val) { case CFG_INITIAL_CLIENTS: m_initial_clients = val; + if (m_clients.size() < val) { + growPool(m_initial_clients); + } break; case CFG_MAX_CLIENTS: m_max_clients = val; @@ -28,13 +40,6 @@ void ClientPool::config(config_options_t opt, int val) { m_max_growth = val; break; default: - if (opt < CLIENT_CONFIG_OPTION_COUNT) { - m_opt_changed[opt] = true; - m_opt_value[opt] = val; - for (auto client : m_clients) { - client.config(opt, val); - } - } break; } } @@ -57,12 +62,12 @@ int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, std::atomic rv = 0; std::lock_guard updating(m_available_lock); - std::for_each(std::execution::par_unseq, m_clients.begin(), m_clients.end(), - [this, &rv](Client& c) { - const int idx = &c - m_clients.data(); - std::lock_guard updating_worker(*m_thread_workers[idx]); - const int err = c.updateServers(m_hosts.data(), m_ports.data(), - m_hosts.size(), m_aliases.data()); + const auto idx = irange(n); + std::for_each(std::execution::par_unseq, idx.begin(), idx.end(), + [this, &rv](int i) { + std::lock_guard updating_worker(*m_thread_workers[i]); + const int err = m_clients[i].c.updateServers( + m_hosts.data(), m_ports.data(), m_hosts.size(), m_aliases.data()); if (err != 0) { rv.store(err, std::memory_order_relaxed); } @@ -85,7 +90,7 @@ int ClientPool::growPool(size_t by) { std::lock_guard growing_pool(m_pool_lock); size_t from = m_clients.size(); m_clients.resize(from + by); - const auto start = m_clients.data() + from; + const auto start = m_clients.begin() + from; std::atomic rv = 0; std::for_each(std::execution::par_unseq, start, start + by, [this, &rv](Client& c) { @@ -113,7 +118,7 @@ int ClientPool::autoGrow() { return 0; } -Client* ClientPool::acquire() { +IndexedClient* ClientPool::_acquire() { m_acquiring_growth.lock_shared(); const auto growing = shouldGrowUnsafe(); m_acquiring_growth.unlock_shared(); @@ -121,16 +126,25 @@ Client* ClientPool::acquire() { std::thread acquire_overflow(&ClientPool::autoGrow, this); } - std::mutex** mux = acquireWorker(); - (**mux).lock(); - return &m_clients[workerIndex(mux)]; + int idx = acquireWorker(); + m_thread_workers[idx]->lock(); + return &m_clients[idx]; } -void ClientPool::release(const Client* ref) { - const int idx = ref - m_clients.data(); - std::mutex** mux = &m_thread_workers[idx]; +void ClientPool::_release(const IndexedClient* idx) { + std::mutex* const * mux = &m_thread_workers[idx->index]; (**mux).unlock(); - releaseWorker(mux); + releaseWorker(idx->index); +} + +Client* ClientPool::acquire() { + return &_acquire()->c; +} + +void ClientPool::release(const Client* ref) { + // C std 6.7.2.1-13 + auto idx = reinterpret_cast(ref); + return _release(idx); } } // namespace mc From b5410a318817060417682dccba99cc2491b7d14d Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sat, 16 Dec 2023 19:32:05 -0800 Subject: [PATCH 11/95] avoiding iota --- include/ClientPool.h | 38 +++++++++++++++++++++++++++++--------- include/LockPool.h | 6 ++++-- src/ClientPool.cpp | 9 ++++----- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/include/ClientPool.h b/include/ClientPool.h index 42cd756b..62863273 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "Client.h" #include "LockPool.h" @@ -24,18 +25,37 @@ void duplicate_strings(const char* const * strs, const size_t n, } class irange { + int i; + public: - irange(int n) : i(0), _end(n) {} + using value_type = int; + using pointer = const int*; + using reference = const int&; + using difference_type = int; + using iterator_category = std::random_access_iterator_tag; - bool operator!=(const irange& other) const { return i != _end; } - irange& operator++() { ++i; return *this; } - int operator*() const { return i; } - irange begin() const { return *this; } - irange end() const { return *this; } + irange() : i(0) {} + irange(int i) : i(i) {} -private: - int i; - int _end; + reference operator*() const { return i; } + pointer operator->() const { return &i; } + value_type operator[](int n) const { return i + n; } + friend bool operator< (const irange& lhs, const irange& rhs) { return lhs.i < rhs.i; } + friend bool operator> (const irange& lhs, const irange& rhs) { return rhs < lhs; } + friend bool operator<=(const irange& lhs, const irange& rhs) { return !(lhs > rhs); } + friend bool operator>=(const irange& lhs, const irange& rhs) { return !(lhs < rhs); } + friend bool operator==(const irange& lhs, const irange& rhs) { return lhs.i == rhs.i; } + friend bool operator!=(const irange& lhs, const irange& rhs) { return !(lhs == rhs); } + irange& operator++() { ++i; return *this; } + irange& operator--() { --i; return *this; } + irange operator++(int) { irange tmp = *this; ++tmp; return tmp; } + irange operator--(int) { irange tmp = *this; --tmp; return tmp; } + irange& operator+=(difference_type n) { i += n; return *this; } + irange& operator-=(difference_type n) { i -= n; return *this; } + friend irange operator+(const irange& lhs, difference_type n) { irange tmp = lhs; tmp += n; return tmp; } + friend irange operator+(difference_type n, const irange& rhs) { return rhs + n; } + friend irange operator-(const irange& lhs, difference_type n) { return lhs.i + (-n); } + friend difference_type operator-(const irange& lhs, const irange& rhs) { return lhs.i - rhs.i; } }; typedef struct { diff --git a/include/LockPool.h b/include/LockPool.h index acff22ad..cffacd3d 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include namespace douban { namespace mc { @@ -36,7 +38,7 @@ class OrderedLock { }; class LockPool : OrderedLock { - std::deque m_available; + std::deque m_available; std::list m_mux_mallocs; protected: @@ -58,7 +60,7 @@ class LockPool : OrderedLock { void addWorkers(size_t n) { std::lock_guard queueing_growth(m_available_lock); const auto from = m_thread_workers.size(); - for (int i = from + n; i > from; i--) { + for (size_t i = from + n; i > from; i--) { m_available.push_back(i); } const auto muxes = new std::mutex[n]; diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 2ddd5eca..8e4dce4d 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -29,7 +29,7 @@ void ClientPool::config(config_options_t opt, int val) { switch (val) { case CFG_INITIAL_CLIENTS: m_initial_clients = val; - if (m_clients.size() < val) { + if (m_clients.size() < m_initial_clients) { growPool(m_initial_clients); } break; @@ -62,8 +62,7 @@ int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, std::atomic rv = 0; std::lock_guard updating(m_available_lock); - const auto idx = irange(n); - std::for_each(std::execution::par_unseq, idx.begin(), idx.end(), + std::for_each(std::execution::par_unseq, irange(), irange(n), [this, &rv](int i) { std::lock_guard updating_worker(*m_thread_workers[i]); const int err = m_clients[i].c.updateServers( @@ -93,8 +92,8 @@ int ClientPool::growPool(size_t by) { const auto start = m_clients.begin() + from; std::atomic rv = 0; std::for_each(std::execution::par_unseq, start, start + by, - [this, &rv](Client& c) { - const int err = setup(&c); + [this, &rv](IndexedClient& c) { + const int err = setup(&c.c); if (err != 0) { rv.store(err, std::memory_order_relaxed); } From 2324b0fea22d7a35f1a1267e4b90e9846e494c10 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sun, 17 Dec 2023 13:38:45 -0800 Subject: [PATCH 12/95] switching machines --- libmc/_client.pyx | 4 +++- tests/test_client_pool.cpp | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 tests/test_client_pool.cpp diff --git a/libmc/_client.pyx b/libmc/_client.pyx index f95f3257..87c61b6a 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -391,7 +391,7 @@ cdef class PyClientSettings: def init(): pass - cpdef _args(): + def _args(): return (self.servers, self.do_split, self.comp_threshold, self.noreply, self.prefix, self.hash_fn, self.failover, self.encoding) @@ -1140,6 +1140,8 @@ class PyPoolClient(PyClient): self._thread_ident = None self._created_stack = traceback.extract_stack() + def _check_thread_ident(self): + pass class PyClientPool(PyClientSettings): diff --git a/tests/test_client_pool.cpp b/tests/test_client_pool.cpp new file mode 100644 index 00000000..7898394b --- /dev/null +++ b/tests/test_client_pool.cpp @@ -0,0 +1,8 @@ +#include "ClientPool.h" + +#include +#include "gtest/gtest.h" + +TEST(test_client_pool, threaded_set_get_del) { + +} \ No newline at end of file From 1e9f94430a7e69e8ce61f902c042408bae115acf Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 19 Dec 2023 20:19:51 -0800 Subject: [PATCH 13/95] still broken... --- include/ClientPool.h | 2 + include/LockPool.h | 95 ++++++++++++++++++++++++------------ src/ClientPool.cpp | 28 ++++++----- tests/test_client_pool.cpp | 98 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 178 insertions(+), 45 deletions(-) diff --git a/include/ClientPool.h b/include/ClientPool.h index 62863273..a171be5b 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -2,6 +2,8 @@ #include #include +#include + #include "Client.h" #include "LockPool.h" diff --git a/include/LockPool.h b/include/LockPool.h index cffacd3d..3ad26aec 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -10,84 +10,117 @@ #include #include +#include +#include +#include + namespace douban { namespace mc { +#define tprintf(str, ...) printf("tid = %lu: " str, syscall(SYS_gettid), ##__VA_ARGS__) + +// https://stackoverflow.com/a/14792685/3476782 class OrderedLock { std::queue m_fifo_locks; +protected: std::mutex m_fifo_access; + bool m_locked; protected: - OrderedLock() {}; - void lock() { + OrderedLock() : m_locked(true) {}; + std::unique_lock lock() { std::unique_lock acquire(m_fifo_access); - std::condition_variable signal; - m_fifo_locks.emplace(&signal); - signal.wait(acquire); + if (m_locked) { + tprintf("locked %lu already waiting\n", m_fifo_locks.size()); + std::condition_variable signal; + m_fifo_locks.emplace(&signal); + signal.wait(acquire); + tprintf("unlocked %lu left waiting lock %p\n", m_fifo_locks.size(), &signal); + assert(acquire.owns_lock()); + } else { + m_locked = true; + tprintf("passed through unlocked fifo\n"); + } + return acquire; } - bool unlock() { - std::unique_lock acquire(m_fifo_access); + void unlock(const bool owns_fifo = false, int i = 0) { + std::unique_lock acquire; + if (!owns_fifo) { + acquire = std::unique_lock(m_fifo_access); + } if (m_fifo_locks.empty()) { - return true; + tprintf("fifo already empty\n"); + m_locked = false; + } else { + const std::condition_variable* lock_ptr = m_fifo_locks.front(); + tprintf("unlocking with %d available %lu waiting lock %p\n", i, m_fifo_locks.size(), lock_ptr); + m_fifo_locks.front()->notify_one(); + m_fifo_locks.pop(); } - m_fifo_locks.front()->notify_one(); - m_fifo_locks.pop(); - return m_fifo_locks.empty(); } }; -class LockPool : OrderedLock { +class LockPool : public OrderedLock { std::deque m_available; - std::list m_mux_mallocs; + std::list m_muxes; + std::list m_mux_mallocs; protected: - std::mutex m_available_lock; std::deque m_thread_workers; - std::atomic m_waiting; - LockPool() : m_waiting(false) {} + LockPool() {} ~LockPool() { - std::lock_guard freeing(m_available_lock); + std::lock_guard freeing(m_fifo_access); for (auto worker : m_thread_workers) { std::lock_guard freeing_worker(*worker); } + for (auto mem : m_muxes) { + mem->std::mutex::~mutex(); + } for (auto mem : m_mux_mallocs) { delete[] mem; } } void addWorkers(size_t n) { - std::lock_guard queueing_growth(m_available_lock); + std::lock_guard growing_pool(m_fifo_access); const auto from = m_thread_workers.size(); - for (size_t i = from + n; i > from; i--) { - m_available.push_back(i); - } const auto muxes = new std::mutex[n]; - m_mux_mallocs.push_back(&muxes); + m_mux_mallocs.push_back(muxes); + for (size_t i = 0; i < n; i++) { + m_available.push_back(from + i); + tprintf("pushing available mux %lu\n", from + i); + m_muxes.push_back(&muxes[i]); + } // static_cast needed for some versions of C++ std::transform( muxes, muxes + n, std::back_inserter(m_thread_workers), static_cast(std::addressof)); - m_waiting = true; - for (int i = n; i-- > 0; !unlock()) {} + tprintf("available size is now %lu\n", m_available.size()); + for (int i = n; i > 0; i--) { + unlock(true, m_available.size()); + } } int acquireWorker() { - if (!m_waiting) { - lock(); - } - std::lock_guard acquiring(m_available_lock); + tprintf("acquire called\n"); + auto fifo_lock = lock(); const auto res = m_available.front(); m_available.pop_front(); - m_waiting = m_available.empty(); + if (!m_available.empty()) { + unlock(true, m_available.size()); + } + tprintf("acquiring %lu; available size is now %lu\n", res, m_available.size()); + assert(m_available.size() <= m_thread_workers.size()); return res; } void releaseWorker(int worker) { - std::lock_guard acquiring(m_available_lock); + std::lock_guard growing_pool(m_fifo_access); m_available.push_front(worker); - m_waiting = unlock(); + tprintf("releasing %d; available size is now %lu\n", worker, m_available.size()); + unlock(true, m_available.size()); } }; diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 8e4dce4d..0892ef2b 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -1,6 +1,6 @@ -#include -#include -#include +//#include +//#include +#include #include "ClientPool.h" namespace douban { @@ -61,8 +61,9 @@ int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, std::copy(ports, ports + n, m_ports.begin()); std::atomic rv = 0; - std::lock_guard updating(m_available_lock); - std::for_each(std::execution::par_unseq, irange(), irange(n), + std::lock_guard updating(m_fifo_access); + //std::for_each(std::execution::par_unseq, irange(), irange(m_clients.size()), + std::for_each(irange(), irange(m_clients.size()), [this, &rv](int i) { std::lock_guard updating_worker(*m_thread_workers[i]); const int err = m_clients[i].c.updateServers( @@ -88,12 +89,14 @@ int ClientPool::growPool(size_t by) { assert(by > 0); std::lock_guard growing_pool(m_pool_lock); size_t from = m_clients.size(); + tprintf("growing from %lu by %lu\n", from, by); m_clients.resize(from + by); - const auto start = m_clients.begin() + from; std::atomic rv = 0; - std::for_each(std::execution::par_unseq, start, start + by, - [this, &rv](IndexedClient& c) { - const int err = setup(&c.c); + //std::for_each(std::execution::par_unseq, irange(from), irange(from + by), + std::for_each(irange(from), irange(from + by), + [this, &rv](int i) { + const int err = setup(&m_clients[i].c); + m_clients[i].index = i; if (err != 0) { rv.store(err, std::memory_order_relaxed); } @@ -101,11 +104,12 @@ int ClientPool::growPool(size_t by) { // adds workers with non-zero return values // if changed, acquire should probably raise rather than hang addWorkers(by); + tprintf("size is now %lu\n", m_clients.size()); return rv; } inline bool ClientPool::shouldGrowUnsafe() { - return m_clients.size() < m_max_clients && !m_waiting; + return m_clients.size() < m_max_clients && m_locked; } int ClientPool::autoGrow() { @@ -122,7 +126,9 @@ IndexedClient* ClientPool::_acquire() { const auto growing = shouldGrowUnsafe(); m_acquiring_growth.unlock_shared(); if (growing) { - std::thread acquire_overflow(&ClientPool::autoGrow, this); + //std::thread acquire_overflow(&ClientPool::autoGrow, this); + //acquire_overflow.detach(); + autoGrow(); } int idx = acquireWorker(); diff --git a/tests/test_client_pool.cpp b/tests/test_client_pool.cpp index 7898394b..24ed7afe 100644 --- a/tests/test_client_pool.cpp +++ b/tests/test_client_pool.cpp @@ -1,8 +1,100 @@ #include "ClientPool.h" +#include "test_common.h" #include #include "gtest/gtest.h" -TEST(test_client_pool, threaded_set_get_del) { - -} \ No newline at end of file +using douban::mc::ClientPool; +using douban::mc::tests::gen_random; + +const unsigned int n_ops = 5; +const unsigned int data_size = 10; +const unsigned int n_servers = 20; +const unsigned int start_port = 21211; +const char host[] = "127.0.0.1"; + +TEST(test_client_pool, simple_set_get) { + uint32_t ports[n_servers]; + const char* hosts[n_servers]; + for (unsigned int i = 0; i < n_servers; i++) { + ports[i] = start_port + i; + hosts[i] = host; + } + + ClientPool* pool = new ClientPool(); + pool->config(CFG_HASH_FUNCTION, OPT_HASH_FNV1A_32); + pool->init(hosts, ports, n_servers); + + retrieval_result_t **r_results = NULL; + message_result_t **m_results = NULL; + size_t nResults = 0; + flags_t flags[] = {}; + size_t data_lens[] = {data_size}; + exptime_t exptime = 0; + char key[data_size + 1]; + char value[data_size + 1]; + const char* keys = &key[0]; + const char* values = &value[0]; + + for (unsigned int j = 0; j < n_ops; j++) { + gen_random(key, data_size); + gen_random(value, data_size); + auto c = pool->acquire(); + c->set(&keys, data_lens, flags, exptime, NULL, 0, &values, data_lens, 1, &m_results, &nResults); + c->destroyMessageResult(); + c->get(&keys, data_lens, 1, &r_results, &nResults); + ASSERT_N_STREQ(r_results[0]->data_block, values, data_size); + c->destroyRetrievalResult(); + pool->release(c); + } + + delete pool; +} + +TEST(test_client_pool, threaded_set_get) { + unsigned int n_threads = 8; + uint32_t ports[n_servers]; + const char* hosts[n_servers]; + for (unsigned int i = 0; i < n_servers; i++) { + ports[i] = start_port + i; + hosts[i] = host; + } + + std::thread* threads = new std::thread[n_threads]; + ClientPool* pool = new ClientPool(); + pool->config(CFG_HASH_FUNCTION, OPT_HASH_FNV1A_32); + pool->init(hosts, ports, n_servers); + + for (unsigned int i = 0; i < n_threads; i++) { + threads[i] = std::thread([&pool]() { + retrieval_result_t **r_results = NULL; + message_result_t **m_results = NULL; + size_t nResults = 0; + flags_t flags[] = {}; + size_t data_lens[] = {data_size}; + exptime_t exptime = 0; + char key[data_size + 1]; + char value[data_size + 1]; + const char* keys = &key[0]; + const char* values = &value[0]; + + for (unsigned int j = 0; j < n_ops; j++) { + gen_random(key, data_size); + gen_random(value, data_size); + auto c = pool->_acquire(); + tprintf("acquired client %d\n", c->index); + c->c.set(&keys, data_lens, flags, exptime, NULL, 0, &values, data_lens, 1, &m_results, &nResults); + c->c.destroyMessageResult(); + c->c.get(&keys, data_lens, 1, &r_results, &nResults); + ASSERT_N_STREQ(r_results[0]->data_block, values, data_size); + c->c.destroyRetrievalResult(); + pool->_release(c); + } + }); + } + for (unsigned int i = 0; i < n_threads; i++) { + threads[i].join(); + } + delete[] threads; + delete pool; +} From 6c860c330c2704933554ff829d679ea57593b46b Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 20 Dec 2023 15:11:18 -0800 Subject: [PATCH 14/95] ...with a different error though --- include/LockPool.h | 28 +++++++++++++--------------- tests/test_client_pool.cpp | 12 ++++++------ 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/include/LockPool.h b/include/LockPool.h index 3ad26aec..6e89f2f5 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -31,11 +31,12 @@ class OrderedLock { std::unique_lock lock() { std::unique_lock acquire(m_fifo_access); if (m_locked) { - tprintf("locked %lu already waiting\n", m_fifo_locks.size()); std::condition_variable signal; + tprintf("locked %lu already waiting signal %p\n", m_fifo_locks.size(), &signal); m_fifo_locks.emplace(&signal); signal.wait(acquire); - tprintf("unlocked %lu left waiting lock %p\n", m_fifo_locks.size(), &signal); + signal.notify_one(); + tprintf("unlocked %lu left waiting signal %p\n", m_fifo_locks.size(), &signal); assert(acquire.owns_lock()); } else { m_locked = true; @@ -44,19 +45,16 @@ class OrderedLock { return acquire; } - void unlock(const bool owns_fifo = false, int i = 0) { - std::unique_lock acquire; - if (!owns_fifo) { - acquire = std::unique_lock(m_fifo_access); - } + void unlock(std::unique_lock& acquire, int i = 0) { if (m_fifo_locks.empty()) { tprintf("fifo already empty\n"); m_locked = false; } else { - const std::condition_variable* lock_ptr = m_fifo_locks.front(); - tprintf("unlocking with %d available %lu waiting lock %p\n", i, m_fifo_locks.size(), lock_ptr); - m_fifo_locks.front()->notify_one(); + std::condition_variable* signal = m_fifo_locks.front(); m_fifo_locks.pop(); + tprintf("unlocking with %d available %lu waiting lock %p\n", i, m_fifo_locks.size(), signal); + signal->notify_one(); + signal->wait(acquire); } } }; @@ -84,7 +82,7 @@ class LockPool : public OrderedLock { } void addWorkers(size_t n) { - std::lock_guard growing_pool(m_fifo_access); + std::unique_lock growing_pool(m_fifo_access); const auto from = m_thread_workers.size(); const auto muxes = new std::mutex[n]; m_mux_mallocs.push_back(muxes); @@ -99,7 +97,7 @@ class LockPool : public OrderedLock { static_cast(std::addressof)); tprintf("available size is now %lu\n", m_available.size()); for (int i = n; i > 0; i--) { - unlock(true, m_available.size()); + unlock(growing_pool, m_available.size()); } } @@ -109,7 +107,7 @@ class LockPool : public OrderedLock { const auto res = m_available.front(); m_available.pop_front(); if (!m_available.empty()) { - unlock(true, m_available.size()); + unlock(fifo_lock, m_available.size()); } tprintf("acquiring %lu; available size is now %lu\n", res, m_available.size()); assert(m_available.size() <= m_thread_workers.size()); @@ -117,10 +115,10 @@ class LockPool : public OrderedLock { } void releaseWorker(int worker) { - std::lock_guard growing_pool(m_fifo_access); + std::unique_lock growing_pool(m_fifo_access); m_available.push_front(worker); tprintf("releasing %d; available size is now %lu\n", worker, m_available.size()); - unlock(true, m_available.size()); + unlock(growing_pool, m_available.size()); } }; diff --git a/tests/test_client_pool.cpp b/tests/test_client_pool.cpp index 24ed7afe..e1323c4e 100644 --- a/tests/test_client_pool.cpp +++ b/tests/test_client_pool.cpp @@ -39,13 +39,13 @@ TEST(test_client_pool, simple_set_get) { for (unsigned int j = 0; j < n_ops; j++) { gen_random(key, data_size); gen_random(value, data_size); - auto c = pool->acquire(); - c->set(&keys, data_lens, flags, exptime, NULL, 0, &values, data_lens, 1, &m_results, &nResults); - c->destroyMessageResult(); - c->get(&keys, data_lens, 1, &r_results, &nResults); + auto c = pool->_acquire(); + c->c.set(&keys, data_lens, flags, exptime, NULL, 0, &values, data_lens, 1, &m_results, &nResults); + c->c.destroyMessageResult(); + c->c.get(&keys, data_lens, 1, &r_results, &nResults); ASSERT_N_STREQ(r_results[0]->data_block, values, data_size); - c->destroyRetrievalResult(); - pool->release(c); + c->c.destroyRetrievalResult(); + pool->_release(c); } delete pool; From 9ce5327d8f0340a17b241e083802af8165c9bc71 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 20 Dec 2023 16:40:25 -0800 Subject: [PATCH 15/95] stranger error --- include/LockPool.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/include/LockPool.h b/include/LockPool.h index 6e89f2f5..7749a5a5 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -13,6 +13,8 @@ #include #include #include +#include +#include namespace douban { namespace mc { @@ -21,7 +23,7 @@ namespace mc { // https://stackoverflow.com/a/14792685/3476782 class OrderedLock { - std::queue m_fifo_locks; + std::list m_fifo_locks; protected: std::mutex m_fifo_access; bool m_locked; @@ -33,9 +35,9 @@ class OrderedLock { if (m_locked) { std::condition_variable signal; tprintf("locked %lu already waiting signal %p\n", m_fifo_locks.size(), &signal); - m_fifo_locks.emplace(&signal); + m_fifo_locks.emplace_back(&signal); signal.wait(acquire); - signal.notify_one(); + signal.notify_all(); tprintf("unlocked %lu left waiting signal %p\n", m_fifo_locks.size(), &signal); assert(acquire.owns_lock()); } else { @@ -51,10 +53,11 @@ class OrderedLock { m_locked = false; } else { std::condition_variable* signal = m_fifo_locks.front(); - m_fifo_locks.pop(); + auto it = m_fifo_locks.begin(); tprintf("unlocking with %d available %lu waiting lock %p\n", i, m_fifo_locks.size(), signal); signal->notify_one(); signal->wait(acquire); + m_fifo_locks.erase(it); } } }; From 07964a8aa49d776f740fb6543624e6043fa87e13 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 20 Dec 2023 17:07:10 -0800 Subject: [PATCH 16/95] well that feels silly after the fact --- include/LockPool.h | 28 ++++++---------------------- src/ClientPool.cpp | 2 -- tests/test_client_pool.cpp | 1 - 3 files changed, 6 insertions(+), 25 deletions(-) diff --git a/include/LockPool.h b/include/LockPool.h index 7749a5a5..ff3b2265 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -19,8 +19,6 @@ namespace douban { namespace mc { -#define tprintf(str, ...) printf("tid = %lu: " str, syscall(SYS_gettid), ##__VA_ARGS__) - // https://stackoverflow.com/a/14792685/3476782 class OrderedLock { std::list m_fifo_locks; @@ -34,30 +32,21 @@ class OrderedLock { std::unique_lock acquire(m_fifo_access); if (m_locked) { std::condition_variable signal; - tprintf("locked %lu already waiting signal %p\n", m_fifo_locks.size(), &signal); m_fifo_locks.emplace_back(&signal); signal.wait(acquire); - signal.notify_all(); - tprintf("unlocked %lu left waiting signal %p\n", m_fifo_locks.size(), &signal); + m_fifo_locks.pop_front(); assert(acquire.owns_lock()); } else { m_locked = true; - tprintf("passed through unlocked fifo\n"); } return acquire; } - void unlock(std::unique_lock& acquire, int i = 0) { + void unlock() { if (m_fifo_locks.empty()) { - tprintf("fifo already empty\n"); m_locked = false; } else { - std::condition_variable* signal = m_fifo_locks.front(); - auto it = m_fifo_locks.begin(); - tprintf("unlocking with %d available %lu waiting lock %p\n", i, m_fifo_locks.size(), signal); - signal->notify_one(); - signal->wait(acquire); - m_fifo_locks.erase(it); + m_fifo_locks.front()->notify_one(); } } }; @@ -91,28 +80,24 @@ class LockPool : public OrderedLock { m_mux_mallocs.push_back(muxes); for (size_t i = 0; i < n; i++) { m_available.push_back(from + i); - tprintf("pushing available mux %lu\n", from + i); m_muxes.push_back(&muxes[i]); } // static_cast needed for some versions of C++ std::transform( muxes, muxes + n, std::back_inserter(m_thread_workers), static_cast(std::addressof)); - tprintf("available size is now %lu\n", m_available.size()); for (int i = n; i > 0; i--) { - unlock(growing_pool, m_available.size()); + unlock(); } } int acquireWorker() { - tprintf("acquire called\n"); auto fifo_lock = lock(); const auto res = m_available.front(); m_available.pop_front(); if (!m_available.empty()) { - unlock(fifo_lock, m_available.size()); + unlock(); } - tprintf("acquiring %lu; available size is now %lu\n", res, m_available.size()); assert(m_available.size() <= m_thread_workers.size()); return res; } @@ -120,8 +105,7 @@ class LockPool : public OrderedLock { void releaseWorker(int worker) { std::unique_lock growing_pool(m_fifo_access); m_available.push_front(worker); - tprintf("releasing %d; available size is now %lu\n", worker, m_available.size()); - unlock(growing_pool, m_available.size()); + unlock(); } }; diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 0892ef2b..a31ca992 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -89,7 +89,6 @@ int ClientPool::growPool(size_t by) { assert(by > 0); std::lock_guard growing_pool(m_pool_lock); size_t from = m_clients.size(); - tprintf("growing from %lu by %lu\n", from, by); m_clients.resize(from + by); std::atomic rv = 0; //std::for_each(std::execution::par_unseq, irange(from), irange(from + by), @@ -104,7 +103,6 @@ int ClientPool::growPool(size_t by) { // adds workers with non-zero return values // if changed, acquire should probably raise rather than hang addWorkers(by); - tprintf("size is now %lu\n", m_clients.size()); return rv; } diff --git a/tests/test_client_pool.cpp b/tests/test_client_pool.cpp index e1323c4e..0c84df3e 100644 --- a/tests/test_client_pool.cpp +++ b/tests/test_client_pool.cpp @@ -82,7 +82,6 @@ TEST(test_client_pool, threaded_set_get) { gen_random(key, data_size); gen_random(value, data_size); auto c = pool->_acquire(); - tprintf("acquired client %d\n", c->index); c->c.set(&keys, data_lens, flags, exptime, NULL, 0, &values, data_lens, 1, &m_results, &nResults); c->c.destroyMessageResult(); c->c.get(&keys, data_lens, 1, &r_results, &nResults); From 550ef713eaeb2236bb1f23751f396eb477dc49ac Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 20 Dec 2023 17:22:38 -0800 Subject: [PATCH 17/95] redundancies --- include/LockPool.h | 22 ++++------------------ src/ClientPool.cpp | 1 + tests/test_client_pool.cpp | 24 ++++++++++++------------ 3 files changed, 17 insertions(+), 30 deletions(-) diff --git a/include/LockPool.h b/include/LockPool.h index ff3b2265..f63a97ab 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -5,23 +5,13 @@ #include #include #include -#include -#include -#include -#include - -#include -#include -#include -#include -#include namespace douban { namespace mc { // https://stackoverflow.com/a/14792685/3476782 class OrderedLock { - std::list m_fifo_locks; + std::queue m_fifo_locks; protected: std::mutex m_fifo_access; bool m_locked; @@ -32,10 +22,9 @@ class OrderedLock { std::unique_lock acquire(m_fifo_access); if (m_locked) { std::condition_variable signal; - m_fifo_locks.emplace_back(&signal); + m_fifo_locks.emplace(&signal); signal.wait(acquire); - m_fifo_locks.pop_front(); - assert(acquire.owns_lock()); + m_fifo_locks.pop(); } else { m_locked = true; } @@ -86,9 +75,7 @@ class LockPool : public OrderedLock { std::transform( muxes, muxes + n, std::back_inserter(m_thread_workers), static_cast(std::addressof)); - for (int i = n; i > 0; i--) { - unlock(); - } + unlock(); } int acquireWorker() { @@ -98,7 +85,6 @@ class LockPool : public OrderedLock { if (!m_available.empty()) { unlock(); } - assert(m_available.size() <= m_thread_workers.size()); return res; } diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index a31ca992..02c982cb 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -1,5 +1,6 @@ //#include //#include +#include #include #include "ClientPool.h" diff --git a/tests/test_client_pool.cpp b/tests/test_client_pool.cpp index 0c84df3e..611b1b52 100644 --- a/tests/test_client_pool.cpp +++ b/tests/test_client_pool.cpp @@ -39,13 +39,13 @@ TEST(test_client_pool, simple_set_get) { for (unsigned int j = 0; j < n_ops; j++) { gen_random(key, data_size); gen_random(value, data_size); - auto c = pool->_acquire(); - c->c.set(&keys, data_lens, flags, exptime, NULL, 0, &values, data_lens, 1, &m_results, &nResults); - c->c.destroyMessageResult(); - c->c.get(&keys, data_lens, 1, &r_results, &nResults); + auto c = pool->acquire(); + c->set(&keys, data_lens, flags, exptime, NULL, 0, &values, data_lens, 1, &m_results, &nResults); + c->destroyMessageResult(); + c->get(&keys, data_lens, 1, &r_results, &nResults); ASSERT_N_STREQ(r_results[0]->data_block, values, data_size); - c->c.destroyRetrievalResult(); - pool->_release(c); + c->destroyRetrievalResult(); + pool->release(c); } delete pool; @@ -81,13 +81,13 @@ TEST(test_client_pool, threaded_set_get) { for (unsigned int j = 0; j < n_ops; j++) { gen_random(key, data_size); gen_random(value, data_size); - auto c = pool->_acquire(); - c->c.set(&keys, data_lens, flags, exptime, NULL, 0, &values, data_lens, 1, &m_results, &nResults); - c->c.destroyMessageResult(); - c->c.get(&keys, data_lens, 1, &r_results, &nResults); + auto c = pool->acquire(); + c->set(&keys, data_lens, flags, exptime, NULL, 0, &values, data_lens, 1, &m_results, &nResults); + c->destroyMessageResult(); + c->get(&keys, data_lens, 1, &r_results, &nResults); ASSERT_N_STREQ(r_results[0]->data_block, values, data_size); - c->c.destroyRetrievalResult(); - pool->_release(c); + c->destroyRetrievalResult(); + pool->release(c); } }); } From f099e6876409319419fa94dc6edfa40fd46820d0 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 20 Dec 2023 19:22:02 -0800 Subject: [PATCH 18/95] partially fix cython integration --- libmc/_client.pyx | 40 +++++++++++++++++++++----------------- src/ClientPool.cpp | 11 +++++------ tests/test_client_pool.cpp | 5 +++-- tests/test_client_pool.py | 38 ++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 26 deletions(-) create mode 100644 tests/test_client_pool.py diff --git a/libmc/_client.pyx b/libmc/_client.pyx index 87c61b6a..14c0533a 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -24,7 +24,7 @@ import threading import zlib import marshal import warnings - +from contextlib import contextmanager cdef extern from "Common.h" namespace "douban::mc": ctypedef enum op_code_t: @@ -388,10 +388,7 @@ cdef class PyClientSettings: self.init() - def init(): - pass - - def _args(): + def _args(self): return (self.servers, self.do_split, self.comp_threshold, self.noreply, self.prefix, self.hash_fn, self.failover, self.encoding) @@ -405,7 +402,7 @@ cdef class PyClient(PyClientSettings): cdef object _thread_ident cdef object _created_stack - def init(): + def init(self): self.last_error = RET_OK self._thread_ident = None self._created_stack = traceback.extract_stack() @@ -1132,10 +1129,10 @@ cdef class PyClient(PyClientSettings): return errCodeToString(self.last_error) -class PyPoolClient(PyClient): +cdef class PyPoolClient(PyClient): cdef IndexedClient* _indexed - def init(): + def init(self): self.last_error = RET_OK self._thread_ident = None self._created_stack = traceback.extract_stack() @@ -1144,29 +1141,36 @@ class PyPoolClient(PyClient): pass -class PyClientPool(PyClientSettings): - worker = PyPoolClient +cdef class PyClientPool(PyClientSettings): cdef list clients + cdef ClientPool* _imp - def init(): + cdef init(self): self._imp = new ClientPool() self._imp.config(CFG_HASH_FUNCTION, self.hash_fn) self.clients = [] - def setup(self, IndexedClientClient* imp): - worker = __class__.worker(*self._args()) + cdef setup(self, IndexedClient* imp): + worker = PyPoolClient(*self._args()) worker._indexed = imp - worker._imp = imp.c + worker._imp = &imp.c return worker def acquire(self): worker = self._imp._acquire() if worker.index >= len(self.clients): - clients += [None] * (worker.index - len(self.clients)) - clients.append(setup(worker)) + self.clients += [None] * (worker.index - len(self.clients)) + self.clients.append(self.setup(worker)) elif self.clients[worker.index] == None: - self.clients[i] = setup(worker); - return self.clients[i] + self.clients[worker.index] = self.setup(worker) + return self.clients[worker.index] def release(self, PyPoolClient worker): self._imp._release(worker._indexed) + + @contextmanager + def client(self): + try: + yield self.acquire() + finally: + self.release() diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 02c982cb..753ba603 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -1,5 +1,5 @@ //#include -//#include +#include #include #include #include "ClientPool.h" @@ -63,8 +63,8 @@ int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, std::atomic rv = 0; std::lock_guard updating(m_fifo_access); - //std::for_each(std::execution::par_unseq, irange(), irange(m_clients.size()), std::for_each(irange(), irange(m_clients.size()), + //std::for_each(std::execution::par_unseq, irange(), irange(m_clients.size()), [this, &rv](int i) { std::lock_guard updating_worker(*m_thread_workers[i]); const int err = m_clients[i].c.updateServers( @@ -92,8 +92,8 @@ int ClientPool::growPool(size_t by) { size_t from = m_clients.size(); m_clients.resize(from + by); std::atomic rv = 0; - //std::for_each(std::execution::par_unseq, irange(from), irange(from + by), std::for_each(irange(from), irange(from + by), + //std::for_each(std::execution::par_unseq, irange(from), irange(from + by), [this, &rv](int i) { const int err = setup(&m_clients[i].c); m_clients[i].index = i; @@ -125,9 +125,8 @@ IndexedClient* ClientPool::_acquire() { const auto growing = shouldGrowUnsafe(); m_acquiring_growth.unlock_shared(); if (growing) { - //std::thread acquire_overflow(&ClientPool::autoGrow, this); - //acquire_overflow.detach(); - autoGrow(); + std::thread acquire_overflow(&ClientPool::autoGrow, this); + acquire_overflow.detach(); } int idx = acquireWorker(); diff --git a/tests/test_client_pool.cpp b/tests/test_client_pool.cpp index 611b1b52..6ae7ed73 100644 --- a/tests/test_client_pool.cpp +++ b/tests/test_client_pool.cpp @@ -12,6 +12,7 @@ const unsigned int data_size = 10; const unsigned int n_servers = 20; const unsigned int start_port = 21211; const char host[] = "127.0.0.1"; +unsigned int n_threads = 8; TEST(test_client_pool, simple_set_get) { uint32_t ports[n_servers]; @@ -36,7 +37,7 @@ TEST(test_client_pool, simple_set_get) { const char* keys = &key[0]; const char* values = &value[0]; - for (unsigned int j = 0; j < n_ops; j++) { + for (unsigned int j = 0; j < n_ops * n_threads; j++) { gen_random(key, data_size); gen_random(value, data_size); auto c = pool->acquire(); @@ -52,7 +53,6 @@ TEST(test_client_pool, simple_set_get) { } TEST(test_client_pool, threaded_set_get) { - unsigned int n_threads = 8; uint32_t ports[n_servers]; const char* hosts[n_servers]; for (unsigned int i = 0; i < n_servers; i++) { @@ -63,6 +63,7 @@ TEST(test_client_pool, threaded_set_get) { std::thread* threads = new std::thread[n_threads]; ClientPool* pool = new ClientPool(); pool->config(CFG_HASH_FUNCTION, OPT_HASH_FNV1A_32); + //pool->config(CFG_INITIAL_CLIENTS, 4); pool->init(hosts, ports, n_servers); for (unsigned int i = 0; i < n_threads; i++) { diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py new file mode 100644 index 00000000..5964bec4 --- /dev/null +++ b/tests/test_client_pool.py @@ -0,0 +1,38 @@ +# coding: utf-8 +import unittest +from threading import Thread +from libmc import ClientPool + + +class ThreadedSingleServerCase(unittest.TestCase): + def setUp(self): + self.pool = ClientPool(["127.0.0.1:21211"]) + + def misc(self): + for i in range(5): + with self.pool.client() as mc: + tid = str(mc._get_current_thread_ident() + (i,)) + f, t = 'foo ' + tid, 'tuiche ' + tid + mc.get_multi([f, t]) + mc.delete(f) + mc.delete(t) + assert mc.get(f) is None + assert mc.get(t) is None + + mc.set(f, 'biu') + mc.set(t, 'bb') + assert mc.get(f) == 'biu' + assert mc.get(t) == 'bb' + assert (mc.get_multi([f, t]) == + {f: 'biu', t: 'bb'}) + mc.set_multi({f: 1024, t: '8964'}) + assert (mc.get_multi([f, t]) == + {f: 1024, t: '8964'}) + + def test_misc(self): + ts = [Thread(target=self.misc) for i in range(8)] + for t in ts: + t.start() + + for t in ts: + t.join() From cfdd1b4725486876aa96e4dbcd2da433e0608a15 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 20 Dec 2023 20:15:24 -0800 Subject: [PATCH 19/95] reset cmake --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ebd9f84..04a4799a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: Debug Release." FORCE) endif (NOT CMAKE_BUILD_TYPE) -set(CMAKE_CXX_FLAGS_COMMON "-Wall -fno-rtti") +set(CMAKE_CXX_FLAGS_COMMON "-Wall -fno-rtti -fno-exceptions") set(CMAKE_CXX_FLAGS_DEBUG "-DDEBUG -g2 ${CMAKE_CXX_FLAGS_COMMON}" CACHE STRING "CXX DEBUG FLAGS" FORCE) set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3 ${CMAKE_CXX_FLAGS_COMMON}" CACHE STRING "CXX RELEASE FLAGS" FORCE) set(CMAKE_INSTALL_INCLUDE include CACHE PATH "Output directory for header files") From e4af40d086722eb09fe237a2c270b0d338468c37 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 20 Dec 2023 20:22:54 -0800 Subject: [PATCH 20/95] python class --- libmc/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libmc/__init__.py b/libmc/__init__.py index ade3209f..2e4a1122 100644 --- a/libmc/__init__.py +++ b/libmc/__init__.py @@ -1,6 +1,6 @@ import os from ._client import ( - PyClient, PyClientPool as ClientPool, ThreadUnsafe, + PyClient, PyClientPool, ThreadUnsafe, encode_value, decode_value, @@ -41,6 +41,9 @@ class Client(PyClient): pass +class ClientPool(PyClientPool): + pass + DYNAMIC_LIBRARIES = [os.path.abspath(_libmc_so_file)] From f538b6edfd7d10845783bb860aba1b2df3332e83 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 20 Dec 2023 20:43:54 -0800 Subject: [PATCH 21/95] remove header... --- src/ClientPool.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 753ba603..abe40971 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -1,7 +1,6 @@ //#include #include #include -#include #include "ClientPool.h" namespace douban { From f0b37f1330361d2f01376d75744b10d8a574a8aa Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Wed, 20 Dec 2023 21:20:17 -0800 Subject: [PATCH 22/95] __cinit__ semantics --- libmc/_client.pyx | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index 14c0533a..7f97a7c5 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -386,8 +386,6 @@ cdef class PyClientSettings: else: self.prefix = None - self.init() - def _args(self): return (self.servers, self.do_split, self.comp_threshold, self.noreply, self.prefix, self.hash_fn, self.failover, self.encoding) @@ -402,7 +400,7 @@ cdef class PyClient(PyClientSettings): cdef object _thread_ident cdef object _created_stack - def init(self): + def __cinit__(self): self.last_error = RET_OK self._thread_ident = None self._created_stack = traceback.extract_stack() @@ -1132,7 +1130,7 @@ cdef class PyClient(PyClientSettings): cdef class PyPoolClient(PyClient): cdef IndexedClient* _indexed - def init(self): + def __cinit__(self): self.last_error = RET_OK self._thread_ident = None self._created_stack = traceback.extract_stack() @@ -1145,7 +1143,7 @@ cdef class PyClientPool(PyClientSettings): cdef list clients cdef ClientPool* _imp - cdef init(self): + def __cinit__(self): self._imp = new ClientPool() self._imp.config(CFG_HASH_FUNCTION, self.hash_fn) self.clients = [] From 4d7ecdbba1a258069f6d72837c54907862d54aeb Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Thu, 21 Dec 2023 00:01:10 -0800 Subject: [PATCH 23/95] split up tests --- tests/test_client_pool.py | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 5964bec4..b8ab68e4 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -10,26 +10,29 @@ def setUp(self): def misc(self): for i in range(5): - with self.pool.client() as mc: - tid = str(mc._get_current_thread_ident() + (i,)) - f, t = 'foo ' + tid, 'tuiche ' + tid - mc.get_multi([f, t]) - mc.delete(f) - mc.delete(t) - assert mc.get(f) is None - assert mc.get(t) is None - - mc.set(f, 'biu') - mc.set(t, 'bb') - assert mc.get(f) == 'biu' - assert mc.get(t) == 'bb' - assert (mc.get_multi([f, t]) == - {f: 'biu', t: 'bb'}) - mc.set_multi({f: 1024, t: '8964'}) - assert (mc.get_multi([f, t]) == - {f: 1024, t: '8964'}) + self.test_misc() def test_misc(self): + with self.pool.client() as mc: + tid = str(mc._get_current_thread_ident() + (i,)) + f, t = 'foo ' + tid, 'tuiche ' + tid + mc.get_multi([f, t]) + mc.delete(f) + mc.delete(t) + assert mc.get(f) is None + assert mc.get(t) is None + + mc.set(f, 'biu') + mc.set(t, 'bb') + assert mc.get(f) == 'biu' + assert mc.get(t) == 'bb' + assert (mc.get_multi([f, t]) == + {f: 'biu', t: 'bb'}) + mc.set_multi({f: 1024, t: '8964'}) + assert (mc.get_multi([f, t]) == + {f: 1024, t: '8964'}) + + def test_misc_threaded(self): ts = [Thread(target=self.misc) for i in range(8)] for t in ts: t.start() From 87b834f5c250510178abd4ff08e86db23c2e2d99 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Thu, 21 Dec 2023 00:08:33 -0800 Subject: [PATCH 24/95] flustered and cygdb can't find debugging logs --- tests/test_client_pool.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index b8ab68e4..8ab2a3fb 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -12,6 +12,9 @@ def misc(self): for i in range(5): self.test_misc() + def test_acquire(self): + assert self.pool.acquire() is not None + def test_misc(self): with self.pool.client() as mc: tid = str(mc._get_current_thread_ident() + (i,)) From df5f3d398ef20048b7d3ded1a668551099b5328e Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Thu, 21 Dec 2023 01:49:12 -0800 Subject: [PATCH 25/95] mild QOL patch --- misc/git/hide-debug.patch | 10 ++++++++++ misc/git/pre-commit.sh | 8 ++++++++ 2 files changed, 18 insertions(+) create mode 100644 misc/git/hide-debug.patch create mode 100755 misc/git/pre-commit.sh diff --git a/misc/git/hide-debug.patch b/misc/git/hide-debug.patch new file mode 100644 index 00000000..0bbb276d --- /dev/null +++ b/misc/git/hide-debug.patch @@ -0,0 +1,10 @@ +--- a/setup.py ++++ b/setup.py +@@ -129,7 +129,6 @@ setup( + include_dirs=include_dirs, + language="c++", + extra_compile_args=COMPILER_FLAGS, +- gdb_debug=True, + ) + ], + tests_require=[ diff --git a/misc/git/pre-commit.sh b/misc/git/pre-commit.sh new file mode 100755 index 00000000..6ff5c834 --- /dev/null +++ b/misc/git/pre-commit.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +DIR="$( dirname `readlink -f "${BASH_SOURCE[0]}"`)" +HOOK="$DIR/../../.git/hooks/pre-commit" + +[[ ! -e "$HOOK" ]] && ln -s "$DIR/pre-commit.sh" "$HOOK" +git apply --cached "$DIR/hide-debug.patch" -q --recount +exit 0 From 5eff5b39cf2ee9503fc4a2aa11dcfa3e3f7e2ee4 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 22 Dec 2023 11:08:47 -0800 Subject: [PATCH 26/95] trailing whitespace --- tests/test_client_pool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 8ab2a3fb..f5d37542 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -39,6 +39,6 @@ def test_misc_threaded(self): ts = [Thread(target=self.misc) for i in range(8)] for t in ts: t.start() - + for t in ts: t.join() From 6285a816709240ae6e8000d90b3bc977f5f64004 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 22 Dec 2023 22:23:36 +0000 Subject: [PATCH 27/95] forgot to init ClientPool --- .gitignore | 2 + libmc/_client.pyx | 126 +++++++++++++++++++++----------------- misc/git/hide-debug.patch | 27 ++++++-- setup.py | 5 +- 4 files changed, 99 insertions(+), 61 deletions(-) diff --git a/.gitignore b/.gitignore index 0e754f62..6d48b7e3 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ tests/resources/keys_*.txt /go.mod /go.sum + +/cython_debug \ No newline at end of file diff --git a/libmc/_client.pyx b/libmc/_client.pyx index 7f97a7c5..d5a97a4d 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -238,6 +238,9 @@ cdef extern from "ClientPool.h" namespace "douban::mc": IndexedClient* _acquire() nogil void _release(const IndexedClient* ref) nogil +ctypedef fused Configurable: + Client + ClientPool cdef uint32_t MC_DEFAULT_PORT = 11211 cdef flags_t _FLAG_EMPTY = 0 @@ -393,8 +396,45 @@ cdef class PyClientSettings: def __reduce__(self): return (self.__class__, self._args()) +cdef _update_servers(Configurable* imp, list servers, bool_t init): + cdef int rv = 0 + cdef size_t n = len(servers) + cdef char** c_hosts = PyMem_Malloc(n * sizeof(char*)) + cdef uint32_t* c_ports = PyMem_Malloc(n * sizeof(uint32_t)) + cdef char** c_aliases = PyMem_Malloc(n * sizeof(char*)) + + servers_ = [] + for srv in servers: + if PY_MAJOR_VERSION > 2: + srv = PyUnicode_AsUTF8String(srv) + srv = PyString_AsString(srv) + servers_.append(srv) + + Py_INCREF(servers_) + for i in range(n): + c_split = splitServerString(servers_[i]) + + c_hosts[i] = c_split.host + c_aliases[i] = c_split.alias + if c_split.port == NULL: + c_ports[i] = MC_DEFAULT_PORT + else: + c_ports[i] = PyInt_AsLong(int(c_split.port)) + + if init: + rv = imp.init(c_hosts, c_ports, n, c_aliases) + else: + rv = imp.updateServers(c_hosts, c_ports, n, c_aliases) + + PyMem_Free(c_hosts) + PyMem_Free(c_ports) + PyMem_Free(c_aliases) + + Py_DECREF(servers_) + + return rv -cdef class PyClient(PyClientSettings): +cdef class PyClientShell(PyClientSettings): cdef Client* _imp cdef err_code_t last_error cdef object _thread_ident @@ -405,58 +445,15 @@ cdef class PyClient(PyClientSettings): self._thread_ident = None self._created_stack = traceback.extract_stack() - self._imp = new Client() - self._imp.config(CFG_HASH_FUNCTION, self.hash_fn) - rv = self._update_servers(self.servers, True) - if self.failover: - self._imp.enableConsistentFailover() - else: - self._imp.disableConsistentFailover() - - cdef _update_servers(self, list servers, bool_t init): - cdef int rv = 0 - cdef size_t n = len(servers) - cdef char** c_hosts = PyMem_Malloc(n * sizeof(char*)) - cdef uint32_t* c_ports = PyMem_Malloc(n * sizeof(uint32_t)) - cdef char** c_aliases = PyMem_Malloc(n * sizeof(char*)) - - servers_ = [] - for srv in servers: - if PY_MAJOR_VERSION > 2: - srv = PyUnicode_AsUTF8String(srv) - srv = PyString_AsString(srv) - servers_.append(srv) - - Py_INCREF(servers_) - for i in range(n): - c_split = splitServerString(servers_[i]) - - c_hosts[i] = c_split.host - c_aliases[i] = c_split.alias - if c_split.port == NULL: - c_ports[i] = MC_DEFAULT_PORT - else: - c_ports[i] = PyInt_AsLong(int(c_split.port)) - - if init: - rv = self._imp.init(c_hosts, c_ports, n, c_aliases) - else: - rv = self._imp.updateServers(c_hosts, c_ports, n, c_aliases) - - PyMem_Free(c_hosts) - PyMem_Free(c_ports) - PyMem_Free(c_aliases) - - Py_DECREF(servers_) - - return rv - def __dealloc__(self): del self._imp def config(self, int opt, int val): self._imp.config(opt, val) + cdef connect(self): + rv = _update_servers(self._imp, self.servers, True) + def get_host_by_key(self, basestring key): cdef bytes key2 = self.normalize_key(key) cdef char* c_key = NULL @@ -1126,35 +1123,54 @@ cdef class PyClient(PyClientSettings): def get_last_strerror(self): return errCodeToString(self.last_error) +cdef class PyClient(PyClientShell): + def __cinit__(self): + self._imp = new Client() + self._imp.config(CFG_HASH_FUNCTION, self.hash_fn) + self.connect() -cdef class PyPoolClient(PyClient): - cdef IndexedClient* _indexed + if self.failover: + self._imp.enableConsistentFailover() + else: + self._imp.disableConsistentFailover() - def __cinit__(self): - self.last_error = RET_OK - self._thread_ident = None - self._created_stack = traceback.extract_stack() +cdef class PyPoolClient(PyClientShell): + cdef IndexedClient* _indexed def _check_thread_ident(self): pass - cdef class PyClientPool(PyClientSettings): cdef list clients cdef ClientPool* _imp def __cinit__(self): self._imp = new ClientPool() - self._imp.config(CFG_HASH_FUNCTION, self.hash_fn) + self.config(CFG_HASH_FUNCTION, self.hash_fn) + self._initialized = False self.clients = [] + if self.failover: + self.config(CFG_SET_FAILOVER, 1) + else: + self.config(CFG_SET_FAILOVER, 0) + + def config(self, int opt, int val): + self._imp.config(opt, val) + cdef setup(self, IndexedClient* imp): worker = PyPoolClient(*self._args()) worker._indexed = imp worker._imp = &imp.c return worker + cdef connect(self): + rv = _update_servers(self._imp, self.servers, True) + def acquire(self): + if not self._initialized: + self.connect() + self._initialized = True worker = self._imp._acquire() if worker.index >= len(self.clients): self.clients += [None] * (worker.index - len(self.clients)) diff --git a/misc/git/hide-debug.patch b/misc/git/hide-debug.patch index 0bbb276d..7931b9c7 100644 --- a/misc/git/hide-debug.patch +++ b/misc/git/hide-debug.patch @@ -1,10 +1,29 @@ +diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py -@@ -129,7 +129,6 @@ setup( - include_dirs=include_dirs, +@@ -10,7 +10,6 @@ from distutils.version import LooseVersion + from glob import glob + from setuptools import setup, Extension + from setuptools.command.test import test as TestCommand +-from Cython.Build import cythonize + + sources = (glob("src/*.cpp") + ["libmc/_client.pyx"]) + include_dirs = ["include"] +@@ -123,7 +122,7 @@ setup( + # Support for the basestring type is new in Cython 0.20. + setup_requires=["Cython >= 0.20"], + cmdclass={"test": PyTest}, +- ext_modules=cythonize([ ++ ext_modules=[ + Extension( + "libmc._client", + sources, +@@ -131,7 +130,7 @@ setup( language="c++", extra_compile_args=COMPILER_FLAGS, -- gdb_debug=True, ) - ], +- ], gdb_debug=True), ++ ], tests_require=[ + "pytest", + "future", diff --git a/setup.py b/setup.py index c9807b95..c2d5ed52 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ from glob import glob from setuptools import setup, Extension from setuptools.command.test import test as TestCommand +from Cython.Build import cythonize sources = (glob("src/*.cpp") + ["libmc/_client.pyx"]) include_dirs = ["include"] @@ -122,7 +123,7 @@ def run_tests(self): # Support for the basestring type is new in Cython 0.20. setup_requires=["Cython >= 0.20"], cmdclass={"test": PyTest}, - ext_modules=[ + ext_modules=cythonize([ Extension( "libmc._client", sources, @@ -130,7 +131,7 @@ def run_tests(self): language="c++", extra_compile_args=COMPILER_FLAGS, ) - ], + ], gdb_debug=True), tests_require=[ "pytest", "future", From 167d123ad203bd50be4455df26a14babe0945689 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 22 Dec 2023 22:37:18 +0000 Subject: [PATCH 28/95] pre-commit still fails on macos --- libmc/_client.pyx | 9 +++++---- tests/test_client_pool.py | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index d5a97a4d..b3bb1e1b 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -1167,7 +1167,7 @@ cdef class PyClientPool(PyClientSettings): cdef connect(self): rv = _update_servers(self._imp, self.servers, True) - def acquire(self): + cdef acquire(self): if not self._initialized: self.connect() self._initialized = True @@ -1179,12 +1179,13 @@ cdef class PyClientPool(PyClientSettings): self.clients[worker.index] = self.setup(worker) return self.clients[worker.index] - def release(self, PyPoolClient worker): + cdef release(self, PyPoolClient worker): self._imp._release(worker._indexed) @contextmanager def client(self): try: - yield self.acquire() + worker = self.acquire() + yield worker finally: - self.release() + self.release(worker) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index f5d37542..d1644700 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -10,12 +10,13 @@ def setUp(self): def misc(self): for i in range(5): - self.test_misc() + self.test_misc(i) def test_acquire(self): - assert self.pool.acquire() is not None + with self.pool.client() as mc: + pass - def test_misc(self): + def test_client_pool_misc(self, i=0): with self.pool.client() as mc: tid = str(mc._get_current_thread_ident() + (i,)) f, t = 'foo ' + tid, 'tuiche ' + tid From 9416571d651c6be3a0b4840aeba1de4ac30eb32d Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 22 Dec 2023 14:53:33 -0800 Subject: [PATCH 29/95] made applying debug patch easier --- misc/git/{hide-debug.patch => debug.patch} | 17 ++++++++--------- misc/git/pre-commit.sh | 2 +- setup.py | 5 ++--- 3 files changed, 11 insertions(+), 13 deletions(-) rename misc/git/{hide-debug.patch => debug.patch} (68%) diff --git a/misc/git/hide-debug.patch b/misc/git/debug.patch similarity index 68% rename from misc/git/hide-debug.patch rename to misc/git/debug.patch index 7931b9c7..5e4ee437 100644 --- a/misc/git/hide-debug.patch +++ b/misc/git/debug.patch @@ -1,29 +1,28 @@ -diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py -@@ -10,7 +10,6 @@ from distutils.version import LooseVersion +@@ -10,6 +10,7 @@ from distutils.version import LooseVersion from glob import glob from setuptools import setup, Extension from setuptools.command.test import test as TestCommand --from Cython.Build import cythonize ++from Cython.Build import cythonize sources = (glob("src/*.cpp") + ["libmc/_client.pyx"]) include_dirs = ["include"] -@@ -123,7 +122,7 @@ setup( +@@ -122,7 +123,7 @@ setup( # Support for the basestring type is new in Cython 0.20. setup_requires=["Cython >= 0.20"], cmdclass={"test": PyTest}, -- ext_modules=cythonize([ -+ ext_modules=[ +- ext_modules=[ ++ ext_modules=cythonize([ Extension( "libmc._client", sources, -@@ -131,7 +130,7 @@ setup( +@@ -130,7 +131,7 @@ setup( language="c++", extra_compile_args=COMPILER_FLAGS, ) -- ], gdb_debug=True), -+ ], +- ], ++ ], gdb_debug=True), tests_require=[ "pytest", "future", diff --git a/misc/git/pre-commit.sh b/misc/git/pre-commit.sh index 6ff5c834..c22e6a72 100755 --- a/misc/git/pre-commit.sh +++ b/misc/git/pre-commit.sh @@ -4,5 +4,5 @@ DIR="$( dirname `readlink -f "${BASH_SOURCE[0]}"`)" HOOK="$DIR/../../.git/hooks/pre-commit" [[ ! -e "$HOOK" ]] && ln -s "$DIR/pre-commit.sh" "$HOOK" -git apply --cached "$DIR/hide-debug.patch" -q --recount +git apply --cached "$DIR/debug.patch" -R --recount -q exit 0 diff --git a/setup.py b/setup.py index c2d5ed52..c9807b95 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ from glob import glob from setuptools import setup, Extension from setuptools.command.test import test as TestCommand -from Cython.Build import cythonize sources = (glob("src/*.cpp") + ["libmc/_client.pyx"]) include_dirs = ["include"] @@ -123,7 +122,7 @@ def run_tests(self): # Support for the basestring type is new in Cython 0.20. setup_requires=["Cython >= 0.20"], cmdclass={"test": PyTest}, - ext_modules=cythonize([ + ext_modules=[ Extension( "libmc._client", sources, @@ -131,7 +130,7 @@ def run_tests(self): language="c++", extra_compile_args=COMPILER_FLAGS, ) - ], gdb_debug=True), + ], tests_require=[ "pytest", "future", From 80d19c6372e33321328ff382e8a8887f115ea278 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 22 Dec 2023 15:03:21 -0800 Subject: [PATCH 30/95] forgot to update method reference --- tests/test_client_pool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index d1644700..95ae994e 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -10,13 +10,13 @@ def setUp(self): def misc(self): for i in range(5): - self.test_misc(i) + self.test_pool_client_misc(i) def test_acquire(self): with self.pool.client() as mc: pass - def test_client_pool_misc(self, i=0): + def test_pool_client_misc(self, i=0): with self.pool.client() as mc: tid = str(mc._get_current_thread_ident() + (i,)) f, t = 'foo ' + tid, 'tuiche ' + tid From c675ec57cebc5f23c6a8e4e464fc4cad90170fa3 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sat, 23 Dec 2023 17:21:52 -0800 Subject: [PATCH 31/95] TCP capture --- misc/memcached_server | 28 ++++++++++++++++++++++++---- tests/test_client_pool.py | 2 +- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/misc/memcached_server b/misc/memcached_server index d47acb13..11b64132 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -55,6 +55,19 @@ function stop() fi } +function capture() +{ + start=`echo $PORTS | cut -d' ' -f1` + host="${1:-${start}}" + proxy=31311 + default=$((proxy+host-start)) + port="${2:-${default}}" + output="$basedir/var/log/$host.log" + $cmd -d -u $USER -l $ip -t $threads -m ${memory} -p $port -P $basedir/var/run/${port}.pid > $basedir/var/log/${port}.log 2>&1 + ncat --sh-exec "ncat localhost ${port}" -l ${host} --keep-open -o "$output" --append-output & + echo $! > "$basedir/var/run/${host}.pid" +} + case "$1" in start) if [ -n "$2" ]; then @@ -103,14 +116,21 @@ case "$1" in ;; stopall) if [ `ls $basedir/var/run/ | grep -c .pid` -ge 1 ]; then - names="`basename $basedir/var/run/*.pid | cut -d. -f1`" - for name in $names; do - stop $name & + # names="`basename $basedir/var/run/*.pid | cut -d. -f1`" + # for name in $names; do + # stop $name & + # done + for f in $basedir/var/run/*.pid; do + cat "$f" | xargs kill done fi - wait + # wait rm -rf $basedir ;; + capture) + shift + capture $@ + ;; *) printf 'Usage: %s {start|stop|restart} \n' "$prog" exit 1 diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 95ae994e..707c43f8 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -36,7 +36,7 @@ def test_pool_client_misc(self, i=0): assert (mc.get_multi([f, t]) == {f: 1024, t: '8964'}) - def test_misc_threaded(self): + def test_pool_client_threaded(self): ts = [Thread(target=self.misc) for i in range(8)] for t in ts: t.start() From a25a7cd5098ec8d16ee8e98efca8ffe924b3f943 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sat, 23 Dec 2023 17:45:47 -0800 Subject: [PATCH 32/95] cython debug script --- misc/git/pre-commit.sh | 2 +- misc/memcached_server | 12 ++++++++++++ setup.py | 5 +++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/misc/git/pre-commit.sh b/misc/git/pre-commit.sh index c22e6a72..39a2a200 100755 --- a/misc/git/pre-commit.sh +++ b/misc/git/pre-commit.sh @@ -1,6 +1,6 @@ #!/bin/sh -DIR="$( dirname `readlink -f "${BASH_SOURCE[0]}"`)" +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) HOOK="$DIR/../../.git/hooks/pre-commit" [[ ! -e "$HOOK" ]] && ln -s "$DIR/pre-commit.sh" "$HOOK" diff --git a/misc/memcached_server b/misc/memcached_server index 11b64132..157f423d 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -131,6 +131,18 @@ case "$1" in shift capture $@ ;; + cy-debug) + DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + cd "$DIR/.." + git apply misc/git/debug.patch -q --recount || true + sh misc/git/pre-commit.sh + if [ -n "$2" ]; then + shift + cygdb . -- --args pythong-gdb setup.py test -a "-k $@" + else + cygdb . -- --args pythong-gdb setup.py test + fi + ;; *) printf 'Usage: %s {start|stop|restart} \n' "$prog" exit 1 diff --git a/setup.py b/setup.py index c9807b95..c2d5ed52 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ from glob import glob from setuptools import setup, Extension from setuptools.command.test import test as TestCommand +from Cython.Build import cythonize sources = (glob("src/*.cpp") + ["libmc/_client.pyx"]) include_dirs = ["include"] @@ -122,7 +123,7 @@ def run_tests(self): # Support for the basestring type is new in Cython 0.20. setup_requires=["Cython >= 0.20"], cmdclass={"test": PyTest}, - ext_modules=[ + ext_modules=cythonize([ Extension( "libmc._client", sources, @@ -130,7 +131,7 @@ def run_tests(self): language="c++", extra_compile_args=COMPILER_FLAGS, ) - ], + ], gdb_debug=True), tests_require=[ "pytest", "future", From 5f2c427f2aea851c1a6504f32fdb4da634a1c368 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sat, 23 Dec 2023 17:53:23 -0800 Subject: [PATCH 33/95] messed up setup.py --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index c2d5ed52..c9807b95 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ from glob import glob from setuptools import setup, Extension from setuptools.command.test import test as TestCommand -from Cython.Build import cythonize sources = (glob("src/*.cpp") + ["libmc/_client.pyx"]) include_dirs = ["include"] @@ -123,7 +122,7 @@ def run_tests(self): # Support for the basestring type is new in Cython 0.20. setup_requires=["Cython >= 0.20"], cmdclass={"test": PyTest}, - ext_modules=cythonize([ + ext_modules=[ Extension( "libmc._client", sources, @@ -131,7 +130,7 @@ def run_tests(self): language="c++", extra_compile_args=COMPILER_FLAGS, ) - ], gdb_debug=True), + ], tests_require=[ "pytest", "future", From 6e40df03259eb1d81f162e14c54d703fa4d6a0cb Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sun, 24 Dec 2023 14:54:23 -0800 Subject: [PATCH 34/95] python versioning and resolve symlinks --- misc/git/pre-commit.sh | 2 +- misc/memcached_server | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/misc/git/pre-commit.sh b/misc/git/pre-commit.sh index 39a2a200..230c718c 100755 --- a/misc/git/pre-commit.sh +++ b/misc/git/pre-commit.sh @@ -1,6 +1,6 @@ #!/bin/sh -DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +DIR=$( cd -- "$( dirname -- `readlink -f "${BASH_SOURCE[0]}"` )" &> /dev/null && pwd ) HOOK="$DIR/../../.git/hooks/pre-commit" [[ ! -e "$HOOK" ]] && ln -s "$DIR/pre-commit.sh" "$HOOK" diff --git a/misc/memcached_server b/misc/memcached_server index 157f423d..fca30dcc 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -136,11 +136,14 @@ case "$1" in cd "$DIR/.." git apply misc/git/debug.patch -q --recount || true sh misc/git/pre-commit.sh + if [ ! -e venv ]; then python -m virtualenv --python="`which python-dbg`" venv; fi + source venv/bin/activate + pip install Cython setuptools future pytest greenify gevent numpy if [ -n "$2" ]; then shift - cygdb . -- --args pythong-gdb setup.py test -a "-k $@" + cygdb . --skip-interpreter -- --args python-dbg setup.py test -a "$*" else - cygdb . -- --args pythong-gdb setup.py test + cygdb . --skip-interpreter -- --args python-dbg setup.py test fi ;; *) From d440b51fb3cb84accb2a8caad945ce8c7657011b Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 26 Dec 2023 07:48:38 +0000 Subject: [PATCH 35/95] ubuntu compat --- misc/git/pre-commit | 7 +++++++ misc/git/pre-commit.sh | 8 -------- misc/memcached_server | 25 ++++++++++++++++--------- 3 files changed, 23 insertions(+), 17 deletions(-) create mode 100755 misc/git/pre-commit delete mode 100755 misc/git/pre-commit.sh diff --git a/misc/git/pre-commit b/misc/git/pre-commit new file mode 100755 index 00000000..92e971df --- /dev/null +++ b/misc/git/pre-commit @@ -0,0 +1,7 @@ +#!/bin/bash + +DIR=$( cd "$( dirname `readlink -f "${BASH_SOURCE[0]}"` )" &> /dev/null && pwd ) +HOOK="$DIR/../../.git/hooks/pre-commit" + +[[ ! -e "$HOOK" ]] && ln -s "$DIR/pre-commit" "$HOOK" +git apply --cached "$DIR/debug.patch" -R --recount &> /dev/null || true diff --git a/misc/git/pre-commit.sh b/misc/git/pre-commit.sh deleted file mode 100755 index 230c718c..00000000 --- a/misc/git/pre-commit.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -DIR=$( cd -- "$( dirname -- `readlink -f "${BASH_SOURCE[0]}"` )" &> /dev/null && pwd ) -HOOK="$DIR/../../.git/hooks/pre-commit" - -[[ ! -e "$HOOK" ]] && ln -s "$DIR/pre-commit.sh" "$HOOK" -git apply --cached "$DIR/debug.patch" -R --recount -q -exit 0 diff --git a/misc/memcached_server b/misc/memcached_server index fca30dcc..47fa7b82 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -132,18 +132,25 @@ case "$1" in capture $@ ;; cy-debug) - DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd "$DIR/.." - git apply misc/git/debug.patch -q --recount || true - sh misc/git/pre-commit.sh - if [ ! -e venv ]; then python -m virtualenv --python="`which python-dbg`" venv; fi - source venv/bin/activate - pip install Cython setuptools future pytest greenify gevent numpy - if [ -n "$2" ]; then + git apply misc/git/debug.patch --recount &> /dev/null || true + source misc/git/pre-commit + if [ ! -e venv ]; then + dbg="`which python-dbg || which python3-dbg`" + python -m virtualenv --python="$dbg" --seeder=pip venv; + source venv/bin/activate + pip install -U pip + pip install Cython setuptools future pytest greenify gevent numpy + else + source venv/bin/activate + fi + dbg="`which python-dbg || which python3-dbg`" + if [ -n "$2" ]; then shift - cygdb . --skip-interpreter -- --args python-dbg setup.py test -a "$*" + cygdb . --skip-interpreter -- --args "$dbg" setup.py test -a "$*" else - cygdb . --skip-interpreter -- --args python-dbg setup.py test + cygdb . --skip-interpreter -- --args "$dbg" setup.py test fi ;; *) From e1a5de4960cbcaf2a54af1f68d16fd61f1872c22 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 26 Dec 2023 14:31:32 -0800 Subject: [PATCH 36/95] missed _update_servers ref --- libmc/_client.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index b3bb1e1b..31226a07 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -1114,7 +1114,7 @@ cdef class PyClientShell(PyClientSettings): return self.last_error def update_servers(self, servers): - rv = self._update_servers(servers, False) + rv = _update_servers(servers, self._imp, False) if rv + len(servers) == 0: self.servers = servers return True From 5369e41cac9879dce608cb8f10b29fc4c5bddf14 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 26 Dec 2023 15:20:45 -0800 Subject: [PATCH 37/95] stricter inheritance --- libmc/_client.pyx | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index 31226a07..6c130297 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -432,7 +432,11 @@ cdef _update_servers(Configurable* imp, list servers, bool_t init): Py_DECREF(servers_) - return rv + if rv + len(servers) == 0: + return True + elif init: + warnings.warn("Client failed to initialize") + return False cdef class PyClientShell(PyClientSettings): cdef Client* _imp @@ -451,9 +455,6 @@ cdef class PyClientShell(PyClientSettings): def config(self, int opt, int val): self._imp.config(opt, val) - cdef connect(self): - rv = _update_servers(self._imp, self.servers, True) - def get_host_by_key(self, basestring key): cdef bytes key2 = self.normalize_key(key) cdef char* c_key = NULL @@ -1113,13 +1114,6 @@ cdef class PyClientShell(PyClientSettings): def get_last_error(self): return self.last_error - def update_servers(self, servers): - rv = _update_servers(servers, self._imp, False) - if rv + len(servers) == 0: - self.servers = servers - return True - return False - def get_last_strerror(self): return errCodeToString(self.last_error) @@ -1134,6 +1128,15 @@ cdef class PyClient(PyClientShell): else: self._imp.disableConsistentFailover() + cdef connect(self): + return _update_servers(self._imp, self.servers, True) + + def update_servers(self, servers): + if _update_servers(self._imp, servers, False): + self.servers = servers + return True + return False + cdef class PyPoolClient(PyClientShell): cdef IndexedClient* _indexed @@ -1164,9 +1167,6 @@ cdef class PyClientPool(PyClientSettings): worker._imp = &imp.c return worker - cdef connect(self): - rv = _update_servers(self._imp, self.servers, True) - cdef acquire(self): if not self._initialized: self.connect() @@ -1189,3 +1189,13 @@ cdef class PyClientPool(PyClientSettings): yield worker finally: self.release(worker) + + # repeated from PyClient because cython can't handle fused types in classes + cdef connect(self): + return _update_servers(self._imp, self.servers, True) + + def update_servers(self, servers): + if _update_servers(self._imp, servers, False): + self.servers = servers + return True + return False From 407933379aeced7df6797cf309ca837e4bc2f965 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 12 Jan 2024 16:42:02 -0800 Subject: [PATCH 38/95] invalid keys in test --- libmc/_client.pyx | 1 + tests/test_client_pool.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index 6c130297..b7162b78 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -1191,6 +1191,7 @@ cdef class PyClientPool(PyClientSettings): self.release(worker) # repeated from PyClient because cython can't handle fused types in classes + # https://github.com/cython/cython/issues/3283 cdef connect(self): return _update_servers(self._imp, self.servers, True) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 707c43f8..6344199f 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -18,8 +18,9 @@ def test_acquire(self): def test_pool_client_misc(self, i=0): with self.pool.client() as mc: - tid = str(mc._get_current_thread_ident() + (i,)) - f, t = 'foo ' + tid, 'tuiche ' + tid + tid = mc._get_current_thread_ident() + (i,) + tid = "_".join(map(str, tid)) + f, t = 'foo_' + tid, 'tuiche_' + tid mc.get_multi([f, t]) mc.delete(f) mc.delete(t) From 2d05017ca887ce5289afe7cec18f1bc34a9aa493 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sun, 14 Jan 2024 13:37:18 -0800 Subject: [PATCH 39/95] removed race conditions --- .gitignore | 2 +- CMakeLists.txt | 2 +- include/LockPool.h | 3 +- libmc/__init__.py | 24 ++++++++++++-- libmc/_client.pyx | 15 ++++++--- misc/memcached_server | 5 +++ src/ClientPool.cpp | 1 - tests/test_client_pool.cpp | 57 ++++++++++++-------------------- tests/test_client_pool.py | 66 ++++++++++++++++++++++++++++++++++++-- 9 files changed, 125 insertions(+), 50 deletions(-) diff --git a/.gitignore b/.gitignore index 6d48b7e3..0949ba73 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,4 @@ tests/resources/keys_*.txt /go.mod /go.sum -/cython_debug \ No newline at end of file +/cython_debug diff --git a/CMakeLists.txt b/CMakeLists.txt index 04a4799a..bddda399 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ if (NOT CMAKE_BUILD_TYPE) endif (NOT CMAKE_BUILD_TYPE) set(CMAKE_CXX_FLAGS_COMMON "-Wall -fno-rtti -fno-exceptions") -set(CMAKE_CXX_FLAGS_DEBUG "-DDEBUG -g2 ${CMAKE_CXX_FLAGS_COMMON}" CACHE STRING "CXX DEBUG FLAGS" FORCE) +set(CMAKE_CXX_FLAGS_DEBUG "-DDEBUG -g2 ${CMAKE_CXX_FLAGS_COMMON} -fsanitize=thread" CACHE STRING "CXX DEBUG FLAGS" FORCE) set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3 ${CMAKE_CXX_FLAGS_COMMON}" CACHE STRING "CXX RELEASE FLAGS" FORCE) set(CMAKE_INSTALL_INCLUDE include CACHE PATH "Output directory for header files") set(CMAKE_INSTALL_LIBDIR lib CACHE PATH "Output directory for libraries") diff --git a/include/LockPool.h b/include/LockPool.h index f63a97ab..7d82b943 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace douban { namespace mc { @@ -14,7 +15,7 @@ class OrderedLock { std::queue m_fifo_locks; protected: std::mutex m_fifo_access; - bool m_locked; + std::atomic m_locked; protected: OrderedLock() : m_locked(true) {}; diff --git a/libmc/__init__.py b/libmc/__init__.py index 2e4a1122..7154b0dc 100644 --- a/libmc/__init__.py +++ b/libmc/__init__.py @@ -1,4 +1,4 @@ -import os +import os, functools from ._client import ( PyClient, PyClientPool, ThreadUnsafe, encode_value, @@ -44,12 +44,32 @@ class Client(PyClient): class ClientPool(PyClientPool): pass +class ThreadedClient(): + @functools.wraps(ClientPool.__init__) + def __init__(self, *args, **kwargs): + self._client_pool = ClientPool(*args, **kwargs) + + def update_servers(self, servers): + return self._client_pool.update_servers(servers) + + def __getattr__(self, key): + if not hasattr(Client, key): + raise AttributeError + result = getattr(Client, key) + if callable(result): + @functools.wraps(result) + def wrapper(*args, **kwargs): + with self._client_pool.client() as mc: + return getattr(mc, key)(*args, **kwargs) + return wrapper + return result + DYNAMIC_LIBRARIES = [os.path.abspath(_libmc_so_file)] __all__ = [ - 'Client', 'ClientPool', 'ThreadUnsafe', '__VERSION__', + 'Client', 'ClientPool', 'ThreadedClient', 'ThreadUnsafe', '__VERSION__', 'encode_value', 'decode_value', 'MC_DEFAULT_EXPTIME', 'MC_POLL_TIMEOUT', 'MC_CONNECT_TIMEOUT', diff --git a/libmc/_client.pyx b/libmc/_client.pyx index b7162b78..c181098d 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -449,9 +449,6 @@ cdef class PyClientShell(PyClientSettings): self._thread_ident = None self._created_stack = traceback.extract_stack() - def __dealloc__(self): - del self._imp - def config(self, int opt, int val): self._imp.config(opt, val) @@ -1109,7 +1106,7 @@ cdef class PyClientShell(PyClientSettings): self._get_current_thread_ident())) def _get_current_thread_ident(self): - return (os.getpid(), threading.current_thread().name) + return (os.getpid(), threading.current_thread().native_id) def get_last_error(self): return self.last_error @@ -1137,6 +1134,9 @@ cdef class PyClient(PyClientShell): return True return False + def __dealloc__(self): + del self._imp + cdef class PyPoolClient(PyClientShell): cdef IndexedClient* _indexed @@ -1172,6 +1172,8 @@ cdef class PyClientPool(PyClientSettings): self.connect() self._initialized = True worker = self._imp._acquire() + return self.setup(worker) + # prone to race conditions pending mux and possibly fails to update if worker.index >= len(self.clients): self.clients += [None] * (worker.index - len(self.clients)) self.clients.append(self.setup(worker)) @@ -1184,8 +1186,8 @@ cdef class PyClientPool(PyClientSettings): @contextmanager def client(self): + worker = self.acquire() try: - worker = self.acquire() yield worker finally: self.release(worker) @@ -1200,3 +1202,6 @@ cdef class PyClientPool(PyClientSettings): self.servers = servers return True return False + + def __dealloc__(self): + del self._imp diff --git a/misc/memcached_server b/misc/memcached_server index 47fa7b82..14496462 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -153,6 +153,11 @@ case "$1" in cygdb . --skip-interpreter -- --args "$dbg" setup.py test fi ;; + run-test) + cd "$( dirname "${BASH_SOURCE[0]}" )/.." &> /dev/null + shift + python setup.py test -a "-k $*" + ;; *) printf 'Usage: %s {start|stop|restart} \n' "$prog" exit 1 diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index abe40971..87a08f3e 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -1,6 +1,5 @@ //#include #include -#include #include "ClientPool.h" namespace douban { diff --git a/tests/test_client_pool.cpp b/tests/test_client_pool.cpp index 6ae7ed73..d77b563c 100644 --- a/tests/test_client_pool.cpp +++ b/tests/test_client_pool.cpp @@ -14,18 +14,7 @@ const unsigned int start_port = 21211; const char host[] = "127.0.0.1"; unsigned int n_threads = 8; -TEST(test_client_pool, simple_set_get) { - uint32_t ports[n_servers]; - const char* hosts[n_servers]; - for (unsigned int i = 0; i < n_servers; i++) { - ports[i] = start_port + i; - hosts[i] = host; - } - - ClientPool* pool = new ClientPool(); - pool->config(CFG_HASH_FUNCTION, OPT_HASH_FNV1A_32); - pool->init(hosts, ports, n_servers); - +void inner_test_loop(ClientPool* pool) { retrieval_result_t **r_results = NULL; message_result_t **m_results = NULL; size_t nResults = 0; @@ -37,7 +26,7 @@ TEST(test_client_pool, simple_set_get) { const char* keys = &key[0]; const char* values = &value[0]; - for (unsigned int j = 0; j < n_ops * n_threads; j++) { + for (unsigned int j = 0; j < n_ops; j++) { gen_random(key, data_size); gen_random(value, data_size); auto c = pool->acquire(); @@ -48,6 +37,23 @@ TEST(test_client_pool, simple_set_get) { c->destroyRetrievalResult(); pool->release(c); } +} + +TEST(test_client_pool, simple_set_get) { + uint32_t ports[n_servers]; + const char* hosts[n_servers]; + for (unsigned int i = 0; i < n_servers; i++) { + ports[i] = start_port + i; + hosts[i] = host; + } + + ClientPool* pool = new ClientPool(); + pool->config(CFG_HASH_FUNCTION, OPT_HASH_FNV1A_32); + pool->init(hosts, ports, n_servers); + + for (unsigned int j = 0; j < n_threads; j++) { + inner_test_loop(pool); + } delete pool; } @@ -67,30 +73,7 @@ TEST(test_client_pool, threaded_set_get) { pool->init(hosts, ports, n_servers); for (unsigned int i = 0; i < n_threads; i++) { - threads[i] = std::thread([&pool]() { - retrieval_result_t **r_results = NULL; - message_result_t **m_results = NULL; - size_t nResults = 0; - flags_t flags[] = {}; - size_t data_lens[] = {data_size}; - exptime_t exptime = 0; - char key[data_size + 1]; - char value[data_size + 1]; - const char* keys = &key[0]; - const char* values = &value[0]; - - for (unsigned int j = 0; j < n_ops; j++) { - gen_random(key, data_size); - gen_random(value, data_size); - auto c = pool->acquire(); - c->set(&keys, data_lens, flags, exptime, NULL, 0, &values, data_lens, 1, &m_results, &nResults); - c->destroyMessageResult(); - c->get(&keys, data_lens, 1, &r_results, &nResults); - ASSERT_N_STREQ(r_results[0]->data_block, values, data_size); - c->destroyRetrievalResult(); - pool->release(c); - } - }); + threads[i] = std::thread([&pool] { inner_test_loop(pool); }); } for (unsigned int i = 0; i < n_threads; i++) { threads[i].join(); diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 6344199f..81dab0b1 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -1,8 +1,7 @@ # coding: utf-8 import unittest from threading import Thread -from libmc import ClientPool - +from libmc import ClientPool#, ThreadedClient class ThreadedSingleServerCase(unittest.TestCase): def setUp(self): @@ -44,3 +43,66 @@ def test_pool_client_threaded(self): for t in ts: t.join() + +''' +class ClientOps: + def client_misc(self, mc, i=0): + tid = mc._get_current_thread_ident() + (i,) + tid = "_".join(map(str, tid)) + f, t = 'foo_' + tid, 'tuiche_' + tid + mc.get_multi([f, t]) + mc.delete(f) + mc.delete(t) + assert mc.get(f) is None + assert mc.get(t) is None + + mc.set(f, 'biu') + mc.set(t, 'bb') + assert mc.get(f) == 'biu' + assert mc.get(t) == 'bb' + assert (mc.get_multi([f, t]) == + {f: 'biu', t: 'bb'}) + mc.set_multi({f: 1024, t: '8964'}) + assert (mc.get_multi([f, t]) == + {f: 1024, t: '8964'}) + + def client_threads(self, target): + ts = [Thread(target=target) for i in range(8)] + for t in ts: + t.start() + + for t in ts: + t.join() + +class ThreadedSingleServerCase(unittest.TestCase, ClientOps): + def setUp(self): + self.pool = ClientPool(["127.0.0.1:21211"]) + + def misc(self): + for i in range(5): + self.test_pool_client_misc(i) + + def test_pool_client_misc(self, i=0): + with self.pool.client() as mc: + self.client_misc(mc, i) + + def test_acquire(self): + with self.pool.client() as mc: + pass + + def test_pool_client_threaded(self): + with open('debug.log', 'a') as f: + f.write("stdout working\n") + self.client_threads(self.misc) + +class ThreadedClientWrapperCheck(unittest.TestCase, ClientOps): + def setUp(self): + self.imp = ThreadedClient(["127.0.0.1:21211"]) + + def misc(self): + for i in range(5): + self.client_misc(self.imp, i) + + def test_many_threads(self): + self.client_threads(self.misc) +''' From 1e9c2bce3dc9e0e05bb227088a754dda4e5820dc Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sun, 14 Jan 2024 17:14:58 -0800 Subject: [PATCH 40/95] hung threads illustration --- libmc/__init__.py | 2 +- libmc/_client.pyx | 20 +++++++++++++--- misc/memcached_server | 4 ++-- tests/test_client_pool.py | 49 ++++----------------------------------- 4 files changed, 24 insertions(+), 51 deletions(-) diff --git a/libmc/__init__.py b/libmc/__init__.py index 7154b0dc..f47f91ba 100644 --- a/libmc/__init__.py +++ b/libmc/__init__.py @@ -44,7 +44,7 @@ class Client(PyClient): class ClientPool(PyClientPool): pass -class ThreadedClient(): +class ThreadedClient: @functools.wraps(ClientPool.__init__) def __init__(self, *args, **kwargs): self._client_pool = ClientPool(*args, **kwargs) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index c181098d..38fd87c1 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -1158,20 +1158,32 @@ cdef class PyClientPool(PyClientSettings): else: self.config(CFG_SET_FAILOVER, 0) + self.connect() + def config(self, int opt, int val): self._imp.config(opt, val) cdef setup(self, IndexedClient* imp): + with open('debug.log', 'a') as f: + f.write("a") worker = PyPoolClient(*self._args()) + with open('debug.log', 'a') as f: + f.write("b") worker._indexed = imp + with open('debug.log', 'a') as f: + f.write("c") worker._imp = &imp.c + with open('debug.log', 'a') as f: + f.write("d") return worker cdef acquire(self): - if not self._initialized: - self.connect() - self._initialized = True + # if not self._initialized: + # self.connect() + # self._initialized = True worker = self._imp._acquire() + with open('debug.log', 'a') as f: + f.write("index: " + repr(worker.index) + "\n") return self.setup(worker) # prone to race conditions pending mux and possibly fails to update if worker.index >= len(self.clients): @@ -1187,6 +1199,8 @@ cdef class PyClientPool(PyClientSettings): @contextmanager def client(self): worker = self.acquire() + with open('debug.log', 'a') as f: + f.write("worker:" + repr(worker) + "\n") try: yield worker finally: diff --git a/misc/memcached_server b/misc/memcached_server index 14496462..11a1a5c7 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -148,9 +148,9 @@ case "$1" in dbg="`which python-dbg || which python3-dbg`" if [ -n "$2" ]; then shift - cygdb . --skip-interpreter -- --args "$dbg" setup.py test -a "$*" + cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" else - cygdb . --skip-interpreter -- --args "$dbg" setup.py test + cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test fi ;; run-test) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 81dab0b1..a9917362 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -1,50 +1,8 @@ # coding: utf-8 import unittest from threading import Thread -from libmc import ClientPool#, ThreadedClient +from libmc import ClientPool, ThreadedClient -class ThreadedSingleServerCase(unittest.TestCase): - def setUp(self): - self.pool = ClientPool(["127.0.0.1:21211"]) - - def misc(self): - for i in range(5): - self.test_pool_client_misc(i) - - def test_acquire(self): - with self.pool.client() as mc: - pass - - def test_pool_client_misc(self, i=0): - with self.pool.client() as mc: - tid = mc._get_current_thread_ident() + (i,) - tid = "_".join(map(str, tid)) - f, t = 'foo_' + tid, 'tuiche_' + tid - mc.get_multi([f, t]) - mc.delete(f) - mc.delete(t) - assert mc.get(f) is None - assert mc.get(t) is None - - mc.set(f, 'biu') - mc.set(t, 'bb') - assert mc.get(f) == 'biu' - assert mc.get(t) == 'bb' - assert (mc.get_multi([f, t]) == - {f: 'biu', t: 'bb'}) - mc.set_multi({f: 1024, t: '8964'}) - assert (mc.get_multi([f, t]) == - {f: 1024, t: '8964'}) - - def test_pool_client_threaded(self): - ts = [Thread(target=self.misc) for i in range(8)] - for t in ts: - t.start() - - for t in ts: - t.join() - -''' class ClientOps: def client_misc(self, mc, i=0): tid = mc._get_current_thread_ident() + (i,) @@ -83,6 +41,8 @@ def misc(self): self.test_pool_client_misc(i) def test_pool_client_misc(self, i=0): + with open('debug.log', 'w') as f: + f.write("stdout working\n") with self.pool.client() as mc: self.client_misc(mc, i) @@ -91,7 +51,7 @@ def test_acquire(self): pass def test_pool_client_threaded(self): - with open('debug.log', 'a') as f: + with open('debug.log', 'w') as f: f.write("stdout working\n") self.client_threads(self.misc) @@ -105,4 +65,3 @@ def misc(self): def test_many_threads(self): self.client_threads(self.misc) -''' From 8362719568cfb38eb576be5c6568f31a2bd76a1b Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sun, 14 Jan 2024 22:27:25 -0800 Subject: [PATCH 41/95] commit for easier copying to a docker environment with a development version of cygdb --- debug.log | 1 + libmc/_client.pyx | 12 ------------ misc/memcached_server | 1 + tests/test_client_pool.py | 4 ---- 4 files changed, 2 insertions(+), 16 deletions(-) create mode 100644 debug.log diff --git a/debug.log b/debug.log new file mode 100644 index 00000000..6636bbca --- /dev/null +++ b/debug.log @@ -0,0 +1 @@ +stdout working diff --git a/libmc/_client.pyx b/libmc/_client.pyx index 38fd87c1..1d2ba183 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -1164,17 +1164,9 @@ cdef class PyClientPool(PyClientSettings): self._imp.config(opt, val) cdef setup(self, IndexedClient* imp): - with open('debug.log', 'a') as f: - f.write("a") worker = PyPoolClient(*self._args()) - with open('debug.log', 'a') as f: - f.write("b") worker._indexed = imp - with open('debug.log', 'a') as f: - f.write("c") worker._imp = &imp.c - with open('debug.log', 'a') as f: - f.write("d") return worker cdef acquire(self): @@ -1182,8 +1174,6 @@ cdef class PyClientPool(PyClientSettings): # self.connect() # self._initialized = True worker = self._imp._acquire() - with open('debug.log', 'a') as f: - f.write("index: " + repr(worker.index) + "\n") return self.setup(worker) # prone to race conditions pending mux and possibly fails to update if worker.index >= len(self.clients): @@ -1199,8 +1189,6 @@ cdef class PyClientPool(PyClientSettings): @contextmanager def client(self): worker = self.acquire() - with open('debug.log', 'a') as f: - f.write("worker:" + repr(worker) + "\n") try: yield worker finally: diff --git a/misc/memcached_server b/misc/memcached_server index 11a1a5c7..9cc519c6 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -163,3 +163,4 @@ case "$1" in exit 1 ;; esac +# cy break libmc._client.PyClientPool.setup diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index a9917362..0ac06114 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -41,8 +41,6 @@ def misc(self): self.test_pool_client_misc(i) def test_pool_client_misc(self, i=0): - with open('debug.log', 'w') as f: - f.write("stdout working\n") with self.pool.client() as mc: self.client_misc(mc, i) @@ -51,8 +49,6 @@ def test_acquire(self): pass def test_pool_client_threaded(self): - with open('debug.log', 'w') as f: - f.write("stdout working\n") self.client_threads(self.misc) class ThreadedClientWrapperCheck(unittest.TestCase, ClientOps): From 7afa87e987275c7ecc5f0f9a259191fa05fea117 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sun, 14 Jan 2024 22:54:58 -0800 Subject: [PATCH 42/95] QoL nit --- misc/memcached_server | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/misc/memcached_server b/misc/memcached_server index 9cc519c6..65dab492 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -138,7 +138,7 @@ case "$1" in source misc/git/pre-commit if [ ! -e venv ]; then dbg="`which python-dbg || which python3-dbg`" - python -m virtualenv --python="$dbg" --seeder=pip venv; + $dbg -m virtualenv --python="$dbg" --seeder=pip venv; source venv/bin/activate pip install -U pip pip install Cython setuptools future pytest greenify gevent numpy @@ -148,7 +148,8 @@ case "$1" in dbg="`which python-dbg || which python3-dbg`" if [ -n "$2" ]; then shift - cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" + cygdb . --skip-interpreter -- -ex "cy break libmc._client.PyClientPool.setup" -ex start --args "$dbg" setup.py test -a "$*" + # cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" else cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test fi From 4673c9093ce6e594c6c821bbadb4b2242655bad3 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sun, 14 Jan 2024 23:10:42 -0800 Subject: [PATCH 43/95] fresh environment fixes --- misc/memcached_server | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misc/memcached_server b/misc/memcached_server index 65dab492..52a90dac 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -138,6 +138,7 @@ case "$1" in source misc/git/pre-commit if [ ! -e venv ]; then dbg="`which python-dbg || which python3-dbg`" + $dbg -m pip install virtualenv $dbg -m virtualenv --python="$dbg" --seeder=pip venv; source venv/bin/activate pip install -U pip @@ -146,10 +147,11 @@ case "$1" in source venv/bin/activate fi dbg="`which python-dbg || which python3-dbg`" + $dbg setup.py build_ext --inplace if [ -n "$2" ]; then shift - cygdb . --skip-interpreter -- -ex "cy break libmc._client.PyClientPool.setup" -ex start --args "$dbg" setup.py test -a "$*" - # cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" + # cygdb . --skip-interpreter -- -ex "cy break libmc._client.PyClientPool.setup" -ex start --args "$dbg" setup.py test -a "$*" + cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" else cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test fi @@ -164,4 +166,3 @@ case "$1" in exit 1 ;; esac -# cy break libmc._client.PyClientPool.setup From 15a705e6e27521a8170604ccce205a3306fa4d66 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 16 Jan 2024 17:21:46 -0800 Subject: [PATCH 44/95] threaded traceback bug --- libmc/_client.pyx | 2 +- misc/memcached_server | 1 - tests/test_client_pool.py | 10 ++++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index 1d2ba183..e83d8090 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -447,7 +447,6 @@ cdef class PyClientShell(PyClientSettings): def __cinit__(self): self.last_error = RET_OK self._thread_ident = None - self._created_stack = traceback.extract_stack() def config(self, int opt, int val): self._imp.config(opt, val) @@ -1116,6 +1115,7 @@ cdef class PyClientShell(PyClientSettings): cdef class PyClient(PyClientShell): def __cinit__(self): + self._created_stack = traceback.extract_stack() self._imp = new Client() self._imp.config(CFG_HASH_FUNCTION, self.hash_fn) self.connect() diff --git a/misc/memcached_server b/misc/memcached_server index 52a90dac..3a8c8013 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -150,7 +150,6 @@ case "$1" in $dbg setup.py build_ext --inplace if [ -n "$2" ]; then shift - # cygdb . --skip-interpreter -- -ex "cy break libmc._client.PyClientPool.setup" -ex start --args "$dbg" setup.py test -a "$*" cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" else cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 0ac06114..d0ac9ccd 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -1,6 +1,6 @@ # coding: utf-8 import unittest -from threading import Thread +import threading from libmc import ClientPool, ThreadedClient class ClientOps: @@ -25,7 +25,13 @@ def client_misc(self, mc, i=0): {f: 1024, t: '8964'}) def client_threads(self, target): - ts = [Thread(target=target) for i in range(8)] + def passthrough(args): + _, e, tb, t = args + e.add_note("Occurred in thread " + str(t)) + raise e.with_traceback(tb) + + threading.excepthook = passthrough + ts = [threading.Thread(target=target) for i in range(8)] for t in ts: t.start() From 03cb6519f8e475f2714d15d281edf0be98d2dce0 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 16 Jan 2024 17:53:08 -0800 Subject: [PATCH 45/95] excepthook behavior --- tests/test_client_pool.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index d0ac9ccd..c26ede67 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -1,8 +1,14 @@ # coding: utf-8 import unittest import threading +import functools from libmc import ClientPool, ThreadedClient +@functools.wraps(print) +def threaded_print(*args, **kwargs): + with open('/tmp/debug.log', 'a+') as fp: + print(*args, **kwargs, file=fp) + class ClientOps: def client_misc(self, mc, i=0): tid = mc._get_current_thread_ident() + (i,) @@ -25,10 +31,11 @@ def client_misc(self, mc, i=0): {f: 1024, t: '8964'}) def client_threads(self, target): + errs = [] def passthrough(args): _, e, tb, t = args e.add_note("Occurred in thread " + str(t)) - raise e.with_traceback(tb) + errs.append(e.with_traceback(tb)) threading.excepthook = passthrough ts = [threading.Thread(target=target) for i in range(8)] @@ -38,6 +45,10 @@ def passthrough(args): for t in ts: t.join() + if errs: + errs[0].add_note(f"Along with {len(errs)} errors in other threads") + raise errs[0] + class ThreadedSingleServerCase(unittest.TestCase, ClientOps): def setUp(self): self.pool = ClientPool(["127.0.0.1:21211"]) From 71d44e430e0f7e6f555c5b3549f772a7332d7b7b Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 16 Jan 2024 18:23:52 -0800 Subject: [PATCH 46/95] debugging tools --- tests/test_client_pool.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index c26ede67..db2cb293 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -4,7 +4,26 @@ import functools from libmc import ClientPool, ThreadedClient +def setup_loging(f): + g = None + + @functools.wraps(f) + def wrapper(*args, **kwargs): + return g(*args, **kwargs) + + @functools.wraps(f) + def begin(*args, **kwargs): + nonlocal g + with open("/tmp/debug.log", "w+") as fp: + fp.write("") + g = f + return wrapper(*args, **kwargs) + + g = begin + return wrapper + @functools.wraps(print) +@setup_loging def threaded_print(*args, **kwargs): with open('/tmp/debug.log', 'a+') as fp: print(*args, **kwargs, file=fp) From 9f3f4ef4368888ad9b334947464498927414794b Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 16 Jan 2024 18:38:23 -0800 Subject: [PATCH 47/95] ln fails if symlink exists but is broken --- misc/git/pre-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/git/pre-commit b/misc/git/pre-commit index 92e971df..a032f1cb 100755 --- a/misc/git/pre-commit +++ b/misc/git/pre-commit @@ -3,5 +3,5 @@ DIR=$( cd "$( dirname `readlink -f "${BASH_SOURCE[0]}"` )" &> /dev/null && pwd ) HOOK="$DIR/../../.git/hooks/pre-commit" -[[ ! -e "$HOOK" ]] && ln -s "$DIR/pre-commit" "$HOOK" +[[ ! -h "$HOOK" ]] && ln -s "$DIR/pre-commit" "$HOOK" git apply --cached "$DIR/debug.patch" -R --recount &> /dev/null || true From 3303ad3077aa6bba494b3b71c93b29fb5a96532f Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 16 Jan 2024 18:57:50 -0800 Subject: [PATCH 48/95] venv for both debug and test --- misc/memcached_server | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/misc/memcached_server b/misc/memcached_server index 3a8c8013..f5f1aa61 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -68,6 +68,20 @@ function capture() echo $! > "$basedir/var/run/${host}.pid" } +function virtualize() +{ + if [ ! -e venv ]; then + dbg="${1:-`which python-dbg || which python3-dbg`}" + $dbg -m pip install virtualenv + $dbg -m virtualenv --python="$dbg" --seeder=pip venv; + source venv/bin/activate + pip install -U pip + pip install Cython setuptools future pytest greenify gevent numpy + else + source venv/bin/activate + fi +} + case "$1" in start) if [ -n "$2" ]; then @@ -136,16 +150,7 @@ case "$1" in cd "$DIR/.." git apply misc/git/debug.patch --recount &> /dev/null || true source misc/git/pre-commit - if [ ! -e venv ]; then - dbg="`which python-dbg || which python3-dbg`" - $dbg -m pip install virtualenv - $dbg -m virtualenv --python="$dbg" --seeder=pip venv; - source venv/bin/activate - pip install -U pip - pip install Cython setuptools future pytest greenify gevent numpy - else - source venv/bin/activate - fi + virtualize dbg="`which python-dbg || which python3-dbg`" $dbg setup.py build_ext --inplace if [ -n "$2" ]; then @@ -157,6 +162,7 @@ case "$1" in ;; run-test) cd "$( dirname "${BASH_SOURCE[0]}" )/.." &> /dev/null + virtualize "`which python 2>/dev/null`" shift python setup.py test -a "-k $*" ;; From 0b2346fa6cd0ad6c16dd98c0f80d2eaf32d45282 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 16 Jan 2024 19:23:26 -0800 Subject: [PATCH 49/95] support for <3.11 --- tests/test_client_pool.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index db2cb293..6cc2350e 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -53,7 +53,8 @@ def client_threads(self, target): errs = [] def passthrough(args): _, e, tb, t = args - e.add_note("Occurred in thread " + str(t)) + if hasattr(e, "add_note"): + e.add_note("Occurred in thread " + str(t)) errs.append(e.with_traceback(tb)) threading.excepthook = passthrough @@ -65,8 +66,10 @@ def passthrough(args): t.join() if errs: - errs[0].add_note(f"Along with {len(errs)} errors in other threads") - raise errs[0] + e = errs[0] + if hasattr(e, "add_note"): + e.add_note(f"Along with {len(errs)} errors in other threads") + raise e class ThreadedSingleServerCase(unittest.TestCase, ClientOps): def setUp(self): From f54eb9b946455218be6aaef0253a55b59b21f476 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Thu, 18 Jan 2024 01:20:19 -0800 Subject: [PATCH 50/95] WIP debug build flags --- Makefile | 3 ++ debug.log | 1 - misc/git/debug.patch | 65 ++++++++++++++++++++++++++++++++++++++++--- misc/memcached_server | 6 +++- 4 files changed, 69 insertions(+), 6 deletions(-) delete mode 100644 debug.log diff --git a/Makefile b/Makefile index 2dff81f3..d1363d8d 100644 --- a/Makefile +++ b/Makefile @@ -9,3 +9,6 @@ gotest: pytest: pytest tests/ + +clean: + rm -fr build cython_debug libmc/*.so libmc/*.cpp diff --git a/debug.log b/debug.log deleted file mode 100644 index 6636bbca..00000000 --- a/debug.log +++ /dev/null @@ -1 +0,0 @@ -stdout working diff --git a/misc/git/debug.patch b/misc/git/debug.patch index 5e4ee437..786e44f7 100644 --- a/misc/git/debug.patch +++ b/misc/git/debug.patch @@ -1,25 +1,82 @@ +diff --git a/setup.py b/setup.py +index c9807b9..359a04c 100644 --- a/setup.py +++ b/setup.py -@@ -10,6 +10,7 @@ from distutils.version import LooseVersion +@@ -10,18 +10,34 @@ from distutils.version import LooseVersion from glob import glob from setuptools import setup, Extension from setuptools.command.test import test as TestCommand ++from setuptools.command.build_ext import build_ext +from Cython.Build import cythonize sources = (glob("src/*.cpp") + ["libmc/_client.pyx"]) include_dirs = ["include"] -@@ -122,7 +123,7 @@ setup( + + COMPILER_FLAGS = [ +- "-fno-strict-aliasing", +- "-fno-exceptions", +- "-fno-rtti", ++ "-DDYNAMIC_ANNOTATIONS_ENABLED=1", ++ "-march=x86-64", ++ "-mtune=generic", ++ "-pipe", ++ "-Wp,-D_FORTIFY_SOURCE=2", ++ "-Werror=format-security", ++ "-fPIC", + "-Wall", ++ "-Wextra", ++ "-fno-rtti", ++ "-fno-exceptions", ++ "-DDEBUG", ++ "-g3", ++ "-Og", ++ "-fno-strict-aliasing", + "-DMC_USE_SMALL_VECTOR", +- "-O3", +- "-DNDEBUG", ++] ++LINKER_FLAGS = [ ++ "-shared", ++ "-g3", ++ "-Wl,-O0,--sort-common,--as-needed,-z,relro,-z,now" + ] + + +@@ -91,6 +107,18 @@ class PyTest(TestCommand): + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + ++# https://shwina.github.io/custom-compiler-linker-extensions/ ++class BlankBuildExt(build_ext): ++ def build_extensions(self): ++ # Override the compiler executables. Importantly, this ++ # removes the "default" compiler flags that would ++ # otherwise get passed on to to the compiler, i.e., ++ # distutils.sysconfig.get_var("CFLAGS"). ++ self.compiler.set_executable("compiler_so", "gcc") ++ self.compiler.set_executable("compiler_cxx", "gcc") ++ self.compiler.set_executable("linker_so", "gcc") ++ build_ext.build_extensions(self) ++ + + setup( + name="libmc", +@@ -121,16 +149,18 @@ setup( + ], # Support for the basestring type is new in Cython 0.20. setup_requires=["Cython >= 0.20"], - cmdclass={"test": PyTest}, +- cmdclass={"test": PyTest}, - ext_modules=[ ++ cmdclass={"test": PyTest, "build_ext": BlankBuildExt}, ++ zip_safe=False, + ext_modules=cythonize([ Extension( "libmc._client", sources, -@@ -130,7 +131,7 @@ setup( + include_dirs=include_dirs, language="c++", extra_compile_args=COMPILER_FLAGS, ++ extra_link_args=LINKER_FLAGS ) - ], + ], gdb_debug=True), diff --git a/misc/memcached_server b/misc/memcached_server index f5f1aa61..c42c760c 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -155,7 +155,11 @@ case "$1" in $dbg setup.py build_ext --inplace if [ -n "$2" ]; then shift - cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" + # cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" + cygdb . --skip-interpreter -- \ + -ex start \ + -ex "cy break libmc._client.PyClientPool.setup" \ + --args "$dbg" setup.py test -a "$*" else cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test fi From e367a3fabe153f8f3133ec80d9b05d7c36557ad8 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Thu, 18 Jan 2024 10:40:38 -0800 Subject: [PATCH 51/95] debug build flags --- misc/git/debug.patch | 55 ++++++++++++++++++++++++++----------------- misc/memcached_server | 1 + 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/misc/git/debug.patch b/misc/git/debug.patch index 786e44f7..07b325cc 100644 --- a/misc/git/debug.patch +++ b/misc/git/debug.patch @@ -1,8 +1,15 @@ diff --git a/setup.py b/setup.py -index c9807b9..359a04c 100644 +index c9807b9..b3a915b 100644 --- a/setup.py +++ b/setup.py -@@ -10,18 +10,34 @@ from distutils.version import LooseVersion +@@ -5,23 +5,43 @@ import sys + import shlex + import pkg_resources + import platform ++import functools ++import inspect + from distutils.sysconfig import get_config_var + from distutils.version import LooseVersion from glob import glob from setuptools import setup, Extension from setuptools.command.test import test as TestCommand @@ -12,28 +19,27 @@ index c9807b9..359a04c 100644 sources = (glob("src/*.cpp") + ["libmc/_client.pyx"]) include_dirs = ["include"] - COMPILER_FLAGS = [ -- "-fno-strict-aliasing", -- "-fno-exceptions", -- "-fno-rtti", ++COMPILER_PREARGS = [ + "-DDYNAMIC_ANNOTATIONS_ENABLED=1", ++ "-g3", ++ "-Og", ++ "-Wall", + "-march=x86-64", + "-mtune=generic", + "-pipe", + "-Wp,-D_FORTIFY_SOURCE=2", + "-Werror=format-security", + "-fPIC", ++] + COMPILER_FLAGS = [ + "-fno-strict-aliasing", + "-fno-exceptions", + "-fno-rtti", "-Wall", -+ "-Wextra", -+ "-fno-rtti", -+ "-fno-exceptions", -+ "-DDEBUG", -+ "-g3", -+ "-Og", -+ "-fno-strict-aliasing", "-DMC_USE_SMALL_VECTOR", - "-O3", - "-DNDEBUG", ++ "-DDEBUG", +] +LINKER_FLAGS = [ + "-shared", @@ -42,26 +48,33 @@ index c9807b9..359a04c 100644 ] -@@ -91,6 +107,18 @@ class PyTest(TestCommand): +@@ -91,6 +111,25 @@ class PyTest(TestCommand): errno = pytest.main(shlex.split(self.pytest_args)) sys.exit(errno) +# https://shwina.github.io/custom-compiler-linker-extensions/ +class BlankBuildExt(build_ext): + def build_extensions(self): -+ # Override the compiler executables. Importantly, this -+ # removes the "default" compiler flags that would -+ # otherwise get passed on to to the compiler, i.e., -+ # distutils.sysconfig.get_var("CFLAGS"). ++ orig = self.compiler.compile ++ ++ @functools.wraps(orig) ++ def prearg_compile(*args, **kwargs): ++ bound = inspect.Signature.from_callable(orig).bind(*args, **kwargs) ++ bound.apply_defaults() ++ bound.arguments["extra_preargs"] = COMPILER_PREARGS ++ return orig(*bound.args, **bound.kwargs) ++ ++ self.compiler.compile = prearg_compile ++ + self.compiler.set_executable("compiler_so", "gcc") -+ self.compiler.set_executable("compiler_cxx", "gcc") -+ self.compiler.set_executable("linker_so", "gcc") ++ #self.compiler.set_executable("compiler_cxx", "gcc") ++ self.compiler.set_executable("linker_so", "g++") + build_ext.build_extensions(self) + setup( name="libmc", -@@ -121,16 +149,18 @@ setup( +@@ -121,16 +160,18 @@ setup( ], # Support for the basestring type is new in Cython 0.20. setup_requires=["Cython >= 0.20"], diff --git a/misc/memcached_server b/misc/memcached_server index c42c760c..c955a67b 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -152,6 +152,7 @@ case "$1" in source misc/git/pre-commit virtualize dbg="`which python-dbg || which python3-dbg`" + make clean $dbg setup.py build_ext --inplace if [ -n "$2" ]; then shift From c3ff9aeed01728a6862d9ce57a7cd07fdff78154 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Thu, 18 Jan 2024 17:40:49 -0800 Subject: [PATCH 52/95] can't assign to python scope variables w/o GIL and underlying functions release it anyway --- libmc/_client.pyx | 146 ++++++++++++++++++------------------------ misc/memcached_server | 8 +-- 2 files changed, 64 insertions(+), 90 deletions(-) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index e83d8090..8556459c 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -458,8 +458,7 @@ cdef class PyClientShell(PyClientSettings): cdef size_t c_key_len = 0 Py_INCREF(key2) PyString_AsStringAndSize(key2, &c_key, &c_key_len) - with nogil: - c_addr = self._imp.getServerAddressByKey(c_key, c_key_len) + c_addr = self._imp.getServerAddressByKey(c_key, c_key_len) cdef basestring server_addr = c_addr Py_DECREF(key2) return server_addr @@ -471,8 +470,7 @@ cdef class PyClientShell(PyClientSettings): cdef size_t c_key_len = 0 Py_INCREF(key2) PyString_AsStringAndSize(key2, &c_key, &c_key_len) - with nogil: - c_addr = self._imp.getRealtimeServerAddressByKey(c_key, c_key_len) + c_addr = self._imp.getRealtimeServerAddressByKey(c_key, c_key_len) Py_DECREF(key2) cdef basestring server_addr if c_addr != NULL: @@ -500,13 +498,12 @@ cdef class PyClientShell(PyClientSettings): PyString_AsStringAndSize(key, &c_key, &c_key_len) # XXX: safe cast? cdef size_t n = 1, n_results = 0 cdef retrieval_result_t** results = NULL - with nogil: - if op == GET_OP: - self.last_error = self._imp.get(&c_key, &c_key_len, n, &results, &n_results) - elif op == GETS_OP: - self.last_error = self._imp.gets(&c_key, &c_key_len, n, &results, &n_results) - else: - pass + if op == GET_OP: + self.last_error = self._imp.get(&c_key, &c_key_len, n, &results, &n_results) + elif op == GETS_OP: + self.last_error = self._imp.gets(&c_key, &c_key_len, n, &results, &n_results) + else: + pass cdef bytes py_value = None if n_results == 1: @@ -515,8 +512,7 @@ cdef class PyClientShell(PyClientSettings): if op == GETS_OP: cas_unique_ptr[0] = results[0].cas_unique Py_DECREF(key) - with nogil: - self._imp.destroyRetrievalResult() + self._imp.destroyRetrievalResult() return py_value def _get_large_raw(self, bytes key, int n_splits, flags_t chuncked_flags): @@ -582,8 +578,7 @@ cdef class PyClientShell(PyClientSettings): cdef retrieval_result_t** results = NULL cdef retrieval_result_t *r = NULL - with nogil: - self.last_error = self._imp.get(c_keys, c_key_lens, n, &results, &n_res) + self.last_error = self._imp.get(c_keys, c_key_lens, n, &results, &n_res) cdef dict rv = {} cdef bytes py_key @@ -597,8 +592,7 @@ cdef class PyClientShell(PyClientSettings): PyMem_Free(c_keys) PyMem_Free(c_key_lens) Py_DECREF(keys) - with nogil: - self._imp.destroyRetrievalResult() + self._imp.destroyRetrievalResult() return rv def get_multi(self, keys): @@ -645,26 +639,24 @@ cdef class PyClientShell(PyClientSettings): cdef message_result_t** results = NULL - with nogil: - if op == SET_OP: - self.last_error = self._imp.set(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - elif op == ADD_OP: - self.last_error = self._imp.add(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - elif op == REPLACE_OP: - self.last_error = self._imp.replace(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - elif op == PREPEND_OP: - self.last_error = self._imp.prepend(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - elif op == APPEND_OP: - self.last_error = self._imp.append(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - elif op == CAS_OP: - self.last_error = self._imp.cas(&c_key, &c_key_len, &flags, exptime, &cas_unique, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - else: - pass + if op == SET_OP: + self.last_error = self._imp.set(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + elif op == ADD_OP: + self.last_error = self._imp.add(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + elif op == REPLACE_OP: + self.last_error = self._imp.replace(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + elif op == PREPEND_OP: + self.last_error = self._imp.prepend(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + elif op == APPEND_OP: + self.last_error = self._imp.append(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + elif op == CAS_OP: + self.last_error = self._imp.cas(&c_key, &c_key_len, &flags, exptime, &cas_unique, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + else: + pass rv = self.last_error == RET_OK and (self.noreply or (n_res == 1 and results[0][0].type_ == MSG_STORED)) - with nogil: - self._imp.destroyMessageResult() + self._imp.destroyMessageResult() Py_DECREF(key) Py_DECREF(val) return rv @@ -794,18 +786,17 @@ cdef class PyClientShell(PyClientSettings): PyString_AsStringAndSize(vals[i], &c_vals[i], &c_val_lens[i]) # XXX: safe cast? cdef message_result_t** results = NULL - with nogil: - if op == SET_OP: - self.last_error = self._imp.set(c_keys, c_key_lens, c_flags, c_exptime, NULL, - self.noreply, c_vals, c_val_lens, n, &results, &n_rst) - elif op == PREPEND_OP: - self.last_error = self._imp.prepend(c_keys, c_key_lens, c_flags, c_exptime, NULL, - self.noreply, c_vals, c_val_lens, n, &results, &n_rst) - elif op == APPEND_OP: - self.last_error = self._imp.append(c_keys, c_key_lens, c_flags, c_exptime, NULL, - self.noreply, c_vals, c_val_lens, n, &results, &n_rst) - else: - pass + if op == SET_OP: + self.last_error = self._imp.set(c_keys, c_key_lens, c_flags, c_exptime, NULL, + self.noreply, c_vals, c_val_lens, n, &results, &n_rst) + elif op == PREPEND_OP: + self.last_error = self._imp.prepend(c_keys, c_key_lens, c_flags, c_exptime, NULL, + self.noreply, c_vals, c_val_lens, n, &results, &n_rst) + elif op == APPEND_OP: + self.last_error = self._imp.append(c_keys, c_key_lens, c_flags, c_exptime, NULL, + self.noreply, c_vals, c_val_lens, n, &results, &n_rst) + else: + pass is_succeed = (self.last_error == RET_OK) and (self.noreply or n_rst == n) @@ -815,8 +806,7 @@ cdef class PyClientShell(PyClientSettings): failed_keys = list(set(keys) - set(succeed_keys)) is_succeed = is_succeed and (len(failed_keys) == 0) - with nogil: - self._imp.destroyMessageResult() + self._imp.destroyMessageResult() PyMem_Free(c_keys) PyMem_Free(c_key_lens) @@ -888,13 +878,11 @@ cdef class PyClientShell(PyClientSettings): PyString_AsStringAndSize(key, &c_key, &c_key_len) cdef message_result_t** results = NULL - with nogil: - self.last_error = self._imp._delete(&c_key, &c_key_len, self.noreply, n, &results, &n_res) + self.last_error = self._imp._delete(&c_key, &c_key_len, self.noreply, n, &results, &n_res) rv = self.last_error == RET_OK and (self.noreply or (n_res == 1 and (results[0][0].type_ == MSG_DELETED or results[0][0].type_ == MSG_NOT_FOUND))) - with nogil: - self._imp.destroyMessageResult() + self._imp.destroyMessageResult() Py_DECREF(key) return rv @@ -914,8 +902,7 @@ cdef class PyClientShell(PyClientSettings): cdef message_result_t** results = NULL cdef message_result_t *r = NULL - with nogil: - self.last_error = self._imp._delete(c_keys, c_key_lens, self.noreply, n, &results, &n_res) + self.last_error = self._imp._delete(c_keys, c_key_lens, self.noreply, n, &results, &n_res) is_succeed = self.last_error == RET_OK and (self.noreply or n_res == n) cdef list failed_keys = [] @@ -926,8 +913,7 @@ cdef class PyClientShell(PyClientSettings): if results[i][0].type_ == MSG_DELETED or results[i][0].type_ == MSG_NOT_FOUND] failed_keys = list(set(keys) - set(succeed_keys)) - with nogil: - self._imp.destroyMessageResult() + self._imp.destroyMessageResult() PyMem_Free(c_key_lens) PyMem_Free(c_keys) Py_DECREF(keys) @@ -948,12 +934,10 @@ cdef class PyClientShell(PyClientSettings): cdef message_result_t** results = NULL - with nogil: - self.last_error = self._imp.touch(&c_key, &c_key_len, exptime, self.noreply, n, &results, &n_res) + self.last_error = self._imp.touch(&c_key, &c_key_len, exptime, self.noreply, n, &results, &n_res) rv = self.last_error == RET_OK and (self.noreply or (n_res == 1 and results[0][0].type_ == MSG_TOUCHED)) - with nogil: - self._imp.destroyMessageResult() + self._imp.destroyMessageResult() Py_DECREF(key) return rv @@ -966,8 +950,7 @@ cdef class PyClientShell(PyClientSettings): self._record_thread_ident() cdef broadcast_result_t* rst = NULL cdef size_t n = 0 - with nogil: - self.last_error = self._imp.version(&rst, &n) + self.last_error = self._imp.version(&rst, &n) rv = {} for i in range(n): @@ -975,8 +958,7 @@ cdef class PyClientShell(PyClientSettings): continue rv[rst[i].host] = rst[i].lines[0][:rst[i].line_lens[0]] - with nogil: - self._imp.destroyBroadcastResult() + self._imp.destroyBroadcastResult() return rv def toggle_flush_all_feature(self, enabled): @@ -986,16 +968,14 @@ cdef class PyClientShell(PyClientSettings): self._record_thread_ident() cdef broadcast_result_t* rst = NULL cdef size_t n = 0 - with nogil: - self.last_error = self._imp.flushAll(&rst, &n) + self.last_error = self._imp.flushAll(&rst, &n) rv = [] for i in range(n): if rst[i].msg_type == MSG_OK: rv.append(rst[i].host) - with nogil: - self._imp.destroyBroadcastResult() + self._imp.destroyBroadcastResult() if self.last_error == RET_PROGRAMMING_ERR: raise RuntimeError( "Please call client.toggle_flush_all_feature(True) first " @@ -1005,9 +985,8 @@ cdef class PyClientShell(PyClientSettings): def quit(self): self._record_thread_ident() - with nogil: - self.last_error = self._imp.quit() - self._imp.destroyBroadcastResult() + self.last_error = self._imp.quit() + self._imp.destroyBroadcastResult() return self.last_error in {RET_CONN_POLL_ERR, RET_OK} def stats(self): @@ -1016,8 +995,7 @@ cdef class PyClientShell(PyClientSettings): cdef broadcast_result_t* r = NULL cdef size_t n = 0 rv = {} - with nogil: - self.last_error = self._imp.stats(&rst, &n) + self.last_error = self._imp.stats(&rst, &n) for i in range(n): r = &rst[i] @@ -1036,8 +1014,7 @@ cdef class PyClientShell(PyClientSettings): except: pass rv[r.host][k] = v - with nogil: - self._imp.destroyBroadcastResult() + self._imp.destroyBroadcastResult() return rv def _incr_decr_raw(self, op_code_t op, bytes key, uint64_t delta): @@ -1050,19 +1027,17 @@ cdef class PyClientShell(PyClientSettings): cdef unsigned_result_t* result = NULL cdef size_t n_res = 0 - with nogil: - if op == INCR_OP: - self.last_error = self._imp.incr(c_key, c_key_len, delta, self.noreply, &result, &n_res) - elif op == DECR_OP: - self.last_error = self._imp.decr(c_key, c_key_len, delta, self.noreply, &result, &n_res) - else: - pass + if op == INCR_OP: + self.last_error = self._imp.incr(c_key, c_key_len, delta, self.noreply, &result, &n_res) + elif op == DECR_OP: + self.last_error = self._imp.decr(c_key, c_key_len, delta, self.noreply, &result, &n_res) + else: + pass rv = None if n_res == 1 and result != NULL: rv = result.value - with nogil: - self._imp.destroyUnsignedResult() + self._imp.destroyUnsignedResult() Py_DECREF(key) return rv @@ -1189,6 +1164,9 @@ cdef class PyClientPool(PyClientSettings): @contextmanager def client(self): worker = self.acquire() + yield worker + self.release(worker) + return try: yield worker finally: diff --git a/misc/memcached_server b/misc/memcached_server index c955a67b..f921e5bc 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -152,15 +152,11 @@ case "$1" in source misc/git/pre-commit virtualize dbg="`which python-dbg || which python3-dbg`" - make clean + #make clean $dbg setup.py build_ext --inplace if [ -n "$2" ]; then shift - # cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" - cygdb . --skip-interpreter -- \ - -ex start \ - -ex "cy break libmc._client.PyClientPool.setup" \ - --args "$dbg" setup.py test -a "$*" + cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" else cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test fi From bdaf276e825169011428a47db058ee35dec9d940 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 19 Jan 2024 17:31:24 -0800 Subject: [PATCH 53/95] sequential benchmark, but measures overhead --- .github/workflows/manual.yml | 29 +++++++++++++++++++++++++++++ CMakeLists.txt | 2 +- misc/runbench.py | 10 +++++++--- setup.py | 2 +- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 49c6e32c..4bac6683 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -87,3 +87,32 @@ jobs: ./misc/travis/gotest.sh - name: Stop memcached servers run: ./misc/memcached_server stop + + reference: + runs-on: ubuntu-latest + strategy: + matrix: + pyver: ["3.12"] + + steps: + - uses: actions/checkout@v2 + - name: Setup system dependencies + run: | + sudo apt-get update + sudo apt-get -y install memcached libmemcached-dev g++ + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.pyver }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools future python-memcached pylibmc + - name: Start memcached servers + run: ./misc/memcached_server start + - name: Run benchmark + run: | + ./misc/travis/benchmark.sh + - name: Stop memcached servers + run: ./misc/memcached_server stop diff --git a/CMakeLists.txt b/CMakeLists.txt index bddda399..04a4799a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ if (NOT CMAKE_BUILD_TYPE) endif (NOT CMAKE_BUILD_TYPE) set(CMAKE_CXX_FLAGS_COMMON "-Wall -fno-rtti -fno-exceptions") -set(CMAKE_CXX_FLAGS_DEBUG "-DDEBUG -g2 ${CMAKE_CXX_FLAGS_COMMON} -fsanitize=thread" CACHE STRING "CXX DEBUG FLAGS" FORCE) +set(CMAKE_CXX_FLAGS_DEBUG "-DDEBUG -g2 ${CMAKE_CXX_FLAGS_COMMON}" CACHE STRING "CXX DEBUG FLAGS" FORCE) set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3 ${CMAKE_CXX_FLAGS_COMMON}" CACHE STRING "CXX RELEASE FLAGS" FORCE) set(CMAKE_INSTALL_INCLUDE include CACHE PATH "Output directory for header files") set(CMAKE_INSTALL_LIBDIR lib CACHE PATH "Output directory for libraries") diff --git a/misc/runbench.py b/misc/runbench.py index 40b2b49a..b8979daa 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -96,7 +96,7 @@ def __init__(self): def __unicode__(self): m = self.mean() d = self.stddev() - fmt = u"%.3gs, σ=%.3g, n=%d, snr=%.3g:%.3g".__mod__ + fmt = u"%.3gs, \u03C3=%.3g, n=%d, snr=%.3g:%.3g".__mod__ return fmt((m, d, len(self.laps)) + ratio(m, d)) __str__ = __unicode__ @@ -139,7 +139,7 @@ def bench_get(mc, key, data): @benchmark_method def bench_set(mc, key, data): - if isinstance(mc.mc, libmc.Client): + if isinstance(mc.mc, libmc.Client) or isinstance(mc.mc, libmc.ThreadedClient): if not mc.set(key, data): logger.warn('%r.set(%r, ...) fail', mc, key) else: @@ -156,7 +156,7 @@ def bench_get_multi(mc, keys, pairs): @benchmark_method def bench_set_multi(mc, keys, pairs): ret = mc.set_multi(pairs) - if isinstance(mc.mc, libmc.Client): + if isinstance(mc.mc, libmc.Client) or isinstance(mc.mc, libmc.ThreadedClient): if not ret: logger.warn('%r.set_multi fail', mc) else: @@ -235,6 +235,10 @@ def make_pylibmc_client(servers, **kw): name='libmc(md5 / ketama / nodelay / nonblocking, from douban)', factory=lambda: Prefix(__import__('libmc').Client(servers, comp_threshold=4000), 'libmc1') ), + Participant( + name='libmc(md5 / ketama / nodelay / nonblocking / threaded, from douban)', + factory=lambda: Prefix(__import__('libmc').ThreadedClient(servers, comp_threshold=4000), 'libmc2') + ), ] diff --git a/setup.py b/setup.py index c9807b95..afc15c1d 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ def run_tests(self): # import here, cause outside the eggs aren't loaded import pytest errno = pytest.main(shlex.split(self.pytest_args)) - sys.exit(errno) + os._exit(errno) setup( From d8d0014feef4f49dba2eb14fe6f411c96bcbc8d0 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 19 Jan 2024 18:12:13 -0800 Subject: [PATCH 54/95] remove debugging statements and shared cython resources --- libmc/_client.pyx | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index 8556459c..4915f874 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -1125,7 +1125,6 @@ cdef class PyClientPool(PyClientSettings): def __cinit__(self): self._imp = new ClientPool() self.config(CFG_HASH_FUNCTION, self.hash_fn) - self._initialized = False self.clients = [] if self.failover: @@ -1145,18 +1144,7 @@ cdef class PyClientPool(PyClientSettings): return worker cdef acquire(self): - # if not self._initialized: - # self.connect() - # self._initialized = True - worker = self._imp._acquire() - return self.setup(worker) - # prone to race conditions pending mux and possibly fails to update - if worker.index >= len(self.clients): - self.clients += [None] * (worker.index - len(self.clients)) - self.clients.append(self.setup(worker)) - elif self.clients[worker.index] == None: - self.clients[worker.index] = self.setup(worker) - return self.clients[worker.index] + return self.setup(self._imp._acquire()) cdef release(self, PyPoolClient worker): self._imp._release(worker._indexed) @@ -1164,9 +1152,6 @@ cdef class PyClientPool(PyClientSettings): @contextmanager def client(self): worker = self.acquire() - yield worker - self.release(worker) - return try: yield worker finally: From f8d4728b1e22c07defaeab1483b4d912c003c099 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 19 Jan 2024 21:50:06 -0800 Subject: [PATCH 55/95] cython threading hangs under load --- misc/git/debug.patch | 2 +- misc/memcached_server | 4 +++- misc/runbench.py | 32 +++++++++++++++++++++++--------- tests/test_client_pool.py | 4 ++-- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/misc/git/debug.patch b/misc/git/debug.patch index 07b325cc..a5360a6e 100644 --- a/misc/git/debug.patch +++ b/misc/git/debug.patch @@ -50,7 +50,7 @@ index c9807b9..b3a915b 100644 @@ -91,6 +111,25 @@ class PyTest(TestCommand): errno = pytest.main(shlex.split(self.pytest_args)) - sys.exit(errno) + os._exit(errno) +# https://shwina.github.io/custom-compiler-linker-extensions/ +class BlankBuildExt(build_ext): diff --git a/misc/memcached_server b/misc/memcached_server index f921e5bc..e074d66a 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -72,7 +72,9 @@ function virtualize() { if [ ! -e venv ]; then dbg="${1:-`which python-dbg || which python3-dbg`}" - $dbg -m pip install virtualenv + if ! $dbg -m pip list | grep -w virtualenv &> /dev/null; then + $dbg -m pip install virtualenv + fi $dbg -m virtualenv --python="$dbg" --seeder=pip venv; source venv/bin/activate pip install -U pip diff --git a/misc/runbench.py b/misc/runbench.py index b8979daa..a37124c5 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -5,6 +5,7 @@ import sys import math import logging +import threading from functools import wraps from collections import namedtuple from contextlib import contextmanager @@ -20,9 +21,10 @@ logger = logging.getLogger('libmc.bench') Benchmark = namedtuple('Benchmark', 'name f args kwargs') -Participant = namedtuple('Participant', 'name factory') +Participant = namedtuple('Participant', 'name factory threads', defaults=(1,)) BENCH_TIME = 1.0 N_SERVERS = 20 +NTHREADS = 5 # 4 max clients plus one to queue class Prefix(object): @@ -232,16 +234,16 @@ def make_pylibmc_client(servers, **kw): ), Participant(name='python-memcached', factory=lambda: Prefix(__import__('memcache').Client(servers), 'memcache1')), Participant( - name='libmc(md5 / ketama / nodelay / nonblocking, from douban)', + name='libmc(md6 / ketama / nodelay / nonblocking, from douban)', factory=lambda: Prefix(__import__('libmc').Client(servers, comp_threshold=4000), 'libmc1') ), Participant( - name='libmc(md5 / ketama / nodelay / nonblocking / threaded, from douban)', - factory=lambda: Prefix(__import__('libmc').ThreadedClient(servers, comp_threshold=4000), 'libmc2') + name='libmc_threaded', + factory=lambda: Prefix(__import__('libmc').ThreadedClient(servers, comp_threshold=4000), 'libmc2'), + threads=NTHREADS ), ] - def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIME): """Do you even lift?""" @@ -261,10 +263,22 @@ def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIM if 'get' in fn.__name__: last_fn(mc, *args, **kwargs) - sw = Stopwatch() - while sw.total() < bench_time: - with sw.timing(): - fn(mc, *args, **kwargs) + def loop(): + while sw.total() < bench_time: + with sw.timing(): + fn(mc, *args, **kwargs) + + if participant.threads == 1: + sw = Stopwatch() + loop() + else: + ts = [threading.Thread(target=loop) for i in range(participant.threads)] + sw = Stopwatch() + for t in ts: + t.start() + + for t in ts: + t.join() means[i].append(sw.mean()) stddevs[i].append(sw.stddev()) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 6cc2350e..417a04cd 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -76,7 +76,7 @@ def setUp(self): self.pool = ClientPool(["127.0.0.1:21211"]) def misc(self): - for i in range(5): + for i in range(100): self.test_pool_client_misc(i) def test_pool_client_misc(self, i=0): @@ -95,7 +95,7 @@ def setUp(self): self.imp = ThreadedClient(["127.0.0.1:21211"]) def misc(self): - for i in range(5): + for i in range(100): self.client_misc(self.imp, i) def test_many_threads(self): From 804a18e902c8003cfd57f082b3605c486b3c68d3 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 22 Jan 2024 16:13:58 -0800 Subject: [PATCH 56/95] Revert "can't assign to python scope variables w/o GIL and underlying functions release it anyway" This reverts commit c3ff9aeed01728a6862d9ce57a7cd07fdff78154. apparently this just hides GIL lock rather than solves it --- libmc/_client.pyx | 143 +++++++++++++++++++++++++----------------- misc/memcached_server | 8 ++- 2 files changed, 90 insertions(+), 61 deletions(-) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index 4915f874..cbef1c7d 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -458,7 +458,8 @@ cdef class PyClientShell(PyClientSettings): cdef size_t c_key_len = 0 Py_INCREF(key2) PyString_AsStringAndSize(key2, &c_key, &c_key_len) - c_addr = self._imp.getServerAddressByKey(c_key, c_key_len) + with nogil: + c_addr = self._imp.getServerAddressByKey(c_key, c_key_len) cdef basestring server_addr = c_addr Py_DECREF(key2) return server_addr @@ -470,7 +471,8 @@ cdef class PyClientShell(PyClientSettings): cdef size_t c_key_len = 0 Py_INCREF(key2) PyString_AsStringAndSize(key2, &c_key, &c_key_len) - c_addr = self._imp.getRealtimeServerAddressByKey(c_key, c_key_len) + with nogil: + c_addr = self._imp.getRealtimeServerAddressByKey(c_key, c_key_len) Py_DECREF(key2) cdef basestring server_addr if c_addr != NULL: @@ -498,12 +500,13 @@ cdef class PyClientShell(PyClientSettings): PyString_AsStringAndSize(key, &c_key, &c_key_len) # XXX: safe cast? cdef size_t n = 1, n_results = 0 cdef retrieval_result_t** results = NULL - if op == GET_OP: - self.last_error = self._imp.get(&c_key, &c_key_len, n, &results, &n_results) - elif op == GETS_OP: - self.last_error = self._imp.gets(&c_key, &c_key_len, n, &results, &n_results) - else: - pass + with nogil: + if op == GET_OP: + self.last_error = self._imp.get(&c_key, &c_key_len, n, &results, &n_results) + elif op == GETS_OP: + self.last_error = self._imp.gets(&c_key, &c_key_len, n, &results, &n_results) + else: + pass cdef bytes py_value = None if n_results == 1: @@ -512,7 +515,8 @@ cdef class PyClientShell(PyClientSettings): if op == GETS_OP: cas_unique_ptr[0] = results[0].cas_unique Py_DECREF(key) - self._imp.destroyRetrievalResult() + with nogil: + self._imp.destroyRetrievalResult() return py_value def _get_large_raw(self, bytes key, int n_splits, flags_t chuncked_flags): @@ -578,7 +582,8 @@ cdef class PyClientShell(PyClientSettings): cdef retrieval_result_t** results = NULL cdef retrieval_result_t *r = NULL - self.last_error = self._imp.get(c_keys, c_key_lens, n, &results, &n_res) + with nogil: + self.last_error = self._imp.get(c_keys, c_key_lens, n, &results, &n_res) cdef dict rv = {} cdef bytes py_key @@ -592,7 +597,8 @@ cdef class PyClientShell(PyClientSettings): PyMem_Free(c_keys) PyMem_Free(c_key_lens) Py_DECREF(keys) - self._imp.destroyRetrievalResult() + with nogil: + self._imp.destroyRetrievalResult() return rv def get_multi(self, keys): @@ -639,24 +645,26 @@ cdef class PyClientShell(PyClientSettings): cdef message_result_t** results = NULL - if op == SET_OP: - self.last_error = self._imp.set(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - elif op == ADD_OP: - self.last_error = self._imp.add(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - elif op == REPLACE_OP: - self.last_error = self._imp.replace(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - elif op == PREPEND_OP: - self.last_error = self._imp.prepend(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - elif op == APPEND_OP: - self.last_error = self._imp.append(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - elif op == CAS_OP: - self.last_error = self._imp.cas(&c_key, &c_key_len, &flags, exptime, &cas_unique, self.noreply, &c_val, &c_val_len, n, &results, &n_res) - else: - pass + with nogil: + if op == SET_OP: + self.last_error = self._imp.set(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + elif op == ADD_OP: + self.last_error = self._imp.add(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + elif op == REPLACE_OP: + self.last_error = self._imp.replace(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + elif op == PREPEND_OP: + self.last_error = self._imp.prepend(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + elif op == APPEND_OP: + self.last_error = self._imp.append(&c_key, &c_key_len, &flags, exptime, NULL, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + elif op == CAS_OP: + self.last_error = self._imp.cas(&c_key, &c_key_len, &flags, exptime, &cas_unique, self.noreply, &c_val, &c_val_len, n, &results, &n_res) + else: + pass rv = self.last_error == RET_OK and (self.noreply or (n_res == 1 and results[0][0].type_ == MSG_STORED)) - self._imp.destroyMessageResult() + with nogil: + self._imp.destroyMessageResult() Py_DECREF(key) Py_DECREF(val) return rv @@ -786,17 +794,18 @@ cdef class PyClientShell(PyClientSettings): PyString_AsStringAndSize(vals[i], &c_vals[i], &c_val_lens[i]) # XXX: safe cast? cdef message_result_t** results = NULL - if op == SET_OP: - self.last_error = self._imp.set(c_keys, c_key_lens, c_flags, c_exptime, NULL, - self.noreply, c_vals, c_val_lens, n, &results, &n_rst) - elif op == PREPEND_OP: - self.last_error = self._imp.prepend(c_keys, c_key_lens, c_flags, c_exptime, NULL, - self.noreply, c_vals, c_val_lens, n, &results, &n_rst) - elif op == APPEND_OP: - self.last_error = self._imp.append(c_keys, c_key_lens, c_flags, c_exptime, NULL, - self.noreply, c_vals, c_val_lens, n, &results, &n_rst) - else: - pass + with nogil: + if op == SET_OP: + self.last_error = self._imp.set(c_keys, c_key_lens, c_flags, c_exptime, NULL, + self.noreply, c_vals, c_val_lens, n, &results, &n_rst) + elif op == PREPEND_OP: + self.last_error = self._imp.prepend(c_keys, c_key_lens, c_flags, c_exptime, NULL, + self.noreply, c_vals, c_val_lens, n, &results, &n_rst) + elif op == APPEND_OP: + self.last_error = self._imp.append(c_keys, c_key_lens, c_flags, c_exptime, NULL, + self.noreply, c_vals, c_val_lens, n, &results, &n_rst) + else: + pass is_succeed = (self.last_error == RET_OK) and (self.noreply or n_rst == n) @@ -806,7 +815,8 @@ cdef class PyClientShell(PyClientSettings): failed_keys = list(set(keys) - set(succeed_keys)) is_succeed = is_succeed and (len(failed_keys) == 0) - self._imp.destroyMessageResult() + with nogil: + self._imp.destroyMessageResult() PyMem_Free(c_keys) PyMem_Free(c_key_lens) @@ -878,11 +888,13 @@ cdef class PyClientShell(PyClientSettings): PyString_AsStringAndSize(key, &c_key, &c_key_len) cdef message_result_t** results = NULL - self.last_error = self._imp._delete(&c_key, &c_key_len, self.noreply, n, &results, &n_res) + with nogil: + self.last_error = self._imp._delete(&c_key, &c_key_len, self.noreply, n, &results, &n_res) rv = self.last_error == RET_OK and (self.noreply or (n_res == 1 and (results[0][0].type_ == MSG_DELETED or results[0][0].type_ == MSG_NOT_FOUND))) - self._imp.destroyMessageResult() + with nogil: + self._imp.destroyMessageResult() Py_DECREF(key) return rv @@ -902,7 +914,8 @@ cdef class PyClientShell(PyClientSettings): cdef message_result_t** results = NULL cdef message_result_t *r = NULL - self.last_error = self._imp._delete(c_keys, c_key_lens, self.noreply, n, &results, &n_res) + with nogil: + self.last_error = self._imp._delete(c_keys, c_key_lens, self.noreply, n, &results, &n_res) is_succeed = self.last_error == RET_OK and (self.noreply or n_res == n) cdef list failed_keys = [] @@ -913,7 +926,8 @@ cdef class PyClientShell(PyClientSettings): if results[i][0].type_ == MSG_DELETED or results[i][0].type_ == MSG_NOT_FOUND] failed_keys = list(set(keys) - set(succeed_keys)) - self._imp.destroyMessageResult() + with nogil: + self._imp.destroyMessageResult() PyMem_Free(c_key_lens) PyMem_Free(c_keys) Py_DECREF(keys) @@ -934,10 +948,12 @@ cdef class PyClientShell(PyClientSettings): cdef message_result_t** results = NULL - self.last_error = self._imp.touch(&c_key, &c_key_len, exptime, self.noreply, n, &results, &n_res) + with nogil: + self.last_error = self._imp.touch(&c_key, &c_key_len, exptime, self.noreply, n, &results, &n_res) rv = self.last_error == RET_OK and (self.noreply or (n_res == 1 and results[0][0].type_ == MSG_TOUCHED)) - self._imp.destroyMessageResult() + with nogil: + self._imp.destroyMessageResult() Py_DECREF(key) return rv @@ -950,7 +966,8 @@ cdef class PyClientShell(PyClientSettings): self._record_thread_ident() cdef broadcast_result_t* rst = NULL cdef size_t n = 0 - self.last_error = self._imp.version(&rst, &n) + with nogil: + self.last_error = self._imp.version(&rst, &n) rv = {} for i in range(n): @@ -958,7 +975,8 @@ cdef class PyClientShell(PyClientSettings): continue rv[rst[i].host] = rst[i].lines[0][:rst[i].line_lens[0]] - self._imp.destroyBroadcastResult() + with nogil: + self._imp.destroyBroadcastResult() return rv def toggle_flush_all_feature(self, enabled): @@ -968,14 +986,16 @@ cdef class PyClientShell(PyClientSettings): self._record_thread_ident() cdef broadcast_result_t* rst = NULL cdef size_t n = 0 - self.last_error = self._imp.flushAll(&rst, &n) + with nogil: + self.last_error = self._imp.flushAll(&rst, &n) rv = [] for i in range(n): if rst[i].msg_type == MSG_OK: rv.append(rst[i].host) - self._imp.destroyBroadcastResult() + with nogil: + self._imp.destroyBroadcastResult() if self.last_error == RET_PROGRAMMING_ERR: raise RuntimeError( "Please call client.toggle_flush_all_feature(True) first " @@ -985,8 +1005,9 @@ cdef class PyClientShell(PyClientSettings): def quit(self): self._record_thread_ident() - self.last_error = self._imp.quit() - self._imp.destroyBroadcastResult() + with nogil: + self.last_error = self._imp.quit() + self._imp.destroyBroadcastResult() return self.last_error in {RET_CONN_POLL_ERR, RET_OK} def stats(self): @@ -995,7 +1016,8 @@ cdef class PyClientShell(PyClientSettings): cdef broadcast_result_t* r = NULL cdef size_t n = 0 rv = {} - self.last_error = self._imp.stats(&rst, &n) + with nogil: + self.last_error = self._imp.stats(&rst, &n) for i in range(n): r = &rst[i] @@ -1014,7 +1036,8 @@ cdef class PyClientShell(PyClientSettings): except: pass rv[r.host][k] = v - self._imp.destroyBroadcastResult() + with nogil: + self._imp.destroyBroadcastResult() return rv def _incr_decr_raw(self, op_code_t op, bytes key, uint64_t delta): @@ -1027,17 +1050,19 @@ cdef class PyClientShell(PyClientSettings): cdef unsigned_result_t* result = NULL cdef size_t n_res = 0 - if op == INCR_OP: - self.last_error = self._imp.incr(c_key, c_key_len, delta, self.noreply, &result, &n_res) - elif op == DECR_OP: - self.last_error = self._imp.decr(c_key, c_key_len, delta, self.noreply, &result, &n_res) - else: - pass + with nogil: + if op == INCR_OP: + self.last_error = self._imp.incr(c_key, c_key_len, delta, self.noreply, &result, &n_res) + elif op == DECR_OP: + self.last_error = self._imp.decr(c_key, c_key_len, delta, self.noreply, &result, &n_res) + else: + pass rv = None if n_res == 1 and result != NULL: rv = result.value - self._imp.destroyUnsignedResult() + with nogil: + self._imp.destroyUnsignedResult() Py_DECREF(key) return rv diff --git a/misc/memcached_server b/misc/memcached_server index e074d66a..d528dc58 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -154,11 +154,15 @@ case "$1" in source misc/git/pre-commit virtualize dbg="`which python-dbg || which python3-dbg`" - #make clean + make clean $dbg setup.py build_ext --inplace if [ -n "$2" ]; then shift - cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" + # cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" + cygdb . --skip-interpreter -- \ + -ex start \ + -ex "cy break libmc._client.PyClientPool.setup" \ + --args "$dbg" setup.py test -a "$*" else cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test fi From 340a8e5b835fd0adcb30ae5368ac02fb18f258cd Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 22 Jan 2024 16:16:04 -0800 Subject: [PATCH 57/95] revert reverted debug setup removal --- misc/memcached_server | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/misc/memcached_server b/misc/memcached_server index d528dc58..e074d66a 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -154,15 +154,11 @@ case "$1" in source misc/git/pre-commit virtualize dbg="`which python-dbg || which python3-dbg`" - make clean + #make clean $dbg setup.py build_ext --inplace if [ -n "$2" ]; then shift - # cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" - cygdb . --skip-interpreter -- \ - -ex start \ - -ex "cy break libmc._client.PyClientPool.setup" \ - --args "$dbg" setup.py test -a "$*" + cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" else cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test fi From f7964e5f4bebbe3b2502f0a8b114956159bf74f3 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 22 Jan 2024 17:57:24 -0800 Subject: [PATCH 58/95] debugging conviences --- Makefile | 2 +- misc/git/debug.patch | 7 +++++-- misc/memcached_server | 32 ++++++++++++++++++++++---------- misc/runbench.py | 2 +- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index d1363d8d..0b6a7ff9 100644 --- a/Makefile +++ b/Makefile @@ -11,4 +11,4 @@ pytest: pytest tests/ clean: - rm -fr build cython_debug libmc/*.so libmc/*.cpp + rm -fr build cython_debug libmc/*.so libmc/*.cpp .eggs diff --git a/misc/git/debug.patch b/misc/git/debug.patch index a5360a6e..b8c94569 100644 --- a/misc/git/debug.patch +++ b/misc/git/debug.patch @@ -1,5 +1,5 @@ diff --git a/setup.py b/setup.py -index c9807b9..b3a915b 100644 +index afc15c1..98f8465 100644 --- a/setup.py +++ b/setup.py @@ -5,23 +5,43 @@ import sys @@ -74,7 +74,7 @@ index c9807b9..b3a915b 100644 setup( name="libmc", -@@ -121,16 +160,18 @@ setup( +@@ -121,18 +160,20 @@ setup( ], # Support for the basestring type is new in Cython 0.20. setup_requires=["Cython >= 0.20"], @@ -96,3 +96,6 @@ index c9807b9..b3a915b 100644 tests_require=[ "pytest", "future", +- "numpy", ++ #"numpy", + ]) diff --git a/misc/memcached_server b/misc/memcached_server index e074d66a..f8bcdf5b 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -71,19 +71,24 @@ function capture() function virtualize() { if [ ! -e venv ]; then - dbg="${1:-`which python-dbg || which python3-dbg`}" - if ! $dbg -m pip list | grep -w virtualenv &> /dev/null; then - $dbg -m pip install virtualenv + py="${1:-`which python 2>/dev/null`}" + if ! $py -m pip list | grep -w virtualenv &> /dev/null; then + $py -m pip install virtualenv fi - $dbg -m virtualenv --python="$dbg" --seeder=pip venv; + $py -m virtualenv --python="$py" --seeder=pip venv; source venv/bin/activate pip install -U pip - pip install Cython setuptools future pytest greenify gevent numpy + pip install Cython setuptools future pytest greenify gevent else source venv/bin/activate fi } +function src_dir() +{ + cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd +} + case "$1" in start) if [ -n "$2" ]; then @@ -148,11 +153,10 @@ case "$1" in capture $@ ;; cy-debug) - DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - cd "$DIR/.." + cd "`src_dir`/.." git apply misc/git/debug.patch --recount &> /dev/null || true source misc/git/pre-commit - virtualize + virtualize "`which python-dbg || which python3-dbg`" dbg="`which python-dbg || which python3-dbg`" #make clean $dbg setup.py build_ext --inplace @@ -164,11 +168,19 @@ case "$1" in fi ;; run-test) - cd "$( dirname "${BASH_SOURCE[0]}" )/.." &> /dev/null - virtualize "`which python 2>/dev/null`" + cd "`src_dir`/.." + virtualize shift python setup.py test -a "-k $*" ;; + bench) + cd "`src_dir`/.." + virtualize + make clean + python setup.py build_ext --inplace + python -m pip install pylibmc python-memcached + python misc/runbench.py + ;; *) printf 'Usage: %s {start|stop|restart} \n' "$prog" exit 1 diff --git a/misc/runbench.py b/misc/runbench.py index a37124c5..b4eef634 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -234,7 +234,7 @@ def make_pylibmc_client(servers, **kw): ), Participant(name='python-memcached', factory=lambda: Prefix(__import__('memcache').Client(servers), 'memcache1')), Participant( - name='libmc(md6 / ketama / nodelay / nonblocking, from douban)', + name='libmc(md5 / ketama / nodelay / nonblocking, from douban)', factory=lambda: Prefix(__import__('libmc').Client(servers, comp_threshold=4000), 'libmc1') ), Participant( From a82f34b1a75145a16828fc3a1499f358be50c6a6 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 22 Jan 2024 18:08:12 -0800 Subject: [PATCH 59/95] cython gil misunderstanding --- libmc/_client.pyx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index cbef1c7d..d3d244d8 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -1169,10 +1169,13 @@ cdef class PyClientPool(PyClientSettings): return worker cdef acquire(self): - return self.setup(self._imp._acquire()) + with nogil: + worker = self._imp._acquire() + return self.setup(worker) cdef release(self, PyPoolClient worker): - self._imp._release(worker._indexed) + with nogil: + self._imp._release(worker._indexed) @contextmanager def client(self): From aff16cf45c6fc406371d4b4720ee8478d6fba701 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 22 Jan 2024 18:19:28 -0800 Subject: [PATCH 60/95] benchmark setup --- Makefile | 2 +- misc/memcached_server | 5 +++-- misc/runbench.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0b6a7ff9..0210df15 100644 --- a/Makefile +++ b/Makefile @@ -11,4 +11,4 @@ pytest: pytest tests/ clean: - rm -fr build cython_debug libmc/*.so libmc/*.cpp .eggs + rm -fr build cython_debug libmc/*.so libmc/*.cpp .eggs libmc.egg-info diff --git a/misc/memcached_server b/misc/memcached_server index f8bcdf5b..e1b6da99 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -158,7 +158,6 @@ case "$1" in source misc/git/pre-commit virtualize "`which python-dbg || which python3-dbg`" dbg="`which python-dbg || which python3-dbg`" - #make clean $dbg setup.py build_ext --inplace if [ -n "$2" ]; then shift @@ -175,10 +174,12 @@ case "$1" in ;; bench) cd "`src_dir`/.." - virtualize + git apply misc/git/debug.patch -R --recount &> /dev/null || true make clean + virtualize python setup.py build_ext --inplace python -m pip install pylibmc python-memcached + python -m pip install -e . python misc/runbench.py ;; *) diff --git a/misc/runbench.py b/misc/runbench.py index b4eef634..667a4061 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -238,7 +238,7 @@ def make_pylibmc_client(servers, **kw): factory=lambda: Prefix(__import__('libmc').Client(servers, comp_threshold=4000), 'libmc1') ), Participant( - name='libmc_threaded', + name='libmc(md5 / ketama / nodelay / nonblocking / threaded, from douban)', factory=lambda: Prefix(__import__('libmc').ThreadedClient(servers, comp_threshold=4000), 'libmc2'), threads=NTHREADS ), From e06cafe871005e7055e4eeb5c2c859579e12858e Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 22 Jan 2024 18:36:15 -0800 Subject: [PATCH 61/95] greenify initial pass --- misc/memcached_server | 1 - misc/runbench.py | 18 ++++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/misc/memcached_server b/misc/memcached_server index e1b6da99..e01e4b7c 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -175,7 +175,6 @@ case "$1" in bench) cd "`src_dir`/.." git apply misc/git/debug.patch -R --recount &> /dev/null || true - make clean virtualize python setup.py build_ext --inplace python -m pip install pylibmc python-memcached diff --git a/misc/runbench.py b/misc/runbench.py index 667a4061..75c82ca8 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -5,7 +5,6 @@ import sys import math import logging -import threading from functools import wraps from collections import namedtuple from contextlib import contextmanager @@ -18,6 +17,21 @@ else: from time import process_time +if False: + import threading + fork = lambda f: threading.Thread(target=f) +else: + import gevent + import gevent.monkey + gevent.monkey.patch_time() + + import greenify + greenify.greenify() + for so_path in libmc.DYNAMIC_LIBRARIES: + assert greenify.patch_lib(so_path) + + fork = gevent.spawn_raw + logger = logging.getLogger('libmc.bench') Benchmark = namedtuple('Benchmark', 'name f args kwargs') @@ -272,7 +286,7 @@ def loop(): sw = Stopwatch() loop() else: - ts = [threading.Thread(target=loop) for i in range(participant.threads)] + ts = [fork(loop) for i in range(participant.threads)] sw = Stopwatch() for t in ts: t.start() From 1def670b403fb09a867b979f7823eb98889ffa0e Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 22 Jan 2024 21:00:11 -0800 Subject: [PATCH 62/95] greenlet misunderstanding --- misc/runbench.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misc/runbench.py b/misc/runbench.py index 75c82ca8..8a1e2622 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -19,7 +19,7 @@ if False: import threading - fork = lambda f: threading.Thread(target=f) + spawn = lambda f: threading.Thread(target=f) else: import gevent import gevent.monkey @@ -30,7 +30,7 @@ for so_path in libmc.DYNAMIC_LIBRARIES: assert greenify.patch_lib(so_path) - fork = gevent.spawn_raw + spawn = gevent.spawn logger = logging.getLogger('libmc.bench') @@ -286,7 +286,7 @@ def loop(): sw = Stopwatch() loop() else: - ts = [fork(loop) for i in range(participant.threads)] + ts = [spawn(loop) for i in range(participant.threads)] sw = Stopwatch() for t in ts: t.start() From ecc5ec7d5956a3c736fafb9f1cee1f96da589b33 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 22 Jan 2024 21:33:38 -0800 Subject: [PATCH 63/95] eventlet unittest --- tests/test_client_pool.py | 41 ++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 417a04cd..28117b96 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -29,6 +29,9 @@ def threaded_print(*args, **kwargs): print(*args, **kwargs, file=fp) class ClientOps: + nthreads=8 + ops = 100 + def client_misc(self, mc, i=0): tid = mc._get_current_thread_ident() + (i,) tid = "_".join(map(str, tid)) @@ -58,7 +61,7 @@ def passthrough(args): errs.append(e.with_traceback(tb)) threading.excepthook = passthrough - ts = [threading.Thread(target=target) for i in range(8)] + ts = [threading.Thread(target=target) for i in range(self.nthreads)] for t in ts: t.start() @@ -76,7 +79,7 @@ def setUp(self): self.pool = ClientPool(["127.0.0.1:21211"]) def misc(self): - for i in range(100): + for i in range(self.ops): self.test_pool_client_misc(i) def test_pool_client_misc(self, i=0): @@ -90,13 +93,37 @@ def test_acquire(self): def test_pool_client_threaded(self): self.client_threads(self.misc) -class ThreadedClientWrapperCheck(unittest.TestCase, ClientOps): - def setUp(self): - self.imp = ThreadedClient(["127.0.0.1:21211"]) - +class ThreadedClientOps(ClientOps): def misc(self): - for i in range(100): + for i in range(self.ops): self.client_misc(self.imp, i) + +class ThreadedClientWrapperCheck(unittest.TestCase, ThreadedClientOps): + def setUp(self): + self.imp = ThreadedClient(["127.0.0.1:21211"]) + def test_many_threads(self): self.client_threads(self.misc) + + +class ThreadedGreenletCompat(unittest.TestCase, ThreadedClientOps): + def setUp(self): + import greenify, libmc + greenify.greenify() + for so_path in libmc.DYNAMIC_LIBRARIES: + assert greenify.patch_lib(so_path) + + self.imp = libmc.ThreadedClient(["127.0.0.1:21211"]) + + def client_threads(self, target): + import gevent + ts = [gevent.spawn(target) for i in range(self.nthreads)] + for t in ts: + t.start() + + for t in ts: + t.join() + + def test_many_eventlets(self): + self.client_threads(self.misc) From 59662fc52dab1d37857673ffe2a2347229025c3e Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 22 Jan 2024 21:46:36 -0800 Subject: [PATCH 64/95] eventlet name instead of thread name --- tests/test_client_pool.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 28117b96..2edfadf8 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -2,6 +2,7 @@ import unittest import threading import functools +import gevent from libmc import ClientPool, ThreadedClient def setup_loging(f): @@ -32,8 +33,11 @@ class ClientOps: nthreads=8 ops = 100 + def tid(self, mc): + return mc._get_current_thread_ident() + def client_misc(self, mc, i=0): - tid = mc._get_current_thread_ident() + (i,) + tid = self.tid(mc) + (i,) tid = "_".join(map(str, tid)) f, t = 'foo_' + tid, 'tuiche_' + tid mc.get_multi([f, t]) @@ -117,13 +121,11 @@ def setUp(self): self.imp = libmc.ThreadedClient(["127.0.0.1:21211"]) def client_threads(self, target): - import gevent ts = [gevent.spawn(target) for i in range(self.nthreads)] - for t in ts: - t.start() + gevent.joinall(ts) - for t in ts: - t.join() + def tid(self, mc): + return (gevent.getcurrent().name,) def test_many_eventlets(self): self.client_threads(self.misc) From 723addc6bc52802666b7b55994db99f157aa9d24 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 26 Jan 2024 23:08:14 -0800 Subject: [PATCH 65/95] threaded python benchmarks --- include/LockPool.h | 2 +- libmc/__init__.py | 10 +- libmc/_client.pyx | 4 + misc/aliases | 99 ++++++++++++++++ misc/memcached_server | 52 +------- misc/runbench.py | 241 ++++++++++++++++++++++++++++++-------- src/ClientPool.cpp | 3 + tests/test_client_pool.py | 14 ++- 8 files changed, 314 insertions(+), 111 deletions(-) create mode 100755 misc/aliases diff --git a/include/LockPool.h b/include/LockPool.h index 7d82b943..89130c00 100644 --- a/include/LockPool.h +++ b/include/LockPool.h @@ -36,7 +36,7 @@ class OrderedLock { if (m_fifo_locks.empty()) { m_locked = false; } else { - m_fifo_locks.front()->notify_one(); + m_fifo_locks.front()->notify_all(); } } }; diff --git a/libmc/__init__.py b/libmc/__init__.py index f47f91ba..9d9e5546 100644 --- a/libmc/__init__.py +++ b/libmc/__init__.py @@ -1,6 +1,6 @@ import os, functools from ._client import ( - PyClient, PyClientPool, ThreadUnsafe, + PyClient, PyClientUnsafe as ClientUnsafe, PyClientPool, ThreadUnsafe, encode_value, decode_value, @@ -45,13 +45,15 @@ class ClientPool(PyClientPool): pass class ThreadedClient: - @functools.wraps(ClientPool.__init__) def __init__(self, *args, **kwargs): self._client_pool = ClientPool(*args, **kwargs) def update_servers(self, servers): return self._client_pool.update_servers(servers) + def config(self, opt, val): + self._client_pool.config(opt, val) + def __getattr__(self, key): if not hasattr(Client, key): raise AttributeError @@ -69,8 +71,8 @@ def wrapper(*args, **kwargs): __all__ = [ - 'Client', 'ClientPool', 'ThreadedClient', 'ThreadUnsafe', '__VERSION__', - 'encode_value', 'decode_value', + 'Client', 'ClientUnsafe', 'ClientPool', 'ThreadedClient', 'ThreadUnsafe', + '__VERSION__', 'encode_value', 'decode_value', 'MC_DEFAULT_EXPTIME', 'MC_POLL_TIMEOUT', 'MC_CONNECT_TIMEOUT', 'MC_RETRY_TIMEOUT', 'MC_SET_FAILOVER', 'MC_INITIAL_CLIENTS', diff --git a/libmc/_client.pyx b/libmc/_client.pyx index d3d244d8..154671ce 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -1137,6 +1137,10 @@ cdef class PyClient(PyClientShell): def __dealloc__(self): del self._imp +cdef class PyClientUnsafe(PyClient): + def _check_thread_ident(self): + pass + cdef class PyPoolClient(PyClientShell): cdef IndexedClient* _indexed diff --git a/misc/aliases b/misc/aliases new file mode 100755 index 00000000..b05ffc89 --- /dev/null +++ b/misc/aliases @@ -0,0 +1,99 @@ +#!/bin/bash + +set -e + +function virtualize() +{ + if [ ! -e venv ]; then + py="${1:-`which python 2>/dev/null`}" + pyenv="virtualenv" + if ! $py -m pip list | grep -w "$pyenv" &> /dev/null; then + if $py -m pip list | grep -w venv &> /dev/null; then + pyenv="venv" + else + $py -m pip install "$pyenv" + fi + fi + $py -m "$pyenv" --python="$py" --seeder=pip venv; + source venv/bin/activate + pip install -U pip + pip install Cython setuptools future pytest + else + source venv/bin/activate + if ! [ venv -ef "$VIRTUAL_ENV" ]; then + echo "virtual environment was created with a different path" + exit 1 + fi + fi +} + +function virtualize_libmc() +{ + if [ ! -e venv ]; then + virtualize "$*" + pip install greenify gevent + else + virtualize "$*" + fi +} + +function src_dir() +{ + cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd +} + +case "$1" in + test) + shift + echo $res + ;; + cy-debug) + cd "`src_dir`/.." + git apply misc/git/debug.patch --recount &> /dev/null || true + source misc/git/pre-commit + virtualize_libmc "`which python-dbg || which python3-dbg`" + python setup.py build_ext --inplace + if [ -n "$2" ]; then + pattern="$2" + shift 2 + cmds=(-ex start) + for arg in "$@"; do cmds+=(-ex "$arg"); done + cygdb . -- "${cmds[@]}" --args python setup.py test -a "-k $pattern" + else + cygdb . -- -ex start --args python setup.py test + fi + ;; + run-test) + cd "`src_dir`/.." + virtualize_libmc + shift + if [ -n "$1" ]; then + python setup.py test -a "-k $*" + else + python setup.py test + fi + ;; + bench) + cd "`src_dir`/.." + git apply misc/git/debug.patch -R --recount &> /dev/null || true + virtualize_libmc + python setup.py build_ext --inplace + # python -m pip install pylibmc python-memcached + if ! python -m pip list | grep -w "libmc" &> /dev/null; then + python -m pip install -e . + fi + python misc/runbench.py + ;; + build-greenify) + virtualize_libmc + greenify=`python -c "import greenify; print(greenify.__file__)"` + deactivate + cd `dirname "$greenify"` + virtualize "`which python-dbg || which python3-dbg`" + python setup.py build_ext --inplace + ;; + *) + printf 'Usage: %s {cy-debug [test pattern [gdb commands...]] | run-test [test pattern] | bench | build-greenify}\n' "$prog" + exit 1 + ;; +esac diff --git a/misc/memcached_server b/misc/memcached_server index e01e4b7c..8fc318b2 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -68,27 +68,6 @@ function capture() echo $! > "$basedir/var/run/${host}.pid" } -function virtualize() -{ - if [ ! -e venv ]; then - py="${1:-`which python 2>/dev/null`}" - if ! $py -m pip list | grep -w virtualenv &> /dev/null; then - $py -m pip install virtualenv - fi - $py -m virtualenv --python="$py" --seeder=pip venv; - source venv/bin/activate - pip install -U pip - pip install Cython setuptools future pytest greenify gevent - else - source venv/bin/activate - fi -} - -function src_dir() -{ - cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd -} - case "$1" in start) if [ -n "$2" ]; then @@ -152,37 +131,8 @@ case "$1" in shift capture $@ ;; - cy-debug) - cd "`src_dir`/.." - git apply misc/git/debug.patch --recount &> /dev/null || true - source misc/git/pre-commit - virtualize "`which python-dbg || which python3-dbg`" - dbg="`which python-dbg || which python3-dbg`" - $dbg setup.py build_ext --inplace - if [ -n "$2" ]; then - shift - cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test -a "$*" - else - cygdb . --skip-interpreter -- -ex start --args "$dbg" setup.py test - fi - ;; - run-test) - cd "`src_dir`/.." - virtualize - shift - python setup.py test -a "-k $*" - ;; - bench) - cd "`src_dir`/.." - git apply misc/git/debug.patch -R --recount &> /dev/null || true - virtualize - python setup.py build_ext --inplace - python -m pip install pylibmc python-memcached - python -m pip install -e . - python misc/runbench.py - ;; *) - printf 'Usage: %s {start|stop|restart} \n' "$prog" + printf 'Usage: %s {start[all] | stop[all] | restart | unix | capture} \n' "$prog" exit 1 ;; esac diff --git a/misc/runbench.py b/misc/runbench.py index 8a1e2622..cf2f0241 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -5,11 +5,13 @@ import sys import math import logging +import threading from functools import wraps from collections import namedtuple from contextlib import contextmanager +from queue import Queue -import pylibmc +#import pylibmc import libmc if sys.version_info.major == 2: @@ -17,13 +19,13 @@ else: from time import process_time -if False: - import threading - spawn = lambda f: threading.Thread(target=f) +if True: + spawn = lambda f, *a: threading.Thread(target=f, args=a) else: + # ThreadedGreenletCompat.test_many_eventlets import gevent import gevent.monkey - gevent.monkey.patch_time() + gevent.monkey.patch_all() import greenify greenify.greenify() @@ -38,8 +40,8 @@ Participant = namedtuple('Participant', 'name factory threads', defaults=(1,)) BENCH_TIME = 1.0 N_SERVERS = 20 -NTHREADS = 5 # 4 max clients plus one to queue - +NTHREADS = 40 +POOL_SIZE = 4 class Prefix(object): '''add prefix for key in mc command''' @@ -137,6 +139,35 @@ def timing(self): self.laps.append(te - t0) +class DelayedStopwatch(Stopwatch): + """ Rollie """ + + def __init__(self, laps=None, bound=0): + super().__init__() + self.laps = laps or [] + self._bound = bound + + @property + def bound(self): + return self._bound or sum(self.laps) + + def timing(self): + self.t0 = process_time() + self.timing = super().timing + return super().timing() + + def __add__(self, other): + bound = ((self.bound or other.bound) + (other.bound or self.bound)) / 2 + return DelayedStopwatch(self.laps + other.laps, bound) + + def mean(self): + return self.bound / len(self.laps) + + def stddev(self): + boundless = DelayedStopwatch(self.laps) if self._bound else super() + return boundless.stddev() + + def benchmark_method(f): "decorator to turn f into a factory of benchmarks" @@ -150,34 +181,40 @@ def inner(name, *args, **kwargs): @benchmark_method def bench_get(mc, key, data): if mc.get(key) != data: - logger.warn('%r.get(%r) fail', mc, key) + # logger.warn('%r.get(%r) fail', mc, key) + raise Exception() @benchmark_method def bench_set(mc, key, data): - if isinstance(mc.mc, libmc.Client) or isinstance(mc.mc, libmc.ThreadedClient): + if any(isinstance(mc.mc, client) for client in libmc_clients): if not mc.set(key, data): - logger.warn('%r.set(%r, ...) fail', mc, key) + # logger.warn('%r.set(%r, ...) fail', mc, key) + raise Exception() else: if not mc.set(key, data, min_compress_len=4001): - logger.warn('%r.set(%r, ...) fail', mc, key) + # logger.warn('%r.set(%r, ...) fail', mc, key) + raise Exception() @benchmark_method def bench_get_multi(mc, keys, pairs): if len(mc.get_multi(keys)) != len(pairs): - logger.warn('%r.get_multi() incomplete', mc) + # logger.warn('%r.get_multi() incomplete', mc) + raise Exception() @benchmark_method def bench_set_multi(mc, keys, pairs): ret = mc.set_multi(pairs) - if isinstance(mc.mc, libmc.Client) or isinstance(mc.mc, libmc.ThreadedClient): + if any(isinstance(mc.mc, client) for client in libmc_clients): if not ret: - logger.warn('%r.set_multi fail', mc) + # logger.warn('%r.set_multi fail', mc) + raise Exception() else: if ret: - logger.warn('%r.set_multi(%r) fail', mc, ret) + # logger.warn('%r.set_multi(%r) fail', mc, ret) + raise Exception() def multi_pairs(n, val_len): @@ -219,9 +256,82 @@ def make_pylibmc_client(servers, **kw): return Prefix(__import__('pylibmc').Client(servers_, **kw), prefix) +class Pool: + ''' adapted from pylibmc ''' + + client = libmc.ClientUnsafe + + def __init__(self, *args, **kwargs): + self.args, self.kwargs = args, kwargs + + def clone(self): + return self.client(*self.args, **self.kwargs) + + def __getattr__(self, key): + if not hasattr(libmc.Client, key): + raise AttributeError + result = getattr(libmc.Client, key) + if callable(result): + @wraps(result) + def wrapper(*args, **kwargs): + with self.reserve() as mc: + return getattr(mc, key)(*args, **kwargs) + return wrapper + return result + + +class ThreadMappedPool(Pool): + client = libmc.Client + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.clients = {} + + @property + def current_key(self): + return threading.current_thread().native_id + + @contextmanager + def reserve(self): + key = self.current_key + mc = self.clients.pop(key, None) + if mc is None: + mc = self.clone() + try: + yield mc + finally: + self.clients[key] = mc + + +class ThreadPool(Pool): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.clients = Queue() + for _ in range(POOL_SIZE): + self.clients.put(self.clone()) + + @contextmanager + def reserve(self): + mc = self.clients.get() + try: + yield mc + finally: + self.clients.put(mc) + + +class BenchmarkThreadedClient(libmc.ThreadedClient): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.config(libmc.MC_INITIAL_CLIENTS, POOL_SIZE) + self.config(libmc.MC_MAX_CLIENTS, POOL_SIZE) + + host = '127.0.0.1' servers = ['%s:%d' % (host, 21211 + i) for i in range(N_SERVERS)] +libmc_clients = (libmc.Client, BenchmarkThreadedClient, ThreadMappedPool, ThreadPool) +libmc_kwargs = {"servers": servers, "comp_threshold": 4000, "hash_fn": libmc.MC_HASH_FNV1_32} + participants = [ Participant( name='pylibmc (md5 / ketama)', @@ -248,22 +358,37 @@ def make_pylibmc_client(servers, **kw): ), Participant(name='python-memcached', factory=lambda: Prefix(__import__('memcache').Client(servers), 'memcache1')), Participant( - name='libmc(md5 / ketama / nodelay / nonblocking, from douban)', - factory=lambda: Prefix(__import__('libmc').Client(servers, comp_threshold=4000), 'libmc1') + # name='libmc(md5 / ketama / nodelay / nonblocking, from douban)', + name='libmc (from douban)', + factory=lambda: Prefix(__import__('libmc').Client(**libmc_kwargs), 'libmc1') ), Participant( - name='libmc(md5 / ketama / nodelay / nonblocking / threaded, from douban)', - factory=lambda: Prefix(__import__('libmc').ThreadedClient(servers, comp_threshold=4000), 'libmc2'), + # name='libmc(md5 / ketama / nodelay / nonblocking / C++ thread pool, from douban)', + name='libmc (C++ thread pool)', + factory=lambda: Prefix(BenchmarkThreadedClient(**libmc_kwargs), 'libmc2'), + threads=NTHREADS + ), + Participant( + # name='libmc(md5 / ketama / nodelay / nonblocking / py thead mapped, from douban)', + name='libmc (py thead mapped)', + factory=lambda: Prefix(ThreadMappedPool(**libmc_kwargs), 'libmc3'), + threads=NTHREADS + ), + Participant( + # name='libmc(md5 / ketama / nodelay / nonblocking / py thead pool, from douban)', + name='libmc (py thead pool)', + factory=lambda: Prefix(ThreadPool(**libmc_kwargs), 'libmc4'), threads=NTHREADS ), ] def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIME): - """Do you even lift?""" + """dyel""" mcs = [p.factory() for p in participants] means = [[] for p in participants] stddevs = [[] for p in participants] + exceptions = [[] for p in participants] # Have each lifter do one benchmark each last_fn = None @@ -272,54 +397,70 @@ def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIM logger.info('%s', benchmark_name) for i, (participant, mc) in enumerate(zip(participants, mcs)): - - # FIXME: set before bench for get - if 'get' in fn.__name__: - last_fn(mc, *args, **kwargs) - - def loop(): - while sw.total() < bench_time: - with sw.timing(): - fn(mc, *args, **kwargs) - - if participant.threads == 1: - sw = Stopwatch() - loop() - else: - ts = [spawn(loop) for i in range(participant.threads)] - sw = Stopwatch() - for t in ts: - t.start() - - for t in ts: - t.join() - - means[i].append(sw.mean()) - stddevs[i].append(sw.stddev()) - - logger.info(u'%76s: %s', participant.name, sw) + failed = False + def loop(sw): + nonlocal failed + try: + while sw.total() < bench_time: + with sw.timing(): + fn(mc, *args, **kwargs) + except Exception as e: + failed = failed or e + + try: + # FIXME: set before bench for get + if 'get' in fn.__name__: + last_fn(mc, *args, **kwargs) + + if participant.threads == 1: + sw = [DelayedStopwatch()] + loop(sw[0]) + else: + sw = [DelayedStopwatch() for i in range(participant.threads)] + ts = [spawn(loop, i) for i in sw] + for t in ts: + t.start() + + for t in ts: + t.join() + + if failed: + raise failed + + total = sum(sw, DelayedStopwatch()) + means[i].append(total.mean()) + stddevs[i].append(total.stddev()) + + logger.info(u'%76s: %s', participant.name, total) + exceptions[i].append(None) + except Exception as e: + logger.info(u'%76s: %s', participant.name, "failed") + exceptions[i].append(e) last_fn = fn - return means, stddevs + return means, stddevs, exceptions def main(args=sys.argv[1:]): - logger.info('pylibmc: %s', pylibmc.__file__) + # logger.info('pylibmc: %s', pylibmc.__file__) logger.info('libmc: %s', libmc.__file__) + logger.info('Running %s servers, %s threads, and a %s client pool', + N_SERVERS, NTHREADS, POOL_SIZE) - ps = [p for p in participants if p.name in args] + ps = [p for p in participants if p.name.startswith("libmc")] ps = ps if ps else participants bs = benchmarks[:] logger.info('%d participants in %d benchmarks', len(ps), len(bs)) - means, stddevs = bench(participants=ps, benchmarks=bs) + means, stddevs, exceptions = bench(participants=ps, benchmarks=bs) print('labels =', [p.name for p in ps]) print('benchmarks =', [b.name for b in bs]) print('means =', means) print('stddevs =', stddevs) + print('exceptions =', exceptions) if __name__ == "__main__": diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 87a08f3e..3a2d23f7 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -28,6 +28,9 @@ void ClientPool::config(config_options_t opt, int val) { switch (val) { case CFG_INITIAL_CLIENTS: m_initial_clients = val; + if (m_initial_clients > m_max_clients) { + m_max_clients = m_initial_clients; + } if (m_clients.size() < m_initial_clients) { growPool(m_initial_clients); } diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 2edfadf8..4dae5002 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -2,7 +2,7 @@ import unittest import threading import functools -import gevent +import os from libmc import ClientPool, ThreadedClient def setup_loging(f): @@ -34,7 +34,7 @@ class ClientOps: ops = 100 def tid(self, mc): - return mc._get_current_thread_ident() + return (os.getpid(), threading.current_thread().native_id) def client_misc(self, mc, i=0): tid = self.tid(mc) + (i,) @@ -110,9 +110,13 @@ def setUp(self): def test_many_threads(self): self.client_threads(self.misc) - class ThreadedGreenletCompat(unittest.TestCase, ThreadedClientOps): def setUp(self): + global gevent + import gevent + import gevent.monkey + gevent.monkey.patch_all() + import greenify, libmc greenify.greenify() for so_path in libmc.DYNAMIC_LIBRARIES: @@ -122,10 +126,10 @@ def setUp(self): def client_threads(self, target): ts = [gevent.spawn(target) for i in range(self.nthreads)] - gevent.joinall(ts) + gevent.joinall(ts, raise_error=True) def tid(self, mc): - return (gevent.getcurrent().name,) + return (os.getpid(), gevent.getcurrent().name) def test_many_eventlets(self): self.client_threads(self.misc) From afa71a0bf194ded80685321d710d15c30de11879 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 26 Jan 2024 23:13:48 -0800 Subject: [PATCH 66/95] typo --- misc/runbench.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misc/runbench.py b/misc/runbench.py index cf2f0241..42276600 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -369,14 +369,14 @@ def __init__(self, *args, **kwargs): threads=NTHREADS ), Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py thead mapped, from douban)', - name='libmc (py thead mapped)', + # name='libmc(md5 / ketama / nodelay / nonblocking / py thread mapped, from douban)', + name='libmc (py thread mapped)', factory=lambda: Prefix(ThreadMappedPool(**libmc_kwargs), 'libmc3'), threads=NTHREADS ), Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py thead pool, from douban)', - name='libmc (py thead pool)', + # name='libmc(md5 / ketama / nodelay / nonblocking / py thread pool, from douban)', + name='libmc (py thread pool)', factory=lambda: Prefix(ThreadPool(**libmc_kwargs), 'libmc4'), threads=NTHREADS ), From 3787008ebf0699eb833bba99b27eb63916953b77 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 26 Jan 2024 23:37:10 -0800 Subject: [PATCH 67/95] missed gevent failure --- libmc/_client.pyx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libmc/_client.pyx b/libmc/_client.pyx index 154671ce..e4b36736 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -1144,6 +1144,9 @@ cdef class PyClientUnsafe(PyClient): cdef class PyPoolClient(PyClientShell): cdef IndexedClient* _indexed + def _record_thread_ident(self): + pass + def _check_thread_ident(self): pass From ad0bd7abe045e50ca74786421f40f3496dcf1e81 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sat, 27 Jan 2024 01:11:38 -0800 Subject: [PATCH 68/95] not sure why this one hangs --- misc/runbench.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/misc/runbench.py b/misc/runbench.py index 42276600..2362f550 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -326,6 +326,37 @@ def __init__(self, *args, **kwargs): self.config(libmc.MC_MAX_CLIENTS, POOL_SIZE) +class FIFOThreadPool(ThreadPool): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.waiting = Queue() + self.semaphore = Queue(1) # sorry + self.semaphore.put(1) + + @contextmanager + def reserve(self): + try: + self.semaphore.get() + mc = self.clients.get(False) + self.semaphore.put(1) + except: + channel = Queue() + self.waiting.put(channel) + self.semaphore.put(1) + channel.get() + mc = self.clients.get(False) + self.semaphore.put(1) + try: + yield mc + finally: + self.clients.put(mc) + try: + self.semaphore.get() + self.waiting.get(False).put(1) + except: + self.semaphore.put(1) + + host = '127.0.0.1' servers = ['%s:%d' % (host, 21211 + i) for i in range(N_SERVERS)] @@ -380,6 +411,12 @@ def __init__(self, *args, **kwargs): factory=lambda: Prefix(ThreadPool(**libmc_kwargs), 'libmc4'), threads=NTHREADS ), + Participant( + # name='libmc(md5 / ketama / nodelay / nonblocking / py thread pool, from douban)', + name='libmc (py FIFO thread pool)', + factory=lambda: Prefix(FIFOThreadPool(**libmc_kwargs), 'libmc5'), + threads=NTHREADS + ), ] def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIME): @@ -434,6 +471,7 @@ def loop(sw): logger.info(u'%76s: %s', participant.name, total) exceptions[i].append(None) except Exception as e: + raise logger.info(u'%76s: %s', participant.name, "failed") exceptions[i].append(e) last_fn = fn From 37eb65c937d27b6579118aa2a3b761f0f26f3813 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sat, 27 Jan 2024 13:05:54 -0800 Subject: [PATCH 69/95] add py FIFO thread pool --- misc/runbench.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/misc/runbench.py b/misc/runbench.py index 2362f550..47fafc8d 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -340,7 +340,7 @@ def reserve(self): mc = self.clients.get(False) self.semaphore.put(1) except: - channel = Queue() + channel = Queue(1) self.waiting.put(channel) self.semaphore.put(1) channel.get() @@ -349,9 +349,9 @@ def reserve(self): try: yield mc finally: + self.semaphore.get() self.clients.put(mc) try: - self.semaphore.get() self.waiting.get(False).put(1) except: self.semaphore.put(1) @@ -412,7 +412,7 @@ def reserve(self): threads=NTHREADS ), Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py thread pool, from douban)', + # name='libmc(md5 / ketama / nodelay / nonblocking / py FIFO thread pool, from douban)', name='libmc (py FIFO thread pool)', factory=lambda: Prefix(FIFOThreadPool(**libmc_kwargs), 'libmc5'), threads=NTHREADS @@ -471,7 +471,6 @@ def loop(sw): logger.info(u'%76s: %s', participant.name, total) exceptions[i].append(None) except Exception as e: - raise logger.info(u'%76s: %s', participant.name, "failed") exceptions[i].append(e) last_fn = fn From 31d856929b92ed26c4cdf67bd4ddf8e9a6a9fcd0 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 2 Feb 2024 15:12:37 -0800 Subject: [PATCH 70/95] pytest stdout --- misc/aliases | 5 +++-- tests/test_client_pool.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/misc/aliases b/misc/aliases index b05ffc89..d06b5d43 100755 --- a/misc/aliases +++ b/misc/aliases @@ -58,9 +58,9 @@ case "$1" in shift 2 cmds=(-ex start) for arg in "$@"; do cmds+=(-ex "$arg"); done - cygdb . -- "${cmds[@]}" --args python setup.py test -a "-k $pattern" + cygdb . -- "${cmds[@]}" --args python setup.py test -a "--full-trace -s -k $pattern" else - cygdb . -- -ex start --args python setup.py test + cygdb . -- -ex start --args python setup.py test -a -s fi ;; run-test) @@ -89,6 +89,7 @@ case "$1" in greenify=`python -c "import greenify; print(greenify.__file__)"` deactivate cd `dirname "$greenify"` + rm -fr build virtualize "`which python-dbg || which python3-dbg`" python setup.py build_ext --inplace ;; diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 4dae5002..87147e6c 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -114,8 +114,8 @@ class ThreadedGreenletCompat(unittest.TestCase, ThreadedClientOps): def setUp(self): global gevent import gevent - import gevent.monkey - gevent.monkey.patch_all() + # import gevent.monkey + # gevent.monkey.patch_all() import greenify, libmc greenify.greenify() From 7a92e7bd7df6b06e895431e11c923d1571af4143 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sat, 3 Feb 2024 17:40:29 -0800 Subject: [PATCH 71/95] config typo --- src/ClientPool.cpp | 2 +- tests/test_client_pool.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 3a2d23f7..5373ac83 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -25,7 +25,7 @@ void ClientPool::config(config_options_t opt, int val) { return; } std::unique_lock initializing(m_acquiring_growth); - switch (val) { + switch (opt) { case CFG_INITIAL_CLIENTS: m_initial_clients = val; if (m_initial_clients > m_max_clients) { diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 87147e6c..667a57b0 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -3,7 +3,7 @@ import threading import functools import os -from libmc import ClientPool, ThreadedClient +from libmc import ClientPool, ThreadedClient, MC_MAX_CLIENTS def setup_loging(f): g = None @@ -123,6 +123,7 @@ def setUp(self): assert greenify.patch_lib(so_path) self.imp = libmc.ThreadedClient(["127.0.0.1:21211"]) + self.imp.config(MC_MAX_CLIENTS, 1) def client_threads(self, target): ts = [gevent.spawn(target) for i in range(self.nthreads)] From f7679319f653c93a32883968cbf635c5d1a2b16a Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Sat, 3 Feb 2024 22:11:59 -0800 Subject: [PATCH 72/95] allow pytest stdout for run-test --- misc/aliases | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/aliases b/misc/aliases index d06b5d43..7042775c 100755 --- a/misc/aliases +++ b/misc/aliases @@ -68,9 +68,9 @@ case "$1" in virtualize_libmc shift if [ -n "$1" ]; then - python setup.py test -a "-k $*" + python setup.py test -a "-s -k $*" else - python setup.py test + python setup.py test -a -s fi ;; bench) From 63ab7f5d0557dcda66d153fa26dd96a59cd1d1a7 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 5 Feb 2024 20:12:27 -0800 Subject: [PATCH 73/95] remove greenify debug setup and internal threading metrics --- misc/aliases | 8 ++---- misc/runbench.py | 52 +++++++++++++++++++-------------------- tests/test_client_pool.py | 9 ++++++- 3 files changed, 36 insertions(+), 33 deletions(-) diff --git a/misc/aliases b/misc/aliases index 7042775c..8887375f 100755 --- a/misc/aliases +++ b/misc/aliases @@ -43,10 +43,6 @@ function src_dir() } case "$1" in - test) - shift - echo $res - ;; cy-debug) cd "`src_dir`/.." git apply misc/git/debug.patch --recount &> /dev/null || true @@ -78,7 +74,7 @@ case "$1" in git apply misc/git/debug.patch -R --recount &> /dev/null || true virtualize_libmc python setup.py build_ext --inplace - # python -m pip install pylibmc python-memcached + python -m pip install pylibmc python-memcached if ! python -m pip list | grep -w "libmc" &> /dev/null; then python -m pip install -e . fi @@ -94,7 +90,7 @@ case "$1" in python setup.py build_ext --inplace ;; *) - printf 'Usage: %s {cy-debug [test pattern [gdb commands...]] | run-test [test pattern] | bench | build-greenify}\n' "$prog" + printf 'Usage: %s {cy-debug [test pattern [gdb commands...]] | run-test [test pattern] | bench | build-greenify}\nNote that cy-debug and run-test don't \`make clean\` so that needs to happen elsewhere for changes to setup.py\n"$prog" exit 1 ;; esac diff --git a/misc/runbench.py b/misc/runbench.py index 47fafc8d..553d1d0e 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -11,7 +11,7 @@ from contextlib import contextmanager from queue import Queue -#import pylibmc +import pylibmc import libmc if sys.version_info.major == 2: @@ -40,9 +40,14 @@ Participant = namedtuple('Participant', 'name factory threads', defaults=(1,)) BENCH_TIME = 1.0 N_SERVERS = 20 -NTHREADS = 40 +NTHREADS = 4 POOL_SIZE = 4 +# setting (eg) NTHREADS to 40 and POOL_SIZE to 4 illustrates a failure case of a +# simpler python solution to thread pools for clients +#NTHREADS = 40 +#POOL_SIZE = 4 + class Prefix(object): '''add prefix for key in mc command''' @@ -389,34 +394,29 @@ def reserve(self): ), Participant(name='python-memcached', factory=lambda: Prefix(__import__('memcache').Client(servers), 'memcache1')), Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking, from douban)', - name='libmc (from douban)', + name='libmc(md5 / ketama / nodelay / nonblocking, from douban)', factory=lambda: Prefix(__import__('libmc').Client(**libmc_kwargs), 'libmc1') ), Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / C++ thread pool, from douban)', - name='libmc (C++ thread pool)', + name='libmc(md5 / ketama / nodelay / nonblocking / C++ thread pool, from douban)', factory=lambda: Prefix(BenchmarkThreadedClient(**libmc_kwargs), 'libmc2'), threads=NTHREADS ), - Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py thread mapped, from douban)', - name='libmc (py thread mapped)', - factory=lambda: Prefix(ThreadMappedPool(**libmc_kwargs), 'libmc3'), - threads=NTHREADS - ), - Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py thread pool, from douban)', - name='libmc (py thread pool)', - factory=lambda: Prefix(ThreadPool(**libmc_kwargs), 'libmc4'), - threads=NTHREADS - ), - Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py FIFO thread pool, from douban)', - name='libmc (py FIFO thread pool)', - factory=lambda: Prefix(FIFOThreadPool(**libmc_kwargs), 'libmc5'), - threads=NTHREADS - ), + # Participant( + # name='libmc(md5 / ketama / nodelay / nonblocking / py thread mapped, from douban)', + # factory=lambda: Prefix(ThreadMappedPool(**libmc_kwargs), 'libmc3'), + # threads=NTHREADS + # ), + # Participant( + # name='libmc(md5 / ketama / nodelay / nonblocking / py thread pool, from douban)', + # factory=lambda: Prefix(ThreadPool(**libmc_kwargs), 'libmc4'), + # threads=NTHREADS + # ), + # Participant( + # name='libmc(md5 / ketama / nodelay / nonblocking / py ordered thread pool, from douban)', + # factory=lambda: Prefix(FIFOThreadPool(**libmc_kwargs), 'libmc5'), + # threads=NTHREADS + # ), ] def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIME): @@ -479,12 +479,12 @@ def loop(sw): def main(args=sys.argv[1:]): - # logger.info('pylibmc: %s', pylibmc.__file__) + logger.info('pylibmc: %s', pylibmc.__file__) logger.info('libmc: %s', libmc.__file__) logger.info('Running %s servers, %s threads, and a %s client pool', N_SERVERS, NTHREADS, POOL_SIZE) - ps = [p for p in participants if p.name.startswith("libmc")] + ps = [p for p in participants if any(p.name.startswith(arg) for arg in args)] ps = ps if ps else participants bs = benchmarks[:] diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 667a57b0..81ebf40f 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -110,7 +110,14 @@ def setUp(self): def test_many_threads(self): self.client_threads(self.misc) -class ThreadedGreenletCompat(unittest.TestCase, ThreadedClientOps): +# class ThreadedGreenletCompat(unittest.TestCase, ThreadedClientOps): +class ThreadedGreenletCompat(ThreadedClientOps): + """ + At the moment, threaded client pools do not support gevent since greenlets' + virtual address space for the stack overlaps. Support may be added in the + future though, so this test is getting left in the codebase but inactive. + """ + def setUp(self): global gevent import gevent From 5cbc847e1c5cf51560905b481a3edc4d130e5037 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Mon, 5 Feb 2024 22:58:34 -0800 Subject: [PATCH 74/95] switch hash function back --- misc/runbench.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/misc/runbench.py b/misc/runbench.py index 553d1d0e..e25a24a2 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -145,8 +145,6 @@ def timing(self): class DelayedStopwatch(Stopwatch): - """ Rollie """ - def __init__(self, laps=None, bound=0): super().__init__() self.laps = laps or [] @@ -366,7 +364,7 @@ def reserve(self): servers = ['%s:%d' % (host, 21211 + i) for i in range(N_SERVERS)] libmc_clients = (libmc.Client, BenchmarkThreadedClient, ThreadMappedPool, ThreadPool) -libmc_kwargs = {"servers": servers, "comp_threshold": 4000, "hash_fn": libmc.MC_HASH_FNV1_32} +libmc_kwargs = {"servers": servers, "comp_threshold": 4000} participants = [ Participant( @@ -420,7 +418,7 @@ def reserve(self): ] def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIME): - """dyel""" + """Do you even lift?""" mcs = [p.factory() for p in participants] means = [[] for p in participants] From 39a87a5afeecfb263f451ecf7e9145443b2c84f6 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 09:43:19 -0800 Subject: [PATCH 75/95] fail fast --- tests/test_client_pool.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_client_pool.cpp b/tests/test_client_pool.cpp index d77b563c..092fe7e4 100644 --- a/tests/test_client_pool.cpp +++ b/tests/test_client_pool.cpp @@ -33,12 +33,23 @@ void inner_test_loop(ClientPool* pool) { c->set(&keys, data_lens, flags, exptime, NULL, 0, &values, data_lens, 1, &m_results, &nResults); c->destroyMessageResult(); c->get(&keys, data_lens, 1, &r_results, &nResults); + EXPECT_EQ(nResults, 1); ASSERT_N_STREQ(r_results[0]->data_block, values, data_size); c->destroyRetrievalResult(); pool->release(c); } } +bool check_availability(ClientPool* pool) { + auto c = pool->acquire(); + broadcast_result_t* results; + size_t nHosts; + int ret = c->version(&results, &nHosts); + c->destroyBroadcastResult(); + pool->release(c); + return ret == 0; +} + TEST(test_client_pool, simple_set_get) { uint32_t ports[n_servers]; const char* hosts[n_servers]; @@ -50,6 +61,7 @@ TEST(test_client_pool, simple_set_get) { ClientPool* pool = new ClientPool(); pool->config(CFG_HASH_FUNCTION, OPT_HASH_FNV1A_32); pool->init(hosts, ports, n_servers); + ASSERT_TRUE(check_availability(pool)); for (unsigned int j = 0; j < n_threads; j++) { inner_test_loop(pool); @@ -71,6 +83,7 @@ TEST(test_client_pool, threaded_set_get) { pool->config(CFG_HASH_FUNCTION, OPT_HASH_FNV1A_32); //pool->config(CFG_INITIAL_CLIENTS, 4); pool->init(hosts, ports, n_servers); + ASSERT_TRUE(check_availability(pool)); for (unsigned int i = 0; i < n_threads; i++) { threads[i] = std::thread([&pool] { inner_test_loop(pool); }); From 299508a6b601542010ef9a00f1325e0e4fab5bb7 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 10:44:23 -0800 Subject: [PATCH 76/95] cppcheck lint --- include/ClientPool.h | 1 - misc/.cppcheck-supp | 3 ++- src/ClientPool.cpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/ClientPool.h b/include/ClientPool.h index a171be5b..9db314a3 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -36,7 +36,6 @@ class irange { using difference_type = int; using iterator_category = std::random_access_iterator_tag; - irange() : i(0) {} irange(int i) : i(i) {} reference operator*() const { return i; } diff --git a/misc/.cppcheck-supp b/misc/.cppcheck-supp index 0973d40e..36e0a410 100644 --- a/misc/.cppcheck-supp +++ b/misc/.cppcheck-supp @@ -15,7 +15,8 @@ *:include/llvm/SmallVector.h:735 *:include/llvm/SmallVector.h:796 constParameter:include/BufferReader.h:99 -unusedFunction:src/Client.cpp:232 +unreadVariable:include/LockPool.h +unusedFunction:src/Client.cpp:239 unusedFunction:src/c_client.cpp:8 unusedFunction:src/c_client.cpp:13 unusedFunction:src/c_client.cpp:25 diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index 5373ac83..fa4ada8d 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -64,8 +64,8 @@ int ClientPool::updateServers(const char* const* hosts, const uint32_t* ports, std::atomic rv = 0; std::lock_guard updating(m_fifo_access); - std::for_each(irange(), irange(m_clients.size()), - //std::for_each(std::execution::par_unseq, irange(), irange(m_clients.size()), + std::for_each(irange(0), irange(m_clients.size()), + //std::for_each(std::execution::par_unseq, irange(0), irange(m_clients.size()), [this, &rv](int i) { std::lock_guard updating_worker(*m_thread_workers[i]); const int err = m_clients[i].c.updateServers( From 2722b56fdd599e670fc446d79615bfd6d21c98cd Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 10:59:30 -0800 Subject: [PATCH 77/95] build fixes --- .github/workflows/golang.yml | 1 + .github/workflows/manual.yml | 59 +----------------------------------- CMakeLists.txt | 2 +- 3 files changed, 3 insertions(+), 59 deletions(-) diff --git a/.github/workflows/golang.yml b/.github/workflows/golang.yml index 56e04387..18784de3 100644 --- a/.github/workflows/golang.yml +++ b/.github/workflows/golang.yml @@ -5,6 +5,7 @@ on: branches: [ master ] pull_request: branches: [ master ] + workflow_dispatch: jobs: diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 4bac6683..1d8cd18d 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - compiler: ["gcc"] + compiler: ["gcc", "clang"] build_type: ["Debug"] steps: @@ -59,60 +59,3 @@ jobs: ./misc/travis/unittest.sh - name: Stop memcached servers run: ./misc/memcached_server stopall - - goclang: - runs-on: ubuntu-latest - strategy: - matrix: - gover: ["1.15"] - compiler: ["clang"] - - steps: - - uses: actions/checkout@v2 - - name: Setup system dependencies - run: | - sudo apt-get update - sudo apt-get -y install memcached - - name: Set up Golang - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.gover }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Start memcached servers - run: ./misc/memcached_server start - - name: Run gotest - run: | - export CC=clang CXX=clang++ - clang++ -v - ./misc/travis/gotest.sh - - name: Stop memcached servers - run: ./misc/memcached_server stop - - reference: - runs-on: ubuntu-latest - strategy: - matrix: - pyver: ["3.12"] - - steps: - - uses: actions/checkout@v2 - - name: Setup system dependencies - run: | - sudo apt-get update - sudo apt-get -y install memcached libmemcached-dev g++ - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.pyver }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools future python-memcached pylibmc - - name: Start memcached servers - run: ./misc/memcached_server start - - name: Run benchmark - run: | - ./misc/travis/benchmark.sh - - name: Stop memcached servers - run: ./misc/memcached_server stop diff --git a/CMakeLists.txt b/CMakeLists.txt index 04a4799a..c0d948f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: Debug Release." FORCE) endif (NOT CMAKE_BUILD_TYPE) -set(CMAKE_CXX_FLAGS_COMMON "-Wall -fno-rtti -fno-exceptions") +set(CMAKE_CXX_FLAGS_COMMON "-Wall -fno-rtti -fno-exceptions -std=c++17") set(CMAKE_CXX_FLAGS_DEBUG "-DDEBUG -g2 ${CMAKE_CXX_FLAGS_COMMON}" CACHE STRING "CXX DEBUG FLAGS" FORCE) set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3 ${CMAKE_CXX_FLAGS_COMMON}" CACHE STRING "CXX RELEASE FLAGS" FORCE) set(CMAKE_INSTALL_INCLUDE include CACHE PATH "Output directory for header files") From af0240fcc2d280bdf0e718c59ce1044a1b6b94f6 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 11:05:07 -0800 Subject: [PATCH 78/95] remove dead code --- misc/memcached_server | 5 ----- 1 file changed, 5 deletions(-) diff --git a/misc/memcached_server b/misc/memcached_server index 8fc318b2..d71f7706 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -116,15 +116,10 @@ case "$1" in ;; stopall) if [ `ls $basedir/var/run/ | grep -c .pid` -ge 1 ]; then - # names="`basename $basedir/var/run/*.pid | cut -d. -f1`" - # for name in $names; do - # stop $name & - # done for f in $basedir/var/run/*.pid; do cat "$f" | xargs kill done fi - # wait rm -rf $basedir ;; capture) From d80eff1af25a1ecf03b26fba75ffd7ad1eccf8f4 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 11:19:31 -0800 Subject: [PATCH 79/95] add c++ std flags --- setup.py | 1 + tests/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index afc15c1d..37dba363 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ "-DMC_USE_SMALL_VECTOR", "-O3", "-DNDEBUG", + "-std=c++17", ] diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9af3d51e..9f301081 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,7 +2,7 @@ file(GLOB TEST_SRC_FILES ${PROJECT_SOURCE_DIR}/tests/test_*.cpp) foreach(SRC ${TEST_SRC_FILES}) get_filename_component(test_name ${SRC} NAME_WE) - set_source_files_properties(${SRC} PROPERTIES COMPILE_FLAGS "-fexceptions") + set_source_files_properties(${SRC} PROPERTIES COMPILE_FLAGS "-fexceptions -std=c++17") add_executable(${test_name} ${SRC}) add_dependencies(${test_name} gtest) target_link_libraries(${test_name} From b1263d15da639ff7dd8477990618f8e2397e94e9 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 14:58:53 -0800 Subject: [PATCH 80/95] update cgo CXXFLAGS --- src/golibmc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/golibmc.go b/src/golibmc.go index 924e98ab..45eb3fe2 100644 --- a/src/golibmc.go +++ b/src/golibmc.go @@ -2,7 +2,7 @@ package golibmc /* #cgo CFLAGS: -I ./../include -#cgo CXXFLAGS: -I ./../include +#cgo CXXFLAGS: -std=c++17 -I ./../include #include "c_client.h" */ import "C" From 36f1644d95e69ed7116d2eb8dcd991f85a022fc9 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 15:21:53 -0800 Subject: [PATCH 81/95] fix double acquire attempt for m_pool_lock --- misc/aliases | 4 +++- src/ClientPool.cpp | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/misc/aliases b/misc/aliases index 8887375f..c0161de7 100755 --- a/misc/aliases +++ b/misc/aliases @@ -49,7 +49,9 @@ case "$1" in source misc/git/pre-commit virtualize_libmc "`which python-dbg || which python3-dbg`" python setup.py build_ext --inplace - if [ -n "$2" ]; then + if [ "$2" == "bench" ]; then + cygdb . -- -ex start --args python misc/runbench.py + elif [ -n "$2" ]; then pattern="$2" shift 2 cmds=(-ex start) diff --git a/src/ClientPool.cpp b/src/ClientPool.cpp index fa4ada8d..a7ff44c0 100644 --- a/src/ClientPool.cpp +++ b/src/ClientPool.cpp @@ -50,6 +50,7 @@ int ClientPool::init(const char* const * hosts, const uint32_t* ports, const size_t n, const char* const * aliases) { updateServers(hosts, ports, n, aliases); std::unique_lock initializing(m_acquiring_growth); + std::lock_guard config_pool(m_pool_lock); return growPool(m_initial_clients); } @@ -86,10 +87,9 @@ int ClientPool::setup(Client* c) { return c->init(m_hosts.data(), m_ports.data(), m_hosts.size(), m_aliases.data()); } -// if called outside acquire, needs to own m_acquiring_growth +// needs to hold both m_acquiring_growth and m_pool_lock int ClientPool::growPool(size_t by) { assert(by > 0); - std::lock_guard growing_pool(m_pool_lock); size_t from = m_clients.size(); m_clients.resize(from + by); std::atomic rv = 0; @@ -115,6 +115,7 @@ inline bool ClientPool::shouldGrowUnsafe() { int ClientPool::autoGrow() { std::unique_lock growing(m_acquiring_growth); if (shouldGrowUnsafe()) { + std::lock_guard growing_pool(m_pool_lock); return growPool(MIN(m_max_clients - m_clients.size(), MIN(m_max_growth, m_clients.size()))); } From ccd20a5743b635e512a0cc2a813f022f903a6efe Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 17:52:14 -0800 Subject: [PATCH 82/95] explicit irange constructor --- include/ClientPool.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/ClientPool.h b/include/ClientPool.h index 9db314a3..eb3fdea6 100644 --- a/include/ClientPool.h +++ b/include/ClientPool.h @@ -36,7 +36,7 @@ class irange { using difference_type = int; using iterator_category = std::random_access_iterator_tag; - irange(int i) : i(i) {} + explicit irange(int i) : i(i) {} reference operator*() const { return i; } pointer operator->() const { return &i; } @@ -55,7 +55,7 @@ class irange { irange& operator-=(difference_type n) { i -= n; return *this; } friend irange operator+(const irange& lhs, difference_type n) { irange tmp = lhs; tmp += n; return tmp; } friend irange operator+(difference_type n, const irange& rhs) { return rhs + n; } - friend irange operator-(const irange& lhs, difference_type n) { return lhs.i + (-n); } + friend irange operator-(const irange& lhs, difference_type n) { irange tmp = lhs; tmp -= n; return tmp; } friend difference_type operator-(const irange& lhs, const irange& rhs) { return lhs.i - rhs.i; } }; From 7e7dfaef5dfd121ac7a9c44477863f095ef68faa Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 19:20:28 -0800 Subject: [PATCH 83/95] README update --- README.rst | 86 ++++++++++++++++++++++++----------------------- libmc/__init__.py | 3 +- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/README.rst b/README.rst index 9c829098..ef15e818 100644 --- a/README.rst +++ b/README.rst @@ -5,15 +5,15 @@ libmc |status| |pypiv| |pyversions| |wheel| |license| libmc is a memcached client library for Python without any other -dependencies in runtime. It's mainly written in C++ and Cython. libmc -can be considered as a drop in replacement for libmemcached and +dependencies at runtime. It's mainly written in C++ and Cython and +can be considered a drop in replacement for libmemcached and `python-libmemcached `__. -libmc is developing and maintaining by Douban Inc. Currently, It is -working in production environment, powering all web traffics in -douban.com. Realtime `benchmark -result `__ is -available on travis. +libmc is developed and maintained by Douban Inc. Currently, it is +working in a production environment, powering all web traffic on +`douban.com `. Realtime +`benchmark results `__ +are available on travis. Build and Installation ---------------------- @@ -37,14 +37,14 @@ Usage: Under the hood -------------- -Under the hood, libmc consists of 2 parts: an internal fully-functional +Under the hood, libmc consists of 2 parts: an internal, fully-functional memcached client implementation in C++ and a Cython wrapper around that implementation. Dynamic memory allocation and memory-copy are slow, so -we tried our best to avoid them. The ``set_multi`` command is not -natively supported by the `memcached +we've tried our best to avoid them. libmc also supports the ``set_multi`` +command, which is not natively supported by the `memcached protocol `__. -Some techniques are applied to make ``set_multi`` command extremely fast -in libmc (compared to some other similiar libraries). +Some techniques have been applied to make ``set_multi`` command extremely fast +in libmc (compared to similiar libraries). Configuration ------------- @@ -77,23 +77,22 @@ Configuration mc.config(MC_RETRY_TIMEOUT, 5) # 5 s -- ``servers``: is a list of memcached server addresses. Each address - can be in format of ``hostname[:port] [alias]``. ``port`` and ``alias`` - are optional. If ``port`` is not given, default port ``11211`` will - be used. ``alias`` will be used to compute server hash if given, - otherwise server hash will be computed based on ``host`` and ``port`` - (i.e.: If ``port`` is not given or it is equal to ``11211``, ``host`` - will be used to compute server hash. If ``port`` is not equal to ``11211``, - ``host:port`` will be used). -- ``do_split``: Memcached server will refuse to store value if size >= - 1MB, if ``do_split`` is enabled, large value (< 10 MB) will be - splitted into several blocks. If the value is too large (>= 10 MB), - it will not be stored. default: ``True`` -- ``comp_threshold``: All kinds of values will be encoded into string - buffer. If ``buffer length > comp_threshold > 0``, it will be - compressed using zlib. If ``comp_threshold = 0``, string buffer will - never be compressed using zlib. default: ``0`` -- ``noreply``: Whether to enable memcached's ``noreply`` behaviour. +- ``servers``: a list of memcached server addresses. Each address + should be formated as ``hostname[:port] [alias]``, where ``port`` and + ``alias`` are optional. If ``port`` is not given, the default port ``11211`` + will be used. If given, ``alias`` will be used to compute the server hash, + which would otherwise be computed based on ``host`` and ``port`` + (i.e. whichever portion is given). +- ``do_split``: splits large values (up to 10MB) into chunks (<1MB). The + memcached server implementation will not store items larger than 1MB, + however in some environments it is beneficial to shard up to 10MB of data. + Attempts to store more than that are ignored. Default: ``True``. +- ``comp_threshold``: compresses large values using zlib. If + ``buffer length > comp_threshold > 0`` (in bytes), the buffer will be + compressed. If ``comp_threshold == 0``, the string buffer will never be + compressed. Default: ``0`` +- ``noreply``: controls memcached's + [``noreply`` feature](https://github.com/memcached/memcached/wiki/CommonFeatures#noreplyquiet). default: ``False`` - ``prefix``: The key prefix. default: ``''`` - ``hash_fn``: hashing function for keys. possible values: @@ -110,16 +109,16 @@ Configuration implementions in libmemcached. - ``failover``: Whether to failover to next server when current server - is not available. default: ``False`` + is not available. Default: ``False`` - ``MC_POLL_TIMEOUT`` Timeout parameter used during set/get procedure. - (default: ``300`` ms) + Default: ``300`` ms - ``MC_CONNECT_TIMEOUT`` Timeout parameter used when connecting to - memcached server on initial phase. (default: ``100`` ms) -- ``MC_RETRY_TIMEOUT`` When a server is not available dur to server-end - error. libmc will try to establish the broken connection in every - ``MC_RETRY_TIMEOUT`` s until the connection is back to live.(default: - ``5`` s) + memcached server in the initial phase. Default: ``100`` ms +- ``MC_RETRY_TIMEOUT`` When a server is not available due to server-end + error, libmc will try to establish the broken connection in every + ``MC_RETRY_TIMEOUT`` s until the connection is back to live. Default: + ``5`` s **NOTE:** The hashing algorithm for host mapping on continuum is always md5. @@ -140,7 +139,7 @@ FAQ Does libmc support PHP? ^^^^^^^^^^^^^^^^^^^^^^^ -No. But if you like, you can write a wrapper for PHP based on the C++ +No, but, if you like, you can write a wrapper for PHP based on the C++ implementation. Is Memcached binary protocol supported ? @@ -151,7 +150,7 @@ No. Only Memcached ASCII protocol is supported currently. Why reinventing the wheel? ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Before libmc, we're using +Before libmc, we were using `python-libmemcached `__, which is a python extention for `libmemcached `__. @@ -161,16 +160,19 @@ still some unsolved bugs. Is libmc thread-safe ? ^^^^^^^^^^^^^^^^^^^^^^ -libmc is a single-threaded memcached client. If you initialize a libmc -client in one thread but reuse that in another thread, a Python -Exception ``ThreadUnsafe`` will raise in Python. +Yes. `libmc.ThreadedClient` is a thread-safe client implementation. To hold +access for more than one request, `libmc.ClientPool` can be used with Python +`with` statements. `libmc.Client`, however, is a single-threaded memcached +client. If you initialize a standard client in one thread but reuse that in +another thread, a Python ``ThreadUnsafe`` Exception will be raised. Is libmc compatible with gevent? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Yes, with the help of `greenify `__, libmc is friendly to gevent. Read ``tests/shabby/gevent_issue.py`` for -details. +details. `libmc.ThreadedClient` and `libmc.ClientPool` are not currently +compatible. **Notice:** diff --git a/libmc/__init__.py b/libmc/__init__.py index 9d9e5546..880f864d 100644 --- a/libmc/__init__.py +++ b/libmc/__init__.py @@ -1,4 +1,5 @@ -import os, functools +import os +import functools from ._client import ( PyClient, PyClientUnsafe as ClientUnsafe, PyClientPool, ThreadUnsafe, encode_value, From 1c8c6bce08c617d114c4986de71f38bd03ad0c84 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 19:24:23 -0800 Subject: [PATCH 84/95] link format --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ef15e818..4c7ea62e 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,7 @@ can be considered a drop in replacement for libmemcached and libmc is developed and maintained by Douban Inc. Currently, it is working in a production environment, powering all web traffic on -`douban.com `. Realtime +`douban.com `__. Realtime `benchmark results `__ are available on travis. @@ -92,7 +92,7 @@ Configuration compressed. If ``comp_threshold == 0``, the string buffer will never be compressed. Default: ``0`` - ``noreply``: controls memcached's - [``noreply`` feature](https://github.com/memcached/memcached/wiki/CommonFeatures#noreplyquiet). + ``noreply`` `feature `__. default: ``False`` - ``prefix``: The key prefix. default: ``''`` - ``hash_fn``: hashing function for keys. possible values: From 399e4e60c65bfdebd0a68a10ad44710afe56def2 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 19:27:32 -0800 Subject: [PATCH 85/95] typo --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 4c7ea62e..e18afc65 100644 --- a/README.rst +++ b/README.rst @@ -83,7 +83,7 @@ Configuration will be used. If given, ``alias`` will be used to compute the server hash, which would otherwise be computed based on ``host`` and ``port`` (i.e. whichever portion is given). -- ``do_split``: splits large values (up to 10MB) into chunks (<1MB). The +- ``do_split``: splits large values (up to 10MB) into chunks (<1MB). The memcached server implementation will not store items larger than 1MB, however in some environments it is beneficial to shard up to 10MB of data. Attempts to store more than that are ignored. Default: ``True``. @@ -93,7 +93,7 @@ Configuration compressed. Default: ``0`` - ``noreply``: controls memcached's ``noreply`` `feature `__. - default: ``False`` + Default: ``False`` - ``prefix``: The key prefix. default: ``''`` - ``hash_fn``: hashing function for keys. possible values: From 728ddcabd804705da32ed0b1480d0498daadec4d Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 19:32:11 -0800 Subject: [PATCH 86/95] md -> rst --- README.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index e18afc65..f3412f7e 100644 --- a/README.rst +++ b/README.rst @@ -160,9 +160,9 @@ still some unsolved bugs. Is libmc thread-safe ? ^^^^^^^^^^^^^^^^^^^^^^ -Yes. `libmc.ThreadedClient` is a thread-safe client implementation. To hold -access for more than one request, `libmc.ClientPool` can be used with Python -`with` statements. `libmc.Client`, however, is a single-threaded memcached +Yes. ``libmc.ThreadedClient`` is a thread-safe client implementation. To hold +access for more than one request, ``libmc.ClientPool`` can be used with Python +``with`` statements. ``libmc.Client``, however, is a single-threaded memcached client. If you initialize a standard client in one thread but reuse that in another thread, a Python ``ThreadUnsafe`` Exception will be raised. @@ -171,13 +171,13 @@ Is libmc compatible with gevent? Yes, with the help of `greenify `__, libmc is friendly to gevent. Read ``tests/shabby/gevent_issue.py`` for -details. `libmc.ThreadedClient` and `libmc.ClientPool` are not currently +details. ``libmc.ThreadedClient`` and ``libmc.ClientPool`` are not currently compatible. **Notice:** -`gevent.monkey.patch_all()` will override -`threading.current_thread().ident` to Greenlet's ID, +``gevent.monkey.patch_all()`` will override +``threading.current_thread().ident`` to Greenlet's ID, this will cause libmc to throw a ThreadUnSafe error or run into dead lock, you should only patch the things that you need, e.g. From d561b8459e0ae8864abc9c06e38c2f2227cd0ac9 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 19:48:56 -0800 Subject: [PATCH 87/95] remove dead link --- README.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.rst b/README.rst index f3412f7e..c32307e9 100644 --- a/README.rst +++ b/README.rst @@ -11,9 +11,7 @@ can be considered a drop in replacement for libmemcached and libmc is developed and maintained by Douban Inc. Currently, it is working in a production environment, powering all web traffic on -`douban.com `__. Realtime -`benchmark results `__ -are available on travis. +`douban.com `__. Build and Installation ---------------------- From 1f8dd888544d1feb0b68ed8ce2fa28dad16b5cb1 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 20:05:46 -0800 Subject: [PATCH 88/95] clarifying statistics --- README.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index c32307e9..5dc22fae 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,7 @@ can be considered a drop in replacement for libmemcached and libmc is developed and maintained by Douban Inc. Currently, it is working in a production environment, powering all web traffic on -`douban.com `__. +`douban.com `__. [#]_ Build and Installation ---------------------- @@ -231,6 +231,12 @@ Documentation https://github.com/douban/libmc/wiki +Footnotes +------- + +.. [#] not an endorsement of the accuracy, just a publicly available reference + point + LICENSE ------- From 87c4e64456a5138803ecbf7c2466b0ac4cf1a50d Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 22:03:17 -0800 Subject: [PATCH 89/95] better footnote about traffic stats --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 5dc22fae..5f7f4de9 100644 --- a/README.rst +++ b/README.rst @@ -234,8 +234,8 @@ https://github.com/douban/libmc/wiki Footnotes ------- -.. [#] not an endorsement of the accuracy, just a publicly available reference - point +.. [#] The link not an endorsement of the site's accuracy, just a publicly + available reference point LICENSE ------- From 78a54317cf1845ae0685058227d9a45af8106e3f Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 6 Feb 2024 22:05:54 -0800 Subject: [PATCH 90/95] wording --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 5f7f4de9..baaba4a7 100644 --- a/README.rst +++ b/README.rst @@ -234,8 +234,8 @@ https://github.com/douban/libmc/wiki Footnotes ------- -.. [#] The link not an endorsement of the site's accuracy, just a publicly - available reference point +.. [#] The link not an endorsement of the traffic estimate site's accuracy, just + a publicly available reference point LICENSE ------- From f8ecc8e19420eb1f4e49c50fc747a13b53355493 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 31 May 2024 13:46:10 -0700 Subject: [PATCH 91/95] gevent support explanation --- README.rst | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index baaba4a7..202c4708 100644 --- a/README.rst +++ b/README.rst @@ -13,6 +13,9 @@ libmc is developed and maintained by Douban Inc. Currently, it is working in a production environment, powering all web traffic on `douban.com `__. [#]_ +.. [#] The link not an endorsement of the traffic estimate site's accuracy, just + a publicly available reference point + Build and Installation ---------------------- @@ -169,8 +172,16 @@ Is libmc compatible with gevent? Yes, with the help of `greenify `__, libmc is friendly to gevent. Read ``tests/shabby/gevent_issue.py`` for -details. ``libmc.ThreadedClient`` and ``libmc.ClientPool`` are not currently -compatible. +details. ``libmc.ThreadedClient`` and ``libmc.ClientPool`` are not compatible. +[#]_ + +.. [#] In order to use a single executable for multiple greenlet contexts, + gevent has to `copy thread memory + `__ + to and from the same stack space. This doesn't affect Python references, + which are handed off through gevent, but makes it impossible for shared + libraries to pass memory addresses across greenlets, which is required for + the worker pool. **Notice:** @@ -231,12 +242,6 @@ Documentation https://github.com/douban/libmc/wiki -Footnotes -------- - -.. [#] The link not an endorsement of the traffic estimate site's accuracy, just - a publicly available reference point - LICENSE ------- @@ -258,3 +263,4 @@ https://github.com/douban/libmc/blob/master/LICENSE.txt .. |pyversions| image:: https://img.shields.io/pypi/pyversions/libmc .. |wheel| image:: https://img.shields.io/pypi/wheel/libmc .. |license| image:: https://img.shields.io/pypi/l/libmc?color=blue + From 51e627f2c7da95446786c1bbbd7c8abf89f90f93 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 31 May 2024 14:15:30 -0700 Subject: [PATCH 92/95] version bump --- CMakeLists.txt | 2 +- libmc/__init__.py | 6 +++--- src/version.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c0d948f3..c27ecce0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ set(CMAKE_MACOSX_RPATH 1) set (MC_VERSION_MAJOR 1) set (MC_VERSION_MINOR 4) -set (MC_VERSION_PATCH 1) +set (MC_VERSION_PATCH 4) set (MC_VERSION ${MC_VERSION_MAJOR}.${MC_VERSION_MINOR}) set (MC_APIVERSION ${MC_VERSION}.${MC_VERSION_PATCH}) diff --git a/libmc/__init__.py b/libmc/__init__.py index 880f864d..0ee2b9f6 100644 --- a/libmc/__init__.py +++ b/libmc/__init__.py @@ -32,11 +32,11 @@ __file__ as _libmc_so_file ) -__VERSION__ = "1.4.3" -__version__ = "v1.4.3" +__VERSION__ = "1.4.4" +__version__ = "v1.4.4" __author__ = "mckelvin" __email__ = "mckelvin@users.noreply.github.com" -__date__ = "Fri Dec 1 07:43:12 2023 +0800" +__date__ = "Sat Jun 1 05:10:05 2024 +0800" class Client(PyClient): diff --git a/src/version.go b/src/version.go index 38f0f52d..978dc32d 100644 --- a/src/version.go +++ b/src/version.go @@ -1,9 +1,9 @@ package golibmc -const _Version = "v1.4.3" +const _Version = "v1.4.4" const _Author = "mckelvin" const _Email = "mckelvin@users.noreply.github.com" -const _Date = "Fri Dec 1 07:43:12 2023 +0800" +const _Date = "Sat Jun 1 05:10:05 2024 +0800" // Version of the package const Version = _Version From 1327704e7c589c6e8d01ff648330f0beb4c5e542 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Fri, 31 May 2024 14:18:41 -0700 Subject: [PATCH 93/95] remove gevent test for ThreadedClient --- tests/test_client_pool.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/tests/test_client_pool.py b/tests/test_client_pool.py index 81ebf40f..d1a0ce45 100644 --- a/tests/test_client_pool.py +++ b/tests/test_client_pool.py @@ -110,34 +110,3 @@ def setUp(self): def test_many_threads(self): self.client_threads(self.misc) -# class ThreadedGreenletCompat(unittest.TestCase, ThreadedClientOps): -class ThreadedGreenletCompat(ThreadedClientOps): - """ - At the moment, threaded client pools do not support gevent since greenlets' - virtual address space for the stack overlaps. Support may be added in the - future though, so this test is getting left in the codebase but inactive. - """ - - def setUp(self): - global gevent - import gevent - # import gevent.monkey - # gevent.monkey.patch_all() - - import greenify, libmc - greenify.greenify() - for so_path in libmc.DYNAMIC_LIBRARIES: - assert greenify.patch_lib(so_path) - - self.imp = libmc.ThreadedClient(["127.0.0.1:21211"]) - self.imp.config(MC_MAX_CLIENTS, 1) - - def client_threads(self, target): - ts = [gevent.spawn(target) for i in range(self.nthreads)] - gevent.joinall(ts, raise_error=True) - - def tid(self, mc): - return (os.getpid(), gevent.getcurrent().name) - - def test_many_eventlets(self): - self.client_threads(self.misc) From f0263ad15946c2166565da58dfc4a2b649a33d33 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 4 Jun 2024 10:21:25 -0700 Subject: [PATCH 94/95] Separation of concerns --- .github/workflows/golang.yml | 2 - .github/workflows/manual.yml | 61 ------------------ .github/workflows/python.yml | 2 - .gitignore | 2 - Makefile | 3 - README.rst | 102 +++++++++++++----------------- include/Common.h | 2 +- libmc/__init__.py | 7 ++- libmc/_client.pyx | 2 +- misc/aliases | 98 ----------------------------- misc/git/debug.patch | 101 ----------------------------- misc/git/pre-commit | 7 --- misc/memcached_server | 25 ++------ misc/runbench.py | 119 ++++++++++------------------------- setup.py | 2 +- src/Common.cpp | 4 +- src/Connection.cpp | 2 +- 17 files changed, 94 insertions(+), 447 deletions(-) delete mode 100644 .github/workflows/manual.yml delete mode 100755 misc/aliases delete mode 100644 misc/git/debug.patch delete mode 100755 misc/git/pre-commit diff --git a/.github/workflows/golang.yml b/.github/workflows/golang.yml index 18784de3..c542e54e 100644 --- a/.github/workflows/golang.yml +++ b/.github/workflows/golang.yml @@ -5,7 +5,6 @@ on: branches: [ master ] pull_request: branches: [ master ] - workflow_dispatch: jobs: @@ -26,7 +25,6 @@ jobs: uses: actions/setup-go@v2 with: go-version: ${{ matrix.gover }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Start memcached servers run: ./misc/memcached_server start - name: Run gotest diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml deleted file mode 100644 index 1d8cd18d..00000000 --- a/.github/workflows/manual.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: manual - -on: - workflow_dispatch: - -jobs: - cpptest: - runs-on: ubuntu-latest - strategy: - matrix: - compiler: ["gcc", "clang"] - build_type: ["Debug"] - - steps: - - uses: actions/checkout@v2 - - name: Setup system dependencies - run: | - sudo apt-get update - sudo apt-get -y install memcached g++ - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools future pytest greenify gevent numpy - - name: Run cpptest - run: | - if [[ ${{ matrix.compiler }} = "gcc" ]]; then export CC=gcc CXX=g++; fi - if [[ ${{ matrix.compiler }} = "clang" ]]; then export CC=clang CXX=clang++; fi - ./misc/travis/cpptest.sh - - pylatest: - runs-on: ubuntu-latest - strategy: - matrix: - pyver: ["3.12"] - compiler: ["gcc"] - build_type: ["Debug"] - - steps: - - uses: actions/checkout@v2 - - name: Setup system dependencies - run: | - sudo apt-get update - sudo apt-get -y install valgrind memcached g++ - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.pyver }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools future pytest greenify gevent numpy - - name: Start memcached servers - run: ./misc/memcached_server startall - - name: Run unittest - run: | - if [[ ${{ matrix.compiler }} = "gcc" ]]; then export CC=gcc CXX=g++; fi - if [[ ${{ matrix.compiler }} = "clang" ]]; then export CC=clang CXX=clang++; fi - ./misc/travis/unittest.sh - - name: Stop memcached servers - run: ./misc/memcached_server stopall diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index be6837ce..f364f304 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -25,7 +25,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.pyver }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install python dependencies run: | python -m pip install --upgrade pip @@ -56,7 +55,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.pyver }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install python dependencies run: | python -m pip install --upgrade pip diff --git a/.gitignore b/.gitignore index 0949ba73..0e754f62 100644 --- a/.gitignore +++ b/.gitignore @@ -25,5 +25,3 @@ tests/resources/keys_*.txt /go.mod /go.sum - -/cython_debug diff --git a/Makefile b/Makefile index 0210df15..2dff81f3 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,3 @@ gotest: pytest: pytest tests/ - -clean: - rm -fr build cython_debug libmc/*.so libmc/*.cpp .eggs libmc.egg-info diff --git a/README.rst b/README.rst index 202c4708..9c829098 100644 --- a/README.rst +++ b/README.rst @@ -5,16 +5,15 @@ libmc |status| |pypiv| |pyversions| |wheel| |license| libmc is a memcached client library for Python without any other -dependencies at runtime. It's mainly written in C++ and Cython and -can be considered a drop in replacement for libmemcached and +dependencies in runtime. It's mainly written in C++ and Cython. libmc +can be considered as a drop in replacement for libmemcached and `python-libmemcached `__. -libmc is developed and maintained by Douban Inc. Currently, it is -working in a production environment, powering all web traffic on -`douban.com `__. [#]_ - -.. [#] The link not an endorsement of the traffic estimate site's accuracy, just - a publicly available reference point +libmc is developing and maintaining by Douban Inc. Currently, It is +working in production environment, powering all web traffics in +douban.com. Realtime `benchmark +result `__ is +available on travis. Build and Installation ---------------------- @@ -38,14 +37,14 @@ Usage: Under the hood -------------- -Under the hood, libmc consists of 2 parts: an internal, fully-functional +Under the hood, libmc consists of 2 parts: an internal fully-functional memcached client implementation in C++ and a Cython wrapper around that implementation. Dynamic memory allocation and memory-copy are slow, so -we've tried our best to avoid them. libmc also supports the ``set_multi`` -command, which is not natively supported by the `memcached +we tried our best to avoid them. The ``set_multi`` command is not +natively supported by the `memcached protocol `__. -Some techniques have been applied to make ``set_multi`` command extremely fast -in libmc (compared to similiar libraries). +Some techniques are applied to make ``set_multi`` command extremely fast +in libmc (compared to some other similiar libraries). Configuration ------------- @@ -78,23 +77,24 @@ Configuration mc.config(MC_RETRY_TIMEOUT, 5) # 5 s -- ``servers``: a list of memcached server addresses. Each address - should be formated as ``hostname[:port] [alias]``, where ``port`` and - ``alias`` are optional. If ``port`` is not given, the default port ``11211`` - will be used. If given, ``alias`` will be used to compute the server hash, - which would otherwise be computed based on ``host`` and ``port`` - (i.e. whichever portion is given). -- ``do_split``: splits large values (up to 10MB) into chunks (<1MB). The - memcached server implementation will not store items larger than 1MB, - however in some environments it is beneficial to shard up to 10MB of data. - Attempts to store more than that are ignored. Default: ``True``. -- ``comp_threshold``: compresses large values using zlib. If - ``buffer length > comp_threshold > 0`` (in bytes), the buffer will be - compressed. If ``comp_threshold == 0``, the string buffer will never be - compressed. Default: ``0`` -- ``noreply``: controls memcached's - ``noreply`` `feature `__. - Default: ``False`` +- ``servers``: is a list of memcached server addresses. Each address + can be in format of ``hostname[:port] [alias]``. ``port`` and ``alias`` + are optional. If ``port`` is not given, default port ``11211`` will + be used. ``alias`` will be used to compute server hash if given, + otherwise server hash will be computed based on ``host`` and ``port`` + (i.e.: If ``port`` is not given or it is equal to ``11211``, ``host`` + will be used to compute server hash. If ``port`` is not equal to ``11211``, + ``host:port`` will be used). +- ``do_split``: Memcached server will refuse to store value if size >= + 1MB, if ``do_split`` is enabled, large value (< 10 MB) will be + splitted into several blocks. If the value is too large (>= 10 MB), + it will not be stored. default: ``True`` +- ``comp_threshold``: All kinds of values will be encoded into string + buffer. If ``buffer length > comp_threshold > 0``, it will be + compressed using zlib. If ``comp_threshold = 0``, string buffer will + never be compressed using zlib. default: ``0`` +- ``noreply``: Whether to enable memcached's ``noreply`` behaviour. + default: ``False`` - ``prefix``: The key prefix. default: ``''`` - ``hash_fn``: hashing function for keys. possible values: @@ -110,16 +110,16 @@ Configuration implementions in libmemcached. - ``failover``: Whether to failover to next server when current server - is not available. Default: ``False`` + is not available. default: ``False`` - ``MC_POLL_TIMEOUT`` Timeout parameter used during set/get procedure. - Default: ``300`` ms + (default: ``300`` ms) - ``MC_CONNECT_TIMEOUT`` Timeout parameter used when connecting to - memcached server in the initial phase. Default: ``100`` ms -- ``MC_RETRY_TIMEOUT`` When a server is not available due to server-end - error, libmc will try to establish the broken connection in every - ``MC_RETRY_TIMEOUT`` s until the connection is back to live. Default: - ``5`` s + memcached server on initial phase. (default: ``100`` ms) +- ``MC_RETRY_TIMEOUT`` When a server is not available dur to server-end + error. libmc will try to establish the broken connection in every + ``MC_RETRY_TIMEOUT`` s until the connection is back to live.(default: + ``5`` s) **NOTE:** The hashing algorithm for host mapping on continuum is always md5. @@ -140,7 +140,7 @@ FAQ Does libmc support PHP? ^^^^^^^^^^^^^^^^^^^^^^^ -No, but, if you like, you can write a wrapper for PHP based on the C++ +No. But if you like, you can write a wrapper for PHP based on the C++ implementation. Is Memcached binary protocol supported ? @@ -151,7 +151,7 @@ No. Only Memcached ASCII protocol is supported currently. Why reinventing the wheel? ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Before libmc, we were using +Before libmc, we're using `python-libmemcached `__, which is a python extention for `libmemcached `__. @@ -161,32 +161,21 @@ still some unsolved bugs. Is libmc thread-safe ? ^^^^^^^^^^^^^^^^^^^^^^ -Yes. ``libmc.ThreadedClient`` is a thread-safe client implementation. To hold -access for more than one request, ``libmc.ClientPool`` can be used with Python -``with`` statements. ``libmc.Client``, however, is a single-threaded memcached -client. If you initialize a standard client in one thread but reuse that in -another thread, a Python ``ThreadUnsafe`` Exception will be raised. +libmc is a single-threaded memcached client. If you initialize a libmc +client in one thread but reuse that in another thread, a Python +Exception ``ThreadUnsafe`` will raise in Python. Is libmc compatible with gevent? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Yes, with the help of `greenify `__, libmc is friendly to gevent. Read ``tests/shabby/gevent_issue.py`` for -details. ``libmc.ThreadedClient`` and ``libmc.ClientPool`` are not compatible. -[#]_ - -.. [#] In order to use a single executable for multiple greenlet contexts, - gevent has to `copy thread memory - `__ - to and from the same stack space. This doesn't affect Python references, - which are handed off through gevent, but makes it impossible for shared - libraries to pass memory addresses across greenlets, which is required for - the worker pool. +details. **Notice:** -``gevent.monkey.patch_all()`` will override -``threading.current_thread().ident`` to Greenlet's ID, +`gevent.monkey.patch_all()` will override +`threading.current_thread().ident` to Greenlet's ID, this will cause libmc to throw a ThreadUnSafe error or run into dead lock, you should only patch the things that you need, e.g. @@ -263,4 +252,3 @@ https://github.com/douban/libmc/blob/master/LICENSE.txt .. |pyversions| image:: https://img.shields.io/pypi/pyversions/libmc .. |wheel| image:: https://img.shields.io/pypi/wheel/libmc .. |license| image:: https://img.shields.io/pypi/l/libmc?color=blue - diff --git a/include/Common.h b/include/Common.h index 9295465a..f7a22de8 100644 --- a/include/Common.h +++ b/include/Common.h @@ -226,7 +226,7 @@ typedef enum { } op_code_t; const char* errCodeToString(err_code_t err); -bool isUnixSocket(const char* host); +bool isLocalSocket(const char* host); server_string_split_t splitServerString(char* input); } // namespace mc diff --git a/libmc/__init__.py b/libmc/__init__.py index 0ee2b9f6..b0f05eaa 100644 --- a/libmc/__init__.py +++ b/libmc/__init__.py @@ -1,9 +1,10 @@ import os import functools from ._client import ( - PyClient, PyClientUnsafe as ClientUnsafe, PyClientPool, ThreadUnsafe, + PyClient, ThreadUnsafe, encode_value, decode_value, + PyClientPool, PyClientUnsafe as ClientUnsafe, MC_DEFAULT_EXPTIME, MC_POLL_TIMEOUT, @@ -72,8 +73,8 @@ def wrapper(*args, **kwargs): __all__ = [ - 'Client', 'ClientUnsafe', 'ClientPool', 'ThreadedClient', 'ThreadUnsafe', - '__VERSION__', 'encode_value', 'decode_value', + 'Client', 'ThreadUnsafe', '__VERSION__', 'encode_value', 'decode_value', + 'ClientUnsafe', 'ClientPool', 'ThreadedClient', 'MC_DEFAULT_EXPTIME', 'MC_POLL_TIMEOUT', 'MC_CONNECT_TIMEOUT', 'MC_RETRY_TIMEOUT', 'MC_SET_FAILOVER', 'MC_INITIAL_CLIENTS', diff --git a/libmc/_client.pyx b/libmc/_client.pyx index e4b36736..1bfd7f71 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -1105,7 +1105,7 @@ cdef class PyClientShell(PyClientSettings): self._get_current_thread_ident())) def _get_current_thread_ident(self): - return (os.getpid(), threading.current_thread().native_id) + return (os.getpid(), threading.current_thread().name) def get_last_error(self): return self.last_error diff --git a/misc/aliases b/misc/aliases deleted file mode 100755 index c0161de7..00000000 --- a/misc/aliases +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - -set -e - -function virtualize() -{ - if [ ! -e venv ]; then - py="${1:-`which python 2>/dev/null`}" - pyenv="virtualenv" - if ! $py -m pip list | grep -w "$pyenv" &> /dev/null; then - if $py -m pip list | grep -w venv &> /dev/null; then - pyenv="venv" - else - $py -m pip install "$pyenv" - fi - fi - $py -m "$pyenv" --python="$py" --seeder=pip venv; - source venv/bin/activate - pip install -U pip - pip install Cython setuptools future pytest - else - source venv/bin/activate - if ! [ venv -ef "$VIRTUAL_ENV" ]; then - echo "virtual environment was created with a different path" - exit 1 - fi - fi -} - -function virtualize_libmc() -{ - if [ ! -e venv ]; then - virtualize "$*" - pip install greenify gevent - else - virtualize "$*" - fi -} - -function src_dir() -{ - cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd -} - -case "$1" in - cy-debug) - cd "`src_dir`/.." - git apply misc/git/debug.patch --recount &> /dev/null || true - source misc/git/pre-commit - virtualize_libmc "`which python-dbg || which python3-dbg`" - python setup.py build_ext --inplace - if [ "$2" == "bench" ]; then - cygdb . -- -ex start --args python misc/runbench.py - elif [ -n "$2" ]; then - pattern="$2" - shift 2 - cmds=(-ex start) - for arg in "$@"; do cmds+=(-ex "$arg"); done - cygdb . -- "${cmds[@]}" --args python setup.py test -a "--full-trace -s -k $pattern" - else - cygdb . -- -ex start --args python setup.py test -a -s - fi - ;; - run-test) - cd "`src_dir`/.." - virtualize_libmc - shift - if [ -n "$1" ]; then - python setup.py test -a "-s -k $*" - else - python setup.py test -a -s - fi - ;; - bench) - cd "`src_dir`/.." - git apply misc/git/debug.patch -R --recount &> /dev/null || true - virtualize_libmc - python setup.py build_ext --inplace - python -m pip install pylibmc python-memcached - if ! python -m pip list | grep -w "libmc" &> /dev/null; then - python -m pip install -e . - fi - python misc/runbench.py - ;; - build-greenify) - virtualize_libmc - greenify=`python -c "import greenify; print(greenify.__file__)"` - deactivate - cd `dirname "$greenify"` - rm -fr build - virtualize "`which python-dbg || which python3-dbg`" - python setup.py build_ext --inplace - ;; - *) - printf 'Usage: %s {cy-debug [test pattern [gdb commands...]] | run-test [test pattern] | bench | build-greenify}\nNote that cy-debug and run-test don't \`make clean\` so that needs to happen elsewhere for changes to setup.py\n"$prog" - exit 1 - ;; -esac diff --git a/misc/git/debug.patch b/misc/git/debug.patch deleted file mode 100644 index b8c94569..00000000 --- a/misc/git/debug.patch +++ /dev/null @@ -1,101 +0,0 @@ -diff --git a/setup.py b/setup.py -index afc15c1..98f8465 100644 ---- a/setup.py -+++ b/setup.py -@@ -5,23 +5,43 @@ import sys - import shlex - import pkg_resources - import platform -+import functools -+import inspect - from distutils.sysconfig import get_config_var - from distutils.version import LooseVersion - from glob import glob - from setuptools import setup, Extension - from setuptools.command.test import test as TestCommand -+from setuptools.command.build_ext import build_ext -+from Cython.Build import cythonize - - sources = (glob("src/*.cpp") + ["libmc/_client.pyx"]) - include_dirs = ["include"] - -+COMPILER_PREARGS = [ -+ "-DDYNAMIC_ANNOTATIONS_ENABLED=1", -+ "-g3", -+ "-Og", -+ "-Wall", -+ "-march=x86-64", -+ "-mtune=generic", -+ "-pipe", -+ "-Wp,-D_FORTIFY_SOURCE=2", -+ "-Werror=format-security", -+ "-fPIC", -+] - COMPILER_FLAGS = [ - "-fno-strict-aliasing", - "-fno-exceptions", - "-fno-rtti", - "-Wall", - "-DMC_USE_SMALL_VECTOR", -- "-O3", -- "-DNDEBUG", -+ "-DDEBUG", -+] -+LINKER_FLAGS = [ -+ "-shared", -+ "-g3", -+ "-Wl,-O0,--sort-common,--as-needed,-z,relro,-z,now" - ] - - -@@ -91,6 +111,25 @@ class PyTest(TestCommand): - errno = pytest.main(shlex.split(self.pytest_args)) - os._exit(errno) - -+# https://shwina.github.io/custom-compiler-linker-extensions/ -+class BlankBuildExt(build_ext): -+ def build_extensions(self): -+ orig = self.compiler.compile -+ -+ @functools.wraps(orig) -+ def prearg_compile(*args, **kwargs): -+ bound = inspect.Signature.from_callable(orig).bind(*args, **kwargs) -+ bound.apply_defaults() -+ bound.arguments["extra_preargs"] = COMPILER_PREARGS -+ return orig(*bound.args, **bound.kwargs) -+ -+ self.compiler.compile = prearg_compile -+ -+ self.compiler.set_executable("compiler_so", "gcc") -+ #self.compiler.set_executable("compiler_cxx", "gcc") -+ self.compiler.set_executable("linker_so", "g++") -+ build_ext.build_extensions(self) -+ - - setup( - name="libmc", -@@ -121,18 +160,20 @@ setup( - ], - # Support for the basestring type is new in Cython 0.20. - setup_requires=["Cython >= 0.20"], -- cmdclass={"test": PyTest}, -- ext_modules=[ -+ cmdclass={"test": PyTest, "build_ext": BlankBuildExt}, -+ zip_safe=False, -+ ext_modules=cythonize([ - Extension( - "libmc._client", - sources, - include_dirs=include_dirs, - language="c++", - extra_compile_args=COMPILER_FLAGS, -+ extra_link_args=LINKER_FLAGS - ) -- ], -+ ], gdb_debug=True), - tests_require=[ - "pytest", - "future", -- "numpy", -+ #"numpy", - ]) diff --git a/misc/git/pre-commit b/misc/git/pre-commit deleted file mode 100755 index a032f1cb..00000000 --- a/misc/git/pre-commit +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -DIR=$( cd "$( dirname `readlink -f "${BASH_SOURCE[0]}"` )" &> /dev/null && pwd ) -HOOK="$DIR/../../.git/hooks/pre-commit" - -[[ ! -h "$HOOK" ]] && ln -s "$DIR/pre-commit" "$HOOK" -git apply --cached "$DIR/debug.patch" -R --recount &> /dev/null || true diff --git a/misc/memcached_server b/misc/memcached_server index d71f7706..d47acb13 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -55,19 +55,6 @@ function stop() fi } -function capture() -{ - start=`echo $PORTS | cut -d' ' -f1` - host="${1:-${start}}" - proxy=31311 - default=$((proxy+host-start)) - port="${2:-${default}}" - output="$basedir/var/log/$host.log" - $cmd -d -u $USER -l $ip -t $threads -m ${memory} -p $port -P $basedir/var/run/${port}.pid > $basedir/var/log/${port}.log 2>&1 - ncat --sh-exec "ncat localhost ${port}" -l ${host} --keep-open -o "$output" --append-output & - echo $! > "$basedir/var/run/${host}.pid" -} - case "$1" in start) if [ -n "$2" ]; then @@ -116,18 +103,16 @@ case "$1" in ;; stopall) if [ `ls $basedir/var/run/ | grep -c .pid` -ge 1 ]; then - for f in $basedir/var/run/*.pid; do - cat "$f" | xargs kill + names="`basename $basedir/var/run/*.pid | cut -d. -f1`" + for name in $names; do + stop $name & done fi + wait rm -rf $basedir ;; - capture) - shift - capture $@ - ;; *) - printf 'Usage: %s {start[all] | stop[all] | restart | unix | capture} \n' "$prog" + printf 'Usage: %s {start|stop|restart} \n' "$prog" exit 1 ;; esac diff --git a/misc/runbench.py b/misc/runbench.py index e25a24a2..89990cf4 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -19,21 +19,6 @@ else: from time import process_time -if True: - spawn = lambda f, *a: threading.Thread(target=f, args=a) -else: - # ThreadedGreenletCompat.test_many_eventlets - import gevent - import gevent.monkey - gevent.monkey.patch_all() - - import greenify - greenify.greenify() - for so_path in libmc.DYNAMIC_LIBRARIES: - assert greenify.patch_lib(so_path) - - spawn = gevent.spawn - logger = logging.getLogger('libmc.bench') Benchmark = namedtuple('Benchmark', 'name f args kwargs') @@ -184,27 +169,23 @@ def inner(name, *args, **kwargs): @benchmark_method def bench_get(mc, key, data): if mc.get(key) != data: - # logger.warn('%r.get(%r) fail', mc, key) - raise Exception() + logger.warn('%r.get(%r) fail', mc, key) @benchmark_method def bench_set(mc, key, data): if any(isinstance(mc.mc, client) for client in libmc_clients): if not mc.set(key, data): - # logger.warn('%r.set(%r, ...) fail', mc, key) - raise Exception() + logger.warn('%r.set(%r, ...) fail', mc, key) else: if not mc.set(key, data, min_compress_len=4001): - # logger.warn('%r.set(%r, ...) fail', mc, key) - raise Exception() + logger.warn('%r.set(%r, ...) fail', mc, key) @benchmark_method def bench_get_multi(mc, keys, pairs): if len(mc.get_multi(keys)) != len(pairs): - # logger.warn('%r.get_multi() incomplete', mc) - raise Exception() + logger.warn('%r.get_multi() incomplete', mc) @benchmark_method @@ -212,12 +193,10 @@ def bench_set_multi(mc, keys, pairs): ret = mc.set_multi(pairs) if any(isinstance(mc.mc, client) for client in libmc_clients): if not ret: - # logger.warn('%r.set_multi fail', mc) - raise Exception() + logger.warn('%r.set_multi fail', mc) else: if ret: - # logger.warn('%r.set_multi(%r) fail', mc, ret) - raise Exception() + logger.warn('%r.set_multi(%r) fail', mc, ret) def multi_pairs(n, val_len): @@ -400,21 +379,6 @@ def reserve(self): factory=lambda: Prefix(BenchmarkThreadedClient(**libmc_kwargs), 'libmc2'), threads=NTHREADS ), - # Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py thread mapped, from douban)', - # factory=lambda: Prefix(ThreadMappedPool(**libmc_kwargs), 'libmc3'), - # threads=NTHREADS - # ), - # Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py thread pool, from douban)', - # factory=lambda: Prefix(ThreadPool(**libmc_kwargs), 'libmc4'), - # threads=NTHREADS - # ), - # Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py ordered thread pool, from douban)', - # factory=lambda: Prefix(FIFOThreadPool(**libmc_kwargs), 'libmc5'), - # threads=NTHREADS - # ), ] def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIME): @@ -423,7 +387,6 @@ def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIM mcs = [p.factory() for p in participants] means = [[] for p in participants] stddevs = [[] for p in participants] - exceptions = [[] for p in participants] # Have each lifter do one benchmark each last_fn = None @@ -432,48 +395,35 @@ def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIM logger.info('%s', benchmark_name) for i, (participant, mc) in enumerate(zip(participants, mcs)): - failed = False def loop(sw): - nonlocal failed - try: - while sw.total() < bench_time: - with sw.timing(): - fn(mc, *args, **kwargs) - except Exception as e: - failed = failed or e - - try: - # FIXME: set before bench for get - if 'get' in fn.__name__: - last_fn(mc, *args, **kwargs) - - if participant.threads == 1: - sw = [DelayedStopwatch()] - loop(sw[0]) - else: - sw = [DelayedStopwatch() for i in range(participant.threads)] - ts = [spawn(loop, i) for i in sw] - for t in ts: - t.start() - - for t in ts: - t.join() - - if failed: - raise failed - - total = sum(sw, DelayedStopwatch()) - means[i].append(total.mean()) - stddevs[i].append(total.stddev()) - - logger.info(u'%76s: %s', participant.name, total) - exceptions[i].append(None) - except Exception as e: - logger.info(u'%76s: %s', participant.name, "failed") - exceptions[i].append(e) + while sw.total() < bench_time: + with sw.timing(): + fn(mc, *args, **kwargs) + + # FIXME: set before bench for get + if 'get' in fn.__name__: + last_fn(mc, *args, **kwargs) + + if participant.threads == 1: + sw = [DelayedStopwatch()] + loop(sw[0]) + else: + sw = [DelayedStopwatch() for i in range(participant.threads)] + ts = [threading.Thread(target=loop, args=i) for i in sw] + for t in ts: + t.start() + + for t in ts: + t.join() + + total = sum(sw, DelayedStopwatch()) + means[i].append(total.mean()) + stddevs[i].append(total.stddev()) + + logger.info(u'%76s: %s', participant.name, total) last_fn = fn - return means, stddevs, exceptions + return means, stddevs def main(args=sys.argv[1:]): @@ -482,20 +432,19 @@ def main(args=sys.argv[1:]): logger.info('Running %s servers, %s threads, and a %s client pool', N_SERVERS, NTHREADS, POOL_SIZE) - ps = [p for p in participants if any(p.name.startswith(arg) for arg in args)] + ps = [p for p in participants if p.name in args] ps = ps if ps else participants bs = benchmarks[:] logger.info('%d participants in %d benchmarks', len(ps), len(bs)) - means, stddevs, exceptions = bench(participants=ps, benchmarks=bs) + means, stddevs = bench(participants=ps, benchmarks=bs) print('labels =', [p.name for p in ps]) print('benchmarks =', [b.name for b in bs]) print('means =', means) print('stddevs =', stddevs) - print('exceptions =', exceptions) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 37dba363..b404f128 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ def run_tests(self): # import here, cause outside the eggs aren't loaded import pytest errno = pytest.main(shlex.split(self.pytest_args)) - os._exit(errno) + sys.exit(errno) setup( diff --git a/src/Common.cpp b/src/Common.cpp index 148bdd92..2e75ffca 100644 --- a/src/Common.cpp +++ b/src/Common.cpp @@ -50,9 +50,9 @@ const char* errCodeToString(err_code_t err) { } } -bool isUnixSocket(const char* host) { +bool isLocalSocket(const char* host) { // errors on the side of false negatives, allowing syntax expansion; - // starting slash syntax is from libmemcached + // starting slash to denote socket paths is from pylibmc return host[0] == '/'; } diff --git a/src/Connection.cpp b/src/Connection.cpp index b44c4963..0b95c8b3 100644 --- a/src/Connection.cpp +++ b/src/Connection.cpp @@ -47,7 +47,7 @@ Connection::~Connection() { int Connection::init(const char* host, uint32_t port, const char* alias) { snprintf(m_host, sizeof m_host, "%s", host); m_port = port; - m_unixSocket = isUnixSocket(m_host); + m_unixSocket = isLocalSocket(m_host); if (alias == NULL) { m_hasAlias = false; if (m_unixSocket) { From 081db1d284f77f68c2de0ab3267ef18b5a1cb322 Mon Sep 17 00:00:00 2001 From: Kent Slaney Date: Tue, 4 Jun 2024 11:09:05 -0700 Subject: [PATCH 95/95] mistake in benchmark thread spawning --- misc/runbench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/runbench.py b/misc/runbench.py index 89990cf4..0135b7d5 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -409,7 +409,7 @@ def loop(sw): loop(sw[0]) else: sw = [DelayedStopwatch() for i in range(participant.threads)] - ts = [threading.Thread(target=loop, args=i) for i in sw] + ts = [threading.Thread(target=loop, args=[i]) for i in sw] for t in ts: t.start()