diff --git a/.github/workflows/golang.yml b/.github/workflows/golang.yml index 18784de3..c542e54e 100644 --- a/.github/workflows/golang.yml +++ b/.github/workflows/golang.yml @@ -5,7 +5,6 @@ on: branches: [ master ] pull_request: branches: [ master ] - workflow_dispatch: jobs: @@ -26,7 +25,6 @@ jobs: uses: actions/setup-go@v2 with: go-version: ${{ matrix.gover }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Start memcached servers run: ./misc/memcached_server start - name: Run gotest diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml deleted file mode 100644 index 1d8cd18d..00000000 --- a/.github/workflows/manual.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: manual - -on: - workflow_dispatch: - -jobs: - cpptest: - runs-on: ubuntu-latest - strategy: - matrix: - compiler: ["gcc", "clang"] - build_type: ["Debug"] - - steps: - - uses: actions/checkout@v2 - - name: Setup system dependencies - run: | - sudo apt-get update - sudo apt-get -y install memcached g++ - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools future pytest greenify gevent numpy - - name: Run cpptest - run: | - if [[ ${{ matrix.compiler }} = "gcc" ]]; then export CC=gcc CXX=g++; fi - if [[ ${{ matrix.compiler }} = "clang" ]]; then export CC=clang CXX=clang++; fi - ./misc/travis/cpptest.sh - - pylatest: - runs-on: ubuntu-latest - strategy: - matrix: - pyver: ["3.12"] - compiler: ["gcc"] - build_type: ["Debug"] - - steps: - - uses: actions/checkout@v2 - - name: Setup system dependencies - run: | - sudo apt-get update - sudo apt-get -y install valgrind memcached g++ - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.pyver }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools future pytest greenify gevent numpy - - name: Start memcached servers - run: ./misc/memcached_server startall - - name: Run unittest - run: | - if [[ ${{ matrix.compiler }} = "gcc" ]]; then export CC=gcc CXX=g++; fi - if [[ ${{ matrix.compiler }} = "clang" ]]; then export CC=clang CXX=clang++; fi - ./misc/travis/unittest.sh - - name: Stop memcached servers - run: ./misc/memcached_server stopall diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index be6837ce..f364f304 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -25,7 +25,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.pyver }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install python dependencies run: | python -m pip install --upgrade pip @@ -56,7 +55,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.pyver }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install python dependencies run: | python -m pip install --upgrade pip diff --git a/.gitignore b/.gitignore index 0949ba73..0e754f62 100644 --- a/.gitignore +++ b/.gitignore @@ -25,5 +25,3 @@ tests/resources/keys_*.txt /go.mod /go.sum - -/cython_debug diff --git a/Makefile b/Makefile index 0210df15..2dff81f3 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,3 @@ gotest: pytest: pytest tests/ - -clean: - rm -fr build cython_debug libmc/*.so libmc/*.cpp .eggs libmc.egg-info diff --git a/README.rst b/README.rst index 202c4708..9c829098 100644 --- a/README.rst +++ b/README.rst @@ -5,16 +5,15 @@ libmc |status| |pypiv| |pyversions| |wheel| |license| libmc is a memcached client library for Python without any other -dependencies at runtime. It's mainly written in C++ and Cython and -can be considered a drop in replacement for libmemcached and +dependencies in runtime. It's mainly written in C++ and Cython. libmc +can be considered as a drop in replacement for libmemcached and `python-libmemcached `__. -libmc is developed and maintained by Douban Inc. Currently, it is -working in a production environment, powering all web traffic on -`douban.com `__. [#]_ - -.. [#] The link not an endorsement of the traffic estimate site's accuracy, just - a publicly available reference point +libmc is developing and maintaining by Douban Inc. Currently, It is +working in production environment, powering all web traffics in +douban.com. Realtime `benchmark +result `__ is +available on travis. Build and Installation ---------------------- @@ -38,14 +37,14 @@ Usage: Under the hood -------------- -Under the hood, libmc consists of 2 parts: an internal, fully-functional +Under the hood, libmc consists of 2 parts: an internal fully-functional memcached client implementation in C++ and a Cython wrapper around that implementation. Dynamic memory allocation and memory-copy are slow, so -we've tried our best to avoid them. libmc also supports the ``set_multi`` -command, which is not natively supported by the `memcached +we tried our best to avoid them. The ``set_multi`` command is not +natively supported by the `memcached protocol `__. -Some techniques have been applied to make ``set_multi`` command extremely fast -in libmc (compared to similiar libraries). +Some techniques are applied to make ``set_multi`` command extremely fast +in libmc (compared to some other similiar libraries). Configuration ------------- @@ -78,23 +77,24 @@ Configuration mc.config(MC_RETRY_TIMEOUT, 5) # 5 s -- ``servers``: a list of memcached server addresses. Each address - should be formated as ``hostname[:port] [alias]``, where ``port`` and - ``alias`` are optional. If ``port`` is not given, the default port ``11211`` - will be used. If given, ``alias`` will be used to compute the server hash, - which would otherwise be computed based on ``host`` and ``port`` - (i.e. whichever portion is given). -- ``do_split``: splits large values (up to 10MB) into chunks (<1MB). The - memcached server implementation will not store items larger than 1MB, - however in some environments it is beneficial to shard up to 10MB of data. - Attempts to store more than that are ignored. Default: ``True``. -- ``comp_threshold``: compresses large values using zlib. If - ``buffer length > comp_threshold > 0`` (in bytes), the buffer will be - compressed. If ``comp_threshold == 0``, the string buffer will never be - compressed. Default: ``0`` -- ``noreply``: controls memcached's - ``noreply`` `feature `__. - Default: ``False`` +- ``servers``: is a list of memcached server addresses. Each address + can be in format of ``hostname[:port] [alias]``. ``port`` and ``alias`` + are optional. If ``port`` is not given, default port ``11211`` will + be used. ``alias`` will be used to compute server hash if given, + otherwise server hash will be computed based on ``host`` and ``port`` + (i.e.: If ``port`` is not given or it is equal to ``11211``, ``host`` + will be used to compute server hash. If ``port`` is not equal to ``11211``, + ``host:port`` will be used). +- ``do_split``: Memcached server will refuse to store value if size >= + 1MB, if ``do_split`` is enabled, large value (< 10 MB) will be + splitted into several blocks. If the value is too large (>= 10 MB), + it will not be stored. default: ``True`` +- ``comp_threshold``: All kinds of values will be encoded into string + buffer. If ``buffer length > comp_threshold > 0``, it will be + compressed using zlib. If ``comp_threshold = 0``, string buffer will + never be compressed using zlib. default: ``0`` +- ``noreply``: Whether to enable memcached's ``noreply`` behaviour. + default: ``False`` - ``prefix``: The key prefix. default: ``''`` - ``hash_fn``: hashing function for keys. possible values: @@ -110,16 +110,16 @@ Configuration implementions in libmemcached. - ``failover``: Whether to failover to next server when current server - is not available. Default: ``False`` + is not available. default: ``False`` - ``MC_POLL_TIMEOUT`` Timeout parameter used during set/get procedure. - Default: ``300`` ms + (default: ``300`` ms) - ``MC_CONNECT_TIMEOUT`` Timeout parameter used when connecting to - memcached server in the initial phase. Default: ``100`` ms -- ``MC_RETRY_TIMEOUT`` When a server is not available due to server-end - error, libmc will try to establish the broken connection in every - ``MC_RETRY_TIMEOUT`` s until the connection is back to live. Default: - ``5`` s + memcached server on initial phase. (default: ``100`` ms) +- ``MC_RETRY_TIMEOUT`` When a server is not available dur to server-end + error. libmc will try to establish the broken connection in every + ``MC_RETRY_TIMEOUT`` s until the connection is back to live.(default: + ``5`` s) **NOTE:** The hashing algorithm for host mapping on continuum is always md5. @@ -140,7 +140,7 @@ FAQ Does libmc support PHP? ^^^^^^^^^^^^^^^^^^^^^^^ -No, but, if you like, you can write a wrapper for PHP based on the C++ +No. But if you like, you can write a wrapper for PHP based on the C++ implementation. Is Memcached binary protocol supported ? @@ -151,7 +151,7 @@ No. Only Memcached ASCII protocol is supported currently. Why reinventing the wheel? ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Before libmc, we were using +Before libmc, we're using `python-libmemcached `__, which is a python extention for `libmemcached `__. @@ -161,32 +161,21 @@ still some unsolved bugs. Is libmc thread-safe ? ^^^^^^^^^^^^^^^^^^^^^^ -Yes. ``libmc.ThreadedClient`` is a thread-safe client implementation. To hold -access for more than one request, ``libmc.ClientPool`` can be used with Python -``with`` statements. ``libmc.Client``, however, is a single-threaded memcached -client. If you initialize a standard client in one thread but reuse that in -another thread, a Python ``ThreadUnsafe`` Exception will be raised. +libmc is a single-threaded memcached client. If you initialize a libmc +client in one thread but reuse that in another thread, a Python +Exception ``ThreadUnsafe`` will raise in Python. Is libmc compatible with gevent? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Yes, with the help of `greenify `__, libmc is friendly to gevent. Read ``tests/shabby/gevent_issue.py`` for -details. ``libmc.ThreadedClient`` and ``libmc.ClientPool`` are not compatible. -[#]_ - -.. [#] In order to use a single executable for multiple greenlet contexts, - gevent has to `copy thread memory - `__ - to and from the same stack space. This doesn't affect Python references, - which are handed off through gevent, but makes it impossible for shared - libraries to pass memory addresses across greenlets, which is required for - the worker pool. +details. **Notice:** -``gevent.monkey.patch_all()`` will override -``threading.current_thread().ident`` to Greenlet's ID, +`gevent.monkey.patch_all()` will override +`threading.current_thread().ident` to Greenlet's ID, this will cause libmc to throw a ThreadUnSafe error or run into dead lock, you should only patch the things that you need, e.g. @@ -263,4 +252,3 @@ https://github.com/douban/libmc/blob/master/LICENSE.txt .. |pyversions| image:: https://img.shields.io/pypi/pyversions/libmc .. |wheel| image:: https://img.shields.io/pypi/wheel/libmc .. |license| image:: https://img.shields.io/pypi/l/libmc?color=blue - diff --git a/include/Common.h b/include/Common.h index 9295465a..f7a22de8 100644 --- a/include/Common.h +++ b/include/Common.h @@ -226,7 +226,7 @@ typedef enum { } op_code_t; const char* errCodeToString(err_code_t err); -bool isUnixSocket(const char* host); +bool isLocalSocket(const char* host); server_string_split_t splitServerString(char* input); } // namespace mc diff --git a/libmc/__init__.py b/libmc/__init__.py index 0ee2b9f6..b0f05eaa 100644 --- a/libmc/__init__.py +++ b/libmc/__init__.py @@ -1,9 +1,10 @@ import os import functools from ._client import ( - PyClient, PyClientUnsafe as ClientUnsafe, PyClientPool, ThreadUnsafe, + PyClient, ThreadUnsafe, encode_value, decode_value, + PyClientPool, PyClientUnsafe as ClientUnsafe, MC_DEFAULT_EXPTIME, MC_POLL_TIMEOUT, @@ -72,8 +73,8 @@ def wrapper(*args, **kwargs): __all__ = [ - 'Client', 'ClientUnsafe', 'ClientPool', 'ThreadedClient', 'ThreadUnsafe', - '__VERSION__', 'encode_value', 'decode_value', + 'Client', 'ThreadUnsafe', '__VERSION__', 'encode_value', 'decode_value', + 'ClientUnsafe', 'ClientPool', 'ThreadedClient', 'MC_DEFAULT_EXPTIME', 'MC_POLL_TIMEOUT', 'MC_CONNECT_TIMEOUT', 'MC_RETRY_TIMEOUT', 'MC_SET_FAILOVER', 'MC_INITIAL_CLIENTS', diff --git a/libmc/_client.pyx b/libmc/_client.pyx index e4b36736..1bfd7f71 100644 --- a/libmc/_client.pyx +++ b/libmc/_client.pyx @@ -1105,7 +1105,7 @@ cdef class PyClientShell(PyClientSettings): self._get_current_thread_ident())) def _get_current_thread_ident(self): - return (os.getpid(), threading.current_thread().native_id) + return (os.getpid(), threading.current_thread().name) def get_last_error(self): return self.last_error diff --git a/misc/aliases b/misc/aliases deleted file mode 100755 index c0161de7..00000000 --- a/misc/aliases +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - -set -e - -function virtualize() -{ - if [ ! -e venv ]; then - py="${1:-`which python 2>/dev/null`}" - pyenv="virtualenv" - if ! $py -m pip list | grep -w "$pyenv" &> /dev/null; then - if $py -m pip list | grep -w venv &> /dev/null; then - pyenv="venv" - else - $py -m pip install "$pyenv" - fi - fi - $py -m "$pyenv" --python="$py" --seeder=pip venv; - source venv/bin/activate - pip install -U pip - pip install Cython setuptools future pytest - else - source venv/bin/activate - if ! [ venv -ef "$VIRTUAL_ENV" ]; then - echo "virtual environment was created with a different path" - exit 1 - fi - fi -} - -function virtualize_libmc() -{ - if [ ! -e venv ]; then - virtualize "$*" - pip install greenify gevent - else - virtualize "$*" - fi -} - -function src_dir() -{ - cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd -} - -case "$1" in - cy-debug) - cd "`src_dir`/.." - git apply misc/git/debug.patch --recount &> /dev/null || true - source misc/git/pre-commit - virtualize_libmc "`which python-dbg || which python3-dbg`" - python setup.py build_ext --inplace - if [ "$2" == "bench" ]; then - cygdb . -- -ex start --args python misc/runbench.py - elif [ -n "$2" ]; then - pattern="$2" - shift 2 - cmds=(-ex start) - for arg in "$@"; do cmds+=(-ex "$arg"); done - cygdb . -- "${cmds[@]}" --args python setup.py test -a "--full-trace -s -k $pattern" - else - cygdb . -- -ex start --args python setup.py test -a -s - fi - ;; - run-test) - cd "`src_dir`/.." - virtualize_libmc - shift - if [ -n "$1" ]; then - python setup.py test -a "-s -k $*" - else - python setup.py test -a -s - fi - ;; - bench) - cd "`src_dir`/.." - git apply misc/git/debug.patch -R --recount &> /dev/null || true - virtualize_libmc - python setup.py build_ext --inplace - python -m pip install pylibmc python-memcached - if ! python -m pip list | grep -w "libmc" &> /dev/null; then - python -m pip install -e . - fi - python misc/runbench.py - ;; - build-greenify) - virtualize_libmc - greenify=`python -c "import greenify; print(greenify.__file__)"` - deactivate - cd `dirname "$greenify"` - rm -fr build - virtualize "`which python-dbg || which python3-dbg`" - python setup.py build_ext --inplace - ;; - *) - printf 'Usage: %s {cy-debug [test pattern [gdb commands...]] | run-test [test pattern] | bench | build-greenify}\nNote that cy-debug and run-test don't \`make clean\` so that needs to happen elsewhere for changes to setup.py\n"$prog" - exit 1 - ;; -esac diff --git a/misc/git/debug.patch b/misc/git/debug.patch deleted file mode 100644 index b8c94569..00000000 --- a/misc/git/debug.patch +++ /dev/null @@ -1,101 +0,0 @@ -diff --git a/setup.py b/setup.py -index afc15c1..98f8465 100644 ---- a/setup.py -+++ b/setup.py -@@ -5,23 +5,43 @@ import sys - import shlex - import pkg_resources - import platform -+import functools -+import inspect - from distutils.sysconfig import get_config_var - from distutils.version import LooseVersion - from glob import glob - from setuptools import setup, Extension - from setuptools.command.test import test as TestCommand -+from setuptools.command.build_ext import build_ext -+from Cython.Build import cythonize - - sources = (glob("src/*.cpp") + ["libmc/_client.pyx"]) - include_dirs = ["include"] - -+COMPILER_PREARGS = [ -+ "-DDYNAMIC_ANNOTATIONS_ENABLED=1", -+ "-g3", -+ "-Og", -+ "-Wall", -+ "-march=x86-64", -+ "-mtune=generic", -+ "-pipe", -+ "-Wp,-D_FORTIFY_SOURCE=2", -+ "-Werror=format-security", -+ "-fPIC", -+] - COMPILER_FLAGS = [ - "-fno-strict-aliasing", - "-fno-exceptions", - "-fno-rtti", - "-Wall", - "-DMC_USE_SMALL_VECTOR", -- "-O3", -- "-DNDEBUG", -+ "-DDEBUG", -+] -+LINKER_FLAGS = [ -+ "-shared", -+ "-g3", -+ "-Wl,-O0,--sort-common,--as-needed,-z,relro,-z,now" - ] - - -@@ -91,6 +111,25 @@ class PyTest(TestCommand): - errno = pytest.main(shlex.split(self.pytest_args)) - os._exit(errno) - -+# https://shwina.github.io/custom-compiler-linker-extensions/ -+class BlankBuildExt(build_ext): -+ def build_extensions(self): -+ orig = self.compiler.compile -+ -+ @functools.wraps(orig) -+ def prearg_compile(*args, **kwargs): -+ bound = inspect.Signature.from_callable(orig).bind(*args, **kwargs) -+ bound.apply_defaults() -+ bound.arguments["extra_preargs"] = COMPILER_PREARGS -+ return orig(*bound.args, **bound.kwargs) -+ -+ self.compiler.compile = prearg_compile -+ -+ self.compiler.set_executable("compiler_so", "gcc") -+ #self.compiler.set_executable("compiler_cxx", "gcc") -+ self.compiler.set_executable("linker_so", "g++") -+ build_ext.build_extensions(self) -+ - - setup( - name="libmc", -@@ -121,18 +160,20 @@ setup( - ], - # Support for the basestring type is new in Cython 0.20. - setup_requires=["Cython >= 0.20"], -- cmdclass={"test": PyTest}, -- ext_modules=[ -+ cmdclass={"test": PyTest, "build_ext": BlankBuildExt}, -+ zip_safe=False, -+ ext_modules=cythonize([ - Extension( - "libmc._client", - sources, - include_dirs=include_dirs, - language="c++", - extra_compile_args=COMPILER_FLAGS, -+ extra_link_args=LINKER_FLAGS - ) -- ], -+ ], gdb_debug=True), - tests_require=[ - "pytest", - "future", -- "numpy", -+ #"numpy", - ]) diff --git a/misc/git/pre-commit b/misc/git/pre-commit deleted file mode 100755 index a032f1cb..00000000 --- a/misc/git/pre-commit +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -DIR=$( cd "$( dirname `readlink -f "${BASH_SOURCE[0]}"` )" &> /dev/null && pwd ) -HOOK="$DIR/../../.git/hooks/pre-commit" - -[[ ! -h "$HOOK" ]] && ln -s "$DIR/pre-commit" "$HOOK" -git apply --cached "$DIR/debug.patch" -R --recount &> /dev/null || true diff --git a/misc/memcached_server b/misc/memcached_server index d71f7706..d47acb13 100755 --- a/misc/memcached_server +++ b/misc/memcached_server @@ -55,19 +55,6 @@ function stop() fi } -function capture() -{ - start=`echo $PORTS | cut -d' ' -f1` - host="${1:-${start}}" - proxy=31311 - default=$((proxy+host-start)) - port="${2:-${default}}" - output="$basedir/var/log/$host.log" - $cmd -d -u $USER -l $ip -t $threads -m ${memory} -p $port -P $basedir/var/run/${port}.pid > $basedir/var/log/${port}.log 2>&1 - ncat --sh-exec "ncat localhost ${port}" -l ${host} --keep-open -o "$output" --append-output & - echo $! > "$basedir/var/run/${host}.pid" -} - case "$1" in start) if [ -n "$2" ]; then @@ -116,18 +103,16 @@ case "$1" in ;; stopall) if [ `ls $basedir/var/run/ | grep -c .pid` -ge 1 ]; then - for f in $basedir/var/run/*.pid; do - cat "$f" | xargs kill + names="`basename $basedir/var/run/*.pid | cut -d. -f1`" + for name in $names; do + stop $name & done fi + wait rm -rf $basedir ;; - capture) - shift - capture $@ - ;; *) - printf 'Usage: %s {start[all] | stop[all] | restart | unix | capture} \n' "$prog" + printf 'Usage: %s {start|stop|restart} \n' "$prog" exit 1 ;; esac diff --git a/misc/runbench.py b/misc/runbench.py index e25a24a2..89990cf4 100644 --- a/misc/runbench.py +++ b/misc/runbench.py @@ -19,21 +19,6 @@ else: from time import process_time -if True: - spawn = lambda f, *a: threading.Thread(target=f, args=a) -else: - # ThreadedGreenletCompat.test_many_eventlets - import gevent - import gevent.monkey - gevent.monkey.patch_all() - - import greenify - greenify.greenify() - for so_path in libmc.DYNAMIC_LIBRARIES: - assert greenify.patch_lib(so_path) - - spawn = gevent.spawn - logger = logging.getLogger('libmc.bench') Benchmark = namedtuple('Benchmark', 'name f args kwargs') @@ -184,27 +169,23 @@ def inner(name, *args, **kwargs): @benchmark_method def bench_get(mc, key, data): if mc.get(key) != data: - # logger.warn('%r.get(%r) fail', mc, key) - raise Exception() + logger.warn('%r.get(%r) fail', mc, key) @benchmark_method def bench_set(mc, key, data): if any(isinstance(mc.mc, client) for client in libmc_clients): if not mc.set(key, data): - # logger.warn('%r.set(%r, ...) fail', mc, key) - raise Exception() + logger.warn('%r.set(%r, ...) fail', mc, key) else: if not mc.set(key, data, min_compress_len=4001): - # logger.warn('%r.set(%r, ...) fail', mc, key) - raise Exception() + logger.warn('%r.set(%r, ...) fail', mc, key) @benchmark_method def bench_get_multi(mc, keys, pairs): if len(mc.get_multi(keys)) != len(pairs): - # logger.warn('%r.get_multi() incomplete', mc) - raise Exception() + logger.warn('%r.get_multi() incomplete', mc) @benchmark_method @@ -212,12 +193,10 @@ def bench_set_multi(mc, keys, pairs): ret = mc.set_multi(pairs) if any(isinstance(mc.mc, client) for client in libmc_clients): if not ret: - # logger.warn('%r.set_multi fail', mc) - raise Exception() + logger.warn('%r.set_multi fail', mc) else: if ret: - # logger.warn('%r.set_multi(%r) fail', mc, ret) - raise Exception() + logger.warn('%r.set_multi(%r) fail', mc, ret) def multi_pairs(n, val_len): @@ -400,21 +379,6 @@ def reserve(self): factory=lambda: Prefix(BenchmarkThreadedClient(**libmc_kwargs), 'libmc2'), threads=NTHREADS ), - # Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py thread mapped, from douban)', - # factory=lambda: Prefix(ThreadMappedPool(**libmc_kwargs), 'libmc3'), - # threads=NTHREADS - # ), - # Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py thread pool, from douban)', - # factory=lambda: Prefix(ThreadPool(**libmc_kwargs), 'libmc4'), - # threads=NTHREADS - # ), - # Participant( - # name='libmc(md5 / ketama / nodelay / nonblocking / py ordered thread pool, from douban)', - # factory=lambda: Prefix(FIFOThreadPool(**libmc_kwargs), 'libmc5'), - # threads=NTHREADS - # ), ] def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIME): @@ -423,7 +387,6 @@ def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIM mcs = [p.factory() for p in participants] means = [[] for p in participants] stddevs = [[] for p in participants] - exceptions = [[] for p in participants] # Have each lifter do one benchmark each last_fn = None @@ -432,48 +395,35 @@ def bench(participants=participants, benchmarks=benchmarks, bench_time=BENCH_TIM logger.info('%s', benchmark_name) for i, (participant, mc) in enumerate(zip(participants, mcs)): - failed = False def loop(sw): - nonlocal failed - try: - while sw.total() < bench_time: - with sw.timing(): - fn(mc, *args, **kwargs) - except Exception as e: - failed = failed or e - - try: - # FIXME: set before bench for get - if 'get' in fn.__name__: - last_fn(mc, *args, **kwargs) - - if participant.threads == 1: - sw = [DelayedStopwatch()] - loop(sw[0]) - else: - sw = [DelayedStopwatch() for i in range(participant.threads)] - ts = [spawn(loop, i) for i in sw] - for t in ts: - t.start() - - for t in ts: - t.join() - - if failed: - raise failed - - total = sum(sw, DelayedStopwatch()) - means[i].append(total.mean()) - stddevs[i].append(total.stddev()) - - logger.info(u'%76s: %s', participant.name, total) - exceptions[i].append(None) - except Exception as e: - logger.info(u'%76s: %s', participant.name, "failed") - exceptions[i].append(e) + while sw.total() < bench_time: + with sw.timing(): + fn(mc, *args, **kwargs) + + # FIXME: set before bench for get + if 'get' in fn.__name__: + last_fn(mc, *args, **kwargs) + + if participant.threads == 1: + sw = [DelayedStopwatch()] + loop(sw[0]) + else: + sw = [DelayedStopwatch() for i in range(participant.threads)] + ts = [threading.Thread(target=loop, args=i) for i in sw] + for t in ts: + t.start() + + for t in ts: + t.join() + + total = sum(sw, DelayedStopwatch()) + means[i].append(total.mean()) + stddevs[i].append(total.stddev()) + + logger.info(u'%76s: %s', participant.name, total) last_fn = fn - return means, stddevs, exceptions + return means, stddevs def main(args=sys.argv[1:]): @@ -482,20 +432,19 @@ def main(args=sys.argv[1:]): logger.info('Running %s servers, %s threads, and a %s client pool', N_SERVERS, NTHREADS, POOL_SIZE) - ps = [p for p in participants if any(p.name.startswith(arg) for arg in args)] + ps = [p for p in participants if p.name in args] ps = ps if ps else participants bs = benchmarks[:] logger.info('%d participants in %d benchmarks', len(ps), len(bs)) - means, stddevs, exceptions = bench(participants=ps, benchmarks=bs) + means, stddevs = bench(participants=ps, benchmarks=bs) print('labels =', [p.name for p in ps]) print('benchmarks =', [b.name for b in bs]) print('means =', means) print('stddevs =', stddevs) - print('exceptions =', exceptions) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 37dba363..b404f128 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ def run_tests(self): # import here, cause outside the eggs aren't loaded import pytest errno = pytest.main(shlex.split(self.pytest_args)) - os._exit(errno) + sys.exit(errno) setup( diff --git a/src/Common.cpp b/src/Common.cpp index 148bdd92..2e75ffca 100644 --- a/src/Common.cpp +++ b/src/Common.cpp @@ -50,9 +50,9 @@ const char* errCodeToString(err_code_t err) { } } -bool isUnixSocket(const char* host) { +bool isLocalSocket(const char* host) { // errors on the side of false negatives, allowing syntax expansion; - // starting slash syntax is from libmemcached + // starting slash to denote socket paths is from pylibmc return host[0] == '/'; } diff --git a/src/Connection.cpp b/src/Connection.cpp index b44c4963..0b95c8b3 100644 --- a/src/Connection.cpp +++ b/src/Connection.cpp @@ -47,7 +47,7 @@ Connection::~Connection() { int Connection::init(const char* host, uint32_t port, const char* alias) { snprintf(m_host, sizeof m_host, "%s", host); m_port = port; - m_unixSocket = isUnixSocket(m_host); + m_unixSocket = isLocalSocket(m_host); if (alias == NULL) { m_hasAlias = false; if (m_unixSocket) {