Skip to content

Commit 5833590

Browse files
rlerdorfrlerdorf
authored
zstd support (#539)
This adds zstd compression support. The current two options, zlib and fastlz is basically a choice between performance and compression ratio. You would choose zlib if you are memory-bound and fastlz if you are cpu-bound. With zstd, you get the performance of fastlz with the compression of zlib. And often it wins on both. See this benchmark I ran on json files of varying sizes: https://gist.github.com/rlerdorf/788f3d0144f9c5514d8fee9477cbe787 Taking just a 40k json blob, we see that zstd at compression level 3 reduces it to 8862 bytes. Our current zlib 1 gets worse compression at 10091 bytes and takes longer both to compress and decompress. C Size ratio% C MB/s D MB/s SCORE Name File 8037 19.9 0.58 2130.89 0.08 zstd 22 file-39.54k-json 8204 20.3 31.85 2381.59 0.01 zstd 10 file-39.54k-json 8371 20.7 47.52 547.12 0.01 zlib 9 file-39.54k-json 8477 20.9 74.84 539.83 0.01 zlib 6 file-39.54k-json 8862 21.9 449.86 2130.89 0.01 zstd 3 file-39.54k-json 9171 22.7 554.62 2381.59 0.01 zstd 1 file-39.54k-json 10091 24.9 153.94 481.99 0.01 zlib 1 file-39.54k-json 10646 26.3 43.39 8097.40 0.01 lz4 16 file-39.54k-json 10658 26.3 72.30 8097.40 0.01 lz4 10 file-39.54k-json 13004 32.1 1396.10 6747.83 0.01 lz4 1 file-39.54k-json 13321 32.9 440.08 1306.03 0.01 fastlz 2 file-39.54k-json 14807 36.6 444.91 1156.77 0.01 fastlz 1 file-39.54k-json 15517 38.3 1190.79 4048.70 0.02 zstd -10 file-39.54k-json The fact that decompression a dramatically faster with zstd is a win for most common memcache uses since they tend to be read-heavy. The PR also adds a `memcache.compression_level` INI switch which currently only applies to zstd compression. It could probably be made to also apply to zlib and fastlz.
1 parent 6926c53 commit 5833590

5 files changed

+125
-3
lines changed

‎config.m4

+10
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ PHP_ARG_ENABLE(memcached-protocol, whether to enable memcached protocol support,
2727
PHP_ARG_WITH(system-fastlz, whether to use system FastLZ library,
2828
[ --with-system-fastlz Use system FastLZ library], no, no)
2929

30+
PHP_ARG_WITH(zstd, whether to use system zstd library,
31+
[ --with-zstd Use system zstd library], no, no)
32+
3033
if test -z "$PHP_ZLIB_DIR"; then
3134
PHP_ARG_WITH(zlib-dir, for ZLIB,
3235
[ --with-zlib-dir=DIR Set the path to ZLIB install prefix.], no)
@@ -345,6 +348,13 @@ if test "$PHP_MEMCACHED" != "no"; then
345348
PHP_MEMCACHED_FILES="${PHP_MEMCACHED_FILES} fastlz/fastlz.c"
346349
fi
347350

351+
if test "$PHP_ZSTD" != "no"; then
352+
AC_CHECK_HEADERS([zstd.h], [ac_cv_have_zstd="yes"], [ac_cv_have_zstd="no"])
353+
PHP_CHECK_LIBRARY(zstd, ZSTD_compress,
354+
[PHP_ADD_LIBRARY(zstd, 1, MEMCACHED_SHARED_LIBADD)],
355+
[AC_MSG_ERROR(zstd library not found)])
356+
fi
357+
348358
if test "$PHP_MEMCACHED_SESSION" != "no"; then
349359
PHP_MEMCACHED_FILES="${PHP_MEMCACHED_FILES} php_memcached_session.c"
350360
fi

‎php_memcached.c

+76-2
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@
3737
#endif
3838
#include <zlib.h>
3939

40+
#ifdef HAVE_ZSTD_H
41+
#include <zstd.h>
42+
#endif
43+
4044
#ifdef HAVE_JSON_API
4145
# include "ext/json/php_json.h"
4246
#endif
@@ -77,6 +81,7 @@ static int php_memc_list_entry(void) {
7781
#define MEMC_OPT_COMPRESSION_TYPE -1004
7882
#define MEMC_OPT_STORE_RETRY_COUNT -1005
7983
#define MEMC_OPT_USER_FLAGS -1006
84+
#define MEMC_OPT_COMPRESSION_LEVEL -1007
8085

8186
/****************************************
8287
Custom result codes
@@ -107,6 +112,7 @@ static int php_memc_list_entry(void) {
107112
#define MEMC_VAL_COMPRESSED (1<<0)
108113
#define MEMC_VAL_COMPRESSION_ZLIB (1<<1)
109114
#define MEMC_VAL_COMPRESSION_FASTLZ (1<<2)
115+
#define MEMC_VAL_COMPRESSION_ZSTD (1<<3)
110116

111117
#define MEMC_VAL_GET_FLAGS(internal_flags) (((internal_flags) & MEMC_MASK_INTERNAL) >> 4)
112118
#define MEMC_VAL_SET_FLAG(internal_flags, internal_flag) ((internal_flags) |= (((internal_flag) << 4) & MEMC_MASK_INTERNAL))
@@ -152,6 +158,7 @@ typedef struct {
152158

153159
zend_long serializer;
154160
zend_long compression_type;
161+
zend_long compression_level;
155162

156163
zend_long store_retry_count;
157164
zend_long set_udf_flags;
@@ -278,6 +285,10 @@ static PHP_INI_MH(OnUpdateCompressionType)
278285
MEMC_G(compression_type) = COMPRESSION_TYPE_FASTLZ;
279286
} else if (!strcmp(ZSTR_VAL(new_value), "zlib")) {
280287
MEMC_G(compression_type) = COMPRESSION_TYPE_ZLIB;
288+
#ifdef HAVE_ZSTD_H
289+
} else if (!strcmp(ZSTR_VAL(new_value), "zstd")) {
290+
MEMC_G(compression_type) = COMPRESSION_TYPE_ZSTD;
291+
#endif
281292
} else {
282293
return FAILURE;
283294
}
@@ -408,6 +419,7 @@ PHP_INI_BEGIN()
408419

409420
MEMC_INI_ENTRY("compression_type", "fastlz", OnUpdateCompressionType, compression_name)
410421
MEMC_INI_ENTRY("compression_factor", "1.3", OnUpdateReal, compression_factor)
422+
MEMC_INI_ENTRY("compression_level", "3", OnUpdateLong, compression_level)
411423
MEMC_INI_ENTRY("compression_threshold", "2000", OnUpdateLong, compression_threshold)
412424
MEMC_INI_ENTRY("serializer", SERIALIZER_DEFAULT_NAME, OnUpdateSerializer, serializer_name)
413425
MEMC_INI_ENTRY("store_retry_count", "0", OnUpdateLong, store_retry_count)
@@ -897,6 +909,19 @@ zend_bool s_compress_value (php_memc_compression_type compression_type, zend_str
897909
}
898910
break;
899911

912+
#ifdef HAVE_ZSTD_H
913+
case COMPRESSION_TYPE_ZSTD:
914+
{
915+
compressed_size = ZSTD_compress((void *)buffer, buffer_size, ZSTR_VAL(payload), ZSTR_LEN(payload), MEMC_G(compression_level));
916+
917+
if (!ZSTD_isError(compressed_size)) {
918+
compress_status = 1;
919+
compression_type_flag = MEMC_VAL_COMPRESSION_ZSTD;
920+
}
921+
}
922+
break;
923+
#endif
924+
900925
case COMPRESSION_TYPE_ZLIB:
901926
{
902927
compressed_size = buffer_size;
@@ -2939,6 +2964,9 @@ static PHP_METHOD(Memcached, getOption)
29392964
case MEMC_OPT_COMPRESSION_TYPE:
29402965
RETURN_LONG(memc_user_data->compression_type);
29412966

2967+
case MEMC_OPT_COMPRESSION_LEVEL:
2968+
RETURN_LONG(memc_user_data->compression_level);
2969+
29422970
case MEMC_OPT_COMPRESSION:
29432971
RETURN_BOOL(memc_user_data->compression_enabled);
29442972

@@ -3001,6 +3029,9 @@ int php_memc_set_option(php_memc_object_t *intern, long option, zval *value)
30013029
case MEMC_OPT_COMPRESSION_TYPE:
30023030
lval = zval_get_long(value);
30033031
if (lval == COMPRESSION_TYPE_FASTLZ ||
3032+
#ifdef HAVE_ZSTD_H
3033+
lval == COMPRESSION_TYPE_ZSTD ||
3034+
#endif
30043035
lval == COMPRESSION_TYPE_ZLIB) {
30053036
memc_user_data->compression_type = lval;
30063037
} else {
@@ -3608,16 +3639,24 @@ zend_string *s_decompress_value (const char *payload, size_t payload_len, uint32
36083639
uint32_t stored_length;
36093640
unsigned long length;
36103641
zend_bool decompress_status = 0;
3611-
zend_bool is_fastlz = 0, is_zlib = 0;
3642+
zend_bool is_fastlz = 0, is_zlib = 0, is_zstd = 0;
36123643

36133644
if (payload_len < sizeof (uint32_t)) {
36143645
return NULL;
36153646
}
36163647

36173648
is_fastlz = MEMC_VAL_HAS_FLAG(flags, MEMC_VAL_COMPRESSION_FASTLZ);
3649+
is_zstd = MEMC_VAL_HAS_FLAG(flags, MEMC_VAL_COMPRESSION_ZSTD);
36183650
is_zlib = MEMC_VAL_HAS_FLAG(flags, MEMC_VAL_COMPRESSION_ZLIB);
36193651

3620-
if (!is_fastlz && !is_zlib) {
3652+
#ifndef HAVE_ZSTD_H
3653+
if (is_zstd) {
3654+
php_error_docref(NULL, E_WARNING, "could not decompress value: value was compressed with zstd but zstd support has not been compiled in");
3655+
return NULL;
3656+
}
3657+
#endif
3658+
3659+
if (!is_fastlz && !is_zlib && !is_zstd) {
36213660
php_error_docref(NULL, E_WARNING, "could not decompress value: unrecognised compression type");
36223661
return NULL;
36233662
}
@@ -3629,6 +3668,23 @@ zend_string *s_decompress_value (const char *payload, size_t payload_len, uint32
36293668

36303669
buffer = zend_string_alloc (stored_length, 0);
36313670

3671+
#ifdef HAVE_ZSTD_H
3672+
if (is_zstd) {
3673+
length = ZSTD_getFrameContentSize(payload, payload_len);
3674+
if (length == ZSTD_CONTENTSIZE_ERROR) {
3675+
php_error_docref(NULL, E_WARNING, "value was not compressed by zstd");
3676+
zend_string_release (buffer);
3677+
return NULL;
3678+
} else if (length == ZSTD_CONTENTSIZE_UNKNOWN) {
3679+
php_error_docref(NULL, E_WARNING, "zstd streaming decompression not supported");
3680+
zend_string_release (buffer);
3681+
return NULL;
3682+
}
3683+
decompress_status = !ZSTD_isError(ZSTD_decompress(&buffer->val, buffer->len, payload, payload_len));
3684+
3685+
}
3686+
else
3687+
#endif
36323688
if (is_fastlz) {
36333689
decompress_status = ((length = fastlz_decompress(payload, payload_len, &buffer->val, buffer->len)) > 0);
36343690
}
@@ -3955,6 +4011,7 @@ PHP_GINIT_FUNCTION(php_memcached)
39554011
php_memcached_globals->memc.compression_threshold = 2000;
39564012
php_memcached_globals->memc.compression_type = COMPRESSION_TYPE_FASTLZ;
39574013
php_memcached_globals->memc.compression_factor = 1.30;
4014+
php_memcached_globals->memc.compression_level = 3;
39584015
php_memcached_globals->memc.store_retry_count = 2;
39594016

39604017
php_memcached_globals->memc.sasl_initialised = 0;
@@ -4000,6 +4057,7 @@ static void php_memc_register_constants(INIT_FUNC_ARGS)
40004057

40014058
REGISTER_MEMC_CLASS_CONST_LONG(OPT_COMPRESSION, MEMC_OPT_COMPRESSION);
40024059
REGISTER_MEMC_CLASS_CONST_LONG(OPT_COMPRESSION_TYPE, MEMC_OPT_COMPRESSION_TYPE);
4060+
REGISTER_MEMC_CLASS_CONST_LONG(OPT_COMPRESSION_LEVEL, MEMC_OPT_COMPRESSION_LEVEL);
40034061
REGISTER_MEMC_CLASS_CONST_LONG(OPT_PREFIX_KEY, MEMC_OPT_PREFIX_KEY);
40044062
REGISTER_MEMC_CLASS_CONST_LONG(OPT_SERIALIZER, MEMC_OPT_SERIALIZER);
40054063

@@ -4015,6 +4073,15 @@ static void php_memc_register_constants(INIT_FUNC_ARGS)
40154073
REGISTER_MEMC_CLASS_CONST_BOOL(HAVE_IGBINARY, 0);
40164074
#endif
40174075

4076+
/*
4077+
* Indicate whether zstd compression is available
4078+
*/
4079+
#ifdef HAVE_ZSTD_H
4080+
REGISTER_MEMC_CLASS_CONST_BOOL(HAVE_ZSTD, 1);
4081+
#else
4082+
REGISTER_MEMC_CLASS_CONST_BOOL(HAVE_ZSTD, 0);
4083+
#endif
4084+
40184085
/*
40194086
* Indicate whether json serializer is available
40204087
*/
@@ -4186,6 +4253,7 @@ static void php_memc_register_constants(INIT_FUNC_ARGS)
41864253
*/
41874254
REGISTER_MEMC_CLASS_CONST_LONG(COMPRESSION_FASTLZ, COMPRESSION_TYPE_FASTLZ);
41884255
REGISTER_MEMC_CLASS_CONST_LONG(COMPRESSION_ZLIB, COMPRESSION_TYPE_ZLIB);
4256+
REGISTER_MEMC_CLASS_CONST_LONG(COMPRESSION_ZSTD, COMPRESSION_TYPE_ZSTD);
41894257

41904258
/*
41914259
* Flags.
@@ -4351,6 +4419,12 @@ PHP_MINFO_FUNCTION(memcached)
43514419
php_info_print_table_row(2, "msgpack support", "no");
43524420
#endif
43534421

4422+
#ifdef HAVE_ZSTD_H
4423+
php_info_print_table_row(2, "zstd support", "yes");
4424+
#else
4425+
php_info_print_table_row(2, "zstd support", "no");
4426+
#endif
4427+
43544428
php_info_print_table_end();
43554429

43564430
DISPLAY_INI_ENTRIES();

‎php_memcached_private.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,8 @@ typedef enum {
9898

9999
typedef enum {
100100
COMPRESSION_TYPE_ZLIB = 1,
101-
COMPRESSION_TYPE_FASTLZ = 2
101+
COMPRESSION_TYPE_FASTLZ = 2,
102+
COMPRESSION_TYPE_ZSTD = 3
102103
} php_memc_compression_type;
103104

104105
typedef struct {
@@ -186,6 +187,7 @@ ZEND_BEGIN_MODULE_GLOBALS(php_memcached)
186187
zend_long compression_threshold;
187188
double compression_factor;
188189
zend_long store_retry_count;
190+
zend_long compression_level;
189191

190192
/* Converted values*/
191193
php_memc_serializer_type serializer_type;

‎tests/compression_conditions.phpt

+2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ function get_compression($name) {
2121
return Memcached::COMPRESSION_ZLIB;
2222
case 'fastlz':
2323
return Memcached::COMPRESSION_FASTLZ;
24+
case 'zstd':
25+
return Memcached::COMPRESSION_ZSTD;
2426
default:
2527
echo "Strange compression type: $name\n";
2628
return 0;

‎tests/compression_types.phpt

+34
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ function get_compression($name) {
1515
return Memcached::COMPRESSION_ZLIB;
1616
case 'fastlz':
1717
return Memcached::COMPRESSION_FASTLZ;
18+
case 'zstd':
19+
if (Memcached::HAVE_ZSTD) {
20+
return Memcached::COMPRESSION_ZSTD;
21+
} else return 0;
1822
default:
1923
echo "Strange compression type: $name\n";
2024
return 0;
@@ -54,6 +58,26 @@ fetch_with_compression($m, 'hello6', $data, '', 'fastlz');
5458
fetch_with_compression($m, 'hello7', $data, 'zlib', '');
5559
fetch_with_compression($m, 'hello8', $data, 'fastlz', '');
5660
fetch_with_compression($m, 'hello9', $data, '', '');
61+
if (Memcached::HAVE_ZSTD) {
62+
fetch_with_compression($m, 'hello10', $data, 'zstd', 'zstd');
63+
fetch_with_compression($m, 'hello11', $data, 'zstd', 'fastlz');
64+
fetch_with_compression($m, 'hello12', $data, 'fastlz', 'zstd');
65+
fetch_with_compression($m, 'hello13', $data, '', 'zstd');
66+
fetch_with_compression($m, 'hello14', $data, 'zstd', '');
67+
} else {
68+
echo <<<EOB
69+
set=[zstd] get=[zstd]
70+
bool(true)
71+
set=[zstd] get=[fastlz]
72+
bool(true)
73+
set=[fastlz] get=[zstd]
74+
bool(true)
75+
set=[] get=[zstd]
76+
bool(true)
77+
set=[zstd] get=[]
78+
bool(true)
79+
EOB;
80+
}
5781
?>
5882
--EXPECT--
5983
set=[zlib] get=[zlib]
@@ -74,3 +98,13 @@ set=[fastlz] get=[]
7498
bool(true)
7599
set=[] get=[]
76100
bool(true)
101+
set=[zstd] get=[zstd]
102+
bool(true)
103+
set=[zstd] get=[fastlz]
104+
bool(true)
105+
set=[fastlz] get=[zstd]
106+
bool(true)
107+
set=[] get=[zstd]
108+
bool(true)
109+
set=[zstd] get=[]
110+
bool(true)

0 commit comments

Comments
 (0)