Skip to content

Commit e39a2e6

Browse files
rposkyrposky
authored
Add option to locally enforce payload size limit (#515)
Add a configuration option to enforce an item size limit on the client side. This avoids sending large items over the wire and getting rejected by the server which can cause delays. The default is 0 for no limit. The same error code RES_E2BIG is used for the client side limit as for the server side limit.
1 parent 7348cc1 commit e39a2e6

8 files changed

+151
-3
lines changed

‎memcached.ini

+6
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,12 @@
133133
; the default is 0
134134
;memcached.store_retry_count = 0
135135

136+
; The maximum payload size in bytes that can be written.
137+
; Writing a payload larger than the limit will result in RES_E2BIG error.
138+
; Specifying 0 means no limit is enforced, though the server may still reject with RES_E2BIG.
139+
; Default is 0.
140+
;memcached.item_size_limit = 1000000
141+
136142
; Sets the default for consistent hashing for new connections.
137143
; (To configure consistent hashing for session connections,
138144
; use memcached.sess_consistent_hash instead)

‎php_memcached.c

+45
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ static int php_memc_list_entry(void) {
8282
#define MEMC_OPT_STORE_RETRY_COUNT -1005
8383
#define MEMC_OPT_USER_FLAGS -1006
8484
#define MEMC_OPT_COMPRESSION_LEVEL -1007
85+
#define MEMC_OPT_ITEM_SIZE_LIMIT -1008
8586

8687
/****************************************
8788
Custom result codes
@@ -162,6 +163,7 @@ typedef struct {
162163

163164
zend_long store_retry_count;
164165
zend_long set_udf_flags;
166+
zend_long item_size_limit;
165167

166168
#ifdef HAVE_MEMCACHED_SASL
167169
zend_bool has_sasl_data;
@@ -423,6 +425,7 @@ PHP_INI_BEGIN()
423425
MEMC_INI_ENTRY("compression_threshold", "2000", OnUpdateLong, compression_threshold)
424426
MEMC_INI_ENTRY("serializer", SERIALIZER_DEFAULT_NAME, OnUpdateSerializer, serializer_name)
425427
MEMC_INI_ENTRY("store_retry_count", "0", OnUpdateLong, store_retry_count)
428+
MEMC_INI_ENTRY("item_size_limit", "0", OnUpdateLongGEZero, item_size_limit)
426429

427430
MEMC_INI_BOOL ("default_consistent_hash", "0", OnUpdateBool, default_behavior.consistent_hash_enabled)
428431
MEMC_INI_BOOL ("default_binary_protocol", "0", OnUpdateBool, default_behavior.binary_protocol_enabled)
@@ -1127,6 +1130,21 @@ zend_string *s_zval_to_payload(php_memc_object_t *intern, zval *value, uint32_t
11271130
return payload;
11281131
}
11291132

1133+
static
1134+
zend_bool s_is_payload_too_big(php_memc_object_t *intern, zend_string *payload)
1135+
{
1136+
php_memc_user_data_t *memc_user_data = memcached_get_user_data(intern->memc);
1137+
1138+
/* An item size limit of 0 implies no limit enforced */
1139+
if (memc_user_data->item_size_limit == 0) {
1140+
return 0;
1141+
}
1142+
if (ZSTR_LEN(payload) > memc_user_data->item_size_limit) {
1143+
return 1;
1144+
}
1145+
return 0;
1146+
}
1147+
11301148
static
11311149
zend_bool s_should_retry_write (php_memc_object_t *intern, memcached_return status)
11321150
{
@@ -1153,6 +1171,12 @@ zend_bool s_memc_write_zval (php_memc_object_t *intern, php_memc_write_op op, ze
11531171
s_memc_set_status(intern, MEMC_RES_PAYLOAD_FAILURE, 0);
11541172
return 0;
11551173
}
1174+
1175+
if (s_is_payload_too_big(intern, payload)) {
1176+
s_memc_set_status(intern, MEMCACHED_E2BIG, 0);
1177+
zend_string_release(payload);
1178+
return 0;
1179+
}
11561180
}
11571181

11581182
#define memc_write_using_fn(fn_name) payload ? fn_name(intern->memc, ZSTR_VAL(key), ZSTR_LEN(key), ZSTR_VAL(payload), ZSTR_LEN(payload), expiration, flags) : MEMC_RES_PAYLOAD_FAILURE;
@@ -1305,6 +1329,7 @@ static PHP_METHOD(Memcached, __construct)
13051329
memc_user_data->encoding_enabled = 0;
13061330
memc_user_data->store_retry_count = MEMC_G(store_retry_count);
13071331
memc_user_data->set_udf_flags = -1;
1332+
memc_user_data->item_size_limit = MEMC_G(item_size_limit);
13081333
memc_user_data->is_persistent = is_persistent;
13091334

13101335
memcached_set_user_data(intern->memc, memc_user_data);
@@ -2145,6 +2170,12 @@ static void php_memc_cas_impl(INTERNAL_FUNCTION_PARAMETERS, zend_bool by_key)
21452170
RETURN_FALSE;
21462171
}
21472172

2173+
if (s_is_payload_too_big(intern, payload)) {
2174+
intern->rescode = MEMCACHED_E2BIG;
2175+
zend_string_release(payload);
2176+
RETURN_FALSE;
2177+
}
2178+
21482179
if (by_key) {
21492180
status = memcached_cas_by_key(intern->memc, ZSTR_VAL(server_key), ZSTR_LEN(server_key), ZSTR_VAL(key), ZSTR_LEN(key), ZSTR_VAL(payload), ZSTR_LEN(payload), expiration, flags, cas);
21502181
} else {
@@ -2970,6 +3001,9 @@ static PHP_METHOD(Memcached, getOption)
29703001
case MEMC_OPT_COMPRESSION:
29713002
RETURN_BOOL(memc_user_data->compression_enabled);
29723003

3004+
case MEMC_OPT_ITEM_SIZE_LIMIT:
3005+
RETURN_LONG(memc_user_data->item_size_limit);
3006+
29733007
case MEMC_OPT_PREFIX_KEY:
29743008
{
29753009
memcached_return retval;
@@ -3041,6 +3075,15 @@ int php_memc_set_option(php_memc_object_t *intern, long option, zval *value)
30413075
}
30423076
break;
30433077

3078+
case MEMC_OPT_ITEM_SIZE_LIMIT:
3079+
lval = zval_get_long(value);
3080+
if (lval < 0) {
3081+
php_error_docref(NULL, E_WARNING, "ITEM_SIZE_LIMIT must be >= 0");
3082+
return 0;
3083+
}
3084+
memc_user_data->item_size_limit = lval;
3085+
break;
3086+
30443087
case MEMC_OPT_PREFIX_KEY:
30453088
{
30463089
zend_string *str;
@@ -4013,6 +4056,7 @@ PHP_GINIT_FUNCTION(php_memcached)
40134056
php_memcached_globals->memc.compression_factor = 1.30;
40144057
php_memcached_globals->memc.compression_level = 3;
40154058
php_memcached_globals->memc.store_retry_count = 2;
4059+
php_memcached_globals->memc.item_size_limit = 0;
40164060

40174061
php_memcached_globals->memc.sasl_initialised = 0;
40184062
php_memcached_globals->no_effect = 0;
@@ -4063,6 +4107,7 @@ static void php_memc_register_constants(INIT_FUNC_ARGS)
40634107

40644108
REGISTER_MEMC_CLASS_CONST_LONG(OPT_USER_FLAGS, MEMC_OPT_USER_FLAGS);
40654109
REGISTER_MEMC_CLASS_CONST_LONG(OPT_STORE_RETRY_COUNT, MEMC_OPT_STORE_RETRY_COUNT);
4110+
REGISTER_MEMC_CLASS_CONST_LONG(OPT_ITEM_SIZE_LIMIT, MEMC_OPT_ITEM_SIZE_LIMIT);
40664111

40674112
/*
40684113
* Indicate whether igbinary serializer is available

‎php_memcached_private.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -186,8 +186,9 @@ ZEND_BEGIN_MODULE_GLOBALS(php_memcached)
186186
char *compression_name;
187187
zend_long compression_threshold;
188188
double compression_factor;
189-
zend_long store_retry_count;
190-
zend_long compression_level;
189+
zend_long store_retry_count;
190+
zend_long compression_level;
191+
zend_long item_size_limit;
191192

192193
/* Converted values*/
193194
php_memc_serializer_type serializer_type;

‎tests/cas_e2big.phpt

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
--TEST--
2+
set data exceeding size limit
3+
--SKIPIF--
4+
<?php include "skipif.inc";?>
5+
--FILE--
6+
<?php
7+
include dirname (__FILE__) . '/config.inc';
8+
$m = memc_get_instance (array (
9+
Memcached::OPT_ITEM_SIZE_LIMIT => 100,
10+
));
11+
12+
$m->delete('cas_e2big_test');
13+
14+
$m->set('cas_e2big_test', 'hello');
15+
$result = $m->get('cas_e2big_test', null, Memcached::GET_EXTENDED);
16+
var_dump(is_array($result) && isset($result['cas']) && isset($result['value']) && $result['value'] == 'hello');
17+
18+
$value = str_repeat('a large payload', 1024 * 1024);
19+
20+
var_dump($m->cas($result['cas'], 'cas_e2big_test', $value, 360));
21+
var_dump($m->getResultCode() == Memcached::RES_E2BIG);
22+
var_dump($m->getResultMessage() == 'ITEM TOO BIG');
23+
var_dump($m->get('cas_e2big_test') == 'hello');
24+
var_dump($m->getResultCode() == Memcached::RES_SUCCESS);
25+
?>
26+
--EXPECT--
27+
bool(true)
28+
bool(false)
29+
bool(true)
30+
bool(true)
31+
bool(true)
32+
bool(true)

‎tests/options.phpt

+32
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,26 @@ var_dump($m->getOption(Memcached::OPT_COMPRESSION_TYPE) == Memcached::COMPRESSIO
2626

2727
var_dump($m->setOption(Memcached::OPT_COMPRESSION_TYPE, 0));
2828
var_dump($m->getOption(Memcached::OPT_COMPRESSION_TYPE) == Memcached::COMPRESSION_FASTLZ);
29+
30+
echo "item_size_limit setOption\n";
31+
var_dump($m->setOption(Memcached::OPT_ITEM_SIZE_LIMIT, 0));
32+
var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) === 0);
33+
var_dump($m->setOption(Memcached::OPT_ITEM_SIZE_LIMIT, -1));
34+
var_dump($m->setOption(Memcached::OPT_ITEM_SIZE_LIMIT, 1000000));
35+
var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) == 1000000);
36+
37+
echo "item_size_limit ini\n";
38+
ini_set('memcached.item_size_limit', '0');
39+
$m = new Memcached();
40+
var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) === 0);
41+
42+
ini_set('memcached.item_size_limit', '1000000');
43+
$m = new Memcached();
44+
var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) == 1000000);
45+
46+
ini_set('memcached.item_size_limit', null);
47+
$m = new Memcached();
48+
var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) === 0);
2949
?>
3050
--EXPECTF--
3151
bool(true)
@@ -41,3 +61,15 @@ bool(true)
4161
bool(true)
4262
bool(false)
4363
bool(true)
64+
item_size_limit setOption
65+
bool(true)
66+
bool(true)
67+
68+
Warning: Memcached::setOption(): ITEM_SIZE_LIMIT must be >= 0 in %s on line %d
69+
bool(false)
70+
bool(true)
71+
bool(true)
72+
item_size_limit ini
73+
bool(true)
74+
bool(true)
75+
bool(true)

‎tests/set_large.phpt

+3-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ set large data
55
--FILE--
66
<?php
77
include dirname (__FILE__) . '/config.inc';
8-
$m = memc_get_instance ();
8+
$m = memc_get_instance (array (
9+
Memcached::OPT_ITEM_SIZE_LIMIT => 0,
10+
));
911

1012
$key = 'foobarbazDEADC0DE';
1113
$value = str_repeat("foo bar", 1024 * 1024);

‎tests/set_large_e2big.phpt

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
--TEST--
2+
set data exceeding size limit
3+
--SKIPIF--
4+
<?php include "skipif.inc";?>
5+
--FILE--
6+
<?php
7+
include dirname (__FILE__) . '/config.inc';
8+
$m = memc_get_instance (array (
9+
Memcached::OPT_ITEM_SIZE_LIMIT => 100,
10+
));
11+
12+
$m->delete('set_large_e2big_test');
13+
14+
$value = str_repeat('a large payload', 1024 * 1024);
15+
16+
var_dump($m->set('set_large_e2big_test', $value));
17+
var_dump($m->getResultCode() == Memcached::RES_E2BIG);
18+
var_dump($m->getResultMessage() == 'ITEM TOO BIG');
19+
var_dump($m->get('set_large_e2big_test') === false);
20+
var_dump($m->getResultCode() == Memcached::RES_NOTFOUND);
21+
?>
22+
--EXPECT--
23+
bool(false)
24+
bool(true)
25+
bool(true)
26+
bool(true)
27+
bool(true)

‎tests/setoptions.phpt

+3
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,14 @@ var_dump($m->setOptions(array(
1313
Memcached::OPT_COMPRESSION => 0,
1414
Memcached::OPT_LIBKETAMA_COMPATIBLE => 1,
1515
Memcached::OPT_CONNECT_TIMEOUT => 5000,
16+
Memcached::OPT_ITEM_SIZE_LIMIT => 1000000,
1617
)));
1718

1819
var_dump($m->getOption(Memcached::OPT_PREFIX_KEY) == 'a_prefix');
1920
var_dump($m->getOption(Memcached::OPT_SERIALIZER) == Memcached::SERIALIZER_PHP);
2021
var_dump($m->getOption(Memcached::OPT_COMPRESSION) == 0);
2122
var_dump($m->getOption(Memcached::OPT_LIBKETAMA_COMPATIBLE) == 1);
23+
var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) == 1000000);
2224

2325
echo "test invalid options\n";
2426

@@ -36,6 +38,7 @@ bool(true)
3638
bool(true)
3739
bool(true)
3840
bool(true)
41+
bool(true)
3942
test invalid options
4043

4144
Warning: Memcached::setOptions(): invalid configuration option in %s on line %d

0 commit comments

Comments
 (0)