Skip to content

Compression conditions adjustments #303

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 25, 2017
Merged

Compression conditions adjustments #303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7283b11
test: compression edge case verification (#256)
dictcp Jun 10, 2016
c85f34c
Make sure that s_compress_value() will always leave a valid payload, …
sodabrew Jan 25, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ Tests
<file role='test' name='check_if_persistent.phpt'/>
<file role='test' name='check_if_pristine.phpt'/>
<file role='test' name='clone.phpt'/>
<file role='test' name='compression_conditions.phpt'/>
<file role='test' name='compression_types.phpt'/>
<file role='test' name='conf_persist.phpt'/>
<file role='test' name='construct.phpt'/>
Expand Down
53 changes: 27 additions & 26 deletions php_memcached.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -830,6 +830,7 @@ zend_bool s_compress_value (php_memc_compression_type compression_type, zend_str
/* status */
zend_bool compress_status = 0;
zend_string *payload = *payload_in;
uint32_t compression_type_flag = 0;

/* Additional 5% for the data */
size_t buffer_size = (size_t) (((double) ZSTR_LEN(payload) * 1.05) + 1.0);
Expand All @@ -847,7 +848,7 @@ zend_bool s_compress_value (php_memc_compression_type compression_type, zend_str

if (compressed_size > 0) {
compress_status = 1;
MEMC_VAL_SET_FLAG(*flags, MEMC_VAL_COMPRESSION_FASTLZ);
compression_type_flag = MEMC_VAL_COMPRESSION_FASTLZ;
}
}
break;
Expand All @@ -859,7 +860,7 @@ zend_bool s_compress_value (php_memc_compression_type compression_type, zend_str

if (status == Z_OK) {
compress_status = 1;
MEMC_VAL_SET_FLAG(*flags, MEMC_VAL_COMPRESSION_ZLIB);
compression_type_flag = MEMC_VAL_COMPRESSION_ZLIB;
}
}
break;
Expand All @@ -869,31 +870,29 @@ zend_bool s_compress_value (php_memc_compression_type compression_type, zend_str
break;
}

if (!compress_status) {
php_error_docref(NULL, E_WARNING, "could not compress value");
efree (buffer);
return 0;
}

/* This means the value was too small to be compressed, still a success */
/* This means the value was too small to be compressed and ended up larger */
if (ZSTR_LEN(payload) <= (compressed_size * MEMC_G(compression_factor))) {
MEMC_VAL_DEL_FLAG(*flags, MEMC_VAL_COMPRESSION_FASTLZ | MEMC_VAL_COMPRESSION_ZLIB);
efree (buffer);
return 1;
compress_status = 0;
}

MEMC_VAL_SET_FLAG(*flags, MEMC_VAL_COMPRESSED);
/* Replace the payload with the compressed copy */
if (compress_status) {
MEMC_VAL_SET_FLAG(*flags, MEMC_VAL_COMPRESSED | compression_type_flag);
payload = zend_string_realloc(payload, compressed_size + sizeof(uint32_t), 0);

payload = zend_string_realloc(payload, compressed_size + sizeof(uint32_t), 0);
/* Copy the uin32_t at the beginning */
memcpy(ZSTR_VAL(payload), &original_size, sizeof(uint32_t));
memcpy(ZSTR_VAL(payload) + sizeof (uint32_t), buffer, compressed_size);
efree(buffer);

/* Copy the uin32_t at the beginning */
memcpy(ZSTR_VAL(payload), &original_size, sizeof(uint32_t));
memcpy(ZSTR_VAL(payload) + sizeof (uint32_t), buffer, compressed_size);
efree(buffer);
zend_string_forget_hash_val(payload);
*payload_in = payload;

zend_string_forget_hash_val(payload);
*payload_in = payload;
return 1;
return 1;
}

/* Original payload was not modified */
return 0;
}

static
Expand Down Expand Up @@ -1043,11 +1042,13 @@ zend_string *s_zval_to_payload(php_memc_object_t *intern, zval *value, uint32_t

/* If we have compression flag, compress the value */
if (should_compress) {
/* status */
if (!s_compress_value (memc_user_data->compression_type, &payload, flags)) {
zend_string_release(payload);
return NULL;
}
/* s_compress_value() will always leave a valid payload, even if that payload
* did not actually get compressed. The flags will be set according to the
* to the compression type or no compression.
*
* No need to check the return value because the payload is always valid.
*/
(void)s_compress_value (memc_user_data->compression_type, &payload, flags);
}

if (memc_user_data->set_udf_flags >= 0) {
Expand Down
123 changes: 123 additions & 0 deletions tests/compression_conditions.phpt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
--TEST--
Memcached compression test
--SKIPIF--
<?php if (!extension_loaded("memcached")) print "skip"; ?>
--FILE--
<?php
include dirname (__FILE__) . '/config.inc';
$m = memc_get_instance ();

$short_data = "abcdefg";
$data = file_get_contents(dirname(__FILE__) . '/testdata.res');

set_error_handler(function($errno, $errstr, $errfile, $errline, array $errcontext) {
echo "$errstr\n";
return true;
}, E_WARNING);

function get_compression($name) {
switch (strtolower($name)) {
case 'zlib':
return Memcached::COMPRESSION_ZLIB;
case 'fastlz':
return Memcached::COMPRESSION_FASTLZ;
default:
echo "Strange compression type: $name\n";
return 0;
}
}

function fetch_with_compression($m, $key, $value, $set_compression = '', $factor = 1.3, $threshold = 2000) {
ini_set("memcached.compression_factor", $factor);
ini_set("memcached.compression_threshold", $threshold);

$len=strlen($value);
echo "len=[$len] set=[$set_compression] factor=[$factor] threshold=[$threshold]\n";

if (!$set_compression) {
$m->setOption(Memcached::OPT_COMPRESSION, false);
} else {
$m->setOption(Memcached::OPT_COMPRESSION, true);
$m->setOption(Memcached::OPT_COMPRESSION_TYPE, get_compression($set_compression));
}

$m->set($key, $value, 1800);

$value_back = $m->get($key);
var_dump($value === $value_back);
}

fetch_with_compression($m, 'hello01', $data, 'zlib', 1.3, 4);
fetch_with_compression($m, 'hello02', $data, 'fastlz', 1.3, 4);
fetch_with_compression($m, 'hello03', $data, '', 1.3, 4);
fetch_with_compression($m, 'hello04', $short_data, 'zlib', 1.3, 4);
fetch_with_compression($m, 'hello05', $short_data, 'fastlz', 1.3, 4);
fetch_with_compression($m, 'hello06', $short_data, '', 1.3, 4);
fetch_with_compression($m, 'hello11', $data, 'zlib', 0.3, 4);
fetch_with_compression($m, 'hello12', $data, 'fastlz', 0.3, 4);
fetch_with_compression($m, 'hello13', $data, '', 0.3, 4);
fetch_with_compression($m, 'hello14', $short_data, 'zlib', 0.3, 4);
fetch_with_compression($m, 'hello15', $short_data, 'fastlz', 0.3, 4);
fetch_with_compression($m, 'hello16', $short_data, '', 0.3, 4);
fetch_with_compression($m, 'hello21', $data, 'zlib', 1.3, 2000);
fetch_with_compression($m, 'hello22', $data, 'fastlz', 1.3, 2000);
fetch_with_compression($m, 'hello23', $data, '', 1.3, 2000);
fetch_with_compression($m, 'hello24', $short_data, 'zlib', 1.3, 2000);
fetch_with_compression($m, 'hello25', $short_data, 'fastlz', 1.3, 2000);
fetch_with_compression($m, 'hello26', $short_data, '', 1.3, 2000);
fetch_with_compression($m, 'hello31', $data, 'zlib', 0.3, 2000);
fetch_with_compression($m, 'hello32', $data, 'fastlz', 0.3, 2000);
fetch_with_compression($m, 'hello33', $data, '', 0.3, 2000);
fetch_with_compression($m, 'hello34', $short_data, 'zlib', 0.3, 2000);
fetch_with_compression($m, 'hello35', $short_data, 'fastlz', 0.3, 2000);
fetch_with_compression($m, 'hello36', $short_data, '', 0.3, 2000);
?>
--EXPECT--
len=[4877] set=[zlib] factor=[1.3] threshold=[4]
bool(true)
len=[4877] set=[fastlz] factor=[1.3] threshold=[4]
bool(true)
len=[4877] set=[] factor=[1.3] threshold=[4]
bool(true)
len=[7] set=[zlib] factor=[1.3] threshold=[4]
bool(true)
len=[7] set=[fastlz] factor=[1.3] threshold=[4]
bool(true)
len=[7] set=[] factor=[1.3] threshold=[4]
bool(true)
len=[4877] set=[zlib] factor=[0.3] threshold=[4]
bool(true)
len=[4877] set=[fastlz] factor=[0.3] threshold=[4]
bool(true)
len=[4877] set=[] factor=[0.3] threshold=[4]
bool(true)
len=[7] set=[zlib] factor=[0.3] threshold=[4]
bool(true)
len=[7] set=[fastlz] factor=[0.3] threshold=[4]
bool(true)
len=[7] set=[] factor=[0.3] threshold=[4]
bool(true)
len=[4877] set=[zlib] factor=[1.3] threshold=[2000]
bool(true)
len=[4877] set=[fastlz] factor=[1.3] threshold=[2000]
bool(true)
len=[4877] set=[] factor=[1.3] threshold=[2000]
bool(true)
len=[7] set=[zlib] factor=[1.3] threshold=[2000]
bool(true)
len=[7] set=[fastlz] factor=[1.3] threshold=[2000]
bool(true)
len=[7] set=[] factor=[1.3] threshold=[2000]
bool(true)
len=[4877] set=[zlib] factor=[0.3] threshold=[2000]
bool(true)
len=[4877] set=[fastlz] factor=[0.3] threshold=[2000]
bool(true)
len=[4877] set=[] factor=[0.3] threshold=[2000]
bool(true)
len=[7] set=[zlib] factor=[0.3] threshold=[2000]
bool(true)
len=[7] set=[fastlz] factor=[0.3] threshold=[2000]
bool(true)
len=[7] set=[] factor=[0.3] threshold=[2000]
bool(true)