Merge pull request #284 from ptweezy/wbits-format-detection

Fix format detection in the `isal_zlib.decompressobj()` function with a `wbits` of 40 to 47

Author: rhpvorderman

CHANGELOG.rst

@@ -48,6 +48,13 @@ version 1.8.1-dev
   input buffer when ``max_length`` split the end of a stream. At the end of a
   stream, ``isal_zlib.Decompress.unconsumed_tail`` is empty, matching the
   CPython fix for gh-158169.
++ Fix ``isal_zlib.decompressobj`` with a ``wbits`` of 40 to 47, which detects
+  zlib or gzip data. It failed on gzip data when the first ``decompress()``
+  call got fewer than 2 bytes, for example, when the data arrived byte by
+  byte.
++ Fix ``isal_zlib.decompressobj`` with a ``wbits`` of 40 to 47 after a
+  ``flush()`` call that came before any data. It then failed on both zlib and
+  gzip data.
 + ``igzip_threaded.open`` raises ``ValueError`` for a ``block_size`` of about
   3.6 GiB or more and ``TypeError`` for a ``compresslevel`` that is not an
   integer. Both raised ``SystemError`` before.

src/isal/isal_zlibmodule.c

@@ -149,10 +149,13 @@ static inline int zlib_mem_level_to_isal(int mem_level) {
     return ZLIB_MEM_LEVEL_TO_ISAL[mem_level];
 }
 
+/* A gzip stream starts with 31, 139. A zlib stream cannot start with 31,
+   because the low 4 bits of its first byte are the compression method 8.
+   So the first byte is enough, even when the data arrives byte by byte. */
 static inline int
 data_is_gzip(Py_buffer *data){
     uint8_t *buf = (uint8_t *)data->buf;
-    return (data->len > 1) && (buf[0] == 31) && (buf[1] == 139);
+    return (data->len > 0) && (buf[0] == 31);
 }
 
 
@@ -1010,6 +1013,11 @@ isal_zlib_Decompress_decompress_impl(decompobject *self, Py_buffer *data,
     ENTER_ZLIB(self);
 
     if (!self->method_set) {
+        if (data->len == 0) {
+            /* Detect the format when the first byte arrives. */
+            LEAVE_ZLIB(self);
+            return PyBytes_FromStringAndSize(NULL, 0);
+        }
         if (data_is_gzip(data)){
             self->zst.crc_flag = ISAL_GZIP;
         }
@@ -1181,6 +1189,14 @@ isal_zlib_Decompress_flush_impl(decompobject *self, Py_ssize_t length)
 
     ENTER_ZLIB(self);
 
+    if (!self->method_set) {
+        /* No data yet, so the format is unknown. isal_inflate would start a
+           raw deflate block, and then decode the zlib or gzip header that
+           comes next as deflate data. */
+        LEAVE_ZLIB(self);
+        return PyBytes_FromStringAndSize(NULL, 0);
+    }
+
     if (PyObject_GetBuffer(self->unconsumed_tail, &data, PyBUF_SIMPLE) == -1) {
         LEAVE_ZLIB(self);
         return NULL;

tests/test_compat.py

@@ -263,6 +263,52 @@ def test_unused_data(unused_size, wbits):
     assert decompressor.unused_data == unused_data
 
 
+@pytest.mark.parametrize("wbits", [15, 31])
+def test_decompressobj_detects_format_byte_by_byte(wbits):
+    data = DATA[:1000]
+    compressor = zlib.compressobj(wbits=wbits)
+    compressed = compressor.compress(data) + compressor.flush()
+    decompressor = isal_zlib.decompressobj(wbits=47)
+    result = b"".join(decompressor.decompress(compressed[i:i + 1])
+                      for i in range(len(compressed)))
+    assert result == data
+    assert decompressor.eof
+
+
+@pytest.mark.parametrize("wbits", [15, 31])
+def test_decompressobj_detects_format_after_empty_input(wbits):
+    data = DATA[:1000]
+    compressor = zlib.compressobj(wbits=wbits)
+    compressed = compressor.compress(data) + compressor.flush()
+    for module in (zlib, isal_zlib):
+        decompressor = module.decompressobj(wbits=47)
+        assert decompressor.decompress(b"") == b""
+        assert decompressor.decompress(compressed[:1]) == b""
+        assert decompressor.decompress(compressed[1:] + b"unused") == data
+        assert decompressor.eof
+        assert decompressor.unused_data == b"unused"
+
+
+@pytest.mark.parametrize("wbits", [15, 31])
+def test_decompressobj_detects_format_after_flush(wbits):
+    data = DATA[:1000]
+    compressor = zlib.compressobj(wbits=wbits)
+    compressed = compressor.compress(data) + compressor.flush()
+    for module in (zlib, isal_zlib):
+        decompressor = module.decompressobj(wbits=47)
+        assert decompressor.flush() == b""
+        assert decompressor.decompress(compressed) == data
+        assert decompressor.eof
+
+
+@pytest.mark.parametrize("data", [b"", b"\x1f", b"\x78"])
+def test_decompress_detect_format_incomplete(data):
+    with pytest.raises(zlib.error):
+        zlib.decompress(data, wbits=47)
+    with pytest.raises(isal_zlib.error):
+        isal_zlib.decompress(data, wbits=47)
+
+
 @pytest.mark.parametrize("wbits", [-15, 15, 31])
 def test_decompressobj_unconsumed_tail_after_eof(wbits):
     source = b"abcdefghijklmnopqrstuvwxyz"