001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.commons.compress.compressors.lz4;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.util.Arrays;
024
025import org.apache.commons.compress.compressors.CompressorInputStream;
026import org.apache.commons.compress.utils.BoundedInputStream;
027import org.apache.commons.compress.utils.ByteUtils;
028import org.apache.commons.compress.utils.ChecksumCalculatingInputStream;
029import org.apache.commons.compress.utils.CountingInputStream;
030import org.apache.commons.compress.utils.IOUtils;
031import org.apache.commons.compress.utils.InputStreamStatistics;
032
033/**
034 * CompressorInputStream for the LZ4 frame format.
035 *
036 * <p>Based on the "spec" in the version "1.5.1 (31/03/2015)"</p>
037 *
038 * @see <a href="http://lz4.github.io/lz4/lz4_Frame_format.html">LZ4 Frame Format Description</a>
039 * @since 1.14
040 * @NotThreadSafe
041 */
042public class FramedLZ4CompressorInputStream extends CompressorInputStream
043    implements InputStreamStatistics {
044
045    // used by FramedLZ4CompressorOutputStream as well
046    static final byte[] LZ4_SIGNATURE = new byte[] { //NOSONAR
047        4, 0x22, 0x4d, 0x18
048    };
049    private static final byte[] SKIPPABLE_FRAME_TRAILER = new byte[] {
050        0x2a, 0x4d, 0x18
051    };
052    private static final byte SKIPPABLE_FRAME_PREFIX_BYTE_MASK = 0x50;
053
054    static final int VERSION_MASK = 0xC0;
055    static final int SUPPORTED_VERSION = 0x40;
056    static final int BLOCK_INDEPENDENCE_MASK = 0x20;
057    static final int BLOCK_CHECKSUM_MASK = 0x10;
058    static final int CONTENT_SIZE_MASK = 0x08;
059    static final int CONTENT_CHECKSUM_MASK = 0x04;
060    static final int BLOCK_MAX_SIZE_MASK = 0x70;
061    static final int UNCOMPRESSED_FLAG_MASK = 0x80000000;
062
063    // used in no-arg read method
064    private final byte[] oneByte = new byte[1];
065
066    private final ByteUtils.ByteSupplier supplier = new ByteUtils.ByteSupplier() {
067        @Override
068        public int getAsByte() throws IOException {
069            return readOneByte();
070        }
071    };
072
073    private final CountingInputStream in;
074    private final boolean decompressConcatenated;
075
076    private boolean expectBlockChecksum;
077    private boolean expectBlockDependency;
078    private boolean expectContentSize;
079    private boolean expectContentChecksum;
080
081    private InputStream currentBlock;
082    private boolean endReached, inUncompressed;
083
084    // used for frame header checksum and content checksum, if present
085    private final XXHash32 contentHash = new XXHash32();
086
087    // used for block checksum, if present
088    private final XXHash32 blockHash = new XXHash32();
089
090    // only created if the frame doesn't set the block independence flag
091    private byte[] blockDependencyBuffer;
092
093    /**
094     * Creates a new input stream that decompresses streams compressed
095     * using the LZ4 frame format and stops after decompressing the
096     * first frame.
097     * @param in  the InputStream from which to read the compressed data
098     * @throws IOException if reading fails
099     */
100    public FramedLZ4CompressorInputStream(InputStream in) throws IOException {
101        this(in, false);
102    }
103
104    /**
105     * Creates a new input stream that decompresses streams compressed
106     * using the LZ4 frame format.
107     * @param in  the InputStream from which to read the compressed data
108     * @param decompressConcatenated if true, decompress until the end
109     *          of the input; if false, stop after the first LZ4 frame
110     *          and leave the input position to point to the next byte
111     *          after the frame stream
112     * @throws IOException if reading fails
113     */
114    public FramedLZ4CompressorInputStream(InputStream in, boolean decompressConcatenated) throws IOException {
115        this.in = new CountingInputStream(in);
116        this.decompressConcatenated = decompressConcatenated;
117        init(true);
118    }
119
120    /** {@inheritDoc} */
121    @Override
122    public int read() throws IOException {
123        return read(oneByte, 0, 1) == -1 ? -1 : oneByte[0] & 0xFF;
124    }
125
126    /** {@inheritDoc} */
127    @Override
128    public void close() throws IOException {
129        try {
130            if (currentBlock != null) {
131                currentBlock.close();
132                currentBlock = null;
133            }
134        } finally {
135            in.close();
136        }
137    }
138
139    /** {@inheritDoc} */
140    @Override
141    public int read(final byte[] b, final int off, final int len) throws IOException {
142        if (endReached) {
143            return -1;
144        }
145        int r = readOnce(b, off, len);
146        if (r == -1) {
147            nextBlock();
148            if (!endReached) {
149                r = readOnce(b, off, len);
150            }
151        }
152        if (r != -1) {
153            if (expectBlockDependency) {
154                appendToBlockDependencyBuffer(b, off, r);
155            }
156            if (expectContentChecksum) {
157                contentHash.update(b, off, r);
158            }
159        }
160        return r;
161    }
162
163    /**
164     * @since 1.17
165     */
166    @Override
167    public long getCompressedCount() {
168        return in.getBytesRead();
169    }
170
171    private void init(boolean firstFrame) throws IOException {
172        if (readSignature(firstFrame)) {
173            readFrameDescriptor();
174            nextBlock();
175        }
176    }
177
178    private boolean readSignature(boolean firstFrame) throws IOException {
179        String garbageMessage = firstFrame ? "Not a LZ4 frame stream" : "LZ4 frame stream followed by garbage";
180        final byte[] b = new byte[4];
181        int read = IOUtils.readFully(in, b);
182        count(read);
183        if (0 == read && !firstFrame) {
184            // good LZ4 frame and nothing after it
185            endReached = true;
186            return false;
187        }
188        if (4 != read) {
189            throw new IOException(garbageMessage);
190        }
191
192        read = skipSkippableFrame(b);
193        if (0 == read && !firstFrame) {
194            // good LZ4 frame with only some skippable frames after it
195            endReached = true;
196            return false;
197        }
198        if (4 != read || !matches(b, 4)) {
199            throw new IOException(garbageMessage);
200        }
201        return true;
202    }
203
204    private void readFrameDescriptor() throws IOException {
205        int flags = readOneByte();
206        if (flags == -1) {
207            throw new IOException("Premature end of stream while reading frame flags");
208        }
209        contentHash.update(flags);
210        if ((flags & VERSION_MASK) != SUPPORTED_VERSION) {
211            throw new IOException("Unsupported version " + (flags >> 6));
212        }
213        expectBlockDependency = (flags & BLOCK_INDEPENDENCE_MASK) == 0;
214        if (expectBlockDependency) {
215            if (blockDependencyBuffer == null) {
216                blockDependencyBuffer = new byte[BlockLZ4CompressorInputStream.WINDOW_SIZE];
217            }
218        } else {
219            blockDependencyBuffer = null;
220        }
221        expectBlockChecksum = (flags & BLOCK_CHECKSUM_MASK) != 0;
222        expectContentSize = (flags & CONTENT_SIZE_MASK) != 0;
223        expectContentChecksum = (flags & CONTENT_CHECKSUM_MASK) != 0;
224        int bdByte = readOneByte();
225        if (bdByte == -1) { // max size is irrelevant for this implementation
226            throw new IOException("Premature end of stream while reading frame BD byte");
227        }
228        contentHash.update(bdByte);
229        if (expectContentSize) { // for now we don't care, contains the uncompressed size
230            byte[] contentSize = new byte[8];
231            int skipped = IOUtils.readFully(in, contentSize);
232            count(skipped);
233            if (8 != skipped) {
234                throw new IOException("Premature end of stream while reading content size");
235            }
236            contentHash.update(contentSize, 0, contentSize.length);
237        }
238        int headerHash = readOneByte();
239        if (headerHash == -1) { // partial hash of header.
240            throw new IOException("Premature end of stream while reading frame header checksum");
241        }
242        int expectedHash = (int) ((contentHash.getValue() >> 8) & 0xff);
243        contentHash.reset();
244        if (headerHash != expectedHash) {
245            throw new IOException("Frame header checksum mismatch");
246        }
247    }
248
249    private void nextBlock() throws IOException {
250        maybeFinishCurrentBlock();
251        long len = ByteUtils.fromLittleEndian(supplier, 4);
252        boolean uncompressed = (len & UNCOMPRESSED_FLAG_MASK) != 0;
253        int realLen = (int) (len & (~UNCOMPRESSED_FLAG_MASK));
254        if (realLen < 0) {
255            throw new IOException("Found illegal block with negative size");
256        }
257        if (realLen == 0) {
258            verifyContentChecksum();
259            if (!decompressConcatenated) {
260                endReached = true;
261            } else {
262                init(false);
263            }
264            return;
265        }
266        InputStream capped = new BoundedInputStream(in, realLen);
267        if (expectBlockChecksum) {
268            capped = new ChecksumCalculatingInputStream(blockHash, capped);
269        }
270        if (uncompressed) {
271            inUncompressed = true;
272            currentBlock = capped;
273        } else {
274            inUncompressed = false;
275            BlockLZ4CompressorInputStream s = new BlockLZ4CompressorInputStream(capped);
276            if (expectBlockDependency) {
277                s.prefill(blockDependencyBuffer);
278            }
279            currentBlock = s;
280        }
281    }
282
283    private void maybeFinishCurrentBlock() throws IOException {
284        if (currentBlock != null) {
285            currentBlock.close();
286            currentBlock = null;
287            if (expectBlockChecksum) {
288                verifyChecksum(blockHash, "block");
289                blockHash.reset();
290            }
291        }
292    }
293
294    private void verifyContentChecksum() throws IOException {
295        if (expectContentChecksum) {
296            verifyChecksum(contentHash, "content");
297        }
298        contentHash.reset();
299    }
300
301    private void verifyChecksum(XXHash32 hash, String kind) throws IOException {
302        byte[] checksum = new byte[4];
303        int read = IOUtils.readFully(in, checksum);
304        count(read);
305        if (4 != read) {
306            throw new IOException("Premature end of stream while reading " + kind + " checksum");
307        }
308        long expectedHash = hash.getValue();
309        if (expectedHash != ByteUtils.fromLittleEndian(checksum)) {
310            throw new IOException(kind + " checksum mismatch.");
311        }
312    }
313
314    private int readOneByte() throws IOException {
315        final int b = in.read();
316        if (b != -1) {
317            count(1);
318            return b & 0xFF;
319        }
320        return -1;
321    }
322
323    private int readOnce(byte[] b, int off, int len) throws IOException {
324        if (inUncompressed) {
325            int cnt = currentBlock.read(b, off, len);
326            count(cnt);
327            return cnt;
328        }
329        BlockLZ4CompressorInputStream l = (BlockLZ4CompressorInputStream) currentBlock;
330        long before = l.getBytesRead();
331        int cnt = currentBlock.read(b, off, len);
332        count(l.getBytesRead() - before);
333        return cnt;
334    }
335
336    private static boolean isSkippableFrameSignature(byte[] b) {
337        if ((b[0] & SKIPPABLE_FRAME_PREFIX_BYTE_MASK) != SKIPPABLE_FRAME_PREFIX_BYTE_MASK) {
338            return false;
339        }
340        for (int i = 1; i < 4; i++) {
341            if (b[i] != SKIPPABLE_FRAME_TRAILER[i - 1]) {
342                return false;
343            }
344        }
345        return true;
346    }
347
348    /**
349     * Skips over the contents of a skippable frame as well as
350     * skippable frames following it.
351     *
352     * <p>It then tries to read four more bytes which are supposed to
353     * hold an LZ4 signature and returns the number of bytes read
354     * while storing the bytes in the given array.</p>
355     */
356    private int skipSkippableFrame(byte[] b) throws IOException {
357        int read = 4;
358        while (read == 4 && isSkippableFrameSignature(b)) {
359            final long len = ByteUtils.fromLittleEndian(supplier, 4);
360            if (len < 0) {
361                throw new IOException("Found illegal skippable frame with negative size");
362            }
363            long skipped = IOUtils.skip(in, len);
364            count(skipped);
365            if (len != skipped) {
366                throw new IOException("Premature end of stream while skipping frame");
367            }
368            read = IOUtils.readFully(in, b);
369            count(read);
370        }
371        return read;
372    }
373
374    private void appendToBlockDependencyBuffer(final byte[] b, final int off, int len) {
375        len = Math.min(len, blockDependencyBuffer.length);
376        if (len > 0) {
377            int keep = blockDependencyBuffer.length - len;
378            if (keep > 0) {
379                // move last keep bytes towards the start of the buffer
380                System.arraycopy(blockDependencyBuffer, len, blockDependencyBuffer, 0, keep);
381            }
382            // append new data
383            System.arraycopy(b, off, blockDependencyBuffer, keep, len);
384        }
385    }
386
387    /**
388     * Checks if the signature matches what is expected for a .lz4 file.
389     *
390     * <p>.lz4 files start with a four byte signature.</p>
391     *
392     * @param signature the bytes to check
393     * @param length    the number of bytes to check
394     * @return          true if this is a .sz stream, false otherwise
395     */
396    public static boolean matches(final byte[] signature, final int length) {
397
398        if (length < LZ4_SIGNATURE.length) {
399            return false;
400        }
401
402        byte[] shortenedSig = signature;
403        if (signature.length > LZ4_SIGNATURE.length) {
404            shortenedSig = new byte[LZ4_SIGNATURE.length];
405            System.arraycopy(signature, 0, shortenedSig, 0, LZ4_SIGNATURE.length);
406        }
407
408        return Arrays.equals(shortenedSig, LZ4_SIGNATURE);
409    }
410}