Require a flag to clobber destination files.

.travis.yml

@@ -7,4 +7,4 @@ compiler:
 os:
 - linux
 - osx
-script: sh autogen.sh && ./configure --enable-custom-allocator && make -j4 && make check && make clean && ./configure --disable-advanced-jpeg-features --enable-custom-allocator  && make -j4 && make check && make clean && CXXFLAGS="-O2 -DALLOW_3_OR_4_SCALING_FACTOR -DALLOW_FOUR_COLORS" ./configure && make -j4 && make check && make clean && ./configure --disable-vectorization --enable-custom-allocator  && make -j4 && make check && make clean && ./configure --enable-ans-experimental && make -j4 && make check
+script: sh autogen.sh && ./configure CXXFLAGS="-O2 -g -mssse3" --enable-custom-allocator && make liblocalzlib.a && make -j4 && make check && make clean && ./configure CXXFLAGS="-O2 -g -mssse3" --disable-advanced-jpeg-features --enable-custom-allocator && make liblocalzlib.a && make -j4 && make check && make clean && CXXFLAGS="-O2 -mssse3 -DALLOW_3_OR_4_SCALING_FACTOR -DALLOW_FOUR_COLORS" ./configure CXXFLAGS="-O2 -mssse3 -g" && make liblocalzlib.a && make -j4 && make check && make clean && ./configure CXXFLAGS="-O2 -mssse3 -g" --disable-vectorization --enable-custom-allocator  && make liblocalzlib.a && make -j4 && make check && make clean && ./configure CXXFLAGS="-O2 -mssse3 -g" --enable-ans-experimental && make liblocalzlib.a && make -j4 && make check

Makefile.am

@@ -360,9 +360,9 @@ test_suite_test_trailing_rst_SOURCES = test_suite/test_harness.cc
 test_suite_test_trailing_rst_CXXFLAGS = $(AM_CXXFLAGS) -DUSE_LEPTON -DTEST_FILE=trailingrst -DTEST_FILE0=trailingrst2
 test_suite_test_trailing_rst_LDADD = libtestdriver.a -lpthread
 
-TESTS = test_suite/test_recode_memory_bound test_suite/test_invariants test_suite/test_baseline_ujg test_suite/test_baseline test_suite/test_misc test_suite/test_iphone test_suite/test_phone_outdoor test_suite/test_truncate test_suite/test_single_row_truncate test_suite/test_android_lowmem test_suite/test_SLR test_suite/test_progressive_ujg test_suite/test_progressive_disallowed test_suite/test_progressive test_suite/test_arithmetic_failfast test_suite/test_hq test_suite/forktester.py test_suite/sockettester.py src/lepton/test_custom_table.sh test_suite/test_baseline_unjailed test_suite/test_baseline_unjailed_thread test_suite/test_baseline_unjailed_decode test_suite/test_baseline_unjailed_decode_thread test_suite/test_seccomp_encode_main test_suite/test_seccomp_encode_thread  test_suite/test_seccomp_decode_main test_suite/test_seccomp_decode_thread test_suite/test_truncate_lowmem test_suite/test_nofsync test_suite/test_colorswap test_suite/test_odd_rst test_suite/test_trailing_header test_suite/test_trailing_rst test_suite/test_legacy.sh test_suite/test_roundtrip.sh test_suite/test_embedded.sh test_suite/test_16threads.sh test_suite/test_future_compat.sh test_suite/test_gray2sf test_suite/test_2nd_block.sh test_suite/test_3rd_block.sh test_suite/test_last_block.sh test_suite/test_truncated_zero_run test_suite/test_bad_zero_run test_suite/test_concat.sh test_suite/test_permissive.sh
+TESTS = test_suite/test_recode_memory_bound test_suite/test_invariants test_suite/test_baseline_ujg test_suite/test_baseline test_suite/test_misc test_suite/test_iphone test_suite/test_phone_outdoor test_suite/test_truncate test_suite/test_single_row_truncate test_suite/test_android_lowmem test_suite/test_SLR test_suite/test_progressive_ujg test_suite/test_progressive_disallowed test_suite/test_progressive test_suite/test_arithmetic_failfast test_suite/test_hq test_suite/forktester.py test_suite/sockettester.py src/lepton/test_custom_table.sh test_suite/test_baseline_unjailed test_suite/test_baseline_unjailed_thread test_suite/test_baseline_unjailed_decode test_suite/test_baseline_unjailed_decode_thread test_suite/test_seccomp_encode_main test_suite/test_seccomp_encode_thread  test_suite/test_seccomp_decode_main test_suite/test_seccomp_decode_thread test_suite/test_truncate_lowmem test_suite/test_nofsync test_suite/test_colorswap test_suite/test_odd_rst test_suite/test_trailing_header test_suite/test_trailing_rst test_suite/test_legacy.sh test_suite/test_roundtrip.sh test_suite/test_embedded.sh test_suite/test_16threads.sh test_suite/test_future_compat.sh test_suite/test_gray2sf test_suite/test_2nd_block.sh test_suite/test_3rd_block.sh test_suite/test_last_block.sh test_suite/test_truncated_zero_run test_suite/test_bad_zero_run test_suite/test_concat.sh test_suite/test_permissive.sh test_suite/test_clobber.sh
 
-dist_check_SCRIPTS = test_suite/test_recode_memory_bound test_suite/test_invariants test_suite/test_baseline_ujg test_suite/test_baseline test_suite/test_misc test_suite/test_iphone test_suite/test_phone_outdoor test_suite/test_truncate test_suite/test_single_row_truncate test_suite/test_android_lowmem test_suite/test_SLR test_suite/test_progressive_ujg test_suite/test_progressive_disallowed test_suite/test_progressive test_suite/test_arithmetic_failfast test_suite/test_hq test_suite/test_baseline_unjailed test_suite/test_baseline_unjailed_thread test_suite/test_baseline_unjailed_decode test_suite/test_baseline_unjailed_decode_thread test_suite/test_seccomp_encode_main test_suite/test_seccomp_encode_thread  test_suite/test_seccomp_decode_main test_suite/test_seccomp_decode_thread test_suite/test_truncate_lowmem test_suite/test_nofsync test_suite/test_colorswap test_suite/test_odd_rst test_suite/test_trailing_header test_suite/test_trailing_rst test_suite/test_legacy.sh test_suite/test_roundtrip.sh test_suite/test_embedded.sh test_suite/test_16threads.sh test_suite/test_future_compat.sh test_suite/test_gray2sf test_suite/test_2nd_block.sh test_suite/test_3rd_block.sh test_suite/test_last_block.sh test_suite/test_truncated_zero_run test_suite/test_bad_zero_run test_suite/test_concat.sh test_suite/test_permissive.sh
+dist_check_SCRIPTS = test_suite/test_recode_memory_bound test_suite/test_invariants test_suite/test_baseline_ujg test_suite/test_baseline test_suite/test_misc test_suite/test_iphone test_suite/test_phone_outdoor test_suite/test_truncate test_suite/test_single_row_truncate test_suite/test_android_lowmem test_suite/test_SLR test_suite/test_progressive_ujg test_suite/test_progressive_disallowed test_suite/test_progressive test_suite/test_arithmetic_failfast test_suite/test_hq test_suite/test_baseline_unjailed test_suite/test_baseline_unjailed_thread test_suite/test_baseline_unjailed_decode test_suite/test_baseline_unjailed_decode_thread test_suite/test_seccomp_encode_main test_suite/test_seccomp_encode_thread  test_suite/test_seccomp_decode_main test_suite/test_seccomp_decode_thread test_suite/test_truncate_lowmem test_suite/test_nofsync test_suite/test_colorswap test_suite/test_odd_rst test_suite/test_trailing_header test_suite/test_trailing_rst test_suite/test_legacy.sh test_suite/test_roundtrip.sh test_suite/test_embedded.sh test_suite/test_16threads.sh test_suite/test_future_compat.sh test_suite/test_gray2sf test_suite/test_2nd_block.sh test_suite/test_3rd_block.sh test_suite/test_last_block.sh test_suite/test_truncated_zero_run test_suite/test_bad_zero_run test_suite/test_concat.sh test_suite/test_permissive.sh test_suite/test_clobber.sh
 
 test:
 	$(MAKE) check

src/ans/rans64.hh

@@ -34,7 +34,8 @@
 
 static inline uint64_t Rans64MulHi(uint64_t a, uint64_t b)
 {
-    return __umulh(a, b);
+//    return __umulh(a, b);
+	abort();
 }
 
 #elif defined(__GNUC__)

src/io/Allocator.hh

@@ -26,8 +26,8 @@
  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-#ifndef _SIRIKATA_JPEG_ARHC_ALLOCATOR_HPP_
-#define _SIRIKATA_JPEG_ARHC_ALLOCATOR_HPP_
+#ifndef SIRIKATA_JPEG_ARHC_ALLOCATOR_HPP_
+#define SIRIKATA_JPEG_ARHC_ALLOCATOR_HPP_
 #include <stdlib.h>
 #include "DecoderPlatform.hh"
 namespace Sirikata {

src/io/DecoderPlatform.hh

@@ -56,8 +56,8 @@ typedef uint8_t byte;
 typedef int8_t int8;
 
 }
-#ifndef _DECODER_PLATFORM_HH_
-#define _DECODER_PLATFORM_HH_
+#ifndef DECODER_PLATFORM_HH_
+#define DECODER_PLATFORM_HH_
 #ifdef _WIN32
 #include <io.h>
 inline int write(int fd, const void*data, unsigned int length) {

src/io/Error.hh

@@ -26,8 +26,8 @@
  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-#ifndef _SIRIKATA_JPEG_ARHC_ERROR_HPP_
-#define _SIRIKATA_JPEG_ARHC_ERROR_HPP_
+#ifndef SIRIKATA_JPEG_ARHC_ERROR_HPP_
+#define SIRIKATA_JPEG_ARHC_ERROR_HPP_
 
 namespace Sirikata {
 

src/io/MemMgrAllocator.hh

@@ -33,8 +33,8 @@
  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-#ifndef _MEM_MGR_ALLOCATOR_HH_
-#define _MEM_MGR_ALLOCATOR_HH_
+#ifndef MEM_MGR_ALLOCATOR_HH_
+#define MEM_MGR_ALLOCATOR_HH_
 
 #include "DecoderPlatform.hh"
 namespace Sirikata {

src/io/MuxReader.hh

@@ -26,8 +26,8 @@
  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-#ifndef _SIRIKATA_MUX_READER_HPP_
-#define _SIRIKATA_MUX_READER_HPP_
+#ifndef SIRIKATA_MUX_READER_HPP_
+#define SIRIKATA_MUX_READER_HPP_
 #include <assert.h>
 #include <algorithm>
 #include "Allocator.hh"

src/io/Reader.hh

@@ -26,8 +26,8 @@
  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-#ifndef _SIRIKATA_READER_HPP_
-#define _SIRIKATA_READER_HPP_
+#ifndef SIRIKATA_READER_HPP_
+#define SIRIKATA_READER_HPP_
 
 #include "Allocator.hh"
 #include "Error.hh"

src/io/ZlibCompression.cc

@@ -114,7 +114,7 @@ std::pair<std::vector<uint8_t, JpegAllocator<uint8_t> >,
             }
             if (strm.avail_out == 0) {
                 retval_size += avail_bytes - strm.avail_out;
-                if (retval.first.size() == max_file_size) {
+                if (retval.first.size() >= max_file_size) {
                     retval.second = JpegError::errShortHuffmanData();
                     break;
                 }

src/io/ioutil.cc

@@ -198,7 +198,7 @@ Sirikata::Array1d<uint8_t, 16> send_and_md5_result(const uint8_t *data,
     MD5_Final(&retval[0], &context);
     return retval;
 }
-void md5_and_copy_to_tee(int copy_to_input_tee, int input_tee, MD5_CTX *context, size_t *input_size, size_t start_byte, size_t end_byte, bool close_input, bool is_socket) {
+void md5_and_copy_to_tee(int copy_to_input_tee, int input_tee, MD5_CTX *context, size_t *input_size, size_t start_byte, size_t end_byte, bool close_input, bool is_socket, std::vector<uint8_t>*byte_return) {
     unsigned char buffer[65536];
     while (true) {
         size_t max_to_read = sizeof(buffer);
@@ -220,10 +220,16 @@ void md5_and_copy_to_tee(int copy_to_input_tee, int input_tee, MD5_CTX *context,
             if (*input_size + del > start_byte) {
                 if (*input_size >= start_byte) {
                     MD5_Update(context, &buffer[0], del);
+					if (byte_return) {
+						byte_return->insert(byte_return->end(), &buffer[0], &buffer[0] + del);
+					}
                 } else {
                     size_t offset = (start_byte - *input_size);
                     MD5_Update(context, &buffer[offset], del - offset);
-                }
+					if (byte_return) {
+						byte_return->insert(byte_return->end(), &buffer[offset], &buffer[offset] + (del-offset));
+					}
+				}
             }
             { // write all to the subprocess
                 size_t cursor = 0;
@@ -242,8 +248,14 @@ void md5_and_copy_to_tee(int copy_to_input_tee, int input_tee, MD5_CTX *context,
                         fflush(stdout);
                         fflush(stderr);
                         del = 0;
+						if (byte_return) {
+							break;
+						}
                         custom_exit(ExitCode::SHORT_READ);
                     } else {
+						if (byte_return) {
+							break;
+						}
                         custom_exit(ExitCode::SHORT_READ);
                     }
                 }
@@ -309,7 +321,7 @@ Sirikata::Array1d<uint8_t, 16> transfer_and_md5(Sirikata::Array1d<uint8_t, 2> he
     uint8_t buffer[65536] = { 0 };
 #ifdef _WIN32
     std::thread worker(std::bind(&md5_and_copy_to_tee,
-        copy_to_input_tee, input_tee, &context, input_size, start_byte, end_byte, close_input, is_socket));
+        copy_to_input_tee, input_tee, &context, input_size, start_byte, end_byte, close_input, is_socket, byte_return));
 #if 1
     while(!failed) {
         auto old_size = storage->size();

src/io/ioutil.hh

@@ -1,5 +1,5 @@
-#ifndef _SIRIKIATA_IO_UTIL_HH_
-#define _SIRIKIATA_IO_UTIL_HH_
+#ifndef SIRIKIATA_IO_UTIL_HH_
+#define SIRIKIATA_IO_UTIL_HH_
 #ifndef _WIN32
 #include <unistd.h>
 #include <sys/errno.h>

src/io/seccomp-bpf.hh

@@ -9,8 +9,8 @@
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
-#ifndef _SECCOMP_BPF_H_
-#define _SECCOMP_BPF_H_
+#ifndef SECCOMP_BPF_H_
+#define SECCOMP_BPF_H_
 
 #define _GNU_SOURCE 1
 #include <stdio.h>

src/lepton/bitops.hh

@@ -31,22 +31,22 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <assert.h>
 #include <cstring>
-#define RBITS( c, n )		( c & ( 0xFF >> (8 - n) ) )
-#define LBITS( c, n )		( c >> (8 - n) )
-#define MBITS( c, l, r )	( RBITS( c,l ) >> r )
-#define RBITS16( c, n )		( c & ( 0xFFFFFFFF >> (16 - n) ) )
-#define LBITS16( c, n )		( c >> (16 - n) )
+#define RBITS( c, n )		( (c) & ( 0xFF >> (8 - (n)) ) )
+#define LBITS( c, n )		( (c) >> (8 - (n)) )
+#define MBITS( c, l, r )	( RBITS( c,l ) >> (r) )
+#define RBITS16( c, n )		( (c) & ( 0xFFFFFFFF >> (16 - (n)) ) )
+#define LBITS16( c, n )		( (c) >> (16 - (n)) )
 #define MBITS16( c, l, r )	( RBITS16( c,l ) >> r )
-#define RBITS32( c, n )		( c & ( 0xFFFFFFFF >> (32 - n) ) )
-#define LBITS32( c, n )		( c >> (32 - n) )
-#define MBITS32( c, l, r )	( RBITS32( c,l ) >> r )
+#define RBITS32( c, n )		( (c) & ( 0xFFFFFFFF >> (32 - (n)) ) )
+#define LBITS32( c, n )		( (c) >> (32 - (n)) )
+#define MBITS32( c, l, r )	( RBITS32( c,l ) >> (r) )
 
-#define RBITS64( c, n )		(n == 0 ? 0ULL : ( (c) & ( 0xFFFFFFFFFFFFFFFFULL >> (64 - (n)) ) ))
-#define LBITS64( c, n )		( c >> (64 - n) )
+#define RBITS64( c, n )		((n) == 0 ? 0ULL : ( (c) & ( 0xFFFFFFFFFFFFFFFFULL >> (64 - (n)) ) ))
+#define LBITS64( c, n )		( (c) >> (64 - (n)) )
 #define MBITS64( c, l, r )	(((r) >= 64) ? 0 : ( RBITS64( c,l ) >> (r) ))
 
-#define BITN( c, n )		( (c >> n) & 0x1 )
-#define FDIV2( v, p )		( ( v < 0 ) ? -( (-v) >> p ) : ( v >> p ) )
+#define BITN( c, n )		( ((c) >> (n)) & 0x1 )
+#define FDIV2( v, p )		( ( (v) < 0 ) ? -( (-v) >> (p) ) : ( (v) >> (p) ) )
 
 #define BTST_BUFF			1024 * 1024
 

src/lepton/component_info.hh

@@ -1,6 +1,6 @@
 /* -*-mode:c++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
-#ifndef _COMPONENT_INFO_HH_
-#define _COMPONENT_INFO_HH_
+#ifndef COMPONENT_INFO_HH_
+#define COMPONENT_INFO_HH_
 struct componentInfo {
 	unsigned short* qtable; // quantization table
 	int huffdc; // no of huffman table (DC)

src/lepton/concat.cc

@@ -1,3 +1,4 @@
+#include <fcntl.h>
 #include "../vp8/util/memory.hh"
 #include "../io/BrotliCompression.hh"
 #include "../io/ioutil.hh"