Merge branch '1.x' into evan.jones/zstd-legacy

.circleci/config.yml

@@ -1,21 +1,45 @@
 version: 2
 
 jobs:
-  "golang-1.14":
+  "golang-1.16":
     docker:
-      - image: circleci/golang:1.14
+      - image: circleci/golang:1.16
     steps:
       - checkout
       - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
       - run: 'unzip mr.zip'
       - run: 'go build'
       - run: 'PAYLOAD=`pwd`/mr go test -v'
       - run: 'PAYLOAD=`pwd`/mr go test -bench .'
-  "golang-1.15":
+  "golang-1.16-external-libzstd":
     docker:
-      - image: circleci/golang:1.15
+      - image: circleci/golang:1.16
     steps:
       - checkout
+      - run: 'sudo apt update'
+      - run: 'sudo apt install libzstd-dev'
+      - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
+      - run: 'unzip mr.zip'
+      - run: 'go build'
+      - run: 'PAYLOAD=`pwd`/mr go test -v'
+      - run: 'PAYLOAD=`pwd`/mr go test -bench .'
+  "golang-1.17":
+    docker:
+      - image: circleci/golang:1.17
+    steps:
+      - checkout
+      - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
+      - run: 'unzip mr.zip'
+      - run: 'go build'
+      - run: 'PAYLOAD=`pwd`/mr go test -v'
+      - run: 'PAYLOAD=`pwd`/mr go test -bench .'
+  "golang-1.17-external-libzstd":
+    docker:
+      - image: circleci/golang:1.17
+    steps:
+      - checkout
+      - run: 'sudo apt update'
+      - run: 'sudo apt install libzstd-dev'
       - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
       - run: 'unzip mr.zip'
       - run: 'go build'
@@ -31,6 +55,18 @@ jobs:
       - run: 'go build'
       - run: 'PAYLOAD=`pwd`/mr go test -v'
       - run: 'PAYLOAD=`pwd`/mr go test -bench .'
+  "golang-latest-external-libzstd":
+    docker:
+      - image: circleci/golang:latest
+    steps:
+      - checkout
+      - run: 'sudo apt update'
+      - run: 'sudo apt install libzstd-dev'
+      - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
+      - run: 'unzip mr.zip'
+      - run: 'go build -tags external_libzstd'
+      - run: 'PAYLOAD=`pwd`/mr go test -tags external_libzstd -v'
+      - run: 'PAYLOAD=`pwd`/mr go test -tags external_libzstd -bench .'
   "golang-efence":
     resource_class: xlarge
     docker:
@@ -41,6 +77,18 @@ jobs:
       - run: 'unzip mr.zip'
       - run: 'go build'
       - run: 'PAYLOAD=`pwd`/mr GODEBUG=efence=1 go test -v'
+  "golang-efence-external-libzstd":
+    resource_class: xlarge
+    docker:
+      - image: circleci/golang:latest
+    steps:
+      - checkout
+      - run: 'sudo apt update'
+      - run: 'sudo apt install libzstd-dev'
+      - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
+      - run: 'unzip mr.zip'
+      - run: 'go build -tags external_libzstd'
+      - run: 'PAYLOAD=`pwd`/mr GODEBUG=efence=1 go test -tags external_libzstd -v'
   "golang-i386":
     docker:
       - image: 32bit/ubuntu:16.04
@@ -52,9 +100,13 @@ workflows:
   version: 2
   build:
     jobs:
-      - "golang-1.14"
-      - "golang-1.15"
+      - "golang-1.16"
+      - "golang-1.16-external-libzstd"
+      - "golang-1.17"
+      - "golang-1.17-external-libzstd"
       - "golang-latest"
+      - "golang-latest-external-libzstd"
       - "golang-efence"
+      - "golang-efence-external-libzstd"
       - "golang-i386"
       - "golang-zstd-legacy-support"

README.md

@@ -6,8 +6,8 @@
 
 [C Zstd Homepage](https://github.com/facebook/zstd)
 
-The current headers and C files are from *v1.4.4* (Commit
-[10f0e699](https://github.com/facebook/zstd/releases/tag/v1.4.4)).
+The current headers and C files are from *v1.5.0* (Commit
+[10f0e699](https://github.com/facebook/zstd/releases/tag/v1.5.0)).
 
 ## Usage
 
@@ -19,6 +19,21 @@ There are two main APIs:
 The compress/decompress APIs mirror that of lz4, while the streaming API was
 designed to be a drop-in replacement for zlib.
 
+### Building against an external libzstd
+
+By default, zstd source code is vendored in this repository and the binding will be built with
+the vendored source code bundled.
+
+If you want to build this binding against an external static or shared libzstd library, you can
+use the `external_libzstd` build tag. This will look for the libzstd pkg-config file and extract
+build and linking parameters from that pkg-config file.
+
+Note that it requires at least libzstd 1.4.0.
+
+```bash
+go build -tags external_libzstd
+```
+
 ### Simple `Compress/Decompress`
 
 

bitstream.h

@@ -1,7 +1,8 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * bitstream
  * Part of FSE library
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Facebook, Inc.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -461,3 +462,5 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
 #endif
 
 #endif /* BITSTREAM_H_MODULE */
+
+#endif /* USE_EXTERNAL_ZSTD */

compiler.h

@@ -1,5 +1,6 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -90,6 +91,7 @@
 #  endif
 #endif
 
+
 /* target attribute */
 #ifndef __has_attribute
   #define __has_attribute(x) 0  /* Compatibility with non-clang compilers. */
@@ -286,3 +288,5 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
 #endif
 
 #endif /* ZSTD_COMPILER_H */
+
+#endif /* USE_EXTERNAL_ZSTD */

cover.c

@@ -1,5 +1,6 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -26,15 +27,16 @@
 #include <string.h> /* memset */
 #include <time.h>   /* clock */
 
+#ifndef ZDICT_STATIC_LINKING_ONLY
+#  define ZDICT_STATIC_LINKING_ONLY
+#endif
+
 #include "mem.h" /* read */
 #include "pool.h"
 #include "threading.h"
-#include "cover.h"
 #include "zstd_internal.h" /* includes zstd.h */
-#ifndef ZDICT_STATIC_LINKING_ONLY
-#define ZDICT_STATIC_LINKING_ONLY
-#endif
 #include "zdict.h"
+#include "cover.h"
 
 /*-*************************************
 *  Constants
@@ -1062,18 +1064,19 @@ typedef struct COVER_tryParameters_data_s {
  * This function is thread safe if zstd is compiled with multithreaded support.
  * It takes its parameters as an *OWNING* opaque pointer to support threading.
  */
-static void COVER_tryParameters(void *opaque) {
+static void COVER_tryParameters(void *opaque)
+{
   /* Save parameters as local variables */
-  COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t *)opaque;
+  COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t*)opaque;
   const COVER_ctx_t *const ctx = data->ctx;
   const ZDICT_cover_params_t parameters = data->parameters;
   size_t dictBufferCapacity = data->dictBufferCapacity;
   size_t totalCompressedSize = ERROR(GENERIC);
   /* Allocate space for hash table, dict, and freqs */
   COVER_map_t activeDmers;
-  BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
+  BYTE* const dict = (BYTE*)malloc(dictBufferCapacity);
   COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
-  U32 *freqs = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
+  U32* const freqs = (U32*)malloc(ctx->suffixSize * sizeof(U32));
   if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
     DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
     goto _cleanup;
@@ -1103,15 +1106,14 @@ static void COVER_tryParameters(void *opaque) {
   free(data);
   COVER_map_destroy(&activeDmers);
   COVER_dictSelectionFree(selection);
-  if (freqs) {
-    free(freqs);
-  }
+  free(freqs);
 }
 
 ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
-    void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
-    const size_t *samplesSizes, unsigned nbSamples,
-    ZDICT_cover_params_t *parameters) {
+    void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer,
+    const size_t* samplesSizes, unsigned nbSamples,
+    ZDICT_cover_params_t* parameters)
+{
   /* constants */
   const unsigned nbThreads = parameters->nbThreads;
   const double splitPoint =
@@ -1243,3 +1245,5 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
     return dictSize;
   }
 }
+
+#endif /* USE_EXTERNAL_ZSTD */

cover.h

@@ -1,5 +1,6 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
- * Copyright (c) 2017-2020, Facebook, Inc.
+ * Copyright (c) Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -8,6 +9,10 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
+#ifndef ZDICT_STATIC_LINKING_ONLY
+#  define ZDICT_STATIC_LINKING_ONLY
+#endif
+
 #include <stdio.h>  /* fprintf */
 #include <stdlib.h> /* malloc, free, qsort */
 #include <string.h> /* memset */
@@ -16,9 +21,6 @@
 #include "pool.h"
 #include "threading.h"
 #include "zstd_internal.h" /* includes zstd.h */
-#ifndef ZDICT_STATIC_LINKING_ONLY
-#define ZDICT_STATIC_LINKING_ONLY
-#endif
 #include "zdict.h"
 
 /**
@@ -155,3 +157,5 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection);
  COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
                        size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
                        size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
+
+#endif /* USE_EXTERNAL_ZSTD */

cpu.h

@@ -1,5 +1,6 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
- * Copyright (c) 2018-2020, Facebook, Inc.
+ * Copyright (c) Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -211,3 +212,5 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
 #undef X
 
 #endif /* ZSTD_COMMON_CPU_H */
+
+#endif /* USE_EXTERNAL_ZSTD */

debug.c

@@ -1,7 +1,8 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Facebook, Inc.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -22,3 +23,5 @@
 #include "debug.h"
 
 int g_debuglevel = DEBUGLEVEL;
+
+#endif /* USE_EXTERNAL_ZSTD */

debug.h

@@ -1,7 +1,8 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Facebook, Inc.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -105,3 +106,5 @@ extern int g_debuglevel; /* the variable is only declared,
 #endif
 
 #endif /* DEBUG_H_12987983217 */
+
+#endif /* USE_EXTERNAL_ZSTD */

divsufsort.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * divsufsort.c for libdivsufsort-lite
  * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
@@ -1576,7 +1577,7 @@ sort_typeBstar(const unsigned char *T, int *SA,
     /* Construct the inverse suffix array of type B* suffixes using trsort. */
     trsort(ISAb, SA, m, 1);
 
-    /* Set the sorted order of tyoe B* suffixes. */
+    /* Set the sorted order of type B* suffixes. */
     for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) {
       for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { }
       if(0 <= i) {
@@ -1911,3 +1912,5 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char *
 
   return pidx;
 }
+
+#endif /* USE_EXTERNAL_ZSTD */

divsufsort.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * divsufsort.h for libdivsufsort-lite
  * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
@@ -65,3 +66,5 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char *
 #endif /* __cplusplus */
 
 #endif /* _DIVSUFSORT_H */
+
+#endif /* USE_EXTERNAL_ZSTD */

entropy_common.c

@@ -1,6 +1,7 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * Common functions of New Generation Entropy library
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Facebook, Inc.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -360,3 +361,5 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
     (void)bmi2;
     return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
 }
+
+#endif /* USE_EXTERNAL_ZSTD */

error_private.c

@@ -1,5 +1,6 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -54,3 +55,5 @@ const char* ERR_getErrorString(ERR_enum code)
     }
 #endif
 }
+
+#endif /* USE_EXTERNAL_ZSTD */