New connector versions and changenotes for 2016-12-16 release:

gcs-connector-1.6.0, bigquery-connector-0.10.1 Change on 2016/12/16 by dhuo <[email protected]> ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=142318117
GoogleCloudDataproc · Dec 17, 2016 · dd48900 · dd48900
1 parent 8c2c4ab
commit dd48900
Show file tree

Hide file tree

Showing 14 changed files with 61 additions and 30 deletions.
diff --git a/bigquery/bigquery-hadoop1.pom.xml b/bigquery/bigquery-hadoop1.pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.google.cloud.bigdataoss</groupId>
     <artifactId>bigquery-connector-parent</artifactId>
-    <version>0.10.1-SNAPSHOT</version>
+    <version>0.10.1</version>
     <relativePath>pom.xml</relativePath>
   </parent>
 
@@ -30,7 +30,7 @@
   <description>
     Hadoop 2 MapReduce input and output formats for use with BigQuery
   </description>
-  <version>0.10.1-hadoop1-SNAPSHOT</version>
+  <version>0.10.1-hadoop1</version>
 
   <dependencies>
     <dependency>

diff --git a/bigquery/bigquery-hadoop2.pom.xml b/bigquery/bigquery-hadoop2.pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.google.cloud.bigdataoss</groupId>
     <artifactId>bigquery-connector-parent</artifactId>
-    <version>0.10.1-SNAPSHOT</version>
+    <version>0.10.1</version>
     <relativePath>pom.xml</relativePath>
   </parent>
 
@@ -30,7 +30,7 @@
   <description>
     Hadoop 2 MapReduce input and output formats for use with BigQuery
   </description>
-  <version>0.10.1-hadoop2-SNAPSHOT</version>
+  <version>0.10.1-hadoop2</version>
 
   <dependencies>
     <dependency>

diff --git a/bigquery/pom.xml b/bigquery/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.google.cloud.bigdataoss</groupId>
     <artifactId>bigdataoss-parent</artifactId>
-    <version>1.5.6-SNAPSHOT</version>
+    <version>1.6.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -30,7 +30,7 @@
   <description>
     MapReduce input and output formats for use with BigQuery
   </description>
-  <version>0.10.1-SNAPSHOT</version>
+  <version>0.10.1</version>
 
   <dependencies>
     <dependency>

diff --git a/bigquery/src/main/java/com/google/cloud/hadoop/io/bigquery/CHANGES.txt b/bigquery/src/main/java/com/google/cloud/hadoop/io/bigquery/CHANGES.txt
@@ -1,8 +1,9 @@
-x.x.x - 2016-xx-xx
+0.10.1 - 2016-12-16
 
   1. Added a configurable write disposition when using 
      IndirectBigQueryOutputFormat with "WRITE_APPEND" as the default.
 
+
 0.10.0 - 2016-11-07
 
   1. Update output configuration keys to conform to the format in

diff --git a/gcs/CHANGES.txt b/gcs/CHANGES.txt
@@ -1,3 +1,33 @@
+1.6.0 - 2016-12-16
+
+  1. Added new PerformanceCachingGoogleCloudStorage; unlike the existing
+     CacheSupplementedGoogleCloudStorage which only serves as an advisory
+     cache for enforcement of list consistency, the new optional caching
+     layer is able to serving certain metadata and listing requests purely
+     out of a short-lived in-memory cache to enhance performance of
+     some workloads. By default this feature is disabled, and can be
+     controlled with the config settings:
+
+       fs.gs.performance.cache.enable=true (default=false)
+       fs.gs.performance.cache.list.caching.enable=true (default=false)
+
+     The first option enables the cache to serve getFileStatus requests,
+     while the second option additionally enables serving listStatus. The
+     duration of cache entries can be controlled with:
+
+       fs.gs.performance.cache.max.entry.age.ms (default=3000)
+
+     It is not recommended to always run with this feature enabled; it should
+     be used specifically to address cases where frameworks perform redundant
+     sequential list/stat operations in a non-distributed manner, and on
+     datasets which are not frequently changing. It is additionally advised
+     to validate data integrity separately whenever using this feature. There
+     is no cooperative cache invalidation between different processes when
+     using this feature, so concurrent mutations to a location from multiple
+     clients *will* produce inconsistent/stale results if this feature is
+     enabled.
+
+
 1.5.5 - 2016-11-04
 
   1. Minor refactoring of logic in CacheSupplementedGoogleCloudStorage to

diff --git a/gcs/gcs-hadoop1.pom.xml b/gcs/gcs-hadoop1.pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.google.cloud.bigdataoss</groupId>
     <artifactId>gcs-connector-parent</artifactId>
-    <version>1.5.6-SNAPSHOT</version>
+    <version>1.6.0</version>
     <relativePath>pom.xml</relativePath>
   </parent>
 
@@ -30,7 +30,7 @@
   <description>
     An implementation of org.apache.hadoop.fs.FileSystem targeting Google Cloud Storage
   </description>
-  <version>1.5.6-hadoop1-SNAPSHOT</version>
+  <version>1.6.0-hadoop1</version>
 
   <dependencies>
     <dependency>

diff --git a/gcs/gcs-hadoop2.pom.xml b/gcs/gcs-hadoop2.pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.google.cloud.bigdataoss</groupId>
     <artifactId>gcs-connector-parent</artifactId>
-    <version>1.5.6-SNAPSHOT</version>
+    <version>1.6.0</version>
     <relativePath>pom.xml</relativePath>
   </parent>
 
@@ -30,7 +30,7 @@
   <description>
     An implementation of org.apache.hadoop.fs.FileSystem targeting Google Cloud Storage
   </description>
-  <version>1.5.6-hadoop2-SNAPSHOT</version>
+  <version>1.6.0-hadoop2</version>
 
   <dependencies>
     <dependency>

diff --git a/gcs/pom.xml b/gcs/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.google.cloud.bigdataoss</groupId>
     <artifactId>bigdataoss-parent</artifactId>
-    <version>1.5.6-SNAPSHOT</version>
+    <version>1.6.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -30,7 +30,7 @@
   <description>
     An implementation of org.apache.hadoop.fs.FileSystem targeting Google Cloud Storage
   </description>
-  <version>1.5.6-SNAPSHOT</version>
+  <version>1.6.0</version>
 
   <profiles>
     <profile>

diff --git a/gcsio/pom.xml b/gcsio/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.google.cloud.bigdataoss</groupId>
     <artifactId>bigdataoss-parent</artifactId>
-    <version>1.5.6-SNAPSHOT</version>
+    <version>1.6.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -30,7 +30,7 @@
   <description>
     An implementation of org.apache.hadoop.fs.FileSystem targeting Google Cloud Storage
   </description>
-  <version>1.5.6-SNAPSHOT</version>
+  <version>1.6.0</version>
   <dependencies>
     <dependency>
       <groupId>com.google.api-client</groupId>

diff --git a/pom.xml b/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>com.google.cloud.bigdataoss</groupId>
   <artifactId>bigdataoss-parent</artifactId>
-  <version>1.5.6-SNAPSHOT</version>
+  <version>1.6.0</version>
   <name>BigDataOSS Parent</name>
   <description>
     Parent project for Google Cloud Platform Big-Data connectors.
@@ -70,12 +70,12 @@
   </prerequisites>
 
   <properties>
-    <bigdataoss.util.version>1.5.6-SNAPSHOT</bigdataoss.util.version>
-    <bigdataoss.gcsio.version>1.5.6-SNAPSHOT</bigdataoss.gcsio.version>
-    <bigdataoss.util-hadoop.one.version>1.5.6-hadoop1-SNAPSHOT</bigdataoss.util-hadoop.one.version>
-    <bigdataoss.util-hadoop.two.version>1.5.6-hadoop2-SNAPSHOT</bigdataoss.util-hadoop.two.version>
-    <bigdataoss.gcs.hadoop.one.version>1.5.6-hadoop1-SNAPSHOT</bigdataoss.gcs.hadoop.one.version>
-    <bigdataoss.gcs.hadoop.two.version>1.5.6-hadoop2-SNAPSHOT</bigdataoss.gcs.hadoop.two.version>
+    <bigdataoss.util.version>1.6.0</bigdataoss.util.version>
+    <bigdataoss.gcsio.version>1.6.0</bigdataoss.gcsio.version>
+    <bigdataoss.util-hadoop.one.version>1.6.0-hadoop1</bigdataoss.util-hadoop.one.version>
+    <bigdataoss.util-hadoop.two.version>1.6.0-hadoop2</bigdataoss.util-hadoop.two.version>
+    <bigdataoss.gcs.hadoop.one.version>1.6.0-hadoop1</bigdataoss.gcs.hadoop.one.version>
+    <bigdataoss.gcs.hadoop.two.version>1.6.0-hadoop2</bigdataoss.gcs.hadoop.two.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <google.api.version>1.20.0</google.api.version>
     <google.guava.version>18.0</google.guava.version>

diff --git a/util-hadoop/pom.xml b/util-hadoop/pom.xml
@@ -20,13 +20,13 @@
   <parent>
     <groupId>com.google.cloud.bigdataoss</groupId>
     <artifactId>bigdataoss-parent</artifactId>
-    <version>1.5.6-SNAPSHOT</version>
+    <version>1.6.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
   <packaging>pom</packaging>
   <artifactId>util-hadoop-parent</artifactId>
-  <version>1.5.6-SNAPSHOT</version>
+  <version>1.6.0</version>
 
   <profiles>
     <profile>

diff --git a/util-hadoop/util-hadoop-hadoop1.pom.xml b/util-hadoop/util-hadoop-hadoop1.pom.xml
@@ -20,13 +20,13 @@
   <parent>
     <groupId>com.google.cloud.bigdataoss</groupId>
     <artifactId>util-hadoop-parent</artifactId>
-    <version>1.5.6-SNAPSHOT</version>
+    <version>1.6.0</version>
     <relativePath>pom.xml</relativePath>
   </parent>
 
   <packaging>jar</packaging>
   <artifactId>util-hadoop</artifactId>
-  <version>1.5.6-hadoop1-SNAPSHOT</version>
+  <version>1.6.0-hadoop1</version>
 
   <dependencies>
     <dependency>

diff --git a/util-hadoop/util-hadoop-hadoop2.pom.xml b/util-hadoop/util-hadoop-hadoop2.pom.xml
@@ -20,13 +20,13 @@
   <parent>
     <groupId>com.google.cloud.bigdataoss</groupId>
     <artifactId>util-hadoop-parent</artifactId>
-    <version>1.5.6-SNAPSHOT</version>
+    <version>1.6.0</version>
     <relativePath>pom.xml</relativePath>
   </parent>
 
   <packaging>jar</packaging>
   <artifactId>util-hadoop</artifactId>
-  <version>1.5.6-hadoop2-SNAPSHOT</version>
+  <version>1.6.0-hadoop2</version>
 
   <dependencies>
     <dependency>

diff --git a/util/pom.xml b/util/pom.xml
@@ -20,13 +20,13 @@
   <parent>
     <groupId>com.google.cloud.bigdataoss</groupId>
     <artifactId>bigdataoss-parent</artifactId>
-    <version>1.5.6-SNAPSHOT</version>
+    <version>1.6.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
   <packaging>jar</packaging>
   <artifactId>util</artifactId>
-  <version>1.5.6-SNAPSHOT</version>
+  <version>1.6.0</version>
 
   <dependencies>
     <dependency>