feat: minor tune after offline testing (#1807)

GaoleMeng · gcf-owl-bot[bot] · web-flow · commit 694a870bac62 · 2022-09-29T15:14:45.000-07:00
* feat: Split writer into connection worker and wrapper, this is a prerequisite for multiplexing client * feat: add connection worker pool skeleton, used for multiplexing client * feat: add Load api for connection worker for multiplexing client * feat: add multiplexing support to connection worker. We will treat every new stream name as a switch of destinationt * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: port the multiplexing client core algorithm and basic tests also fixed a tiny bug inside fake bigquery write impl for getting thre response from offset * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: wire multiplexing connection pool to stream writer * feat: some fixes for multiplexing client * feat: fix some todos, and reject the mixed behavior of passed in client or not * feat: fix the bug that we may peek into the write_stream field but it's possible the proto schema does not contain this field * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: fix the bug that we may peek into the write_stream field but it's possible the proto schema does not contain this field Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java
@@ -735,8 +735,8 @@ public Load getLoad() {
   public abstract static class Load {
     // Consider the load on this worker to be overwhelmed when above some percentage of
     // in-flight bytes or in-flight requests count.
-    private static double overwhelmedInflightCount = 0.5;
-    private static double overwhelmedInflightBytes = 0.6;
+    private static double overwhelmedInflightCount = 0.2;
+    private static double overwhelmedInflightBytes = 0.2;
 
     // Number of in-flight requests bytes in the worker.
     abstract long inFlightRequestsBytes();
diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPool.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPool.java
@@ -163,7 +163,7 @@ public abstract static class Settings {
     public static Builder builder() {
       return new AutoValue_ConnectionWorkerPool_Settings.Builder()
           .setMinConnectionsPerRegion(2)
-          .setMaxConnectionsPerRegion(10);
+          .setMaxConnectionsPerRegion(20);
     }
 
     /** Builder for the options to config {@link ConnectionWorkerPool}. */
diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java
@@ -213,15 +213,16 @@ private StreamWriter(Builder builder) throws IOException {
                   String fetchedLocation = writeStream.getLocation();
                   log.info(
                       String.format(
-                          "Fethed location %s for stream name %s", fetchedLocation, streamName));
+                          "Fethed location %s for stream name %s, extracted project and dataset name: %s\"",
+                          fetchedLocation, streamName, datasetAndProjectName));
                   return fetchedLocation;
                 });
         if (location.isEmpty()) {
           throw new IllegalStateException(
               String.format(
                   "The location is empty for both user passed in value and looked up value for "
-                      + "stream: %s",
-                  streamName));
+                      + "stream: %s, extracted project and dataset name: %s",
+                  streamName, datasetAndProjectName));
         }
       }
       this.location = location;
diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPoolTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPoolTest.java
@@ -68,6 +68,8 @@ public void setUp() throws Exception {
                 .setCredentialsProvider(NoCredentialsProvider.create())
                 .setTransportChannelProvider(serviceHelper.createChannelProvider())
                 .build());
+    ConnectionWorker.Load.setOverwhelmedCountsThreshold(0.5);
+    ConnectionWorker.Load.setOverwhelmedBytesThreshold(0.6);
   }
 
   @Test

Original file line number	Diff line number	Diff line change
`@@ -163,7 +163,7 @@ public abstract static class Settings {`
`163`	`163`	`public static Builder builder() {`
`164`	`164`	`return new AutoValue_ConnectionWorkerPool_Settings.Builder()`
`165`	`165`	`.setMinConnectionsPerRegion(2)`
`166`		`- .setMaxConnectionsPerRegion(10);`
	`166`	`+ .setMaxConnectionsPerRegion(20);`
`167`	`167`	`}`
`168`	`168`
`169`	`169`	`/** Builder for the options to config {@link ConnectionWorkerPool}. */`
Original file line number	Diff line number	Diff line change
`@@ -68,6 +68,8 @@ public void setUp() throws Exception {`
`68`	`68`	`.setCredentialsProvider(NoCredentialsProvider.create())`
`69`	`69`	`.setTransportChannelProvider(serviceHelper.createChannelProvider())`
`70`	`70`	`.build());`
	`71`	`+ ConnectionWorker.Load.setOverwhelmedCountsThreshold(0.5);`
	`72`	`+ ConnectionWorker.Load.setOverwhelmedBytesThreshold(0.6);`
`71`	`73`	`}`
`72`	`74`
`73`	`75`	`@Test`