[#9826][test] Fix for timeout error in org.yb.cql.TestBigNumShards#te…

…stDropTableTimeout.

Summary:
The test `org.yb.cql.TestBigNumShards#testDropTableTimeout` sets `NumShardsPerTServer `==32.
For 1 table + 5 indexes it created 192 tablets. For the many number of tablets Jenkins can occasionally fail with timeout.

The big number of tablets was a way to reproduce some cases when the DROP TABLE happens when the CREATE TABLE/INDEX is not finished.
See for details:
- GH: #3032
- Diff: https://phabricator.dev.yugabyte.com/D7670
- Commit: a70ab64

Current fix keeps the test for big number of shards, but the number of tables (and indexes) reduced to 2.
Slow CREATE TABLE test-case is re-implemented via `TEST_simulate_slow_table_create_secs`:  `org.yb.cql.TestMasterLatency#testSlowCreateDropIndex `.

Test Plan:
ybd --java-test org.yb.cql.TestBigNumShards#testCreateDropTable --tp 1 -n 10
ybd --java-test org.yb.cql.TestMasterLatency#testSlowCreateDropTable --tp 1 -n 10
ybd --java-test org.yb.cql.TestMasterLatency#testSlowCreateDropIndex --tp 1 -n 10
ybd --java-test org.yb.cql.TestSlowCreateTable#testCreateTableTimeout --tp 1 -n 10

Reviewers: timur, amitanand

Reviewed By: amitanand

Subscribers: yql

Differential Revision: https://phabricator.dev.yugabyte.com/D12427

java/yb-cql/src/test/java/org/yb/cql/TestBigNumShards.java

@@ -29,24 +29,18 @@ protected int getNumShardsPerTServer() {
   }
 
   @Test
-  public void testDropTableTimeout() throws Exception {
+  public void testCreateDropTable() throws Exception {
     LOG.info("Start test: " + getCurrentTestMethodName());
 
     // Create test table.
     session.execute("create table test_drop (h1 int primary key, " +
                     "c1 int, c2 int, c3 int, c4 int, c5 int) " +
                     "with transactions = {'enabled' : true};");
 
-    // Create test indexes.
+    // Create test index.
     session.execute("create index i1 on test_drop (c1);");
-    session.execute("create index i2 on test_drop (c2);");
-    session.execute("create index i3 on test_drop (c3);");
-    session.execute("create index i4 on test_drop (c4);");
-    session.execute("create index i5 on test_drop (c5);");
 
     // Drop test table.
     session.execute("drop table test_drop;");
-
-    LOG.info("End test: " + getCurrentTestMethodName());
   }
 }

java/yb-cql/src/test/java/org/yb/cql/TestMasterLatency.java

@@ -18,9 +18,9 @@
 import org.slf4j.LoggerFactory;
 
 import org.yb.minicluster.MiniYBClusterBuilder;
-import org.yb.YBTestRunner;
+import org.yb.util.YBTestRunnerNonTsanOnly;
 
-@RunWith(value=YBTestRunner.class)
+@RunWith(value=YBTestRunnerNonTsanOnly.class)
 public class TestMasterLatency extends BaseCQLTest {
   private static final Logger LOG = LoggerFactory.getLogger(TestMasterLatency.class);
 
@@ -35,15 +35,33 @@ protected void customizeMiniClusterBuilder(MiniYBClusterBuilder builder) {
   }
 
   @Test
-  public void testCreateDropTableTimeout() throws Exception {
+  public void testSlowCreateDropTable() throws Exception {
     LOG.info("Start test: " + getCurrentTestMethodName());
 
     // Create test table.
     session.execute("create table test_table (h1 int primary key, c1 int) " +
                     "with transactions = {'enabled' : true};");
     // Drop test table.
     session.execute("drop table test_table;");
+  }
 
-    LOG.info("End test: " + getCurrentTestMethodName());
+  @Test
+  public void testSlowCreateDropIndex() throws Exception {
+    LOG.info("Start test: " + getCurrentTestMethodName());
+
+    // Create test table.
+    session.execute("create table test_drop (h1 int primary key, " +
+                    "c1 int, c2 int, c3 int, c4 int, c5 int) " +
+                    "with transactions = {'enabled' : true};");
+
+    // Create test indexes.
+    session.execute("create index i1 on test_drop (c1);");
+    session.execute("create index i2 on test_drop (c2);");
+    session.execute("create index i3 on test_drop (c3);");
+    session.execute("create index i4 on test_drop (c4);");
+    session.execute("create index i5 on test_drop (c5);");
+
+    // Drop test table.
+    session.execute("drop table test_drop;");
   }
 }

java/yb-cql/src/test/java/org/yb/cql/TestSlowCreateTable.java

@@ -0,0 +1,62 @@
+// Copyright (c) YugaByte, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied.  See the License for the specific language governing permissions and limitations
+// under the License.
+//
+package org.yb.cql;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.yb.minicluster.MiniYBClusterBuilder;
+import org.yb.util.YBTestRunnerNonTsanOnly;
+
+import static org.yb.AssertionWrappers.assertTrue;
+import static org.yb.AssertionWrappers.fail;
+
+@RunWith(value=YBTestRunnerNonTsanOnly.class)
+public class TestSlowCreateTable extends BaseCQLTest {
+  private static final Logger LOG = LoggerFactory.getLogger(TestSlowCreateTable.class);
+
+  @Override
+  protected void customizeMiniClusterBuilder(MiniYBClusterBuilder builder) {
+    super.customizeMiniClusterBuilder(builder);
+    // Set latency in yb::master::CatalogManager::CreateTable().
+    builder.addMasterFlag("TEST_simulate_slow_table_create_secs", "20");
+    // Set default CREATE/DROP TABLE operation timeout.
+    builder.addCommonTServerFlag("yb_client_admin_operation_timeout_sec", "10");
+  }
+
+  @Test
+  public void testCreateTableTimeout() throws Exception {
+    LOG.info("Start test: " + getCurrentTestMethodName());
+
+    // Create test table.
+    try {
+      session.execute("create table test_table (h1 int primary key, c1 int) " +
+                      "with transactions = {'enabled' : true};");
+      fail("Create Table did not fail due to timeout as expected");
+    } catch (com.datastax.driver.core.exceptions.ServerError ex) {
+      LOG.info("Expected exception:", ex);
+      String errorMsg = ex.getCause().getMessage();
+      if (!errorMsg.contains("Timed out waiting for Table Creation") &&
+          !errorMsg.contains("CreateTable timed out after deadline expired.")) {
+        fail("Unexpected error message '" + errorMsg + "'");
+      }
+    }
+
+    Thread.sleep(15 * 1000); // Let CREATE TABLE finish.
+
+    // Drop test table.
+    session.execute("drop table test_table;");
+  }
+}