Merge remote-tracking branch 'origin/master' into SNAP-2366

Author: Sumedh Wale

NOTICE

@@ -128,10 +128,11 @@ dependencies {
     exclude(group: 'com.sun.jersey.contribs')
     exclude(group: 'com.google.protobuf', module: 'protobuf-java')
     exclude(group: 'com.jcraft', module: 'jsch')
+    exclude(group: 'org.apache.directory.server', module: 'apacheds-kerberos-codec')
   }
 
   testCompile project(':dunit')
-  testCompile 'it.unimi.dsi:fastutil:8.2.2'
+  testCompile "it.unimi.dsi:fastutil:${fastutilVersion}"
   testCompile "org.scalatest:scalatest_${scalaBinaryVersion}:${scalatestVersion}"
 
   if (new File(rootDir, 'aqp/build.gradle').exists() && rootProject.hasProperty('snappydata.enterprise')) {
@@ -143,7 +144,7 @@ dependencies {
 
 // Creates the version properties file and writes it to the resources dir
 task createVersionPropertiesFile(dependsOn: 'processResources') {
-  def propertiesDir = file("${buildDir}/classes/main/io/snappydata")
+  def propertiesDir = file("${sourceSets.main.scala.outputDir}/io/snappydata")
   outputs.file "${propertiesDir}/SnappyDataVersion.properties"
   inputs.file "${rootProject.projectDir}/build.gradle"
 

build.gradle

@@ -19,10 +19,12 @@ package io.snappydata.cluster
 import java.sql.{Connection, DriverManager, SQLException}
 
 import com.pivotal.gemfirexd.internal.engine.{GfxdConstants, Misc}
+import io.snappydata.SnappyFunSuite.resultSetToDataset
 import io.snappydata.test.dunit.{AvailablePortHelper, SerializableRunnable}
 
-import org.apache.spark.sql.SnappyContext
 import org.apache.spark.sql.collection.Utils
+import org.apache.spark.sql.store.ViewTest
+import org.apache.spark.sql.{Dataset, Row, SnappyContext, SnappySession}
 
 class DDLRoutingDUnitTest(val s: String) extends ClusterManagerTestBase(s) {
 
@@ -72,12 +74,12 @@ class DDLRoutingDUnitTest(val s: String) extends ClusterManagerTestBase(s) {
     val conn = getANetConnection(netPort1)
 
     // first fail a statement
-    failCreateTableXD(conn, tableName, true, " row ")
+    failCreateTableXD(conn, tableName, doFail = true, " row ")
 
     createTableXD(conn, tableName, " row ")
     tableMetadataAssertRowTable("APP", tableName)
     // Test create table - error for recreate
-    failCreateTableXD(conn, tableName, false, " row ")
+    failCreateTableXD(conn, tableName, doFail = false, " row ")
 
     // Drop Table and Recreate
     dropTableXD(conn, tableName)
@@ -167,7 +169,7 @@ class DDLRoutingDUnitTest(val s: String) extends ClusterManagerTestBase(s) {
 
     vm2.invoke(classOf[ClusterManagerTestBase], "stopAny")
     val props = bootProps.clone().asInstanceOf[java.util.Properties]
-    props.put("distributed-system-id" , "1")
+    props.put("distributed-system-id", "1")
     props.put("server-groups", "sg1")
 
     val restartServer = new SerializableRunnable() {
@@ -185,7 +187,7 @@ class DDLRoutingDUnitTest(val s: String) extends ClusterManagerTestBase(s) {
     var s = conn.createStatement()
     s.execute(s"CREATE TABLE $tableName (Col1 INT, Col2 INT, Col3 STRING)")
     insertDataXD(conn, tableName)
-    var snc = org.apache.spark.sql.SnappyContext(sc)
+    val snc = org.apache.spark.sql.SnappyContext(sc)
     verifyResultAndSchema(snc, tableName, 3)
 
     s.execute(s"ALTER TABLE $tableName ADD Col4 INT")
@@ -207,21 +209,21 @@ class DDLRoutingDUnitTest(val s: String) extends ClusterManagerTestBase(s) {
     s.execute(s"insert into $tableName values (1,1)")
     s.execute(s"ALTER TABLE $tableName add constraint emp_uk unique (Col1)")
     try {
-    s.execute(s"insert into $tableName values (1,1)")
+      s.execute(s"insert into $tableName values (1,1)")
     } catch {
       case sqle: SQLException =>
         if (sqle.getSQLState != "23505" ||
-          !sqle.getMessage.contains("duplicate key value in a unique or" +
-            " primary key constraint or unique index")) {
+            !sqle.getMessage.contains("duplicate key value in a unique or" +
+                " primary key constraint or unique index")) {
           throw sqle
         }
     }
 
     // asynceventlistener
     s.execute("CREATE ASYNCEVENTLISTENER myListener (" +
-      " listenerclass 'com.pivotal.gemfirexd.callbacks.DBSynchronizer'" +
-      " initparams 'org.apache.derby.jdbc.EmbeddedDriver,jdbc:derby:newDB;create=true')" +
-      " server groups(sg1)")
+        " listenerclass 'com.pivotal.gemfirexd.callbacks.DBSynchronizer'" +
+        " initparams 'org.apache.derby.jdbc.EmbeddedDriver,jdbc:derby:newDB;create=true')" +
+        " server groups(sg1)")
 
     s.execute(s"ALTER TABLE $tableName SET ASYNCEVENTLISTENER (myListener) ")
     var rs = s.executeQuery(s"select * from SYS.SYSTABLES where tablename='$tableName'")
@@ -287,7 +289,8 @@ class DDLRoutingDUnitTest(val s: String) extends ClusterManagerTestBase(s) {
     var cnt = 0
     while (rs.next()) {
       cnt += 1
-      rs.getInt(1); rs.getInt(2);
+      rs.getInt(1)
+      rs.getInt(2)
     }
     assert(cnt == 5, cnt)
 
@@ -296,7 +299,9 @@ class DDLRoutingDUnitTest(val s: String) extends ClusterManagerTestBase(s) {
     cnt = 0
     while (rs2.next()) {
       cnt += 1
-      rs2.getInt(1); rs2.getInt(2); rs2.getInt(3);
+      rs2.getInt(1)
+      rs2.getInt(2)
+      rs2.getInt(3)
     }
     assert(cnt == 5, cnt)
 
@@ -324,6 +329,36 @@ class DDLRoutingDUnitTest(val s: String) extends ClusterManagerTestBase(s) {
     dropTableXD(conn, tableName)
   }
 
+  def testViews(): Unit = {
+    val netPort1 = AvailablePortHelper.getRandomAvailableTCPPort
+    vm2.invoke(classOf[ClusterManagerTestBase], "startNetServer", netPort1)
+
+    val session = new SnappySession(sc)
+    ViewTest.createTables(session)
+
+    def newExecution(): String => Dataset[Row] = {
+      val session = new SnappySession(sc)
+      val conn = getANetConnection(netPort1)
+      val stmt = conn.createStatement()
+      resultSetToDataset(session, stmt)
+    }
+
+    val conn = getANetConnection(netPort1)
+    val stmt = conn.createStatement()
+    ViewTest.testTemporaryView(resultSetToDataset(session, stmt), newExecution)
+    ViewTest.testGlobalTemporaryView(resultSetToDataset(session, stmt), newExecution)
+    ViewTest.testTemporaryViewUsing(resultSetToDataset(session, stmt), newExecution)
+    ViewTest.testGlobalTemporaryViewUsing(resultSetToDataset(session, stmt), newExecution)
+    ViewTest.testPersistentView(resultSetToDataset(session, stmt), checkPlans = false,
+      newExecution, restartSpark)
+    ViewTest.dropTables(new SnappySession(sc))
+  }
+
+  private def restartSpark(): Unit = {
+    ClusterManagerTestBase.stopAny()
+    ClusterManagerTestBase.startSnappyLead(ClusterManagerTestBase.locatorPort, bootProps)
+  }
+
   def createTableXD(conn: Connection, tableName: String,
       usingStr: String): Unit = {
     val s = conn.createStatement()
@@ -421,7 +456,7 @@ class DDLRoutingDUnitTest(val s: String) extends ClusterManagerTestBase(s) {
       s.execute("CREATE EXTERNAL TABLE airlineRef_temp(Code VARCHAR(25), " +
           "Description VARCHAR(25)) USING parquet OPTIONS()")
     } catch {
-      case e: java.sql.SQLException =>
+      case _: java.sql.SQLException =>
       // println("Exception stack. create. ex=" + e.getMessage +
       //   " ,stack=" + ExceptionUtils.getFullStackTrace(e))
     }

cluster/build.gradle

@@ -114,7 +114,7 @@ class DistributedIndexDUnitTest(s: String) extends ClusterManagerTestBase(s) {
 //    }
 
 //    executeQ(s"select * from $tableName where col2 = 'aaa' ") {
-//      CreateIndexTest.validateIndex(Seq.empty, tableName)(_)
+//      CreateIndexTest.validateIndex(Nil, tableName)(_)
 //    }
 
     executeQ(s"select * from $tableName where col2 = 'bbb' and col3 = 'halo' ") {
@@ -164,7 +164,7 @@ class DistributedIndexDUnitTest(s: String) extends ClusterManagerTestBase(s) {
     //    }
 
     //    executeQ(s"select * from $tableName where col2 = 'aaa' ") {
-    //      CreateIndexTest.validateIndex(Seq.empty, tableName)(_)
+    //      CreateIndexTest.validateIndex(Nil, tableName)(_)
     //    }
 
     System.setProperty("LOG-NOW", "xxx")
@@ -228,7 +228,7 @@ class DistributedIndexDUnitTest(s: String) extends ClusterManagerTestBase(s) {
     //    }
 
     //    executeQ(s"select * from $tableName where col2 = 'aaa' ") {
-    //      CreateIndexTest.validateIndex(Seq.empty, tableName)(_)
+    //      CreateIndexTest.validateIndex(Nil, tableName)(_)
     //    }
 
     System.setProperty("LOG-NOW", "xxx")

cluster/src/dunit/scala/io/snappydata/cluster/DDLRoutingDUnitTest.scala

@@ -761,6 +761,7 @@ class QueryRoutingDUnitTest(val s: String)
 
       TPCHUtils.createAndLoadTables(snc, true)
 
+      snc.setConf(Property.EnableExperimentalFeatures.name, "true")
       snc.sql(
         s"""CREATE INDEX idx_orders_cust ON orders(o_custkey)
              options (COLOCATE_WITH 'customer')

cluster/src/dunit/scala/io/snappydata/cluster/DistributedIndexDUnitTest.scala

@@ -435,7 +435,7 @@ object SplitSnappyClusterDUnitTest
     snc.dropTable("splitModeTable1", ifExists = true)
 
     // recreate the dropped table
-    var expected = Seq.empty[ComplexData]
+    var expected: Seq[ComplexData] = Nil
     if (isComplex) {
       expected = createComplexTableUsingDataSourceAPI(snc, "splitModeTable1",
         tableType, props)

cluster/src/dunit/scala/io/snappydata/cluster/QueryRoutingDUnitTest.scala

@@ -67,7 +67,7 @@ class DynamicJarInstallationDUnitTest(val s: String)
     var testJar = DynamicJarInstallationDUnitTest.createJarWithClasses(
       classNames = Seq("FakeJobClass", "FakeJobClass1"),
       toStringValue = "1",
-      Seq.empty, Seq.empty,
+      Nil, Nil,
       "testJar_SNAPPY_JOB_SERVER_JAR_%s.jar".format(System.currentTimeMillis()))
 
     var jobCompleted = false
@@ -106,7 +106,7 @@ class DynamicJarInstallationDUnitTest(val s: String)
     testJar = DynamicJarInstallationDUnitTest.createJarWithClasses(
       classNames = Seq("FakeJobClass", "FakeJobClass1"),
       toStringValue = "2",
-      Seq.empty, Seq.empty,
+      Nil, Nil,
       "testJar_SNAPPY_JOB_SERVER_JAR_%s.jar".format(System.currentTimeMillis()))
 
     localProperty = (Seq("app1", DateTime.now) ++ Array[URL](testJar)).mkString(",")
@@ -169,4 +169,4 @@ object DynamicJarInstallationDUnitTest {
         else false
     }
   }
-}
+}

cluster/src/dunit/scala/io/snappydata/cluster/SplitSnappyClusterDUnitTest.scala

@@ -279,7 +279,7 @@ object UserDefinedFunctionsDUnitTest {
 
   def createUDFClass(name: String, code: String): File = {
     TestUtils.createCompiledClass(name, destDir,
-      getJavaSourceFromString(name, code), Seq.empty[URL])
+      getJavaSourceFromString(name, code), Nil)
   }
 
   def createJarFile(files: Seq[File]): String = {

cluster/src/dunit/scala/org/apache/spark/DynamicJarInstallationDUnitTest.scala

@@ -32,7 +32,7 @@
 
 public class SnappySystemAdmin extends GfxdSystemAdmin {
 
-  SnappySystemAdmin() {
+  private SnappySystemAdmin() {
     super();
     UTIL_Tools_DSProps = "UTIL_Snappy_Tools_DSProps";
     UTIL_DSProps_HelpPost = "UTIL_Snappy_Tools_DSProps_HelpPost";
@@ -91,31 +91,28 @@ public void invoke(String[] args) {
 
       super.invoke(args);
     } finally {
-      // remove zero-sized log-file
-      if (this.defaultLogFileName != null) {
-        try {
-          File logFile = new File(this.defaultLogFileName);
-          if (logFile.exists() && logFile.isFile() && logFile.length() == 0) {
-            logFile.delete();
-          }
-        } catch (Throwable t) {
-          // ignore at this point
+      // remove zero-sized generatedcode.log file
+      try {
+        File codeLogFile = new File("generatedcode.log");
+        if (codeLogFile.exists() && codeLogFile.isFile() && codeLogFile.length() == 0) {
+          codeLogFile.delete();
         }
+      } catch (Throwable t) {
+        // ignore at this point
       }
     }
   }
 
   public boolean handleVersion(String[] args) {
-    String cmd = null;
-    final ArrayList<String> cmdLine = new ArrayList<String>(Arrays.asList(args));
+    String cmd;
+    final ArrayList<String> cmdLine = new ArrayList<>(Arrays.asList(args));
     try {
       Iterator<String> it = cmdLine.iterator();
       while (it.hasNext()) {
         String arg = it.next();
         if (arg.startsWith("-")) {
           checkDashArg(null, arg, it);
-        }
-        else {
+        } else {
           break;
         }
       }
@@ -159,9 +156,8 @@ public boolean handleVersion(String[] args) {
     }
 
     if (cmd.equalsIgnoreCase("version")) {
-      Boolean optionOK = (cmdLine.size() == 0);
+      boolean optionOK = (cmdLine.size() == 0);
       if (cmdLine.size() == 1) {
-        optionOK = false;
         String option = cmdLine.get(0);
         if ("CREATE".equals(option) || "FULL".equalsIgnoreCase(option)) {
           optionOK = true;