Issue #8212 : Add a Database perspective as an alternative to the SQL editor dialog

NOTICE

@@ -16,6 +16,9 @@ See: https://github.com/delta-io/delta
 This product bundles Apache Iceberg under the Apache License, Version 2.0.
 See: https://iceberg.apache.org/
 
+This product bundles Apache Calcite under the Apache License, Version 2.0.
+See: https://calcite.apache.org/
+
 This product bundles Unity Catalog client libraries (transitive dependency of
 Delta Lake) under the Apache License, Version 2.0.
 See: https://github.com/unitycatalog/unitycatalog

assemblies/plugins/pom.xml

@@ -714,6 +714,12 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hop</groupId>
+            <artifactId>hop-tech-calcite</artifactId>
+            <version>${project.version}</version>
+            <type>zip</type>
+        </dependency>
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-tech-ftp</artifactId>

core/src/main/java/org/apache/hop/core/database/BaseDatabaseMeta.java

@@ -924,6 +924,56 @@ public String getSqlQueryFields(String tableName) {
     return "SELECT * FROM " + tableName;
   }
 
+  /**
+   * SQL-standard catalog lookup. Dialects without {@code INFORMATION_SCHEMA.VIEWS} override this
+   * (Oracle, SQLite) or {@link #getSqlObjectDdl(String, String)} (MySQL {@code SHOW CREATE TABLE}).
+   */
+  @Override
+  public String getSqlViewDefinition(String schemaName, String viewName) {
+    if (Utils.isEmpty(viewName)) {
+      return null;
+    }
+    StringBuilder sql = new StringBuilder();
+    sql.append("SELECT VIEW_DEFINITION FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_NAME = ");
+    sql.append(quoteSqlString(viewName));
+    if (!Utils.isEmpty(schemaName)) {
+      sql.append(" AND TABLE_SCHEMA = ").append(quoteSqlString(schemaName));
+    }
+    return sql.toString();
+  }
+
+  /**
+   * Quote an identifier for catalog SQL such as {@code SHOW CREATE TABLE}. Always quotes, unlike
+   * {@link DatabaseMeta#quoteField(String)}.
+   */
+  protected String quoteIdentifierAlways(String name) {
+    if (name == null) {
+      return null;
+    }
+    String start = Const.NVL(getStartQuote(), "");
+    String end = Const.NVL(getEndQuote(), start);
+    if (start.isEmpty()) {
+      return name;
+    }
+    return start + name.replace(end, end + end) + end;
+  }
+
+  /**
+   * {@code SHOW CREATE TABLE} for MySQL, Hive and similar. Quotes schema and object names.
+   *
+   * @return the statement, or {@code null} when {@code objectName} is empty
+   */
+  protected String showCreateTableSql(String schemaName, String objectName) {
+    if (Utils.isEmpty(objectName)) {
+      return null;
+    }
+    String qualified =
+        Utils.isEmpty(schemaName)
+            ? quoteIdentifierAlways(objectName)
+            : quoteIdentifierAlways(schemaName) + "." + quoteIdentifierAlways(objectName);
+    return "SHOW CREATE TABLE " + qualified;
+  }
+
   /**
    * Most databases round number(7,2) 17.29999999 to 17.30, but some don't.
    *
@@ -1648,12 +1698,7 @@ public List<SqlScriptStatement> getSqlScriptStatements(String sqlScript) {
         String stat = all.substring(from, to);
         if (!onlySpaces(stat)) {
           String s = Const.trim(stat);
-          statements.add(
-              new SqlScriptStatement(
-                  s,
-                  from,
-                  to,
-                  s.toUpperCase().startsWith("SELECT") || s.toLowerCase().startsWith("show")));
+          statements.add(new SqlScriptStatement(s, from, to, SqlQueryClassifier.isQuery(s)));
         }
         to++;
         from = to;

core/src/main/java/org/apache/hop/core/database/Database.java

@@ -130,8 +130,9 @@ public class Database implements IVariables, ILoggingObject, AutoCloseable {
 
   /**
    * When positive, applied to statements created in {@link #openQuery(String, IRowMeta, Object[],
-   * int, boolean)} via {@link Statement#setQueryTimeout(int)} (whole seconds). Zero leaves the JDBC
-   * driver default (typically unlimited). Intended for short-lived GUI preview connections.
+   * int, boolean)} and {@link #execStatement(String, IRowMeta, Object[])} via {@link
+   * Statement#setQueryTimeout(int)} (whole seconds). Zero leaves the JDBC driver default (typically
+   * unlimited). Intended for short-lived GUI preview connections.
    */
   private int statementQueryTimeoutSeconds;
 
@@ -270,8 +271,8 @@ public void setQueryLimit(int rows) {
 
   /**
    * Sets the JDBC {@link Statement#setQueryTimeout(int)} (seconds) for statements opened by {@link
-   * #openQuery(String, IRowMeta, Object[], int, boolean)} until {@link #disconnect()}. Use {@code
-   * 0} to use the driver default.
+   * #openQuery(String, IRowMeta, Object[], int, boolean)} and {@link #execStatement(String,
+   * IRowMeta, Object[])} until {@link #disconnect()}. Use {@code 0} to use the driver default.
    *
    * @param seconds query timeout in whole seconds; values {@code < 0} are treated as {@code 0}
    */
@@ -1525,12 +1526,14 @@ public Result execStatement(String rawsql, IRowMeta params, Object[] data)
       if (params != null) {
         PreparedStatement prepStmt = connection.prepareStatement(databaseMeta.stripCR(sql));
         setValues(params, data, prepStmt); // set the parameters!
+        applyStatementQueryTimeout(prepStmt);
         resultSet = prepStmt.execute();
         count = prepStmt.getUpdateCount();
         prepStmt.close();
       } else {
         String sqlStripped = databaseMeta.stripCR(sql);
         try (Statement stmt = connection.createStatement()) {
+          applyStatementQueryTimeout(stmt);
           resultSet = stmt.execute(sqlStripped);
           count = stmt.getUpdateCount();
         }
@@ -1614,8 +1617,7 @@ public Result execStatements(String script, IRowMeta params, Object[] data)
 
         if (!Const.onlySpaces(stat)) {
           String sql = Const.trim(stat);
-          if (sql.toUpperCase().startsWith("SELECT")
-              && !sql.toUpperCase().matches("(?is)^(select\\s.*\\sinto\\s).*")) {
+          if (SqlQueryClassifier.isQuery(sql)) {
             // A Query
             if (log.isDetailed()) {
               log.logDetailed("launch SELECT statement: " + Const.CR + sql);
@@ -3275,6 +3277,70 @@ public String getCreateTableStatement(
     return retval.toString();
   }
 
+  /**
+   * DDL to recreate a table or view: catalog text when the dialect can read it, otherwise a {@code
+   * CREATE TABLE} (or a column-only {@code CREATE VIEW}) from field metadata.
+   *
+   * @param schemaName schema or catalog, may be {@code null}
+   * @param objectName table or view name
+   * @param view {@code true} to emit {@code CREATE VIEW}
+   * @param fields columns already loaded, or {@code null} to look them up
+   */
+  public String getObjectDdl(String schemaName, String objectName, boolean view, IRowMeta fields)
+      throws HopDatabaseException {
+    String qualified = databaseMeta.getQuotedSchemaTableCombination(this, schemaName, objectName);
+    String catalog = readCatalogDdl(schemaName, objectName, view);
+    if (!Utils.isEmpty(catalog)) {
+      if (view && !DatabaseObjectDdl.startsWithCreate(catalog)) {
+        return DatabaseObjectDdl.asCreateViewStatement(qualified, catalog);
+      }
+      return DatabaseObjectDdl.ensureSemicolon(catalog);
+    }
+    IRowMeta layout = fields;
+    if (layout == null || layout.isEmpty()) {
+      layout = loadObjectFields(schemaName, objectName, qualified);
+    }
+    if (view) {
+      return DatabaseObjectDdl.synthesizeCreateView(
+          qualified, layout, "View definition is not available from the catalog");
+    }
+    if (layout == null || layout.isEmpty()) {
+      return "";
+    }
+    databaseMeta.quoteReservedWords(layout);
+    return getCreateTableStatement(qualified, layout, null, false, null, true);
+  }
+
+  private IRowMeta loadObjectFields(String schemaName, String objectName, String qualified)
+      throws HopDatabaseException {
+    try {
+      IRowMeta meta = getTableFieldsMeta(schemaName, objectName);
+      if (meta != null && meta.size() > 0) {
+        return meta;
+      }
+    } catch (Exception ignored) {
+      // Fall back to the query-based layout.
+    }
+    return getTableFields(qualified);
+  }
+
+  private String readCatalogDdl(String schemaName, String objectName, boolean view) {
+    String sql = databaseMeta.getSqlObjectDdl(schemaName, objectName);
+    if (Utils.isEmpty(sql) && view) {
+      sql = databaseMeta.getSqlViewDefinition(schemaName, objectName);
+    }
+    if (Utils.isEmpty(sql)) {
+      return null;
+    }
+    try {
+      RowMetaAndData row = getOneRow(sql);
+      return DatabaseObjectDdl.extractDefinition(row);
+    } catch (Exception e) {
+      log.logDebug("Unable to read object DDL from the catalog", e);
+      return null;
+    }
+  }
+
   public String getAlterTableStatement(
       String tableName,
       IRowMeta fields,
@@ -3694,9 +3760,18 @@ public List<Object[]> getRows(
     if (monitor != null) {
       monitor.setTaskName("Opening query...");
     }
-    ResultSet rset = openQuery(sql, params, data, fetchMode, lazyConversion);
-
-    return getRows(rset, limit, monitor);
+    // openQuery honours setQueryLimit via Statement.setMaxRows. The limit argument used to only
+    // stop the client read loop, so drivers that buffer the result still fetched the whole table.
+    int previousLimit = rowlimit;
+    if (limit > 0) {
+      setQueryLimit(limit);
+    }
+    try {
+      ResultSet rset = openQuery(sql, params, data, fetchMode, lazyConversion);
+      return getRows(rset, limit, monitor);
+    } finally {
+      rowlimit = previousLimit;
+    }
   }
 
   /**

core/src/main/java/org/apache/hop/core/database/DatabaseMeta.java

@@ -1179,6 +1179,24 @@ public String getSqlQueryFields(String tableName) {
     return iDatabase.getSqlQueryFields(tableName);
   }
 
+  /**
+   * @param schemaName schema or catalog, or {@code null}
+   * @param viewName view name
+   * @return catalog SQL for the view definition, or {@code null}
+   */
+  public String getSqlViewDefinition(String schemaName, String viewName) {
+    return iDatabase.getSqlViewDefinition(schemaName, viewName);
+  }
+
+  /**
+   * @param schemaName schema or catalog, or {@code null}
+   * @param objectName table or view name
+   * @return catalog SQL for {@code CREATE TABLE}/{@code CREATE VIEW}, or {@code null}
+   */
+  public String getSqlObjectDdl(String schemaName, String objectName) {
+    return iDatabase.getSqlObjectDdl(schemaName, objectName);
+  }
+
   public String getAddColumnStatement(
       String tableName,
       IValueMeta v,

core/src/main/java/org/apache/hop/core/database/DatabaseObjectDdl.java

@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.database;
+
+import java.util.Locale;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.RowMetaAndData;
+import org.apache.hop.core.exception.HopValueException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.util.Utils;
+
+/**
+ * Helpers for CREATE TABLE / CREATE VIEW text shown in the Database perspective object-information
+ * tab.
+ */
+public final class DatabaseObjectDdl {
+
+  private DatabaseObjectDdl() {}
+
+  /**
+   * Pick the catalog definition from a query row. Prefers columns named like {@code CREATE}, {@code
+   * VIEW_DEFINITION}, {@code sql} or {@code TEXT}; otherwise the last non-empty string.
+   */
+  public static String extractDefinition(RowMetaAndData row) throws HopValueException {
+    if (row == null || row.getRowMeta() == null || row.getData() == null) {
+      return null;
+    }
+    IRowMeta meta = row.getRowMeta();
+    int fallback = -1;
+    for (int i = 0; i < meta.size(); i++) {
+      String value = row.getString(i, null);
+      if (Utils.isEmpty(value)) {
+        continue;
+      }
+      fallback = i;
+      String column = Const.NVL(meta.getValueMeta(i).getName(), "").toUpperCase(Locale.ROOT);
+      if (column.contains("CREATE")
+          || column.contains("DEFINITION")
+          || column.contains("DDL")
+          || "SQL".equals(column)
+          || "TEXT".equals(column)) {
+        return value;
+      }
+    }
+    return fallback >= 0 ? row.getString(fallback, null) : null;
+  }
+
+  /**
+   * If {@code definition} is already a CREATE statement, return it (with a trailing semicolon).
+   * Otherwise wrap it as {@code CREATE VIEW qualified AS ...}.
+   */
+  public static String asCreateViewStatement(String qualifiedName, String definition) {
+    if (Utils.isEmpty(definition)) {
+      return "";
+    }
+    String trimmed = definition.trim();
+    if (startsWithCreate(trimmed)) {
+      return ensureSemicolon(trimmed);
+    }
+    String name = Utils.isEmpty(qualifiedName) ? "view" : qualifiedName;
+    return "CREATE VIEW " + name + " AS" + Const.CR + trimmed + ";";
+  }
+
+  public static boolean startsWithCreate(String sql) {
+    if (Utils.isEmpty(sql)) {
+      return false;
+    }
+    return sql.trim().toUpperCase(Locale.ROOT).startsWith("CREATE");
+  }
+
+  public static String ensureSemicolon(String sql) {
+    if (Utils.isEmpty(sql)) {
+      return sql;
+    }
+    String trimmed = sql.trim();
+    if (trimmed.endsWith(";")) {
+      return trimmed;
+    }
+    return trimmed + ";";
+  }
+
+  /**
+   * Last-resort view DDL when the catalog has no SELECT text: column list only.
+   *
+   * @param comment optional leading SQL comment (without {@code --})
+   */
+  public static String synthesizeCreateView(String qualifiedName, IRowMeta fields, String comment) {
+    StringBuilder buffer = new StringBuilder();
+    if (!Utils.isEmpty(comment)) {
+      buffer.append("-- ").append(comment).append(Const.CR);
+    }
+    buffer.append("CREATE VIEW ").append(qualifiedName).append(" AS").append(Const.CR);
+    if (fields == null || fields.isEmpty()) {
+      buffer.append("SELECT *");
+    } else {
+      buffer.append("SELECT").append(Const.CR);
+      for (int i = 0; i < fields.size(); i++) {
+        IValueMeta value = fields.getValueMeta(i);
+        if (i > 0) {
+          buffer.append(",").append(Const.CR);
+        }
+        buffer.append("  ").append(value.getName());
+      }
+    }
+    buffer.append(';');
+    return buffer.toString();
+  }
+}