Skip to content

Optimizer chooses wrong execution plan in some cases when index-sorted optimization is possible #4116

@forchid

Description

@forchid

Version info

H2-2.3.232, java 21, and windows 10 x64.

Data batcher in the standalone test

java -Xmx256m org.h2.tools.Server -baseDir D:\database\h2db -tcp -ifNotExists
import java.sql.*;
import java.io.*;
import java.text.*;

public class PerfTest implements Closeable {

  static final String  URL = System.getProperty("jdbc.url", "jdbc:hsqldb:test2");
  static final String  USR = System.getProperty("jdbc.user", "sa");
  static final String  PWD = System.getProperty("jdbc.password", "");
  static final int     MAX = Integer.getInteger("rows", 25000000);
  static final boolean MEM = Boolean.getBoolean("memory");
  static final boolean DWR = Boolean.getBoolean("delayWrite");

  Connection connection;
  Statement statement;
  String url, user, password;

  final int max;
  final boolean persistent;
  final boolean delayWrite;

  public static void main(String[] args) throws Exception {
    if (args.length > 0 && ("-?".equals(args[0]) || "-h".equals(args[0]) 
      || "-help".equals(args[0]) || "--help".equals(args[0]))) {
      String usage = 
        "java -Djdbc.url=URL -Djdbc.user=USR -Djdbc.password=PWD -Drows=MAX PerfTest [-?|-h]%n" + 
        "  URL    the jdbc connection url, default jdbc:hsqldb:test2%n" +
        "  USR    the jdbc connection user, default sa%n" + 
        "  PWD    the jdbc connection password, default \"\"%n" +
        "  MAX    the performance test max rows, default 25000000";
      System.out.println(usage);
      System.exit(0);
    }

    String  url = URL, usr = USR, pwd = PWD;
    boolean pss = !MEM;
    // Load JDBC driver for the old hsqldb/h2 version
    if (url.startsWith("jdbc:hsqldb:")) {
      Class.forName("org.hsqldb.jdbcDriver");
    } else if (url.startsWith("jdbc:h2:")) {
      Class.forName("org.h2.Driver");
    }

    Connection connection = DriverManager.getConnection(url, usr, pwd);
    Statement  statement  = connection.createStatement();
    PerfTest   perf       = new PerfTest(connection, statement, url, usr, pwd, MAX, pss, DWR);
    try {
      perf.test();
    } finally {
      perf.close();
    }
  }

  public PerfTest(Connection connection, Statement statement, 
    String url, String user, String password,
    int maxRows, boolean persistent, boolean delayWrite) {
    this.connection = connection;
    this.statement  = statement;
    this.url        = url;
    this.user       = user;
    this.password   = password;
    this.max        = maxRows;
    this.persistent = persistent;
    this.delayWrite = delayWrite;
  }

  public void test() throws Exception {
    final long staMillis = System.currentTimeMillis();
    boolean hsql = this.url.startsWith("jdbc:hsqldb:");
    boolean h2   = false;
    if (!hsql) hsql = h2 = this.url.startsWith("jdbc:h2:");

    if (hsql) print("Cache, index and performance tests");
    else print("Table, index and performance tests");

    String s;
    if (hsql) s = "CREATE CACHED TABLE Addr(ID INT PRIMARY KEY, First VARCHAR(50), Name VARCHAR(50), ZIP INT, Create_At datetime not null)";
    else s = "CREATE TABLE Addr(ID INT PRIMARY KEY, First VARCHAR(50), Name VARCHAR(50), ZIP INT, Create_At datetime not null)";
    statement.execute(s);

    s = "CREATE INDEX addr_index_name ON Addr(Name asc, Name desc)"; // h2 default desc?
    statement.execute(s);
    s = "CREATE INDEX addr_index_create_at ON Addr(Create_At asc, Create_At desc)";
    statement.execute(s);
    if (hsql) {
      if (h2) s = "SET WRITE_DELAY "+ (delayWrite? "100": "0");
      else s = "SET WRITE_DELAY "+ (delayWrite? "TRUE": "FALSE");
      statement.execute(s);
    }

    long start = System.currentTimeMillis();
    int  max   = this.max;
    int statSpan = Math.max(max / 5, 100);
    DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    connection.setAutoCommit(false);
    for (int i = 0; i < max; i++) {
      s = "INSERT INTO Addr VALUES(" + i + ",'Marcel" + i + "',"
        + "'Renggli" + (max - i - (i % 31)) + "',"
        + (3000 + i % statSpan) + ",'" 
        + df.format(new java.util.Date()) + "')";

      if (statement.executeUpdate(s) != 1) {
        throw new Exception("Insert failed");
      }

      if (i % statSpan == 0) printStatus("Inserted  ", i, max, start);
      if ((i + 1) % 1000 == 0) connection.commit();
    }
    
    printStatus("Inserted  ", max, max, start);
    print("");
            
    connection.setAutoCommit(true);
    s = "SELECT COUNT(*) FROM Addr";
    ResultSet r = statement.executeQuery(s);
    r.next();
    int c = r.getInt(1);
    if (c != max) {
      throw new Exception("Count should be " + (max) + " but is "+ c);
    }

    if (persistent && hsql) {
      // close & reopen to test backup
      connection.close();
      connection = DriverManager.getConnection(url, user, password);
      statement = connection.createStatement();
      if (hsql) {
        s = "SET WRITE_DELAY "+ (delayWrite? "TRUE": "FALSE");
        if (h2) s = "SET WRITE_DELAY "+ (delayWrite? "100": "0"); 
        statement.execute(s);
      }
    }

    start = System.currentTimeMillis();
    connection.setAutoCommit(false);
    for (int i = 0; i < this.max; i++) {
      s = "UPDATE Addr SET Name='Robert" + (i + (i % 31)) + "' WHERE ID=" + i;
      if (statement.executeUpdate(s) != 1) {
        throw new Exception("Update failed");
      }

      if (i % statSpan == 0) printStatus("Updated   ", i, max, start);
      if ((i + 1) % 1000 == 0) connection.commit();
    }

    printStatus("Updated   ", max, max, start);
    print("");

    if (persistent && hsql) {
      s = "SHUTDOWN IMMEDIATELY";
      statement.execute(s);
      try {
        connection.close();
      } catch (SQLException e) {
        String sqlstate = e.getSQLState();
        if (!"90098".equals(sqlstate)) throw e;
        // Here h2-1.3.176
      }
      // open the database; it must be restored after shutdown
      connection = DriverManager.getConnection(url, user, password);
      statement = connection.createStatement();
      if (hsql) {
        s = "SET WRITE_DELAY "+ (delayWrite? "TRUE": "FALSE");
        if (h2) s = "SET WRITE_DELAY "+ (delayWrite? "100": "0"); 
        statement.execute(s);
      }
    }

    start = System.currentTimeMillis();
    connection.setAutoCommit(false);
    for (int i = 0; i < max; i++) {
      s = "DELETE FROM Addr WHERE ID=" + (max - 1 - i);
      if (statement.executeUpdate(s) != 1) {
        throw new Exception("Delete failed");
      }

      if (i % statSpan == 0) printStatus("Deleted   ", i, max, start);
      if ((i + 1) % 1000 == 0) connection.commit();
    }

    printStatus("Deleted   ", max, max, start);
    print("Cleaning ..");
    statement.execute("DROP TABLE Addr");
    final long endMillis = System.currentTimeMillis();
    print("OK(total "+ (endMillis - staMillis) + "ms).");
  }
  
  @Override
  public void close() {
    this.url  = null;
    this.user = this.password = null;
    try {
      try {
        statement.close();
      } finally {
        connection.close();
      }
    } catch (SQLException e) {
      // Ignore
    }
  }

  static void printStatus(String s, int i, int max, long start) {
    String status = String.format("%s: %d/%d %d%% ", 
        s, i, max, (100L * i / max));

    long now = System.currentTimeMillis();
    if (now > start) {
        status += (i * 1000L / (now - start));
    } else {
        status += "--";
    }
    
    System.out.printf("%s rows/s%n", status);
  }

  static void print(String s) {
    System.out.println(s);
  }
  
}

The h2 shell test

java -Xmx64m -Djdbc.url=jdbc:h2:tcp://127.0.0.1:9092/test -Djdbc.password=123456 PerfTest
sql> select *from addr a join addr b on a.name = b.name order by a.name desc limit 10;
ID | FIRST   | NAME            | ZIP  | CREATE_AT           | ID | FIRST   | NAME            | ZIP  | CREATE_AT
0  | Marcel0 | Renggli25000000 | 3000 | 2024-08-15 11:44:15 | 0  | Marcel0 | Renggli25000000 | 3000 | 2024-08-15 11:44:15
1  | Marcel1 | Renggli24999998 | 3001 | 2024-08-15 11:44:15 | 1  | Marcel1 | Renggli24999998 | 3001 | 2024-08-15 11:44:15
2  | Marcel2 | Renggli24999996 | 3002 | 2024-08-15 11:44:15 | 2  | Marcel2 | Renggli24999996 | 3002 | 2024-08-15 11:44:15
3  | Marcel3 | Renggli24999994 | 3003 | 2024-08-15 11:44:15 | 3  | Marcel3 | Renggli24999994 | 3003 | 2024-08-15 11:44:15
4  | Marcel4 | Renggli24999992 | 3004 | 2024-08-15 11:44:15 | 4  | Marcel4 | Renggli24999992 | 3004 | 2024-08-15 11:44:15
5  | Marcel5 | Renggli24999990 | 3005 | 2024-08-15 11:44:15 | 5  | Marcel5 | Renggli24999990 | 3005 | 2024-08-15 11:44:15
6  | Marcel6 | Renggli24999988 | 3006 | 2024-08-15 11:44:15 | 6  | Marcel6 | Renggli24999988 | 3006 | 2024-08-15 11:44:15
7  | Marcel7 | Renggli24999986 | 3007 | 2024-08-15 11:44:16 | 7  | Marcel7 | Renggli24999986 | 3007 | 2024-08-15 11:44:16
8  | Marcel8 | Renggli24999984 | 3008 | 2024-08-15 11:44:16 | 8  | Marcel8 | Renggli24999984 | 3008 | 2024-08-15 11:44:16
9  | Marcel9 | Renggli24999982 | 3009 | 2024-08-15 11:44:16 | 9  | Marcel9 | Renggli24999982 | 3009 | 2024-08-15 11:44:16
(10 rows, 8 ms)
sql> select *from addr a join addr b on a.name = b.name order by a.name limit 10;
ID       | FIRST          | NAME            | ZIP     | CREATE_AT           | ID       | FIRST          | NAME            | ZIP     | CREATE_AT
13524989 | Marcel13524989 | Renggli11474981 | 3527989 | 2024-08-15 12:10:49 | 13524989 | Marcel13524989 | Renggli11474981 | 3527989 | 2024-08-15 12:10:49
13524988 | Marcel13524988 | Renggli11474983 | 3527988 | 2024-08-15 12:10:49 | 13524988 | Marcel13524988 | Renggli11474983 | 3527988 | 2024-08-15 12:10:49
13524987 | Marcel13524987 | Renggli11474985 | 3527987 | 2024-08-15 12:10:49 | 13524987 | Marcel13524987 | Renggli11474985 | 3527987 | 2024-08-15 12:10:49
13524986 | Marcel13524986 | Renggli11474987 | 3527986 | 2024-08-15 12:10:49 | 13524986 | Marcel13524986 | Renggli11474987 | 3527986 | 2024-08-15 12:10:49
13524985 | Marcel13524985 | Renggli11474989 | 3527985 | 2024-08-15 12:10:49 | 13524985 | Marcel13524985 | Renggli11474989 | 3527985 | 2024-08-15 12:10:49
13524984 | Marcel13524984 | Renggli11474991 | 3527984 | 2024-08-15 12:10:49 | 13524984 | Marcel13524984 | Renggli11474991 | 3527984 | 2024-08-15 12:10:49
13524999 | Marcel13524999 | Renggli11474992 | 3527999 | 2024-08-15 12:10:49 | 13524999 | Marcel13524999 | Renggli11474992 | 3527999 | 2024-08-15 12:10:49
13524983 | Marcel13524983 | Renggli11474993 | 3527983 | 2024-08-15 12:10:49 | 13524983 | Marcel13524983 | Renggli11474993 | 3527983 | 2024-08-15 12:10:49
13524998 | Marcel13524998 | Renggli11474994 | 3527998 | 2024-08-15 12:10:49 | 13524998 | Marcel13524998 | Renggli11474994 | 3527998 | 2024-08-15 12:10:49
13524982 | Marcel13524982 | Renggli11474995 | 3527982 | 2024-08-15 12:10:49 | 13524982 | Marcel13524982 | Renggli11474995 | 3527982 | 2024-08-15 12:10:49
(10 rows, 117576  ms)

And these queries order by create_at asc or desc always are very quick! eg.

sql> select *from addr a join addr b on a.name = b.name order by a.create_at asc limit 10;
ID | FIRST   | NAME            | ZIP  | CREATE_AT           | ID | FIRST   | NAME            | ZIP  | CREATE_AT
0  | Marcel0 | Renggli25000000 | 3000 | 2024-08-15 11:44:15 | 0  | Marcel0 | Renggli25000000 | 3000 | 2024-08-15 11:44:15
1  | Marcel1 | Renggli24999998 | 3001 | 2024-08-15 11:44:15 | 1  | Marcel1 | Renggli24999998 | 3001 | 2024-08-15 11:44:15
2  | Marcel2 | Renggli24999996 | 3002 | 2024-08-15 11:44:15 | 2  | Marcel2 | Renggli24999996 | 3002 | 2024-08-15 11:44:15
3  | Marcel3 | Renggli24999994 | 3003 | 2024-08-15 11:44:15 | 3  | Marcel3 | Renggli24999994 | 3003 | 2024-08-15 11:44:15
4  | Marcel4 | Renggli24999992 | 3004 | 2024-08-15 11:44:15 | 4  | Marcel4 | Renggli24999992 | 3004 | 2024-08-15 11:44:15
5  | Marcel5 | Renggli24999990 | 3005 | 2024-08-15 11:44:15 | 5  | Marcel5 | Renggli24999990 | 3005 | 2024-08-15 11:44:15
6  | Marcel6 | Renggli24999988 | 3006 | 2024-08-15 11:44:15 | 6  | Marcel6 | Renggli24999988 | 3006 | 2024-08-15 11:44:15
7  | Marcel7 | Renggli24999986 | 3007 | 2024-08-15 11:44:16 | 7  | Marcel7 | Renggli24999986 | 3007 | 2024-08-15 11:44:16
8  | Marcel8 | Renggli24999984 | 3008 | 2024-08-15 11:44:16 | 8  | Marcel8 | Renggli24999984 | 3008 | 2024-08-15 11:44:16
9  | Marcel9 | Renggli24999982 | 3009 | 2024-08-15 11:44:16 | 9  | Marcel9 | Renggli24999982 | 3009 | 2024-08-15 11:44:16
(10 rows, 6 ms)
sql> select *from addr a join addr b on a.name = b.name order by a.create_at desc limit 10;
ID       | FIRST          | NAME            | ZIP     | CREATE_AT           | ID       | FIRST          | NAME            | ZIP     | CREATE_AT
12402999 | Marcel12402999 | Renggli12596978 | 2405999 | 2024-08-15 12:08:25 | 12402999 | Marcel12402999 | Renggli12596978 | 2405999 | 2024-08-15 12:08:25
12402998 | Marcel12402998 | Renggli12596980 | 2405998 | 2024-08-15 12:08:25 | 12402998 | Marcel12402998 | Renggli12596980 | 2405998 | 2024-08-15 12:08:25
12402997 | Marcel12402997 | Renggli12596982 | 2405997 | 2024-08-15 12:08:25 | 12402997 | Marcel12402997 | Renggli12596982 | 2405997 | 2024-08-15 12:08:25
12402996 | Marcel12402996 | Renggli12596984 | 2405996 | 2024-08-15 12:08:25 | 12402996 | Marcel12402996 | Renggli12596984 | 2405996 | 2024-08-15 12:08:25
12402995 | Marcel12402995 | Renggli12596986 | 2405995 | 2024-08-15 12:08:25 | 12402995 | Marcel12402995 | Renggli12596986 | 2405995 | 2024-08-15 12:08:25
12402994 | Marcel12402994 | Renggli12596988 | 2405994 | 2024-08-15 12:08:25 | 12402994 | Marcel12402994 | Renggli12596988 | 2405994 | 2024-08-15 12:08:25
12402993 | Marcel12402993 | Renggli12596990 | 2405993 | 2024-08-15 12:08:25 | 12402993 | Marcel12402993 | Renggli12596990 | 2405993 | 2024-08-15 12:08:25
12402992 | Marcel12402992 | Renggli12596992 | 2405992 | 2024-08-15 12:08:25 | 12402992 | Marcel12402992 | Renggli12596992 | 2405992 | 2024-08-15 12:08:25
12402991 | Marcel12402991 | Renggli12596994 | 2405991 | 2024-08-15 12:08:25 | 12402991 | Marcel12402991 | Renggli12596994 | 2405991 | 2024-08-15 12:08:25
12402990 | Marcel12402990 | Renggli12596996 | 2405990 | 2024-08-15 12:08:25 | 12402990 | Marcel12402990 | Renggli12596996 | 2405990 | 2024-08-15 12:08:25
(10 rows, 5 ms)

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions