From 3095ec5f2a5b2005f885ae2d3b95513446767043 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 11:40:49 +0800 Subject: [PATCH 01/14] Add TLPWhereGenerator interface --- .../common/gen/PartitionGenerator.java | 27 ++++++++++++++++++ .../common/gen/TLPWhereGenerator.java | 28 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 src/sqlancer/common/gen/PartitionGenerator.java create mode 100644 src/sqlancer/common/gen/TLPWhereGenerator.java diff --git a/src/sqlancer/common/gen/PartitionGenerator.java b/src/sqlancer/common/gen/PartitionGenerator.java new file mode 100644 index 00000000..affc62c4 --- /dev/null +++ b/src/sqlancer/common/gen/PartitionGenerator.java @@ -0,0 +1,27 @@ +package sqlancer.common.gen; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.schema.AbstractTableColumn; + +public interface PartitionGenerator, C extends AbstractTableColumn> { + + /** + * Negates a predicate (i.e., uses a NOT operator). + * + * @param predicate + * the boolean predicate. + * + * @return the negated predicate. + */ + E negatePredicate(E predicate); + + /** + * Checks if an expression evaluates to NULL (i.e., implements the IS NULL operator). + * + * @param expr + * the expression + * + * @return an expression that checks whether the expression evaluates to NULL. + */ + E isNull(E expr); +} diff --git a/src/sqlancer/common/gen/TLPWhereGenerator.java b/src/sqlancer/common/gen/TLPWhereGenerator.java new file mode 100644 index 00000000..095a878f --- /dev/null +++ b/src/sqlancer/common/gen/TLPWhereGenerator.java @@ -0,0 +1,28 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public interface TLPWhereGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> + extends PartitionGenerator { + + TLPWhereGenerator setTablesAndColumns(AbstractTables tables); + + E generateBooleanExpression(); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + List generateOrderBys(); +} From 307713ac59675bae6286e86eda8290db63e1ddb8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 11:41:04 +0800 Subject: [PATCH 02/14] Add generic TLPWhere oracle --- .../common/oracle/TLPWhereOracle.java | 84 +++++++++++++++++++ .../common/oracle/TestOracleUtils.java | 33 ++++++++ 2 files changed, 117 insertions(+) create mode 100644 src/sqlancer/common/oracle/TLPWhereOracle.java diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java new file mode 100644 index 00000000..5f7b3feb --- /dev/null +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -0,0 +1,84 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public class TLPWhereOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + + private TLPWhereGenerator gen; + private final ExpectedErrors errors; + + private String generatedQueryString; + + public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + S s = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(s); + gen = gen.setTablesAndColumns(targetTables); + + Select select = gen.generateSelect(); + + boolean shouldCreateDummy = true; + select.setFetchColumns(gen.generateFetchColumns(shouldCreateDummy)); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + select.setWhereClause(null); + + String originalQueryString = select.asString(); + generatedQueryString = originalQueryString; + List firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + boolean orderBy = Randomly.getBooleanWithSmallProbability(); + if (orderBy) { + select.setOrderByClauses(gen.generateOrderBys()); + } + + TestOracleUtils.PredicateVariants predicates = TestOracleUtils.initializeTernaryPredicateVariants(gen, + gen.generateBooleanExpression()); + select.setWhereClause(predicates.predicate); + String firstQueryString = select.asString(); + select.setWhereClause(predicates.negatedPredicate); + String secondQueryString = select.asString(); + select.setWhereClause(predicates.isNullPredicate); + String thirdQueryString = select.asString(); + + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + + ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java index b22cc89f..bab2e26c 100644 --- a/src/sqlancer/common/oracle/TestOracleUtils.java +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -2,6 +2,8 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.gen.PartitionGenerator; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; @@ -12,6 +14,18 @@ public final class TestOracleUtils { private TestOracleUtils() { } + public static final class PredicateVariants, C extends AbstractTableColumn> { + public E predicate; + public E negatedPredicate; + public E isNullPredicate; + + PredicateVariants(E predicate, E negatedPredicate, E isNullPredicate) { + this.predicate = predicate; + this.negatedPredicate = negatedPredicate; + this.isNullPredicate = isNullPredicate; + } + } + public static , C extends AbstractTableColumn> AbstractTables getRandomTableNonEmptyTables( AbstractSchema schema) { if (schema.getDatabaseTables().isEmpty()) { @@ -19,4 +33,23 @@ private TestOracleUtils() { } return new AbstractTables<>(Randomly.nonEmptySubset(schema.getDatabaseTables())); } + + public static , T extends AbstractTable, C extends AbstractTableColumn> PredicateVariants initializeTernaryPredicateVariants( + PartitionGenerator gen, E predicate) { + if (gen == null) { + throw new IllegalStateException(); + } + if (predicate == null) { + throw new IllegalStateException(); + } + E negatedPredicate = gen.negatePredicate(predicate); + if (negatedPredicate == null) { + throw new IllegalStateException(); + } + E isNullPredicate = gen.isNull(predicate); + if (isNullPredicate == null) { + throw new IllegalStateException(); + } + return new PredicateVariants<>(predicate, negatedPredicate, isNullPredicate); + } } From 6e4204830d61292333e3ac58621f3dfbc294fde9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 11:59:25 +0800 Subject: [PATCH 03/14] Implement TLPWhereGenerator interfaces for SQLite3 --- .../sqlite3/gen/SQLite3ExpressionGenerator.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 0ac0cd27..e4f0741f 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -9,6 +9,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Aggregate; @@ -50,7 +51,8 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; public class SQLite3ExpressionGenerator implements ExpressionGenerator, - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private SQLite3RowValue rw; private final SQLite3GlobalState globalState; @@ -133,6 +135,7 @@ public static SQLite3Expression getRandomLiteralValue(SQLite3GlobalState globalS return new SQLite3ExpressionGenerator(globalState).getRandomLiteralValueInternal(globalState.getRandomly()); } + @Override public List generateOrderBys() { List expressions = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { @@ -747,6 +750,18 @@ public List getTableRefs() { return tableRefs; } + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + List columns = new ArrayList<>(); + if (shouldCreateDummy && Randomly.getBoolean()) { + columns.add(new SQLite3ColumnName(SQLite3Column.createDummy("*"), null)); + } else { + columns = Randomly.nonEmptySubset(this.columns).stream().map(c -> new SQLite3ColumnName(c, null)) + .collect(Collectors.toList()); + } + return columns; + } + @Override public String generateOptimizedQueryString(SQLite3Select select, SQLite3Expression whereCondition, boolean shouldUseAggregate) { From 17f7d3ee09f267267c08cbdd1120482afe4d7715 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 12:00:14 +0800 Subject: [PATCH 04/14] Use generic TLPWhere oracle for SQLite3 --- .../oracle/tlp/SQLite3TLPWhereOracle.java | 49 +++++++------------ 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java index b9580ddd..8d850fdc 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java @@ -1,50 +1,37 @@ package sqlancer.sqlite3.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Expression; +import sqlancer.sqlite3.ast.SQLite3Select; +import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.schema.SQLite3Schema; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3TLPWhereOracle extends SQLite3TLPBase { +public class SQLite3TLPWhereOracle implements TestOracle { - private String generatedQueryString; + private final TLPWhereOracle oracle; public SQLite3TLPWhereOracle(SQLite3GlobalState state) { - super(state); + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) + .build(); + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = SQLite3Visitor.asString(select); - generatedQueryString = originalQueryString; - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithSmallProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = SQLite3Visitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = SQLite3Visitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = SQLite3Visitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); } @Override public String getLastQueryString() { - return generatedQueryString; + return oracle.getLastQueryString(); } - } From 3989d762ede46b8251a104fa66a294aa27897271 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 29 Jun 2024 16:27:43 +0800 Subject: [PATCH 05/14] [Databend] Refactor tests --- .github/workflows/main.yml | 4 +- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestDatabend.java | 108 ---------------------- test/sqlancer/dbms/TestDatabendNoREC.java | 23 +++++ test/sqlancer/dbms/TestDatabendPQS.java | 23 +++++ test/sqlancer/dbms/TestDatabendTLP.java | 21 +++++ 6 files changed, 71 insertions(+), 109 deletions(-) delete mode 100644 test/sqlancer/dbms/TestDatabend.java create mode 100644 test/sqlancer/dbms/TestDatabendNoREC.java create mode 100644 test/sqlancer/dbms/TestDatabendPQS.java create mode 100644 test/sqlancer/dbms/TestDatabendTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e5269fb..6b920cd6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -197,7 +197,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test + DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendTLP test + DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendNoREC test + DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendPQS test duckdb: name: DBMS Tests (DuckDB) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index baab9822..67c406fb 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -4,6 +4,7 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java deleted file mode 100644 index 707d7b8a..00000000 --- a/test/sqlancer/dbms/TestDatabend.java +++ /dev/null @@ -1,108 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; -import sqlancer.Randomly; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class TestDatabend { - - @Test - public void testDatabendNoREC() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); - } - - @Test - public void testDatabendPQS() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); - } - - @Test - public void testDatabendTLPQueryPartitioning() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); - } - - // @Test - // public void testDatabendTLPWhere() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "WHERE")); - // } - // - // @Test - // public void testDatabendTLPGroupBy() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "GROUP_BY")); - // } - // - // @Test - // public void testDatabendTLPHaving() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "HAVING")); - // } - // - // @Test - // public void testDatabendTLPDistinct() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "DISTINCT")); - // } - // - // @Test - // public void testDatabendTLPAggregate() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "AGGREGATE")); - // } - -} diff --git a/test/sqlancer/dbms/TestDatabendNoREC.java b/test/sqlancer/dbms/TestDatabendNoREC.java new file mode 100644 index 00000000..679f8c16 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendNoREC.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.Randomly; + +public class TestDatabendNoREC { + + @Test + public void testDatabendNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); + } + +} diff --git a/test/sqlancer/dbms/TestDatabendPQS.java b/test/sqlancer/dbms/TestDatabendPQS.java new file mode 100644 index 00000000..fba733d4 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendPQS.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.Randomly; + +public class TestDatabendPQS { + + @Test + public void testDatabendPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); + } + +} diff --git a/test/sqlancer/dbms/TestDatabendTLP.java b/test/sqlancer/dbms/TestDatabendTLP.java new file mode 100644 index 00000000..27ba5341 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendTLP.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDatabendTLP { + + @Test + public void testDatabendTLPQueryPartitioning() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); + } +} From 37de4a2cf8b90ce7c22f3db37d85590d6dca873f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 29 Jun 2024 16:27:54 +0800 Subject: [PATCH 06/14] [Databend] Update to v1.2.542 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b920cd6..10cd3ace 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -176,7 +176,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend:v1.2.452 + image: datafuselabs/databend:v1.2.542-nightly env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer From 92fb68865daa18d1ca6e99ef7da1da6614a0abe3 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Mon, 8 Jul 2024 14:42:43 +0800 Subject: [PATCH 07/14] Remove duplicated logs in NoREC oracle (#953) --- src/sqlancer/common/oracle/NoRECOracle.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index acc7cffa..53048829 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -123,10 +123,6 @@ public Reproducer getLastReproducer() { private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState state) { SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); - if (state.getOptions().logEachSelect()) { - state.getLogger().writeCurrent(queryString); - } - int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { @@ -151,10 +147,6 @@ private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState< private int extractCounts(String queryString, ExpectedErrors errors, SQLGlobalState state) { SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); - if (state.getOptions().logEachSelect()) { - state.getLogger().writeCurrent(queryString); - } - int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { From 852c1e4fecc853dc6246b45a3cc3045da9d8e920 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:18:44 +0200 Subject: [PATCH 08/14] [ClickHouse] Added pattern for changed error message (#955) * [ClickHouse] Added pattern for changed error message https://fiddle.clickhouse.com/a3a95024-4da7-4275-baff-86f116014744 ``` Received exception from server (version 24.2.3): Code: 403. DB::Exception: Received from localhost:9000. DB::Exception: Cannot get JOIN keys from JOIN ON section: '476505718 = `_--right_2.c0`', found keys: [Left keys: [] Right keys [] Condition columns: '', 'equals(476505718, _--right_2.c0)']. (INVALID_JOIN_ON_EXPRESSION) (query: SELECT SUM(check <> 0) FROM ((SELECT right_0.c0 AS `check` FROM t0 AS left FULL OUTER JOIN t0 AS right_0 ON ((left.c0)=(right_0.c0)) LEFT OUTER JOIN t0 AS right_1 ON ((left.c0)=(right_1.c0)) LEFT ANTI JOIN t0 AS right_2 ON ((476505718)=(right_2.c0)))) as res;) ``` * formatter --- src/sqlancer/clickhouse/ClickHouseErrors.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index c4ccd7e4..0354a617 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -45,8 +45,8 @@ public static List getExpectedExpressionErrors() { "Positional argument numeric constant expression is not representable as", "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", "with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)", - "Unexpected inf or nan to integer conversion", "Unsigned type must not contain", - "Unexpected inf or nan to integer conversion", + "Cannot get JOIN keys from JOIN ON section", "Unexpected inf or nan to integer conversion", + "Unsigned type must not contain", "Unexpected inf or nan to integer conversion", // The way we generate JOINs we can have ambiguous left table column without // alias From ec00b2d19115dc9ee080835b65b53d6aa7a93848 Mon Sep 17 00:00:00 2001 From: Robins Date: Tue, 16 Jul 2024 11:01:01 +0930 Subject: [PATCH 09/14] Avoid storage parameters during CREATE TABLE for partitioned tables (#956) CREATE TABLE WITH() allows storage parameters, but partitioned tables emit an error (given below) if attempted. This is because partitioned non-leaf tables are virutal tables [1] and don't accept storage parameters. Sample Error: "ERROR: cannot specify storage parameters for a partitioned table" This patch skips storage parameter generation for partitioned tables. Ref: 1. https://www.postgresql.org/docs/current/ddl-partitioning.html --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 7d65b8eb..9ae649ae 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -113,7 +113,9 @@ private void createStandard() throws AssertionError { generateInherits(); generatePartitionBy(); generateUsing(); - PostgresCommon.generateWith(sb, globalState, errors); + if (!isPartitionedTable) { + PostgresCommon.generateWith(sb, globalState, errors); + } if (Randomly.getBoolean() && isTemporaryTable) { sb.append(" ON COMMIT "); sb.append(Randomly.fromOptions("PRESERVE ROWS", "DELETE ROWS", "DROP")); From 6d2b92a5b72a5904d5f7827df6ff9488668f55e6 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Tue, 16 Jul 2024 17:44:17 +0800 Subject: [PATCH 10/14] Review feedback --- .github/workflows/main.yml | 2 +- .../datafusion/DataFusionProvider.java | 7 +++--- src/sqlancer/datafusion/DataFusionUtil.java | 25 ++++++++----------- .../server/datafusion_server/Cargo.toml | 13 ++++------ .../server/datafusion_server/src/main.rs | 9 ++----- 5 files changed, 22 insertions(+), 34 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f2a12c6..bd65c989 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -214,7 +214,7 @@ jobs: - name: Run DataFusion Server run: | cd src/sqlancer/datafusion/server/datafusion_server - cargo run --features "datafusion_stable" & sleep 300 + cargo run & sleep 300 - name: Set up JDK 11 uses: actions/setup-java@v3 with: diff --git a/src/sqlancer/datafusion/DataFusionProvider.java b/src/sqlancer/datafusion/DataFusionProvider.java index 161c324d..37328e4b 100644 --- a/src/sqlancer/datafusion/DataFusionProvider.java +++ b/src/sqlancer/datafusion/DataFusionProvider.java @@ -1,7 +1,7 @@ package sqlancer.datafusion; -import static java.lang.System.exit; import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.DML; +import static sqlancer.datafusion.DataFusionUtil.dfAssert; import static sqlancer.datafusion.DataFusionUtil.displayTables; import java.sql.Connection; @@ -52,13 +52,12 @@ public void generateDatabase(DataFusionGlobalState globalState) throws Exception List allTables = globalState.getSchema().getDatabaseTables(); List allTablesName = allTables.stream().map(t -> t.getName()).collect(Collectors.toList()); if (allTablesName.isEmpty()) { - System.out.println("Generate database failed"); - exit(1); + dfAssert(false, "Generate Database failed."); } // Randomly insert some data into existing tables for (DataFusionTable table : allTables) { - int nInsertQuery = globalState.getRandomly().getInteger(0, 8); // [0, 10) + int nInsertQuery = globalState.getRandomly().getInteger(0, globalState.getOptions().getMaxNumberInserts()); for (int i = 0; i < nInsertQuery; i++) { SQLQueryAdapter insertQuery = null; diff --git a/src/sqlancer/datafusion/DataFusionUtil.java b/src/sqlancer/datafusion/DataFusionUtil.java index e37ad8d6..8761bec9 100644 --- a/src/sqlancer/datafusion/DataFusionUtil.java +++ b/src/sqlancer/datafusion/DataFusionUtil.java @@ -1,7 +1,5 @@ package sqlancer.datafusion; -import static java.lang.System.exit; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -66,11 +64,15 @@ public static String displayTables(DataFusionGlobalState state, List fro return resultStringBuilder.toString(); } + // During development, you might want to manually let this function call exit(1) to fail fast public static void dfAssert(boolean condition, String message) { if (!condition) { - String methodName = Thread.currentThread().getStackTrace()[2].getMethodName(); - System.err.println("DataFusion assertion failed in function '" + methodName + "': " + message); - exit(1); + // // Development mode assertion failure + // String methodName = Thread.currentThread().getStackTrace()[2]// .getMethodName(); + // System.err.println("DataFusion assertion failed in function '" + methodName + "': " + message); + // exit(1); + + throw new AssertionError(message); } } @@ -149,9 +151,7 @@ public void appendToLog(DataFusionLogType logType, String logContent) { try { logFileWriter = new FileWriter(errorLogFile, true); } catch (IOException e) { - System.out.println("Failed to create FileWriter for errorLogFIle"); - e.printStackTrace(); - exit(1); + dfAssert(false, "Failed to create FileWriter for errorLogFIle"); } DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); String formattedDateTime = LocalDateTime.now().format(formatter); @@ -175,14 +175,11 @@ public void appendToLog(DataFusionLogType logType, String logContent) { logFileWriter.write(logContent); logFileWriter.flush(); } catch (IOException e) { - System.out.println("Failed to write to " + logType + " log: " + e.getMessage()); - e.printStackTrace(); - exit(1); + String err = "Failed to write to " + logType + " log: " + e.getMessage(); + dfAssert(false, err); } } else { - System.out.println("appending to log failed"); - Thread.currentThread().getStackTrace(); - exit(1); + dfAssert(false, "appending to log failed"); } } diff --git a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml index ebff15b3..cd8b85e1 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml +++ b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml @@ -18,8 +18,10 @@ async-trait = "0.1.73" bytes = "1.4" chrono = { version = "0.4.34", default-features = false } dashmap = "5.5.0" -datafusion_stable = { package = "datafusion", git = "https://github.com/apache/datafusion.git", rev = "e693ed7", optional = true } -datafusion_dev = { package = "datafusion", git = "https://github.com/apache/datafusion.git", branch = "main", optional = true } +# This version is for SQLancer CI run +datafusion = { version = "40.0.0" } +# Use following line if you want to test against the latest main branch of DataFusion +# datafusion = { git = "https://github.com/apache/datafusion.git", branch = "main" } env_logger = "0.11" futures = "0.3" half = { version = "2.2.1", default-features = false } @@ -41,9 +43,4 @@ mimalloc = { version = "0.1", default-features = false } [[bin]] name = "datafusion-server" -path = "src/main.rs" - -[features] -default = ["datafusion_stable_feature"] -datafusion_stable_feature = ["datafusion_stable"] -datafusion_dev_feature = ["datafusion_dev"] +path = "src/main.rs" \ No newline at end of file diff --git a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs index c19728bb..13ec73e9 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs +++ b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs @@ -1,8 +1,3 @@ -#[cfg(feature = "datafusion_dev")] -extern crate datafusion_dev as datafusion; -#[cfg(feature = "datafusion_stable")] -extern crate datafusion_stable as datafusion; - use arrow::array::{ArrayRef, StringArray}; use arrow::ipc::writer::IpcWriteOptions; use arrow::record_batch::RecordBatch; @@ -21,8 +16,8 @@ use arrow_flight::{ }; use arrow_schema::{DataType, Field, Schema}; use dashmap::DashMap; -use datafusion_stable::logical_expr::LogicalPlan; -use datafusion_stable::prelude::{DataFrame, ParquetReadOptions, SessionConfig, SessionContext}; +use datafusion::logical_expr::LogicalPlan; +use datafusion::prelude::{DataFrame, ParquetReadOptions, SessionConfig, SessionContext}; use futures::{Stream, StreamExt, TryStreamExt}; use log::info; use mimalloc::MiMalloc; From 0316c1c9c2fa2ad2a82977cfe2587faaba78cec8 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Wed, 17 Jul 2024 12:36:27 +0930 Subject: [PATCH 11/14] Ensure INHERITS() only uses tables (not views) In Postgres, CREATE TABLE INHERITS () throws an error when VIEWs are provided, sample given below. ERROR: inherited relation "pg_buffercache" is not a table or foreign table Ref: 1. https://www.postgresql.org/docs/current/ddl-inherit.html --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 9ae649ae..29ccfcf2 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -207,7 +207,7 @@ private void generateUsing() { } private void generateInherits() { - if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { + if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) .collect(Collectors.joining(", "))); From cc189b4d32961c09916a8870a0c0df3c5ce57c59 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Wed, 17 Jul 2024 12:40:42 +0930 Subject: [PATCH 12/14] Revert "Ensure INHERITS() only uses tables (not views)" This reverts commit 0316c1c9c2fa2ad2a82977cfe2587faaba78cec8. --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 29ccfcf2..9ae649ae 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -207,7 +207,7 @@ private void generateUsing() { } private void generateInherits() { - if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { + if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) .collect(Collectors.joining(", "))); From 96adb5446e05e52a8aff88b4aee7a5e42ff2c6a2 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Wed, 17 Jul 2024 12:42:23 +0930 Subject: [PATCH 13/14] Ensure INHERITS() only uses tables (not views) In Postgres, CREATE TABLE INHERITS () throws an error when VIEWs are provided, sample given below. ERROR: inherited relation "pg_buffercache" is not a table or foreign table Ref: 1. https://www.postgresql.org/docs/current/ddl-inherit.html --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 9ae649ae..29ccfcf2 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -207,7 +207,7 @@ private void generateUsing() { } private void generateInherits() { - if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { + if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) .collect(Collectors.joining(", "))); From 51e7ae87daba53699d1e9318f827b612d871f510 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:22:03 +0200 Subject: [PATCH 14/14] Another fix --- src/sqlancer/clickhouse/ClickHouseErrors.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 0354a617..e6a4613d 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -46,7 +46,8 @@ public static List getExpectedExpressionErrors() { "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", "with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)", "Cannot get JOIN keys from JOIN ON section", "Unexpected inf or nan to integer conversion", - "Unsigned type must not contain", "Unexpected inf or nan to integer conversion", + "Cannot determine join keys in", "Unsigned type must not contain", + "Unexpected inf or nan to integer conversion", // The way we generate JOINs we can have ambiguous left table column without // alias