[perf] introduce option to set JDBC fetch size

For some workloads and databases it can be beneficial to set a JDBC fetch size
to increase performance and decrease memory usage. This can help a lot (5x and
more in terms of speed) with result sets withs thousands of rows or more.

The feature is disabled by default and can be enabled by adding the config
option "jdbc_fetch_size", set to the desired number of rows (a value between
100 and 1000 is usually best, but depends on the workload). Since this
is just a hint to the underlying JDBC driver, the option might only work
in combination with other properties or not at all.

While at it, always close ResultSet right after use. This shouldn't matter
but there is no reason to special case anything.

Author: dr-itz

src/java/arjdbc/jdbc/RubyJdbcConnection.java

@@ -128,6 +128,7 @@ public class RubyJdbcConnection extends RubyObject {
     private boolean lazy = false; // final once set on initialize
     private boolean jndi; // final once set on initialize
     private boolean configureConnection = true; // final once initialized
+    private int fetchSize = 0; // 0 = JDBC default
 
     protected RubyJdbcConnection(Ruby runtime, RubyClass metaClass) {
         super(runtime, metaClass);
@@ -571,6 +572,11 @@ protected void doInitialize(final ThreadContext context, final IRubyObject confi
         else {
             this.configureConnection = value != context.runtime.getFalse();
         }
+
+        IRubyObject jdbcFetchSize = getConfigValue(context, "jdbc_fetch_size");
+        if (jdbcFetchSize != context.nil) {
+            this.fetchSize = RubyNumeric.fix2int(jdbcFetchSize);
+        }
     }
 
     @JRubyMethod(name = "adapter")
@@ -818,6 +824,7 @@ public IRubyObject call(final Connection connection) throws SQLException {
                             // is called, so we have to process the result sets as we get them
                             // this shouldn't be an issue in most cases since we're only getting 1 result set anyways
                             result = mapExecuteResult(context, connection, resultSet);
+                            resultSet.close();
                         } else {
                             result = context.runtime.newFixnum(updateCount);
                         }
@@ -851,6 +858,7 @@ protected Statement createStatement(final ThreadContext context, final Connectio
         else {
             statement.setEscapeProcessing(escapeProcessing.isTrue());
         }
+        if (fetchSize != 0) statement.setFetchSize(fetchSize);
         return statement;
     }
 
@@ -1045,6 +1053,7 @@ public IRubyObject call(final Connection connection) throws SQLException {
                     else {
                         final PreparedStatement prepStatement;
                         statement = prepStatement = connection.prepareStatement(query);
+                        if (fetchSize != 0) statement.setFetchSize(fetchSize);
                         statement.setMaxRows(maxRows); // zero means there is no limit
                         setStatementParameters(context, connection, prepStatement, binds);
                         hasResult = prepStatement.execute();
@@ -1123,7 +1132,9 @@ public IRubyObject prepare_statement(final ThreadContext context, final IRubyObj
         return withConnection(context, new Callable<IRubyObject>() {
             public IRubyObject call(Connection connection) throws SQLException {
                 final String query = sql.convertToString().getUnicodeValue();
-                return JavaUtil.convertJavaToRuby(context.runtime, connection.prepareStatement(query));
+                PreparedStatement statement = connection.prepareStatement(query);
+                if (fetchSize != 0) statement.setFetchSize(fetchSize);
+                return JavaUtil.convertJavaToRuby(context.runtime, statement);
             }
         });
     }
@@ -1158,19 +1169,15 @@ public IRubyObject call(final Connection connection) throws SQLException {
                         statement = (PreparedStatement) JavaEmbedUtils.rubyToJava(cachedStatement);
                     } else {
                         statement = connection.prepareStatement(query);
+                        if (fetchSize != 0) statement.setFetchSize(fetchSize);
                     }
 
                     setStatementParameters(context, connection, statement, (RubyArray) binds);
 
                     if (statement.execute()) {
                         ResultSet resultSet = statement.getResultSet();
                         IRubyObject results = mapQueryResult(context, connection, resultSet);
-
-                        if (cached) {
-                            // Make sure we free the result set if we are caching the statement
-                            // It gets closed automatically when the statement is closed if we aren't caching
-                            resultSet.close();
-                        }
+                        resultSet.close();
 
                         return results;
                     } else {
@@ -2501,6 +2508,8 @@ protected IRubyObject arrayToRuby(final ThreadContext context,
             while ( arrayResult.next() ) {
                 array.append( jdbcToRuby(context, runtime, 2, baseType, arrayResult) );
             }
+            arrayResult.close();
+
             return array;
         }
         finally { if ( value != null ) value.free(); }

test/rails/config.yml

@@ -62,6 +62,7 @@ connections:
       driver: <%= 'org.mariadb.jdbc.Driver' %>
       <% end %>
       <% if defined? JRUBY_VERSION %>
+      jdbc_fetch_size: 100
       properties:
         serverTimezone: <%= java.util.TimeZone.getDefault.getID %>
       <% end %>
@@ -80,6 +81,7 @@ connections:
       prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || ENV['PS'] %> # should leave it off to its default if not set
       <% end %>
       username: <%= ENV['PGUSER'] || ENV['user'] || 'rails' %>
+      jdbc_fetch_size: 100
       properties:
         # Postgres' JDBC driver does not prepare statements until executed 5 times by default, let's lower:
         prepareThreshold: <%= ENV['PREPARE_THRESHOLD'] || 1 %>
@@ -96,9 +98,11 @@ connections:
     arunit:
       database: <%= FIXTURES_ROOT %>/fixture_database.sqlite3
       timeout:  5000
+      jdbc_fetch_size: 100
     arunit2:
       database: <%= FIXTURES_ROOT %>/fixture_database_2.sqlite3
       timeout:  5000
+      jdbc_fetch_size: 100
 
   sqlite3_mem:
     arunit: