From 06b0faaf5770da47681d9d32e06ea204519a3846 Mon Sep 17 00:00:00 2001 From: Steve Ebersole Date: Fri, 2 Nov 2012 13:27:54 -0500 Subject: [PATCH] HHH-7746 - Investigate alternative batch loading algorithms --- .../org/hibernate/cfg/AvailableSettings.java | 7 + .../main/java/org/hibernate/cfg/Settings.java | 10 + .../org/hibernate/cfg/SettingsFactory.java | 6 + .../hibernate/internal/util/StringHelper.java | 86 ++++++ .../util/collections/ArrayHelper.java | 39 ++- .../loader/AbstractEntityJoinWalker.java | 9 +- .../org/hibernate/loader/BatchFetchStyle.java | 90 ++++++ .../java/org/hibernate/loader/Loader.java | 17 +- .../org/hibernate/loader/OuterJoinLoader.java | 3 +- .../BatchingCollectionInitializer.java | 99 +------ .../BatchingCollectionInitializerBuilder.java | 112 ++++++++ .../collection/CollectionJoinWalker.java | 2 +- .../loader/collection/CollectionLoader.java | 4 + ...cBatchingCollectionInitializerBuilder.java | 265 +++++++++++++++++ ...yBatchingCollectionInitializerBuilder.java | 103 +++++++ ...dBatchingCollectionInitializerBuilder.java | 119 ++++++++ .../hibernate/loader/custom/CustomLoader.java | 2 +- .../loader/entity/BatchingEntityLoader.java | 162 +++++------ .../entity/BatchingEntityLoaderBuilder.java | 117 ++++++++ .../DynamicBatchingEntityLoaderBuilder.java | 266 ++++++++++++++++++ .../LegacyBatchingEntityLoaderBuilder.java | 125 ++++++++ .../PaddedBatchingEntityLoaderBuilder.java | 140 +++++++++ .../loader/entity/UniqueEntityLoader.java | 1 + .../org/hibernate/loader/hql/QueryLoader.java | 2 +- .../collection/BasicCollectionPersister.java | 4 +- .../collection/OneToManyPersister.java | 4 +- .../entity/AbstractEntityPersister.java | 19 +- .../test/batchfetch/BatchFetchTest.java | 64 +++++ .../test/batchfetch/BatchLoadableEntity.java | 64 +++++ .../batchload/BatchedManyToManyTest.java | 29 +- 30 files changed, 1734 insertions(+), 236 deletions(-) create mode 100644 hibernate-core/src/main/java/org/hibernate/loader/BatchFetchStyle.java create mode 100644 hibernate-core/src/main/java/org/hibernate/loader/collection/BatchingCollectionInitializerBuilder.java create mode 100644 hibernate-core/src/main/java/org/hibernate/loader/collection/DynamicBatchingCollectionInitializerBuilder.java create mode 100644 hibernate-core/src/main/java/org/hibernate/loader/collection/LegacyBatchingCollectionInitializerBuilder.java create mode 100644 hibernate-core/src/main/java/org/hibernate/loader/collection/PaddedBatchingCollectionInitializerBuilder.java create mode 100644 hibernate-core/src/main/java/org/hibernate/loader/entity/BatchingEntityLoaderBuilder.java create mode 100644 hibernate-core/src/main/java/org/hibernate/loader/entity/DynamicBatchingEntityLoaderBuilder.java create mode 100644 hibernate-core/src/main/java/org/hibernate/loader/entity/LegacyBatchingEntityLoaderBuilder.java create mode 100644 hibernate-core/src/main/java/org/hibernate/loader/entity/PaddedBatchingEntityLoaderBuilder.java create mode 100644 hibernate-core/src/test/java/org/hibernate/test/batchfetch/BatchLoadableEntity.java diff --git a/hibernate-core/src/main/java/org/hibernate/cfg/AvailableSettings.java b/hibernate-core/src/main/java/org/hibernate/cfg/AvailableSettings.java index 657619f4b47a..346daaa28ded 100644 --- a/hibernate-core/src/main/java/org/hibernate/cfg/AvailableSettings.java +++ b/hibernate-core/src/main/java/org/hibernate/cfg/AvailableSettings.java @@ -579,4 +579,11 @@ public interface AvailableSettings { public static final String ENABLE_LAZY_LOAD_NO_TRANS = "hibernate.enable_lazy_load_no_trans"; public static final String HQL_BULK_ID_STRATEGY = "hibernate.hql.bulk_id_strategy"; + + /** + * Names the {@link org.hibernate.loader.BatchFetchStyle} to use. Can specify either the + * {@link org.hibernate.loader.BatchFetchStyle} name (insensitively), or a + * {@link org.hibernate.loader.BatchFetchStyle} instance. + */ + public static final String BATCH_FETCH_STYLE = "hibernate.batch_fetch_style"; } diff --git a/hibernate-core/src/main/java/org/hibernate/cfg/Settings.java b/hibernate-core/src/main/java/org/hibernate/cfg/Settings.java index b11b15e78883..c648e02c1dde 100644 --- a/hibernate-core/src/main/java/org/hibernate/cfg/Settings.java +++ b/hibernate-core/src/main/java/org/hibernate/cfg/Settings.java @@ -33,6 +33,7 @@ import org.hibernate.hql.spi.MultiTableBulkIdStrategy; import org.hibernate.hql.spi.QueryTranslatorFactory; import org.hibernate.engine.transaction.jta.platform.spi.JtaPlatform; +import org.hibernate.loader.BatchFetchStyle; import org.hibernate.tuple.entity.EntityTuplizerFactory; /** @@ -91,6 +92,7 @@ public final class Settings { private JtaPlatform jtaPlatform; private MultiTableBulkIdStrategy multiTableBulkIdStrategy; + private BatchFetchStyle batchFetchStyle; /** @@ -480,4 +482,12 @@ public MultiTableBulkIdStrategy getMultiTableBulkIdStrategy() { void setMultiTableBulkIdStrategy(MultiTableBulkIdStrategy multiTableBulkIdStrategy) { this.multiTableBulkIdStrategy = multiTableBulkIdStrategy; } + + public BatchFetchStyle getBatchFetchStyle() { + return batchFetchStyle; + } + + void setBatchFetchStyle(BatchFetchStyle batchFetchStyle) { + this.batchFetchStyle = batchFetchStyle; + } } diff --git a/hibernate-core/src/main/java/org/hibernate/cfg/SettingsFactory.java b/hibernate-core/src/main/java/org/hibernate/cfg/SettingsFactory.java index 71bf2a1f849a..59da16e9ded1 100644 --- a/hibernate-core/src/main/java/org/hibernate/cfg/SettingsFactory.java +++ b/hibernate-core/src/main/java/org/hibernate/cfg/SettingsFactory.java @@ -49,6 +49,7 @@ import org.hibernate.internal.CoreMessageLogger; import org.hibernate.internal.util.StringHelper; import org.hibernate.internal.util.config.ConfigurationHelper; +import org.hibernate.loader.BatchFetchStyle; import org.hibernate.service.ServiceRegistry; import org.hibernate.boot.registry.classloading.spi.ClassLoaderService; import org.hibernate.engine.jdbc.connections.spi.ConnectionProvider; @@ -199,6 +200,11 @@ public Settings buildSettings(Properties props, ServiceRegistry serviceRegistry) } settings.setConnectionReleaseMode( releaseMode ); + final BatchFetchStyle batchFetchStyle = BatchFetchStyle.interpret( properties.get( AvailableSettings.BATCH_FETCH_STYLE ) ); + LOG.debugf( "Using BatchFetchStyle : " + batchFetchStyle.name() ); + settings.setBatchFetchStyle( batchFetchStyle ); + + //SQL Generation settings: String defaultSchema = properties.getProperty( AvailableSettings.DEFAULT_SCHEMA ); diff --git a/hibernate-core/src/main/java/org/hibernate/internal/util/StringHelper.java b/hibernate-core/src/main/java/org/hibernate/internal/util/StringHelper.java index 10bafd49e78d..19f3e6483397 100644 --- a/hibernate-core/src/main/java/org/hibernate/internal/util/StringHelper.java +++ b/hibernate-core/src/main/java/org/hibernate/internal/util/StringHelper.java @@ -24,6 +24,7 @@ */ package org.hibernate.internal.util; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -66,6 +67,17 @@ public static String join(String seperator, String[] strings) { return buf.toString(); } + public static String joinWithQualifier(String[] values, String qualifier, String deliminator) { + int length = values.length; + if ( length == 0 ) return ""; + StringBuilder buf = new StringBuilder( length * values[0].length() ) + .append( qualify( qualifier, values[0] ) ); + for ( int i = 1; i < length; i++ ) { + buf.append( deliminator ).append( qualify( qualifier, values[i] ) ); + } + return buf.toString(); + } + public static String join(String seperator, Iterator objects) { StringBuilder buf = new StringBuilder(); if ( objects.hasNext() ) buf.append( objects.next() ); @@ -89,6 +101,15 @@ public static String repeat(String string, int times) { return buf.toString(); } + public static String repeat(String string, int times, String deliminator) { + StringBuilder buf = new StringBuilder( ( string.length() * times ) + ( deliminator.length() * (times-1) ) ) + .append( string ); + for ( int i = 1; i < times; i++ ) { + buf.append( deliminator ).append( string ); + } + return buf.toString(); + } + public static String repeat(char character, int times) { char[] buffer = new char[times]; Arrays.fill( buffer, character ); @@ -661,4 +682,69 @@ public static String[] unquote(String[] names, Dialect dialect) { } return unquoted; } + + + public static final String BATCH_ID_PLACEHOLDER = "$$BATCH_ID_PLACEHOLDER$$"; + + public static StringBuilder buildBatchFetchRestrictionFragment( + String alias, + String[] columnNames, + Dialect dialect) { + // the general idea here is to just insert a placeholder that we can easily find later... + if ( columnNames.length == 1 ) { + // non-composite key + return new StringBuilder( StringHelper.qualify( alias, columnNames[0] ) ) + .append( " in (" ).append( BATCH_ID_PLACEHOLDER ).append( ")" ); + } + else { + // composite key - the form to use here depends on what the dialect supports. + if ( dialect.supportsRowValueConstructorSyntaxInInList() ) { + // use : (col1, col2) in ( (?,?), (?,?), ... ) + StringBuilder builder = new StringBuilder(); + builder.append( "(" ); + boolean firstPass = true; + String deliminator = ""; + for ( String columnName : columnNames ) { + builder.append( deliminator ).append( StringHelper.qualify( alias, columnName ) ); + if ( firstPass ) { + firstPass = false; + deliminator = ","; + } + } + builder.append( ") in (" ); + builder.append( BATCH_ID_PLACEHOLDER ); + builder.append( ")" ); + return builder; + } + else { + // use : ( (col1 = ? and col2 = ?) or (col1 = ? and col2 = ?) or ... ) + // unfortunately most of this building needs to be held off until we know + // the exact number of ids :( + return new StringBuilder( "(" ).append( BATCH_ID_PLACEHOLDER ).append( ")" ); + } + } + } + + public static String expandBatchIdPlaceholder( + String sql, + Serializable[] ids, + String alias, + String[] keyColumnNames, + Dialect dialect) { + if ( keyColumnNames.length == 1 ) { + // non-composite + return StringHelper.replace( sql, BATCH_ID_PLACEHOLDER, repeat( "?", ids.length, "," ) ); + } + else { + // composite + if ( dialect.supportsRowValueConstructorSyntaxInInList() ) { + final String tuple = "(" + StringHelper.repeat( "?", keyColumnNames.length, "," ); + return StringHelper.replace( sql, BATCH_ID_PLACEHOLDER, repeat( tuple, ids.length, "," ) ); + } + else { + final String keyCheck = joinWithQualifier( keyColumnNames, alias, " and " ); + return replace( sql, BATCH_ID_PLACEHOLDER, repeat( keyCheck, ids.length, " or " ) ); + } + } + } } diff --git a/hibernate-core/src/main/java/org/hibernate/internal/util/collections/ArrayHelper.java b/hibernate-core/src/main/java/org/hibernate/internal/util/collections/ArrayHelper.java index f2eb8cb6afa5..f43e35b6b431 100644 --- a/hibernate-core/src/main/java/org/hibernate/internal/util/collections/ArrayHelper.java +++ b/hibernate-core/src/main/java/org/hibernate/internal/util/collections/ArrayHelper.java @@ -24,6 +24,7 @@ */ package org.hibernate.internal.util.collections; +import java.io.Serializable; import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Arrays; @@ -31,6 +32,7 @@ import java.util.Iterator; import java.util.List; +import org.hibernate.HibernateException; import org.hibernate.LockMode; import org.hibernate.LockOptions; import org.hibernate.type.Type; @@ -372,10 +374,43 @@ public static boolean isEquals(byte[] b1, byte[] b2) { } return true; } -} - + public static Serializable[] extractNonNull(Serializable[] array) { + final int nonNullCount = countNonNull( array ); + final Serializable[] result = new Serializable[nonNullCount]; + int i = 0; + for ( Serializable element : array ) { + if ( element != null ) { + result[i++] = element; + } + } + if ( i != nonNullCount ) { + throw new HibernateException( "Number of non-null elements varied between iterations" ); + } + return result; + } + public static int countNonNull(Serializable[] array) { + int i = 0; + for ( Serializable element : array ) { + if ( element != null ) { + i++; + } + } + return i; + } + public static void main(String... args) { + int[] batchSizes = ArrayHelper.getBatchSizes( 32 ); + System.out.println( "Forward ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" ); + for ( int i = 0; i < batchSizes.length; i++ ) { + System.out.println( "[" + i + "] -> " + batchSizes[i] ); + } + System.out.println( "Backward ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" ); + for ( int i = batchSizes.length-1; i >= 0; i-- ) { + System.out.println( "[" + i + "] -> " + batchSizes[i] ); + } + } +} diff --git a/hibernate-core/src/main/java/org/hibernate/loader/AbstractEntityJoinWalker.java b/hibernate-core/src/main/java/org/hibernate/loader/AbstractEntityJoinWalker.java index 23c5a7b01aad..673741e8fbb7 100755 --- a/hibernate-core/src/main/java/org/hibernate/loader/AbstractEntityJoinWalker.java +++ b/hibernate-core/src/main/java/org/hibernate/loader/AbstractEntityJoinWalker.java @@ -187,10 +187,7 @@ protected final boolean isJoinFetchEnabledByProfile(OuterJoinLoadable persister, public abstract String getComment(); @Override - protected boolean isDuplicateAssociation( - final String foreignKeyTable, - final String[] foreignKeyColumns - ) { + protected boolean isDuplicateAssociation(final String foreignKeyTable, final String[] foreignKeyColumns) { //disable a join back to this same association final boolean isSameJoin = persister.getTableName().equals( foreignKeyTable ) && @@ -201,11 +198,11 @@ protected boolean isDuplicateAssociation( - protected final Loadable getPersister() { + public final Loadable getPersister() { return persister; } - protected final String getAlias() { + public final String getAlias() { return alias; } diff --git a/hibernate-core/src/main/java/org/hibernate/loader/BatchFetchStyle.java b/hibernate-core/src/main/java/org/hibernate/loader/BatchFetchStyle.java new file mode 100644 index 000000000000..a429408cec5a --- /dev/null +++ b/hibernate-core/src/main/java/org/hibernate/loader/BatchFetchStyle.java @@ -0,0 +1,90 @@ +/* + * Hibernate, Relational Persistence for Idiomatic Java + * + * Copyright (c) 2012, Red Hat Inc. or third-party contributors as + * indicated by the @author tags or express copyright attribution + * statements applied by the authors. All third-party contributions are + * distributed under license by Red Hat Inc. + * + * This copyrighted material is made available to anyone wishing to use, modify, + * copy, or redistribute it subject to the terms and conditions of the GNU + * Lesser General Public License, as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this distribution; if not, write to: + * Free Software Foundation, Inc. + * 51 Franklin Street, Fifth Floor + * Boston, MA 02110-1301 USA + */ +package org.hibernate.loader; + +import org.jboss.logging.Logger; + +/** + * Defines the style that should be used to perform batch loading. Which style to use is declared using + * the "{@value org.hibernate.cfg.AvailableSettings#BATCH_FETCH_STYLE}" + * ({@link org.hibernate.cfg.AvailableSettings#BATCH_FETCH_STYLE}) setting + * + * @author Steve Ebersole + */ +public enum BatchFetchStyle { + /** + * The legacy algorithm where we keep a set of pre-built batch sizes based on + * {@link org.hibernate.internal.util.collections.ArrayHelper#getBatchSizes}. Batches are performed + * using the next-smaller pre-built batch size from the number of existing batchable identifiers. + *

+ * For example, with a batch-size setting of 32 the pre-built batch sizes would be [32, 16, 10, 9, 8, 7, .., 1]. + * An attempt to batch load 31 identifiers would result in batches of 16, 10, and 5. + */ + LEGACY, + /** + * Still keeps the concept of pre-built batch sizes, but uses the next-bigger batch size and pads the extra + * identifier placeholders. + *

+ * Using the same example of a batch-size setting of 32 the pre-built batch sizes would be the same. However, the + * attempt to batch load 31 identifiers would result just a single batch of size 32. The identifiers to load would + * be "padded" (aka, repeated) to make up the difference. + */ + PADDED, + /** + * Dynamically builds its SQL based on the actual number of available ids. Does still limit to the batch-size + * defined on the entity/collection + */ + DYNAMIC; + + private static final Logger log = Logger.getLogger( BatchFetchStyle.class ); + + public static BatchFetchStyle byName(String name) { + return valueOf( name.toUpperCase() ); + } + + public static BatchFetchStyle interpret(Object setting) { + log.tracef( "Interpreting BatchFetchStyle from setting : %s", setting ); + + if ( setting == null ) { + return LEGACY; // as default + } + + if ( BatchFetchStyle.class.isInstance( setting ) ) { + return (BatchFetchStyle) setting; + } + + try { + final BatchFetchStyle byName = byName( setting.toString() ); + if ( byName != null ) { + return byName; + } + } + catch (Exception ignore) { + } + + log.debugf( "Unable to interpret given setting [%s] as BatchFetchStyle", setting ); + + return LEGACY; // again as default. + } +} diff --git a/hibernate-core/src/main/java/org/hibernate/loader/Loader.java b/hibernate-core/src/main/java/org/hibernate/loader/Loader.java index 043663946bed..2370a7c921d4 100644 --- a/hibernate-core/src/main/java/org/hibernate/loader/Loader.java +++ b/hibernate-core/src/main/java/org/hibernate/loader/Loader.java @@ -119,7 +119,7 @@ public Loader(SessionFactoryImplementor factory) { * * @return The sql command this loader should use to get its {@link ResultSet}. */ - protected abstract String getSQLString(); + public abstract String getSQLString(); /** * An array of persisters of entity classes contained in each row of results; @@ -256,7 +256,7 @@ private String prependComment(String sql, QueryParameters parameters) { * persister from each row of the ResultSet. If an object is supplied, will attempt to * initialize that object. If a collection is supplied, attempt to initialize that collection. */ - private List doQueryAndInitializeNonLazyCollections( + public List doQueryAndInitializeNonLazyCollections( final SessionImplementor session, final QueryParameters queryParameters, final boolean returnProxies) throws HibernateException, SQLException { @@ -268,7 +268,7 @@ private List doQueryAndInitializeNonLazyCollections( ); } - private List doQueryAndInitializeNonLazyCollections( + public List doQueryAndInitializeNonLazyCollections( final SessionImplementor session, final QueryParameters queryParameters, final boolean returnProxies, @@ -1722,8 +1722,17 @@ protected ResultSet executeQueryStatement( final QueryParameters queryParameters, final boolean scroll, final SessionImplementor session) throws SQLException { + return executeQueryStatement( getSQLString(), queryParameters, scroll, session ); + } + + protected ResultSet executeQueryStatement( + final String sqlStatement, + final QueryParameters queryParameters, + final boolean scroll, + final SessionImplementor session) throws SQLException { + // Processing query filters. - queryParameters.processFilters( getSQLString(), session ); + queryParameters.processFilters( sqlStatement, session ); // Applying LIMIT clause. final LimitHandler limitHandler = getLimitHandler( diff --git a/hibernate-core/src/main/java/org/hibernate/loader/OuterJoinLoader.java b/hibernate-core/src/main/java/org/hibernate/loader/OuterJoinLoader.java index 6b41233f1050..9d295cbe2571 100644 --- a/hibernate-core/src/main/java/org/hibernate/loader/OuterJoinLoader.java +++ b/hibernate-core/src/main/java/org/hibernate/loader/OuterJoinLoader.java @@ -75,7 +75,8 @@ protected String[] getCollectionSuffixes() { return collectionSuffixes; } - protected final String getSQLString() { + @Override + public final String getSQLString() { return sql; } diff --git a/hibernate-core/src/main/java/org/hibernate/loader/collection/BatchingCollectionInitializer.java b/hibernate-core/src/main/java/org/hibernate/loader/collection/BatchingCollectionInitializer.java index 413e74d084d5..e659d2b5de6f 100644 --- a/hibernate-core/src/main/java/org/hibernate/loader/collection/BatchingCollectionInitializer.java +++ b/hibernate-core/src/main/java/org/hibernate/loader/collection/BatchingCollectionInitializer.java @@ -1,10 +1,10 @@ /* * Hibernate, Relational Persistence for Idiomatic Java * - * Copyright (c) 2008, Red Hat Middleware LLC or third-party contributors as + * Copyright (c) 2008, 2012, Red Hat Inc. or third-party contributors as * indicated by the @author tags or express copyright attribution * statements applied by the authors. All third-party contributions are - * distributed under license by Red Hat Middleware LLC. + * distributed under license by Red Hat Inc. * * This copyrighted material is made available to anyone wishing to use, modify, * copy, or redistribute it subject to the terms and conditions of the GNU @@ -20,106 +20,35 @@ * Free Software Foundation, Inc. * 51 Franklin Street, Fifth Floor * Boston, MA 02110-1301 USA - * */ package org.hibernate.loader.collection; -import java.io.Serializable; -import org.hibernate.HibernateException; -import org.hibernate.MappingException; -import org.hibernate.engine.spi.LoadQueryInfluencers; -import org.hibernate.engine.spi.SessionFactoryImplementor; -import org.hibernate.engine.spi.SessionImplementor; -import org.hibernate.internal.util.collections.ArrayHelper; -import org.hibernate.loader.Loader; import org.hibernate.persister.collection.CollectionPersister; import org.hibernate.persister.collection.QueryableCollection; /** - * "Batch" loads collections, using multiple foreign key values in the - * SQL where clause. + * The base contract for loaders capable of performing batch-fetch loading of collections using multiple foreign key + * values in the SQL WHERE clause. + * + * @author Gavin King + * @author Steve Ebersole * + * @see BatchingCollectionInitializerBuilder * @see BasicCollectionLoader * @see OneToManyLoader - * @author Gavin King */ -public class BatchingCollectionInitializer implements CollectionInitializer { - private final Loader[] loaders; - private final int[] batchSizes; - private final CollectionPersister collectionPersister; +public abstract class BatchingCollectionInitializer implements CollectionInitializer { + private final QueryableCollection collectionPersister; - public BatchingCollectionInitializer(CollectionPersister collPersister, int[] batchSizes, Loader[] loaders) { - this.loaders = loaders; - this.batchSizes = batchSizes; - this.collectionPersister = collPersister; + public BatchingCollectionInitializer(QueryableCollection collectionPersister) { + this.collectionPersister = collectionPersister; } public CollectionPersister getCollectionPersister() { return collectionPersister; } - public Loader[] getLoaders() { - return loaders; - } - - public int[] getBatchSizes() { - return batchSizes; - } - - public void initialize(Serializable id, SessionImplementor session) - throws HibernateException { - - Serializable[] batch = session.getPersistenceContext().getBatchFetchQueue() - .getCollectionBatch( collectionPersister, id, batchSizes[0] ); - - for ( int i=0; i 1 ) { - int[] batchSizesToCreate = ArrayHelper.getBatchSizes(maxBatchSize); - Loader[] loadersToCreate = new Loader[ batchSizesToCreate.length ]; - for ( int i=0; i 1 ) { - int[] batchSizesToCreate = ArrayHelper.getBatchSizes( maxBatchSize ); - Loader[] loadersToCreate = new Loader[ batchSizesToCreate.length ]; - for ( int i=0; i= numberOfIds ) { + indexToUse = i; + } + else { + break; + } + } + + final Serializable[] idsToLoad = new Serializable[ batchSizes[indexToUse] ]; + System.arraycopy( batch, 0, idsToLoad, 0, numberOfIds ); + for ( int i = numberOfIds; i < batchSizes[indexToUse]; i++ ) { + idsToLoad[i] = id; + } + + loaders[indexToUse].loadCollectionBatch( session, idsToLoad, collectionPersister().getKeyType() ); + } + } +} diff --git a/hibernate-core/src/main/java/org/hibernate/loader/custom/CustomLoader.java b/hibernate-core/src/main/java/org/hibernate/loader/custom/CustomLoader.java index 948a2c7d5cd8..c3b241805987 100755 --- a/hibernate-core/src/main/java/org/hibernate/loader/custom/CustomLoader.java +++ b/hibernate-core/src/main/java/org/hibernate/loader/custom/CustomLoader.java @@ -294,7 +294,7 @@ protected String getQueryIdentifier() { } @Override - protected String getSQLString() { + public String getSQLString() { return sql; } diff --git a/hibernate-core/src/main/java/org/hibernate/loader/entity/BatchingEntityLoader.java b/hibernate-core/src/main/java/org/hibernate/loader/entity/BatchingEntityLoader.java index ef7469aa8fc3..bde9a8c24af4 100644 --- a/hibernate-core/src/main/java/org/hibernate/loader/entity/BatchingEntityLoader.java +++ b/hibernate-core/src/main/java/org/hibernate/loader/entity/BatchingEntityLoader.java @@ -1,10 +1,10 @@ /* * Hibernate, Relational Persistence for Idiomatic Java * - * Copyright (c) 2008, Red Hat Middleware LLC or third-party contributors as + * Copyright (c) 2008, 2012, Red Hat Inc. or third-party contributors as * indicated by the @author tags or express copyright attribution * statements applied by the authors. All third-party contributions are - * distributed under license by Red Hat Middleware LLC. + * distributed under license by Red Hat Inc. * * This copyrighted material is made available to anyone wishing to use, modify, * copy, or redistribute it subject to the terms and conditions of the GNU @@ -20,135 +20,109 @@ * Free Software Foundation, Inc. * 51 Franklin Street, Fifth Floor * Boston, MA 02110-1301 USA - * */ package org.hibernate.loader.entity; import java.io.Serializable; -import java.util.Iterator; +import java.sql.SQLException; +import java.util.Arrays; import java.util.List; -import org.hibernate.LockMode; +import org.jboss.logging.Logger; + import org.hibernate.LockOptions; -import org.hibernate.MappingException; -import org.hibernate.engine.spi.LoadQueryInfluencers; -import org.hibernate.engine.spi.SessionFactoryImplementor; +import org.hibernate.engine.spi.QueryParameters; import org.hibernate.engine.spi.SessionImplementor; -import org.hibernate.internal.util.collections.ArrayHelper; import org.hibernate.loader.Loader; import org.hibernate.persister.entity.EntityPersister; -import org.hibernate.persister.entity.OuterJoinLoadable; +import org.hibernate.pretty.MessageHelper; import org.hibernate.type.Type; /** - * "Batch" loads entities, using multiple primary key values in the - * SQL where clause. + * The base contract for loaders capable of performing batch-fetch loading of entities using multiple primary key + * values in the SQL WHERE clause. * - * @see EntityLoader * @author Gavin King + * @author Steve Ebersole + * + * @see BatchingEntityLoaderBuilder + * @see UniqueEntityLoader */ -public class BatchingEntityLoader implements UniqueEntityLoader { +public abstract class BatchingEntityLoader implements UniqueEntityLoader { + private static final Logger log = Logger.getLogger( BatchingEntityLoader.class ); - private final Loader[] loaders; - private final int[] batchSizes; private final EntityPersister persister; - private final Type idType; - public BatchingEntityLoader(EntityPersister persister, int[] batchSizes, Loader[] loaders) { - this.batchSizes = batchSizes; - this.loaders = loaders; + public BatchingEntityLoader(EntityPersister persister) { this.persister = persister; - idType = persister.getIdentifierType(); } - private Object getObjectFromList(List results, Serializable id, SessionImplementor session) { - // get the right object from the list ... would it be easier to just call getEntity() ?? - Iterator iter = results.iterator(); - while ( iter.hasNext() ) { - Object obj = iter.next(); - final boolean equal = idType.isEqual( - id, - session.getContextEntityIdentifier(obj), - session.getFactory() - ); - if ( equal ) return obj; - } - return null; + public EntityPersister persister() { + return persister; } - /** - * {@inheritDoc} - */ + @Override + @Deprecated public Object load(Serializable id, Object optionalObject, SessionImplementor session) { - // this form is deprecated! return load( id, optionalObject, session, LockOptions.NONE ); } - public Object load(Serializable id, Object optionalObject, SessionImplementor session, LockOptions lockOptions) { - Serializable[] batch = session.getPersistenceContext() - .getBatchFetchQueue() - .getEntityBatch( persister, id, batchSizes[0], persister.getEntityMode() ); + protected QueryParameters buildQueryParameters( + Serializable id, + Serializable[] ids, + Object optionalObject, + LockOptions lockOptions) { + Type[] types = new Type[ids.length]; + Arrays.fill( types, persister().getIdentifierType() ); + + QueryParameters qp = new QueryParameters(); + qp.setPositionalParameterTypes( types ); + qp.setPositionalParameterValues( ids ); + qp.setOptionalObject( optionalObject ); + qp.setOptionalEntityName( persister().getEntityName() ); + qp.setOptionalId( id ); + qp.setLockOptions( lockOptions ); + return qp; + } - for ( int i=0; i1 ) { - int[] batchSizesToCreate = ArrayHelper.getBatchSizes(maxBatchSize); - Loader[] loadersToCreate = new Loader[ batchSizesToCreate.length ]; - for ( int i=0; i1 ) { - int[] batchSizesToCreate = ArrayHelper.getBatchSizes(maxBatchSize); - Loader[] loadersToCreate = new Loader[ batchSizesToCreate.length ]; - for ( int i=0; i= numberOfIds ) { + indexToUse = i; + } + else { + break; + } + } + + final Serializable[] idsToLoad = new Serializable[ batchSizes[indexToUse] ]; + System.arraycopy( batch, 0, idsToLoad, 0, numberOfIds ); + for ( int i = numberOfIds; i < batchSizes[indexToUse]; i++ ) { + idsToLoad[i] = id; + } + + return doBatchLoad( id, loaders[indexToUse], session, idsToLoad, optionalObject, lockOptions ); + } + } + +} diff --git a/hibernate-core/src/main/java/org/hibernate/loader/entity/UniqueEntityLoader.java b/hibernate-core/src/main/java/org/hibernate/loader/entity/UniqueEntityLoader.java index 4e701507332c..f3e7982ffd65 100644 --- a/hibernate-core/src/main/java/org/hibernate/loader/entity/UniqueEntityLoader.java +++ b/hibernate-core/src/main/java/org/hibernate/loader/entity/UniqueEntityLoader.java @@ -43,6 +43,7 @@ public interface UniqueEntityLoader { * @deprecated use {@link #load(java.io.Serializable, Object, SessionImplementor, LockOptions)} instead. */ @SuppressWarnings( {"JavaDoc"}) + @Deprecated public Object load(Serializable id, Object optionalObject, SessionImplementor session) throws HibernateException; /** diff --git a/hibernate-core/src/main/java/org/hibernate/loader/hql/QueryLoader.java b/hibernate-core/src/main/java/org/hibernate/loader/hql/QueryLoader.java index 7e845d9b0486..18ee1d5f95e4 100644 --- a/hibernate-core/src/main/java/org/hibernate/loader/hql/QueryLoader.java +++ b/hibernate-core/src/main/java/org/hibernate/loader/hql/QueryLoader.java @@ -239,7 +239,7 @@ protected String getQueryIdentifier() { /** * The SQL query string to be called. */ - protected String getSQLString() { + public String getSQLString() { return queryTranslator.getSQLString(); } diff --git a/hibernate-core/src/main/java/org/hibernate/persister/collection/BasicCollectionPersister.java b/hibernate-core/src/main/java/org/hibernate/persister/collection/BasicCollectionPersister.java index 58647cdc9b21..55619991771e 100644 --- a/hibernate-core/src/main/java/org/hibernate/persister/collection/BasicCollectionPersister.java +++ b/hibernate-core/src/main/java/org/hibernate/persister/collection/BasicCollectionPersister.java @@ -45,6 +45,7 @@ import org.hibernate.jdbc.Expectation; import org.hibernate.jdbc.Expectations; import org.hibernate.loader.collection.BatchingCollectionInitializer; +import org.hibernate.loader.collection.BatchingCollectionInitializerBuilder; import org.hibernate.loader.collection.CollectionInitializer; import org.hibernate.loader.collection.SubselectCollectionLoader; import org.hibernate.mapping.Collection; @@ -332,7 +333,8 @@ private String manyToManySelectFragment( @Override protected CollectionInitializer createCollectionInitializer(LoadQueryInfluencers loadQueryInfluencers) throws MappingException { - return BatchingCollectionInitializer.createBatchingCollectionInitializer( this, batchSize, getFactory(), loadQueryInfluencers ); + return BatchingCollectionInitializerBuilder.getBuilder( getFactory() ) + .createBatchingCollectionInitializer( this, batchSize, getFactory(), loadQueryInfluencers ); } public String fromJoinFragment(String alias, boolean innerJoin, boolean includeSubclasses) { diff --git a/hibernate-core/src/main/java/org/hibernate/persister/collection/OneToManyPersister.java b/hibernate-core/src/main/java/org/hibernate/persister/collection/OneToManyPersister.java index 488e631e5d7b..977134a52a1c 100644 --- a/hibernate-core/src/main/java/org/hibernate/persister/collection/OneToManyPersister.java +++ b/hibernate-core/src/main/java/org/hibernate/persister/collection/OneToManyPersister.java @@ -44,6 +44,7 @@ import org.hibernate.jdbc.Expectation; import org.hibernate.jdbc.Expectations; import org.hibernate.loader.collection.BatchingCollectionInitializer; +import org.hibernate.loader.collection.BatchingCollectionInitializerBuilder; import org.hibernate.loader.collection.CollectionInitializer; import org.hibernate.loader.collection.SubselectOneToManyLoader; import org.hibernate.loader.entity.CollectionElementLoader; @@ -359,7 +360,8 @@ public String selectFragment( @Override protected CollectionInitializer createCollectionInitializer(LoadQueryInfluencers loadQueryInfluencers) throws MappingException { - return BatchingCollectionInitializer.createBatchingOneToManyInitializer( this, batchSize, getFactory(), loadQueryInfluencers ); + return BatchingCollectionInitializerBuilder.getBuilder( getFactory() ) + .createBatchingOneToManyInitializer( this, batchSize, getFactory(), loadQueryInfluencers ); } public String fromJoinFragment(String alias, diff --git a/hibernate-core/src/main/java/org/hibernate/persister/entity/AbstractEntityPersister.java b/hibernate-core/src/main/java/org/hibernate/persister/entity/AbstractEntityPersister.java index 6d8ed821db82..8d31ca988aee 100644 --- a/hibernate-core/src/main/java/org/hibernate/persister/entity/AbstractEntityPersister.java +++ b/hibernate-core/src/main/java/org/hibernate/persister/entity/AbstractEntityPersister.java @@ -90,6 +90,7 @@ import org.hibernate.jdbc.Expectations; import org.hibernate.jdbc.TooManyRowsAffectedException; import org.hibernate.loader.entity.BatchingEntityLoader; +import org.hibernate.loader.entity.BatchingEntityLoaderBuilder; import org.hibernate.loader.entity.CascadeEntityLoader; import org.hibernate.loader.entity.EntityLoader; import org.hibernate.loader.entity.UniqueEntityLoader; @@ -2440,26 +2441,16 @@ protected UniqueEntityLoader createEntityLoader( LockMode lockMode, LoadQueryInfluencers loadQueryInfluencers) throws MappingException { //TODO: disable batch loading if lockMode > READ? - return BatchingEntityLoader.createBatchingEntityLoader( - this, - batchSize, - lockMode, - getFactory(), - loadQueryInfluencers - ); + return BatchingEntityLoaderBuilder.getBuilder( getFactory() ) + .buildLoader( this, batchSize, lockMode, getFactory(), loadQueryInfluencers ); } protected UniqueEntityLoader createEntityLoader( LockOptions lockOptions, LoadQueryInfluencers loadQueryInfluencers) throws MappingException { //TODO: disable batch loading if lockMode > READ? - return BatchingEntityLoader.createBatchingEntityLoader( - this, - batchSize, - lockOptions, - getFactory(), - loadQueryInfluencers - ); + return BatchingEntityLoaderBuilder.getBuilder( getFactory() ) + .buildLoader( this, batchSize, lockOptions, getFactory(), loadQueryInfluencers ); } protected UniqueEntityLoader createEntityLoader(LockMode lockMode) throws MappingException { diff --git a/hibernate-core/src/test/java/org/hibernate/test/batchfetch/BatchFetchTest.java b/hibernate-core/src/test/java/org/hibernate/test/batchfetch/BatchFetchTest.java index b0071d0394af..994ca6cf46b6 100755 --- a/hibernate-core/src/test/java/org/hibernate/test/batchfetch/BatchFetchTest.java +++ b/hibernate-core/src/test/java/org/hibernate/test/batchfetch/BatchFetchTest.java @@ -31,6 +31,10 @@ import org.hibernate.Hibernate; import org.hibernate.Session; import org.hibernate.Transaction; +import org.hibernate.cfg.AvailableSettings; +import org.hibernate.cfg.Configuration; +import org.hibernate.loader.BatchFetchStyle; + import org.hibernate.testing.junit4.BaseCoreFunctionalTestCase; import static org.junit.Assert.assertEquals; @@ -46,6 +50,18 @@ public String[] getMappings() { return new String[] { "batchfetch/ProductLine.hbm.xml" }; } + @Override + protected Class[] getAnnotatedClasses() { + return new Class[] { BatchLoadableEntity.class }; + } + + @Override + protected void configure(Configuration configuration) { + super.configure( configuration ); + configuration.setProperty( AvailableSettings.GENERATE_STATISTICS, "true" ); + configuration.setProperty( AvailableSettings.USE_SECOND_LEVEL_CACHE, "false" ); + } + @SuppressWarnings( {"unchecked"}) @Test public void testBatchFetch() { @@ -136,5 +152,53 @@ public void testBatchFetch() { s.close(); } + @Test + @SuppressWarnings( {"unchecked"}) + public void testBatchFetch2() { + Session s = openSession(); + s.beginTransaction(); + int size = 32+14; + for ( int i = 0; i < size; i++ ) { + s.save( new BatchLoadableEntity( i ) ); + } + s.getTransaction().commit(); + s.close(); + + s = openSession(); + s.beginTransaction(); + // load them all as proxies + for ( int i = 0; i < size; i++ ) { + BatchLoadableEntity entity = (BatchLoadableEntity) s.load( BatchLoadableEntity.class, i ); + assertFalse( Hibernate.isInitialized( entity ) ); + } + sessionFactory().getStatistics().clear(); + // now start initializing them... + for ( int i = 0; i < size; i++ ) { + BatchLoadableEntity entity = (BatchLoadableEntity) s.load( BatchLoadableEntity.class, i ); + Hibernate.initialize( entity ); + assertTrue( Hibernate.isInitialized( entity ) ); + } + // so at this point, all entities are initialized. see how many fetches were performed. + final int expectedFetchCount; + if ( sessionFactory().getSettings().getBatchFetchStyle() == BatchFetchStyle.LEGACY ) { + expectedFetchCount = 3; // (32 + 10 + 4) + } + else if ( sessionFactory().getSettings().getBatchFetchStyle() == BatchFetchStyle.DYNAMIC ) { + expectedFetchCount = 2; // (32 + 14) : because we limited batch-size to 32 + } + else { + // PADDED + expectedFetchCount = 2; // (32 + 16*) with the 16 being padded + } + assertEquals( expectedFetchCount, sessionFactory().getStatistics().getEntityStatistics( BatchLoadableEntity.class.getName() ).getFetchCount() ); + s.getTransaction().commit(); + s.close(); + + s = openSession(); + s.beginTransaction(); + s.createQuery( "delete BatchLoadableEntity" ).executeUpdate(); + s.getTransaction().commit(); + s.close(); + } } diff --git a/hibernate-core/src/test/java/org/hibernate/test/batchfetch/BatchLoadableEntity.java b/hibernate-core/src/test/java/org/hibernate/test/batchfetch/BatchLoadableEntity.java new file mode 100644 index 000000000000..2d15f95b30b1 --- /dev/null +++ b/hibernate-core/src/test/java/org/hibernate/test/batchfetch/BatchLoadableEntity.java @@ -0,0 +1,64 @@ +/* + * Hibernate, Relational Persistence for Idiomatic Java + * + * Copyright (c) 2012, Red Hat Inc. or third-party contributors as + * indicated by the @author tags or express copyright attribution + * statements applied by the authors. All third-party contributions are + * distributed under license by Red Hat Inc. + * + * This copyrighted material is made available to anyone wishing to use, modify, + * copy, or redistribute it subject to the terms and conditions of the GNU + * Lesser General Public License, as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this distribution; if not, write to: + * Free Software Foundation, Inc. + * 51 Franklin Street, Fifth Floor + * Boston, MA 02110-1301 USA + */ +package org.hibernate.test.batchfetch; + +import javax.persistence.Entity; +import javax.persistence.Id; + +import org.hibernate.annotations.BatchSize; + +/** + * @author Steve Ebersole + */ +@Entity +@BatchSize( size = 32 ) +public class BatchLoadableEntity { + private Integer id; + private String name; + + public BatchLoadableEntity() { + } + + public BatchLoadableEntity(int id) { + this.id = id; + this.name = "Entity #" + id; + } + + @Id + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/hibernate-core/src/test/java/org/hibernate/test/manytomany/batchload/BatchedManyToManyTest.java b/hibernate-core/src/test/java/org/hibernate/test/manytomany/batchload/BatchedManyToManyTest.java index 262bf68454c4..5ee397b82bb3 100644 --- a/hibernate-core/src/test/java/org/hibernate/test/manytomany/batchload/BatchedManyToManyTest.java +++ b/hibernate-core/src/test/java/org/hibernate/test/manytomany/batchload/BatchedManyToManyTest.java @@ -25,9 +25,6 @@ import java.util.List; -import junit.framework.Assert; -import org.junit.Test; - import org.hibernate.EmptyInterceptor; import org.hibernate.Hibernate; import org.hibernate.Interceptor; @@ -39,12 +36,13 @@ import org.hibernate.engine.jdbc.batch.spi.Batch; import org.hibernate.engine.jdbc.batch.spi.BatchKey; import org.hibernate.engine.jdbc.spi.JdbcCoordinator; -import org.hibernate.loader.collection.BatchingCollectionInitializer; -import org.hibernate.persister.collection.AbstractCollectionPersister; import org.hibernate.stat.CollectionStatistics; + +import org.junit.Test; +import junit.framework.Assert; + import org.hibernate.testing.junit4.BaseCoreFunctionalTestCase; -import static org.hibernate.testing.junit4.ExtraAssertions.assertClassAssignability; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -80,25 +78,6 @@ public TestingBatch(BatchKey key, JdbcCoordinator jdbcCoordinator) { } } - @Test - public void testProperLoaderSetup() { - AbstractCollectionPersister cp = ( AbstractCollectionPersister ) - sessionFactory().getCollectionPersister( User.class.getName() + ".groups" ); - assertClassAssignability( BatchingCollectionInitializer.class, cp.getInitializer().getClass() ); - BatchingCollectionInitializer initializer = ( BatchingCollectionInitializer ) cp.getInitializer(); - assertEquals( 50, findMaxBatchSize( initializer.getBatchSizes() ) ); - } - - private int findMaxBatchSize(int[] batchSizes) { - int max = 0; - for ( int size : batchSizes ) { - if ( size > max ) { - max = size; - } - } - return max; - } - @Test public void testLoadingNonInverseSide() { prepareTestData();