/src/com/google/appengine/datanucleus/DatastorePersistenceHandler.java

  1/**********************************************************************
  2Copyright (c) 2009 Google Inc.
  3
  4Licensed under the Apache License, Version 2.0 (the "License");
  5you may not use this file except in compliance with the License.
  6You may obtain a copy of the License at
  7
  8http://www.apache.org/licenses/LICENSE-2.0
  9
 10Unless required by applicable law or agreed to in writing, software
 11distributed under the License is distributed on an "AS IS" BASIS,
 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13See the License for the specific language governing permissions and
 14limitations under the License.
 15**********************************************************************/
 16package com.google.appengine.datanucleus;
 17
 18import com.google.appengine.api.datastore.Entity;
 19import com.google.appengine.api.datastore.EntityNotFoundException;
 20import com.google.appengine.api.datastore.Key;
 21import com.google.appengine.api.datastore.KeyFactory;
 22import com.google.appengine.datanucleus.mapping.DatastoreTable;
 23import com.google.appengine.datanucleus.mapping.DependentDeleteRequest;
 24import com.google.appengine.datanucleus.mapping.FetchMappingConsumer;
 25
 26import org.datanucleus.ClassLoaderResolver;
 27import org.datanucleus.PropertyNames;
 28import org.datanucleus.exceptions.NucleusObjectNotFoundException;
 29import org.datanucleus.exceptions.NucleusOptimisticException;
 30import org.datanucleus.metadata.AbstractClassMetaData;
 31import org.datanucleus.metadata.AbstractMemberMetaData;
 32import org.datanucleus.metadata.ColumnMetaData;
 33import org.datanucleus.metadata.DiscriminatorMetaData;
 34import org.datanucleus.metadata.IdentityType;
 35import org.datanucleus.metadata.VersionMetaData;
 36import org.datanucleus.metadata.VersionStrategy;
 37import org.datanucleus.store.AbstractPersistenceHandler;
 38import org.datanucleus.ExecutionContext;
 39import org.datanucleus.state.ObjectProvider;
 40import org.datanucleus.store.PersistenceBatchType;
 41import org.datanucleus.store.StoreManager;
 42import org.datanucleus.store.VersionHelper;
 43import org.datanucleus.store.mapped.DatastoreClass;
 44import org.datanucleus.store.mapped.DatastoreField;
 45import org.datanucleus.store.mapped.mapping.ArrayMapping;
 46import org.datanucleus.store.mapped.mapping.CollectionMapping;
 47import org.datanucleus.store.mapped.mapping.IndexMapping;
 48import org.datanucleus.store.mapped.mapping.JavaTypeMapping;
 49import org.datanucleus.store.mapped.mapping.MapMapping;
 50import org.datanucleus.store.mapped.mapping.MappingCallbacks;
 51import org.datanucleus.store.schema.naming.ColumnType;
 52import org.datanucleus.store.types.SCO;
 53import org.datanucleus.util.Localiser;
 54import org.datanucleus.util.NucleusLogger;
 55import org.datanucleus.util.StringUtils;
 56
 57import java.sql.Timestamp;
 58import java.util.Collection;
 59import java.util.Collections;
 60import java.util.Iterator;
 61import java.util.List;
 62import java.util.Map;
 63import java.util.Set;
 64import java.util.concurrent.ConcurrentHashMap;
 65
 66/**
 67 * Handler for persistence requests for GAE/J datastore. Lifecycle management processes persists, updates, deletes
 68 * and field access and hands them off here to interface with the datastore.
 69 * No method in here should be called from anywhere other than DataNucleus core.
 70 * <h3>Persistence Process</h3>
 71 * Receive calls to the following from DataNucleus core for persistence events. All persistence events arrive in 
 72 * the store plugin in the order they are performed by the user. Optimistic operations are queued until flush().
 73 * <p>
 74 * <b>PersistenceHandler.insertObject</b><br/>
 75 * <ol>
 76 * <li>CREATE Entity belonging to the appropriate entity group to represent the object.</li>
 77 * 
 78 * <li>If the entity is “owned” the entity group must be established before the Entity is initially put(),
 79 * there is no way to adjust it after. So when persisting an Entity that is a child of some other Entity, you need to
 80 * figure out who its parent is before you can do this first put.
 81 * <ol>
 82 * <li>Key 'id' can be assigned by datastore (long, Long)</li>
 83 * <li>Key 'name' can be assigned by the application (or value-generator) (String)</li>
 84 * </ol></li>
 85 * 
 86 * <li>Create StoreFieldManager, and set properties in Entity for all fields which have values ready.
 87 * <ol>
 88 * <li>If “identity” not set on this related object, make note of and skip relation fields.</li>
 89 * <li>If related object is detached, note its field number</li>
 90 * <li>If “identity” set on this related object, and related object not persistent, flush the related object(s) to 
 91 * get their Key(s).</li>
 92 * </ol></li>
 93 * 
 94 * <li>PUT the Entity in datastore</li>
 95 * <li>Set any generated id back on the Entity</li>
 96 * <li>If fields noted in step 3.1
 97 * <ol>
 98 * <li>Reuse StoreFieldManager from above, and process fields noted earlier.
 99 * <ol>
100 * <li>Attach any detached related objects</li>
101 * <li>Persist and flush new related object(s), and add property(s) for relation fields to the Entity</li>
102 * </ol></li>
103 * <li>PUT the updated Entity in datastore</li>
104 * </ol></li>
105 * 
106 * </ol>
107 * </p>
108 * 
109 * <p>
110 * <b>PersistenceHandler.updateObject</b><br/>
111 * <ol>
112 * <li>GET Entity that represents the object</li>
113 * <li>Populate all updated fields that have values ready, forcing the flush of any relation fields that don't have 
114 * their id present, and attach any detached related objects</li>
115 * <li>PUT the updated Entity in datastore</li>
116 * </ol>
117 * </p>
118 * <p>
119 * <b>PersistenceHandler.deleteObject</b><br/>
120 * <ol>
121 * <li>GET Entity that represents the object</li>
122 * <li>Handle any cascade deletion</li>
123 * <li>DELETE the Entity from datastore</li>
124 * </ol>
125 * </p>
126 * 
127 * @author Max Ross <maxr@google.com>
128 * @author Andy Jefferson
129 */
130public class DatastorePersistenceHandler extends AbstractPersistenceHandler {
131  protected static final Localiser GAE_LOCALISER = Localiser.getInstance(
132      "com.google.appengine.datanucleus.Localisation", DatastoreManager.class.getClassLoader());
133
134  private final Map<ExecutionContext, BatchPutManager> batchPutManagerByExecutionContext = new ConcurrentHashMap();
135
136  private final Map<ExecutionContext, BatchDeleteManager> batchDeleteManagerByExecutionContext = new ConcurrentHashMap();
137
138  private final DatastoreManager datastoreMgr;
139
140  /**
141   * Constructor.
142   * @param storeMgr The StoreManager to use.
143   */
144  public DatastorePersistenceHandler(StoreManager storeMgr) {
145    super(storeMgr);
146    this.datastoreMgr = (DatastoreManager) storeMgr;
147  }
148
149  public void close() {}
150
151  protected BatchPutManager getBatchPutManager(ExecutionContext ec) {
152    BatchPutManager putMgr = batchPutManagerByExecutionContext.get(ec);
153    if (putMgr == null) {
154      putMgr = new BatchPutManager();
155      batchPutManagerByExecutionContext.put(ec, putMgr);
156    }
157    return putMgr;
158  }
159
160  protected BatchDeleteManager getBatchDeleteManager(ExecutionContext ec) {
161    BatchDeleteManager deleteMgr = batchDeleteManagerByExecutionContext.get(ec);
162    if (deleteMgr == null) {
163      deleteMgr = new BatchDeleteManager(ec);
164      batchDeleteManagerByExecutionContext.put(ec, deleteMgr);
165    }
166    return deleteMgr;
167  }
168
169  /* (non-Javadoc)
170   * @see org.datanucleus.store.AbstractPersistenceHandler#batchStart(org.datanucleus.store.ExecutionContext, org.datanucleus.store.PersistenceBatchType)
171   */
172  @Override
173  public void batchStart(ExecutionContext ec, PersistenceBatchType batchType) {
174    if (batchType == PersistenceBatchType.PERSIST) {
175      getBatchPutManager(ec).start();
176    }
177    else if (batchType == PersistenceBatchType.DELETE) {
178      getBatchDeleteManager(ec).start();
179    }
180  }
181
182  /* (non-Javadoc)
183   * @see org.datanucleus.store.AbstractPersistenceHandler#batchEnd(org.datanucleus.store.ExecutionContext, org.datanucleus.store.PersistenceBatchType)
184   */
185  @Override
186  public void batchEnd(ExecutionContext ec, PersistenceBatchType batchType) {
187    if (batchType == PersistenceBatchType.PERSIST) {
188      getBatchPutManager(ec).finish(this);
189      batchPutManagerByExecutionContext.remove(ec);
190    }
191    else if (batchType == PersistenceBatchType.DELETE) {
192      getBatchDeleteManager(ec).finish(this);
193      batchDeleteManagerByExecutionContext.remove(ec);
194    }
195  }
196
197  /**
198   * Method to insert the specified managed object into the datastore.
199   * @param op ObjectProvider for the managed object
200   */
201  public void insertObject(ObjectProvider op) {
202    // Make sure writes are permitted
203    assertReadOnlyForUpdateOfObject(op);
204    datastoreMgr.validateMetaDataForClass(op.getClassMetaData());
205
206    // If we're in the middle of a batch operation just register the ObjectProvider that needs the insertion
207    BatchPutManager batchPutMgr = getBatchPutManager(op.getExecutionContext());
208    if (batchPutMgr.batchOperationInProgress()) {
209      batchPutMgr.add(op);
210      return;
211    }
212
213    insertObjectsInternal(Collections.singletonList(op));
214  }
215
216  /**
217   * Method to perform the work of inserting the specified objects. If multiple are to be inserted then
218   * performs it initially as a batch PUT. If some of these need subsequent work (e.g forcing the persist of 
219   * children followed by a repersist to link to the children) then this is done one-by-one.
220   */
221  void insertObjectsInternal(List<ObjectProvider> opsToInsert) {
222    if (opsToInsert == null || opsToInsert.isEmpty()) {
223      return;
224    }
225
226    // All must be in same ExecutionContext
227    ExecutionContext ec = opsToInsert.get(0).getExecutionContext();
228    List<PutState> putStateList = Utils.newArrayList();
229    for (ObjectProvider op : opsToInsert) {
230      AbstractClassMetaData cmd = op.getClassMetaData();
231
232      // Create the Entity, and populate all fields that can be populated (this will omit any owned child objects 
233      // if we don't have the key of this object yet).
234      StoreFieldManager fieldMgr =
235        new StoreFieldManager(op, EntityUtils.determineKind(cmd, ec));
236      op.provideFields(op.getClassMetaData().getAllMemberPositions(), fieldMgr);
237
238      // Make sure the Entity parent is set (if any)
239      Object assignedParentPk = fieldMgr.establishEntityGroup();
240      Entity entity = fieldMgr.getEntity();
241
242      if (!datastoreMgr.storageVersionAtLeast(StorageVersion.READ_OWNED_CHILD_KEYS_FROM_PARENTS)) {
243        // Older storage versions : store list positions in the element
244        DatastoreTable table = datastoreMgr.getDatastoreClass(op.getClassMetaData().getFullClassName(),
245            ec.getClassLoaderResolver());
246        Collection<JavaTypeMapping> orderMappings = table.getExternalOrderMappings().values();
247        for (JavaTypeMapping orderMapping : orderMappings) {
248          if (orderMapping instanceof IndexMapping) {
249            Object orderValue = op.getAssociatedValue(orderMapping);
250            if (orderValue != null) {
251              // Set order index on the entity
252              DatastoreField indexProp = orderMapping.getDatastoreMapping(0).getDatastoreField();
253              entity.setProperty(indexProp.getIdentifier().toString(), orderValue); // Is this indexed in the datastore?
254            } else {
255              // Element has been persisted and has the owner set, but not positioned, so leave til user does it
256            }
257          }
258        }
259      }
260
261      // Set version
262      handleVersioningBeforeWrite(op, entity, true, "inserting");
263
264      // Set discriminator
265      if (op.getClassMetaData().hasDiscriminatorStrategy()) {
266        DiscriminatorMetaData dismd = op.getClassMetaData().getDiscriminatorMetaDataRoot();
267        EntityUtils.setEntityProperty(entity, dismd, 
268            EntityUtils.getDiscriminatorPropertyName(datastoreMgr.getIdentifierFactory(), dismd), 
269            op.getClassMetaData().getDiscriminatorValue());
270      }
271
272      // Add Multi-tenancy discriminator if applicable
273      if (storeMgr.getStringProperty(PropertyNames.PROPERTY_TENANT_ID) != null) {
274          if ("true".equalsIgnoreCase(cmd.getValueForExtension("multitenancy-disable"))) {
275              // Don't bother with multitenancy for this class
276          }
277          else {
278            String name = storeMgr.getNamingFactory().getColumnName(cmd, ColumnType.MULTITENANCY_COLUMN);
279            EntityUtils.setEntityProperty(entity, cmd, name, storeMgr.getStringProperty(PropertyNames.PROPERTY_TENANT_ID));
280          }
281      }
282
283      // Update parent PK field on pojo
284      AbstractMemberMetaData parentPkMmd = datastoreMgr.getMetaDataForParentPK(cmd);
285      if (assignedParentPk != null) {
286        // we automatically assigned a parent to the entity so make sure that makes it back on to the pojo
287        op.replaceField(parentPkMmd.getAbsoluteFieldNumber(), assignedParentPk);
288      }
289
290      // Add the "state" for this put to the list.
291      putStateList.add(new PutState(op, fieldMgr, entity));
292    }
293
294    // PUT all entities in single call
295    if (!putStateList.isEmpty()) {
296      DatastoreTransaction txn = null;
297      AbstractClassMetaData acmd = null;
298      List<Entity> entityList = Utils.newArrayList();
299      for (PutState putState : putStateList) {
300        if (txn == null) {
301          txn = datastoreMgr.getDatastoreTransaction(ec);
302        }
303        if (acmd == null) {
304          acmd = putState.op.getClassMetaData();
305        }
306        entityList.add(putState.entity);
307      }
308
309      EntityUtils.putEntitiesIntoDatastore(ec, entityList);
310      for (PutState putState : putStateList) {
311        putState.op.setAssociatedValue(txn, putState.entity);
312      }
313    }
314
315    // Post-processing for all puts
316    for (PutState putState : putStateList) {
317      AbstractClassMetaData cmd = putState.op.getClassMetaData();
318
319      // Set the generated key back on the pojo.  If the pk field is a Key just set it on the field directly. 
320      // If the pk field is a String, convert the Key to a String, similarly for long.
321      // Assumes we only have a single pk member position
322      Object newId = null;
323      Class pkType = null;
324      boolean identityStrategyUsed = false;
325      if (cmd.pkIsDatastoreAttributed(storeMgr)) {
326        if (cmd.getIdentityType() == IdentityType.APPLICATION) {
327          // Assume only 1 PK field
328          identityStrategyUsed = true;
329          pkType = cmd.getMetaDataForManagedMemberAtAbsolutePosition(cmd.getPKMemberPositions()[0]).getType();
330        } else if (cmd.getIdentityType() == IdentityType.DATASTORE) {
331          identityStrategyUsed = true;
332          pkType = Key.class;
333          ColumnMetaData colmd = cmd.getIdentityMetaData().getColumnMetaData();
334          if (colmd != null) {
335            if ("varchar".equalsIgnoreCase(colmd.getJdbcType()) || "char".equalsIgnoreCase(colmd.getJdbcType())) {
336              pkType = String.class;
337            } else if ("integer".equalsIgnoreCase(colmd.getJdbcType()) || "numeric".equalsIgnoreCase(colmd.getJdbcType())) {
338              pkType = Long.class;
339            }
340          }
341        }
342      }
343
344      if (identityStrategyUsed) {
345        // Update the identity of the object with the datastore-assigned id
346        if (pkType.equals(Key.class)) {
347          newId = putState.entity.getKey();
348        } else if (pkType.equals(String.class)) {
349          if (MetaDataUtils.hasEncodedPKField(cmd)) {
350            newId = KeyFactory.keyToString(putState.entity.getKey());
351          } else {
352            newId = putState.entity.getKey().getName();
353          }
354        } else if (pkType.equals(Long.class) || pkType.equals(long.class)) {
355          newId = putState.entity.getKey().getId();
356        }
357
358        putState.op.setPostStoreNewObjectId(newId);
359      }
360
361      // Update relation fields (including cascade-persist etc)
362      if (putState.fieldMgr.storeRelations(KeyRegistry.getKeyRegistry(ec))) {
363        // PUT Entity into datastore with these changes
364        EntityUtils.putEntityIntoDatastore(ec, putState.entity);
365      }
366
367      putState.op.replaceAllLoadedSCOFieldsWithWrappers();
368
369      if (ec.getStatistics() != null) {
370        ec.getStatistics().incrementInsertCount();
371      }
372    }
373  }
374
375  /**
376   * Method to update the specified fields of the managed object in the datastore.
377   * @param op ObjectProvider of the managed object
378   * @param fieldNumbers Fields to be updated in the datastore
379   */
380  public void updateObject(ObjectProvider op, int fieldNumbers[]) {
381    if (op.getLifecycleState().isDeleted()) {
382      // don't perform updates on objects that are already deleted - this will cause them to be recreated
383      // This happens with JPAOneToOneTest/JDOOneToOneTest when deleting, called from DependentDeleteRequest
384      return;
385    }
386
387    // Make sure writes are permitted
388    assertReadOnlyForUpdateOfObject(op);
389    datastoreMgr.validateMetaDataForClass(op.getClassMetaData());
390
391    AbstractClassMetaData cmd = op.getClassMetaData();
392    long startTime = System.currentTimeMillis();
393    if (NucleusLogger.DATASTORE_PERSIST.isDebugEnabled()) {
394      StringBuffer fieldStr = new StringBuffer();
395      for (int i=0;i<fieldNumbers.length;i++) {
396        if (i > 0) {
397          fieldStr.append(",");
398        }
399        fieldStr.append(cmd.getMetaDataForManagedMemberAtAbsolutePosition(fieldNumbers[i]).getName());
400      }
401      NucleusLogger.DATASTORE_PERSIST.debug(GAE_LOCALISER.msg("AppEngine.Update.Start", 
402          StringUtils.toJVMIDString(op.getObject()), op.getInternalObjectId(), fieldStr.toString()));
403    }
404
405    ExecutionContext ec = op.getExecutionContext();
406    Entity entity = (Entity) op.getAssociatedValue(datastoreMgr.getDatastoreTransaction(ec));
407    if (entity == null) {
408      // Corresponding entity hasn't been fetched yet, so get it.
409      Key key = EntityUtils.getPkAsKey(op);
410      entity = EntityUtils.getEntityFromDatastore(datastoreMgr.getDatastoreServiceForReads(ec), op, key);
411    }
412
413    // Update the Entity with the specified fields
414    StoreFieldManager fieldMgr = new StoreFieldManager(op, entity, fieldNumbers);
415    op.provideFields(fieldNumbers, fieldMgr);
416
417    // Check and update the version
418    handleVersioningBeforeWrite(op, entity, true, "updating");
419
420    // Update relation fields (including cascade-persist etc)
421    fieldMgr.storeRelations(KeyRegistry.getKeyRegistry(op.getExecutionContext()));
422
423    // PUT Entity into datastore
424    DatastoreTransaction txn = EntityUtils.putEntityIntoDatastore(ec, entity);
425    op.setAssociatedValue(txn, entity);
426
427    op.replaceAllLoadedSCOFieldsWithWrappers();
428
429    if (NucleusLogger.DATASTORE_PERSIST.isDebugEnabled()) {
430      NucleusLogger.DATASTORE_PERSIST.debug(GAE_LOCALISER.msg("AppEngine.ExecutionTime", 
431        (System.currentTimeMillis() - startTime)));
432    }
433    if (ec.getStatistics() != null) {
434      ec.getStatistics().incrementUpdateCount();
435    }
436  }
437
438  /**
439   * Method to delete the specified managed object from the datastore.
440   * @param op ObjectProvider of the managed object
441   */
442  public void deleteObject(ObjectProvider op) {
443    // Make sure writes are permitted
444    assertReadOnlyForUpdateOfObject(op);
445    datastoreMgr.validateMetaDataForClass(op.getClassMetaData());
446
447    long startTime = System.currentTimeMillis();
448    if (NucleusLogger.DATASTORE_PERSIST.isDebugEnabled()) {
449      NucleusLogger.DATASTORE_PERSIST.debug(GAE_LOCALISER.msg("AppEngine.Delete.Start", 
450          StringUtils.toJVMIDString(op.getObject()), op.getInternalObjectId()));
451    }
452
453    ExecutionContext ec = op.getExecutionContext();
454    Entity entity = (Entity) op.getAssociatedValue(datastoreMgr.getDatastoreTransaction(ec));
455    if (entity == null) {
456      // Corresponding entity hasn't been fetched yet, so get it.
457      Key key = EntityUtils.getPkAsKey(op);
458      entity = EntityUtils.getEntityFromDatastore(datastoreMgr.getDatastoreServiceForReads(ec), op, key);
459    }
460
461    DatastoreTransaction txn = datastoreMgr.getDatastoreTransaction(ec);
462    if (txn != null) {
463      txn.addDeletedKey(entity.getKey());
464    }
465
466    // Check the version is valid to delete; any updates since read?
467    handleVersioningBeforeWrite(op, entity, false, "deleting");
468
469    // first handle any dependent deletes that need deleting before we delete this object
470    ClassLoaderResolver clr = ec.getClassLoaderResolver();
471    DatastoreClass dc = datastoreMgr.getDatastoreClass(op.getObject().getClass().getName(), clr);
472    DependentDeleteRequest req = new DependentDeleteRequest(dc, op.getClassMetaData(), clr);
473    Set relatedObjectsToDelete = req.execute(op, entity);
474
475    Key keyToDelete = EntityUtils.getPkAsKey(op);
476
477    // If we're in the middle of a batch operation just register the key that needs the delete
478    BatchDeleteManager bdm = getBatchDeleteManager(ec);
479    if (bdm.batchOperationInProgress()) {
480      bdm.add(new BatchDeleteManager.BatchDeleteState(txn, keyToDelete));
481
482      if (relatedObjectsToDelete != null && !relatedObjectsToDelete.isEmpty()) {
483        // Delete any related objects that need deleting after the delete of this object
484        Iterator iter = relatedObjectsToDelete.iterator();
485        while (iter.hasNext()) {
486          Object relatedObject = iter.next();
487          ec.deleteObjectInternal(relatedObject);
488        }
489      }
490      if (ec.getStatistics() != null) {
491        ec.getStatistics().incrementDeleteCount();
492      }
493
494      return;
495    }
496
497    // Delete this object
498    EntityUtils.deleteEntitiesFromDatastore(ec, Collections.singletonList(keyToDelete));
499
500    if (relatedObjectsToDelete != null && !relatedObjectsToDelete.isEmpty()) {
501      // Delete any related objects that need deleting after the delete of this object
502      Iterator iter = relatedObjectsToDelete.iterator();
503      while (iter.hasNext()) {
504        Object relatedObject = iter.next();
505        ec.deleteObjectInternal(relatedObject);
506      }
507    }
508    if (ec.getStatistics() != null) {
509      ec.getStatistics().incrementDeleteCount();
510    }
511
512    if (NucleusLogger.DATASTORE_PERSIST.isDebugEnabled()) {
513      NucleusLogger.DATASTORE_PERSIST.debug(GAE_LOCALISER.msg("AppEngine.ExecutionTime", 
514        (System.currentTimeMillis() - startTime)));
515    }
516  }
517
518  /**
519   * Method to fetch the specified fields of the managed object from the datastore.
520   * @param op ObjectProvider of the object whose fields need fetching
521   * @param fieldNumbers Fields to fetch
522   */
523  public void fetchObject(ObjectProvider op, int fieldNumbers[]) {
524    if (fieldNumbers == null || fieldNumbers.length == 0) {
525      return;
526    }
527
528    AbstractClassMetaData cmd = op.getClassMetaData();
529    datastoreMgr.validateMetaDataForClass(cmd);
530
531    // We always fetch the entire object, so if the state manager
532    // already has an associated Entity we know that associated
533    // Entity has all the fields.
534    ExecutionContext ec = op.getExecutionContext();
535    Entity entity = (Entity) op.getAssociatedValue(datastoreMgr.getDatastoreTransaction(ec));
536    if (entity == null) {
537      Key pk = EntityUtils.getPkAsKey(op);
538      entity = EntityUtils.getEntityFromDatastore(datastoreMgr.getDatastoreServiceForReads(ec), op, pk); // Throws NucleusObjectNotFoundException if necessary
539    }
540
541    if (NucleusLogger.DATASTORE_RETRIEVE.isDebugEnabled()) {
542      // Debug information about what we are retrieving
543      StringBuffer str = new StringBuffer("Fetching object \"");
544      str.append(StringUtils.toJVMIDString(op.getObject())).append("\" (id=");
545      str.append(op.getInternalObjectId()).append(")").append(" fields [");
546      for (int i=0;i<fieldNumbers.length;i++) {
547        if (i > 0) {
548          str.append(",");
549        }
550        str.append(cmd.getMetaDataForManagedMemberAtAbsolutePosition(fieldNumbers[i]).getName());
551      }
552      str.append("]");
553      NucleusLogger.DATASTORE_RETRIEVE.debug(str);
554    }
555
556    long startTime = System.currentTimeMillis();
557    if (NucleusLogger.DATASTORE_RETRIEVE.isDebugEnabled()) {
558      NucleusLogger.DATASTORE_RETRIEVE.debug(GAE_LOCALISER.msg("AppEngine.Fetch.Start", 
559          StringUtils.toJVMIDString(op.getObject()), op.getInternalObjectId()));
560    }
561
562    op.replaceFields(fieldNumbers, new FetchFieldManager(op, entity, fieldNumbers));
563
564    // Refresh version in case not yet set (e.g created HOLLOW object, and this is first fetch)
565    VersionMetaData vmd = cmd.getVersionMetaDataForClass();
566    if (cmd.isVersioned()) {
567      Object versionValue = entity.getProperty(EntityUtils.getVersionPropertyName(datastoreMgr.getIdentifierFactory(), vmd));
568      if (vmd.getVersionStrategy() == VersionStrategy.DATE_TIME) {
569        versionValue = new Timestamp((Long)versionValue);
570      }
571      op.setVersion(versionValue);
572    }
573
574    // Run post-fetch mapping callbacks. What is this actually achieving?
575    AbstractMemberMetaData[] fmds = new AbstractMemberMetaData[fieldNumbers.length];
576    for (int i = 0; i < fmds.length; i++) {
577      fmds[i] = op.getClassMetaData().getMetaDataForManagedMemberAtAbsolutePosition(fieldNumbers[i]);
578    }
579    ClassLoaderResolver clr = ec.getClassLoaderResolver();
580    DatastoreClass dc = datastoreMgr.getDatastoreClass(op.getObject().getClass().getName(), clr);
581    FetchMappingConsumer consumer = new FetchMappingConsumer(op.getClassMetaData());
582    dc.provideMappingsForMembers(consumer, fmds, true);
583    dc.provideDatastoreIdMappings(consumer);
584    dc.providePrimaryKeyMappings(consumer);
585    for (MappingCallbacks callback : consumer.getMappingCallbacks()) {
586      // Arrays and Maps don't use backing stores
587      if (callback instanceof ArrayMapping || callback instanceof MapMapping) {
588        // Do nothing since arrays and maps are stored in the parent property and loaded above using FetchFieldManager
589      } else if (callback instanceof CollectionMapping) {
590        CollectionMapping m = (CollectionMapping)callback;
591        Object val = op.provideField(m.getMemberMetaData().getAbsoluteFieldNumber());
592        if (val == null || !(val instanceof SCO)) {
593          // Not yet wrapped, so make sure we wrap it
594          callback.postFetch(op);
595        }
596      } else {
597        callback.postFetch(op);
598      }
599    }
600
601    if (NucleusLogger.DATASTORE_RETRIEVE.isDebugEnabled()) {
602      NucleusLogger.DATASTORE_RETRIEVE.debug(GAE_LOCALISER.msg("AppEngine.ExecutionTime",
603            (System.currentTimeMillis() - startTime)));
604    }
605    if (ec.getStatistics() != null) {
606      ec.getStatistics().incrementFetchCount();
607    }
608  }
609
610  /**
611   * Method to locate the specified managed objects in the datastore.
612   * @param ops ObjectProviders for the managed objects
613   * @throws NucleusObjectNotFoundException if any of the objects aren't found in the datastore
614   */
615  public void locateObjects(ObjectProvider[] ops) {
616    if (ops == null) {
617      return;
618    }
619
620    List<Key> keysToLocate = Utils.newArrayList();
621    for (int i=0;i<ops.length;i++) {
622      Key key = EntityUtils.getPkAsKey(ops[i]);
623      keysToLocate.add(key);
624    }
625    EntityUtils.getEntitiesFromDatastore(datastoreMgr.getDatastoreServiceForReads(ops[0].getExecutionContext()),
626        keysToLocate, ops[0].getExecutionContext());
627  }
628
629  /**
630   * Method to locate the specified managed object in the datastore.
631   * @param op ObjectProvider for the managed object
632   * @throws NucleusObjectNotFoundException if the object isn't found in the datastore
633   */
634  public void locateObject(ObjectProvider op) {
635    datastoreMgr.validateMetaDataForClass(op.getClassMetaData());
636    EntityUtils.getEntityFromDatastore(datastoreMgr.getDatastoreServiceForReads(op.getExecutionContext()), op, 
637        EntityUtils.getPkAsKey(op));
638  }
639
640  /**
641   * Implementation of this operation is optional and is intended for
642   * datastores that instantiate the model objects themselves (as opposed
643   * to letting datanucleus do it).  The App Engine datastore lets
644   * datanucleus instantiate the model objects so we just return null.
645   */
646  public Object findObject(ExecutionContext ec, Object id) {
647    return null;
648  }
649
650  /**
651   * Method to check optimistic versioning, and to set the version on the entity when required.
652   * @param op ObjectProvider for the object
653   * @param entity The entity being updated
654   * @param versionBehavior Behaviour required for versioning here
655   * @param operation Convenience string for messages
656   */
657  private void handleVersioningBeforeWrite(ObjectProvider op, Entity entity, boolean increment, String operation) {
658    AbstractClassMetaData cmd = op.getClassMetaData();
659    VersionMetaData vmd = cmd.getVersionMetaDataForClass();
660    if (cmd.isVersioned()) {
661      ExecutionContext ec = op.getExecutionContext();
662      String versionPropertyName = EntityUtils.getVersionPropertyName(datastoreMgr.getIdentifierFactory(), vmd);
663      Object curVersion = op.getVersion();
664      if (curVersion != null) {
665        // Fetch the latest and greatest version of the entity from the datastore
666        // to see if anyone has made a change underneath us.  We need to execute
667        // the fetch outside a txn to guarantee that we see the latest version.
668        if (NucleusLogger.DATASTORE_NATIVE.isDebugEnabled()) {
669          NucleusLogger.DATASTORE_NATIVE.debug("Getting entity with key " + entity.getKey());
670        }
671        Entity refreshedEntity;
672        try {
673          if (ec.getStatistics() != null) {
674            ec.getStatistics().incrementNumReads();
675          }
676          refreshedEntity = datastoreMgr.getDatastoreServiceForReads(op.getExecutionContext()).get(entity.getKey());
677        } catch (EntityNotFoundException e) {
678          // someone deleted out from under us
679          throw new NucleusOptimisticException(GAE_LOCALISER.msg("AppEngine.OptimisticError.EntityHasBeenDeleted", operation,
680              cmd.getFullClassName(), entity.getKey()));
681        }
682
683        Object datastoreVersion = refreshedEntity.getProperty(versionPropertyName);
684        if (vmd.getVersionStrategy() == VersionStrategy.DATE_TIME) {
685          datastoreVersion = new Timestamp((Long) datastoreVersion);
686        }
687
688        if (!datastoreVersion.equals(curVersion)) {
689          throw new NucleusOptimisticException(GAE_LOCALISER.msg("AppEngine.OptimisticError.EntityHasBeenUpdated", operation,
690              cmd.getFullClassName(), entity.getKey()));
691        }
692      }
693
694      Object nextVersion = VersionHelper.getNextVersion(vmd.getVersionStrategy(), curVersion);
695      op.setTransactionalVersion(nextVersion);
696      if (vmd.getVersionStrategy() == VersionStrategy.DATE_TIME) {
697        EntityUtils.setEntityProperty(entity, vmd, versionPropertyName, ((Timestamp)nextVersion).getTime());
698      } else {
699        EntityUtils.setEntityProperty(entity, vmd, versionPropertyName, nextVersion);
700      }
701
702      // Version field - update the version on the object
703      if (increment && vmd.getFieldName() != null) {
704        AbstractMemberMetaData verfmd =
705            ((AbstractClassMetaData)vmd.getParent()).getMetaDataForMember(vmd.getFieldName());
706        if (nextVersion instanceof Number) {
707          // Version can be long, Long, int, Integer, short, Short (or Timestamp).
708          Number nextNumber = (Number) nextVersion;
709          if (verfmd.getType().equals(Long.class) || verfmd.getType().equals(Long.TYPE)) {
710            nextVersion = nextNumber.longValue();
711          } else if (verfmd.getType().equals(Integer.class) || verfmd.getType().equals(Integer.TYPE)) {
712            nextVersion = nextNumber.intValue();
713          } else if (verfmd.getType().equals(Short.class) || verfmd.getType().equals(Short.TYPE)) {
714            nextVersion = nextNumber.shortValue();
715          }
716        }
717        op.replaceField(verfmd.getAbsoluteFieldNumber(), nextVersion);
718      }
719    }
720  }
721
722  /**
723   * All the information needed to perform a put on an Entity.
724   */
725  private static final class PutState {
726    private final ObjectProvider op;
727    private final StoreFieldManager fieldMgr;
728    private final Entity entity;
729
730    private PutState(ObjectProvider op, StoreFieldManager fieldMgr, Entity entity) {
731      this.op = op;
732      this.fieldMgr = fieldMgr;
733      this.entity = entity;
734    }
735  }
736}