New feature: Reprocess missing lookups after import

SandboxberryLib/ObjectTransformer.cs

@@ -38,6 +38,12 @@ public class ObjectTransformer
 
         public Dictionary<string, string> ObjectRelationships { get; set; }
 
+        /// <summary>
+        /// Lookups that will need to be reprocessed later as the referenced objects didn't exist
+        /// yet when the wrapped object was created
+        /// </summary>
+        public List<LookupInfo> LookupsToReprocess { get; set; }
+
         public List<string> InactiveUserIds { get; set; }
 
         public List<string> MissingUserIds { get; set; }
@@ -51,6 +57,12 @@ public void ApplyTransformations(sObjectWrapper wrap)
             CorrectInactiveUser(wrap.sObj, this.InactiveUserIds, this.CurrentUserId);
             if (this.RecursiveRelationshipField != null)
                 RememberRecursiveId(wrap, this.RecursiveRelationshipField);
+
+            foreach(var lookup in this.LookupsToReprocess)
+            {
+                RemeberLookupIdsToReprocess(lookup, wrap.sObj);
+            }
+
             FixRelatedIds(wrap.sObj, this.ObjectRelationships);
             RemoveIdFromSObject(wrap.sObj);
 
@@ -169,7 +181,19 @@ private void RememberRecursiveId(ObjectTransformer.sObjectWrapper wrap, string r
 
         }
 
+        /// <summary>
+        /// Remember IDs to reprocess later for this lookup relationship
+        /// </summary>
+        private void RemeberLookupIdsToReprocess(LookupInfo lookup, sObject obj)
+        {
+            var lookupField = obj.Any.FirstOrDefault(e => e.LocalName == lookup.FieldName);
+            var relatedId = lookupField.InnerText;
 
+            if (!String.IsNullOrEmpty(relatedId))
+            {
+                lookup.IdPairs.Add(new KeyValuePair<string, string>(obj.Id, relatedId));
+            }
+        }
 
         public class sObjectWrapper
         {

SandboxberryLib/PopulateSandbox.cs

@@ -86,6 +86,8 @@ public PopulateSandboxResult Start(IProgress<string> progress)
             var targetUserIds = _targetTasks.GetAllUsers();
             var missingUserIds = sourceUserIds.Except(targetUserIds).ToList();
             logger.DebugFormat("Found {0} users in Source that are not in Target", missingUserIds.Count());
+            var processedObjects = new List<String> { "User" }; // user is copied already
+            var objectsToReprocess = new List<ObjectTransformer>();
 
             foreach (SbbObject objLoop in _instructions.SbbObjects)
             {
@@ -103,10 +105,35 @@ public PopulateSandboxResult Start(IProgress<string> progress)
 
                 transformer.ObjectRelationships = _sourceTasks.GetObjectRelationships(objLoop.ApiName);
                 transformer.RecursiveRelationshipField = transformer.ObjectRelationships.FirstOrDefault(d => d.Value == objLoop.ApiName).Key;
+
                 if (transformer.RecursiveRelationshipField != null)
                     logger.DebugFormat("Object {0} has a recurive relation to iteself in field {1}",
                         objLoop.ApiName, transformer.RecursiveRelationshipField);
 
+                // find lookups on this object that can't be populated, to reprocess later
+                transformer.LookupsToReprocess = transformer.ObjectRelationships
+                    .Where(d => d.Value != objLoop.ApiName)          // where it's not a recursive relationship
+                    .Where(d => !processedObjects.Contains(d.Value)) // and the referenced record doesn't exist yet
+                    .Where(d => !objLoop.SbbFieldOptions.Any(        // and it's not one of the skipped fields
+                                e => e.ApiName.Equals(d.Key)
+                                     && e.Skip))
+                    // TODO: but is still one of the included object types (e.g. not Contact -> "rh2__PS_Describe__c")
+                    .Select(d => new LookupInfo
+                    {
+                        FieldName = d.Key,
+                        ObjectName = objLoop.ApiName,
+                        RelatedObjectName = d.Value
+                    })
+                    .ToList();
+
+                if (transformer.LookupsToReprocess.Count > 0)
+                {
+                    objectsToReprocess.Add(transformer);
+                    var fields = transformer.LookupsToReprocess.Select(lookup => lookup.FieldName);
+                    logger.DebugFormat("Object {0} has lookups that will need to be reprocessed: {1}",
+                        objLoop.ApiName,
+                        String.Join(", ", fields));
+                }
 
                 List<sObject> sourceData = null;
                 try
@@ -179,9 +206,12 @@ public PopulateSandboxResult Start(IProgress<string> progress)
 
                 ProgressUpdate(string.Format("Summary for {0}: Success {1} Fail {2}",
                     objLoop.ApiName, objres.SuccessCount, objres.FailCount));
-
+                processedObjects.Add(objLoop.ApiName);
             }
 
+            // reprocess lookup relationships that can be populated now that the inserts are done
+            var reprocessingResults = ReprocessObjects(objectsToReprocess);
+
             // log summary
             ProgressUpdate("************************************************");
             foreach (var resLoop in res.ObjectResults)
@@ -190,14 +220,18 @@ public PopulateSandboxResult Start(IProgress<string> progress)
                  resLoop.ApiName, resLoop.SuccessCount, resLoop.FailCount));
 
             }
+
+            // log reprocssing summary
+            foreach (var resLoop in reprocessingResults.ObjectResults)
+            {
+                ProgressUpdate(string.Format("Reprocessing summary for {0}: Success {1} Fail {2}",
+                    resLoop.ApiName, resLoop.SuccessCount, resLoop.FailCount));
+            }
             ProgressUpdate("************************************************");
-            
+
             return res;
         }
 
-
-
-
 
 
         private void UpdateRecursiveField(string apiName, List<ObjectTransformer.sObjectWrapper> workingList, string recursiveRelationshipField)
@@ -211,39 +245,107 @@ private void UpdateRecursiveField(string apiName, List<ObjectTransformer.sObject
             {
                 if (!string.IsNullOrEmpty(wrapLoop.RecursiveRelationshipOriginalId))
                 {
-                    var upd = new sObject();
+                    var updateObject = CreateSobjectWithLookup(apiName, apiName, recursiveRelationshipField,
+                        new KeyValuePair<string, string>(wrapLoop.NewId, wrapLoop.RecursiveRelationshipOriginalId));
 
-                    upd.type = wrapLoop.sObj.type;
-                    upd.Id = wrapLoop.NewId;
-                    XmlDocument dummydoc = new XmlDocument();
-                    XmlElement recursiveEl = dummydoc.CreateElement(recursiveRelationshipField);
+                    if (updateObject != null)
+                    {
+                        updateList.Add(updateObject);
+                    }
+                }
+            }
 
-                    string replaceValue = _relationMapper.RecallNewId(apiName, wrapLoop.RecursiveRelationshipOriginalId);
+            logger.DebugFormat("{0} rows in Object {1} have recursive relation {2} to update ....",
+                updateList.Count(), apiName, recursiveRelationshipField);
+
+            var result = UpdateRecords(apiName, updateList);
+        }
 
-                    if (replaceValue == null)
+        /// <summary>
+        /// Reprocesses lookup relationship fields that were missed during the initial import and
+        /// populates them, once all the records are loaded into the sandbox. Similar to 
+        /// UpdateRecursiveField.
+        /// </summary>
+        private PopulateSandboxResult ReprocessObjects(List<ObjectTransformer> objectsToReprocess)
+        {
+            var results = new PopulateSandboxResult();
+
+            foreach (var obj in objectsToReprocess)
+            {
+                foreach (var field in obj.LookupsToReprocess)
+                {
+                    ProgressUpdate(string.Format("Reprocessing referenced objects for {0} field {1}",
+                        field.ObjectName, field.FieldName));
+
+                    List<sObject> updateList = new List<sObject>();
+                    foreach (var idPair in field.IdPairs)
                     {
-                        logger.DebugFormat("Object {0} {1} recursive field {2} have value {3} could not translate - will ignore",
-                           apiName, wrapLoop.OriginalId, recursiveRelationshipField, wrapLoop.RecursiveRelationshipOriginalId);
+                        var updateObj = CreateSobjectWithLookup(field.ObjectName,
+                            field.RelatedObjectName, field.FieldName, idPair);
+
+                        if (updateObj != null)
+                        {
+                            updateList.Add(updateObj);
+                        }
                     }
-                    else
-                    {
 
-                        recursiveEl.InnerText = replaceValue;
+                    // switch the IDs with the new ones in the sandbox
+                    foreach (sObject rowLoop in updateList)
+                    {
+                        rowLoop.Id = _relationMapper.RecallNewId(field.ObjectName, rowLoop.Id);
+                    }
 
-                        upd.Any = new XmlElement[] { recursiveEl };
+                    ProgressUpdate(string.Format("Updating {0} {1} records",
+                        updateList.Count, field.ObjectName));
 
-                        updateList.Add(upd);
-                    }
+                    var result = UpdateRecords(field.ObjectName, updateList);
+                    results.ObjectResults.Add(result);
                 }
+            }
+
+            return results;
+        }
+
+        /// <summary>
+        /// Creates a new sObject record that only contains the specified lookup relationship field
+        /// and replaces IDs for the referenced objects
+        /// </summary>
+        /// <returns>The newly-constructed sObject with the correct referenced ID, otherwise null</returns>
+        private sObject CreateSobjectWithLookup(string objectName, string relatedObjectName,
+            string fieldName, KeyValuePair<string, string> idPair)
+        {
+            var newObject = new sObject
+            {
+                type = objectName,
+                Id = idPair.Key
+            };
 
+            XmlDocument dummydoc = new XmlDocument();
+            XmlElement recursiveEl = dummydoc.CreateElement(fieldName);
+            string replaceValue = _relationMapper.RecallNewId(relatedObjectName, idPair.Value);
+
+            if (replaceValue == null)
+            {
+                logger.DebugFormat("Object {0} {1} relationship field {2} have value {3} could not translate - will ignore",
+                   objectName, idPair.Key, fieldName, idPair.Value);
+                return null;
             }
+            else
+            {
+                recursiveEl.InnerText = replaceValue;
+                newObject.Any = new XmlElement[] { recursiveEl };
+                return newObject;
+            }
+        }
 
-            logger.DebugFormat("{0} rows in Object {1} have recursive relation {2} to update ....",
-                updateList.Count(), apiName, recursiveRelationshipField);
+        /// <summary>
+        /// Updates a list of sobject records
+        /// </summary>
+        private PopulateObjectResult UpdateRecords(string objectName, List<sObject> updateList)
+        {
+            var result = new PopulateObjectResult();
+            result.ApiName = objectName;
 
-            // update objects in batches
-            int successCount = 0;
-            int failCount = 0;
             int done = 0;
             bool allDone = false;
             if (updateList.Count == 0)
@@ -255,29 +357,31 @@ private void UpdateRecursiveField(string apiName, List<ObjectTransformer.sObject
                 if (done >= updateList.Count)
                     allDone = true;
 
-                var updateRes = _targetTasks.UpdateSObjects(apiName,
+                var updateRes = _targetTasks.UpdateSObjects(objectName,
                        batch.ToArray());
 
                 for (int i = 0; i < updateRes.Length; i++)
                 {
                     if (updateRes[i].Success)
                     {
-                        successCount+=1;
+                        result.SuccessCount += 1;
                     }
                     else
                     {
-                        
+
                         logger.WarnFormat("Error when updating {0} {1} in target: {2}",
-                            apiName, batch[i].Id, updateRes[i].ErrorMessage);
-                        failCount += 1;
+                            objectName, batch[i].Id, updateRes[i].ErrorMessage);
+                        result.FailCount += 1;
                     }
 
                 }
-                   
+
             }
-            logger.DebugFormat("Object {0} recursive relation {1} updated. Attempted {2} Success {3} Failed {4}",
-               apiName, recursiveRelationshipField, updateList.Count, successCount, failCount);
 
+            logger.DebugFormat("Object {0} updated. Attempted {1} Success {2} Failed {3}",
+               objectName, updateList.Count, result.SuccessCount, result.FailCount);
+
+            return result;
         }
 
         private void LoginToBoth()

SandboxberryLib/ResultsModel/LookupInfo.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace SandboxberryLib.ResultsModel
+{
+    /// <summary>
+    /// Stores information about lookup relationships and the referenced IDs, sometimes they can't
+    /// be populated during the initial import (when the referenced object doesn't exist yet) and
+    /// need to be reprocessed after
+    /// </summary>
+    public class LookupInfo
+    {
+        public LookupInfo()
+        {
+            this.IdPairs = new List<KeyValuePair<string, string>>();
+        }
+
+        /// <summary>
+        /// API Name of the object that owns the lookup relationship field
+        /// </summary>
+        public String ObjectName { get; set; }
+
+        /// <summary>
+        /// API Name of the object that the lookup relationship field references
+        /// </summary>
+        public String RelatedObjectName { get; set; }
+
+        /// <summary>
+        /// API Name of the lookup relationship field
+        /// </summary>
+        public String FieldName { get; set; }
+
+        /// <summary>
+        /// IDs used by this lookup relationship in the form of key=object, value=referenced object,
+        /// saved during initial processing for reprocessing later
+        /// </summary>
+        public List<KeyValuePair<string,string>> IdPairs { get; set; }
+    }
+}

SandboxberryLib/SalesforceTasks.cs

@@ -166,11 +166,25 @@ public string BuildQuery(string sobjectName, List<string> colList, string filter
             return sb.ToString();
         }
 
+        /// <summary>
+        /// Gets data for all columns of an sobject
+        /// </summary>
         public List<sObject> GetDataFromSObject(string sobjectName, string filter)
         {
-            LoginIfRequired();
             List<string> colNames = RemoveSystemColumns(GetObjectColumns(sobjectName));
             string soql = BuildQuery(sobjectName, colNames, filter);
+            List<sObject> allResults = GetDataFromSObject(sobjectName, colNames, filter);
+            return allResults;
+        }
+
+        /// <summary>
+        /// Gets data for specified columns of an sobject
+        /// </summary>
+        public List<sObject> GetDataFromSObject(string sobjectName, List<string> colList, string filter)
+        {
+            LoginIfRequired();
+            colList = RemoveSystemColumns(colList);
+            string soql = BuildQuery(sobjectName, colList, filter);
 
             bool allResultsReturned = false;
             List<sObject> allResults = new List<sObject>();

SandboxberryLib/SandboxberryLib.csproj

@@ -58,6 +58,7 @@
       <DependentUpon>Settings.settings</DependentUpon>
     </Compile>
     <Compile Include="RelationMapper.cs" />
+    <Compile Include="ResultsModel\LookupInfo.cs" />
     <Compile Include="ResultsModel\PopulateObjectResult.cs" />
     <Compile Include="ResultsModel\PopulateSandboxResult.cs" />
     <Compile Include="ResultsModel\RowFailInfo.cs" />