diff --git a/SandboxberryLib/ObjectTransformer.cs b/SandboxberryLib/ObjectTransformer.cs index 28fd6e3..890f1b6 100644 --- a/SandboxberryLib/ObjectTransformer.cs +++ b/SandboxberryLib/ObjectTransformer.cs @@ -38,6 +38,12 @@ public class ObjectTransformer public Dictionary ObjectRelationships { get; set; } + /// + /// Lookups that will need to be reprocessed later as the referenced objects didn't exist + /// yet when the wrapped object was created + /// + public List LookupsToReprocess { get; set; } + public List InactiveUserIds { get; set; } public List MissingUserIds { get; set; } @@ -51,6 +57,12 @@ public void ApplyTransformations(sObjectWrapper wrap) CorrectInactiveUser(wrap.sObj, this.InactiveUserIds, this.CurrentUserId); if (this.RecursiveRelationshipField != null) RememberRecursiveId(wrap, this.RecursiveRelationshipField); + + foreach(var lookup in this.LookupsToReprocess) + { + RemeberLookupIdsToReprocess(lookup, wrap.sObj); + } + FixRelatedIds(wrap.sObj, this.ObjectRelationships); RemoveIdFromSObject(wrap.sObj); @@ -169,7 +181,19 @@ private void RememberRecursiveId(ObjectTransformer.sObjectWrapper wrap, string r } + /// + /// Remember IDs to reprocess later for this lookup relationship + /// + private void RemeberLookupIdsToReprocess(LookupInfo lookup, sObject obj) + { + var lookupField = obj.Any.FirstOrDefault(e => e.LocalName == lookup.FieldName); + var relatedId = lookupField.InnerText; + if (!String.IsNullOrEmpty(relatedId)) + { + lookup.IdPairs.Add(new KeyValuePair(obj.Id, relatedId)); + } + } public class sObjectWrapper { diff --git a/SandboxberryLib/PopulateSandbox.cs b/SandboxberryLib/PopulateSandbox.cs index 920a56f..9d0f2c2 100644 --- a/SandboxberryLib/PopulateSandbox.cs +++ b/SandboxberryLib/PopulateSandbox.cs @@ -86,6 +86,8 @@ public PopulateSandboxResult Start(IProgress progress) var targetUserIds = _targetTasks.GetAllUsers(); var missingUserIds = sourceUserIds.Except(targetUserIds).ToList(); logger.DebugFormat("Found {0} users in Source that are not in Target", missingUserIds.Count()); + var processedObjects = new List { "User" }; // user is copied already + var objectsToReprocess = new List(); foreach (SbbObject objLoop in _instructions.SbbObjects) { @@ -103,10 +105,35 @@ public PopulateSandboxResult Start(IProgress progress) transformer.ObjectRelationships = _sourceTasks.GetObjectRelationships(objLoop.ApiName); transformer.RecursiveRelationshipField = transformer.ObjectRelationships.FirstOrDefault(d => d.Value == objLoop.ApiName).Key; + if (transformer.RecursiveRelationshipField != null) logger.DebugFormat("Object {0} has a recurive relation to iteself in field {1}", objLoop.ApiName, transformer.RecursiveRelationshipField); + // find lookups on this object that can't be populated, to reprocess later + transformer.LookupsToReprocess = transformer.ObjectRelationships + .Where(d => d.Value != objLoop.ApiName) // where it's not a recursive relationship + .Where(d => !processedObjects.Contains(d.Value)) // and the referenced record doesn't exist yet + .Where(d => !objLoop.SbbFieldOptions.Any( // and it's not one of the skipped fields + e => e.ApiName.Equals(d.Key) + && e.Skip)) + // TODO: but is still one of the included object types (e.g. not Contact -> "rh2__PS_Describe__c") + .Select(d => new LookupInfo + { + FieldName = d.Key, + ObjectName = objLoop.ApiName, + RelatedObjectName = d.Value + }) + .ToList(); + + if (transformer.LookupsToReprocess.Count > 0) + { + objectsToReprocess.Add(transformer); + var fields = transformer.LookupsToReprocess.Select(lookup => lookup.FieldName); + logger.DebugFormat("Object {0} has lookups that will need to be reprocessed: {1}", + objLoop.ApiName, + String.Join(", ", fields)); + } List sourceData = null; try @@ -179,9 +206,12 @@ public PopulateSandboxResult Start(IProgress progress) ProgressUpdate(string.Format("Summary for {0}: Success {1} Fail {2}", objLoop.ApiName, objres.SuccessCount, objres.FailCount)); - + processedObjects.Add(objLoop.ApiName); } + // reprocess lookup relationships that can be populated now that the inserts are done + var reprocessingResults = ReprocessObjects(objectsToReprocess); + // log summary ProgressUpdate("************************************************"); foreach (var resLoop in res.ObjectResults) @@ -190,14 +220,18 @@ public PopulateSandboxResult Start(IProgress progress) resLoop.ApiName, resLoop.SuccessCount, resLoop.FailCount)); } + + // log reprocssing summary + foreach (var resLoop in reprocessingResults.ObjectResults) + { + ProgressUpdate(string.Format("Reprocessing summary for {0}: Success {1} Fail {2}", + resLoop.ApiName, resLoop.SuccessCount, resLoop.FailCount)); + } ProgressUpdate("************************************************"); - + return res; } - - - private void UpdateRecursiveField(string apiName, List workingList, string recursiveRelationshipField) @@ -211,39 +245,107 @@ private void UpdateRecursiveField(string apiName, List(wrapLoop.NewId, wrapLoop.RecursiveRelationshipOriginalId)); - upd.type = wrapLoop.sObj.type; - upd.Id = wrapLoop.NewId; - XmlDocument dummydoc = new XmlDocument(); - XmlElement recursiveEl = dummydoc.CreateElement(recursiveRelationshipField); + if (updateObject != null) + { + updateList.Add(updateObject); + } + } + } - string replaceValue = _relationMapper.RecallNewId(apiName, wrapLoop.RecursiveRelationshipOriginalId); + logger.DebugFormat("{0} rows in Object {1} have recursive relation {2} to update ....", + updateList.Count(), apiName, recursiveRelationshipField); + + var result = UpdateRecords(apiName, updateList); + } - if (replaceValue == null) + /// + /// Reprocesses lookup relationship fields that were missed during the initial import and + /// populates them, once all the records are loaded into the sandbox. Similar to + /// UpdateRecursiveField. + /// + private PopulateSandboxResult ReprocessObjects(List objectsToReprocess) + { + var results = new PopulateSandboxResult(); + + foreach (var obj in objectsToReprocess) + { + foreach (var field in obj.LookupsToReprocess) + { + ProgressUpdate(string.Format("Reprocessing referenced objects for {0} field {1}", + field.ObjectName, field.FieldName)); + + List updateList = new List(); + foreach (var idPair in field.IdPairs) { - logger.DebugFormat("Object {0} {1} recursive field {2} have value {3} could not translate - will ignore", - apiName, wrapLoop.OriginalId, recursiveRelationshipField, wrapLoop.RecursiveRelationshipOriginalId); + var updateObj = CreateSobjectWithLookup(field.ObjectName, + field.RelatedObjectName, field.FieldName, idPair); + + if (updateObj != null) + { + updateList.Add(updateObj); + } } - else - { - recursiveEl.InnerText = replaceValue; + // switch the IDs with the new ones in the sandbox + foreach (sObject rowLoop in updateList) + { + rowLoop.Id = _relationMapper.RecallNewId(field.ObjectName, rowLoop.Id); + } - upd.Any = new XmlElement[] { recursiveEl }; + ProgressUpdate(string.Format("Updating {0} {1} records", + updateList.Count, field.ObjectName)); - updateList.Add(upd); - } + var result = UpdateRecords(field.ObjectName, updateList); + results.ObjectResults.Add(result); } + } + + return results; + } + + /// + /// Creates a new sObject record that only contains the specified lookup relationship field + /// and replaces IDs for the referenced objects + /// + /// The newly-constructed sObject with the correct referenced ID, otherwise null + private sObject CreateSobjectWithLookup(string objectName, string relatedObjectName, + string fieldName, KeyValuePair idPair) + { + var newObject = new sObject + { + type = objectName, + Id = idPair.Key + }; + XmlDocument dummydoc = new XmlDocument(); + XmlElement recursiveEl = dummydoc.CreateElement(fieldName); + string replaceValue = _relationMapper.RecallNewId(relatedObjectName, idPair.Value); + + if (replaceValue == null) + { + logger.DebugFormat("Object {0} {1} relationship field {2} have value {3} could not translate - will ignore", + objectName, idPair.Key, fieldName, idPair.Value); + return null; } + else + { + recursiveEl.InnerText = replaceValue; + newObject.Any = new XmlElement[] { recursiveEl }; + return newObject; + } + } - logger.DebugFormat("{0} rows in Object {1} have recursive relation {2} to update ....", - updateList.Count(), apiName, recursiveRelationshipField); + /// + /// Updates a list of sobject records + /// + private PopulateObjectResult UpdateRecords(string objectName, List updateList) + { + var result = new PopulateObjectResult(); + result.ApiName = objectName; - // update objects in batches - int successCount = 0; - int failCount = 0; int done = 0; bool allDone = false; if (updateList.Count == 0) @@ -255,29 +357,31 @@ private void UpdateRecursiveField(string apiName, List= updateList.Count) allDone = true; - var updateRes = _targetTasks.UpdateSObjects(apiName, + var updateRes = _targetTasks.UpdateSObjects(objectName, batch.ToArray()); for (int i = 0; i < updateRes.Length; i++) { if (updateRes[i].Success) { - successCount+=1; + result.SuccessCount += 1; } else { - + logger.WarnFormat("Error when updating {0} {1} in target: {2}", - apiName, batch[i].Id, updateRes[i].ErrorMessage); - failCount += 1; + objectName, batch[i].Id, updateRes[i].ErrorMessage); + result.FailCount += 1; } } - + } - logger.DebugFormat("Object {0} recursive relation {1} updated. Attempted {2} Success {3} Failed {4}", - apiName, recursiveRelationshipField, updateList.Count, successCount, failCount); + logger.DebugFormat("Object {0} updated. Attempted {1} Success {2} Failed {3}", + objectName, updateList.Count, result.SuccessCount, result.FailCount); + + return result; } private void LoginToBoth() diff --git a/SandboxberryLib/ResultsModel/LookupInfo.cs b/SandboxberryLib/ResultsModel/LookupInfo.cs new file mode 100644 index 0000000..9202bd3 --- /dev/null +++ b/SandboxberryLib/ResultsModel/LookupInfo.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace SandboxberryLib.ResultsModel +{ + /// + /// Stores information about lookup relationships and the referenced IDs, sometimes they can't + /// be populated during the initial import (when the referenced object doesn't exist yet) and + /// need to be reprocessed after + /// + public class LookupInfo + { + public LookupInfo() + { + this.IdPairs = new List>(); + } + + /// + /// API Name of the object that owns the lookup relationship field + /// + public String ObjectName { get; set; } + + /// + /// API Name of the object that the lookup relationship field references + /// + public String RelatedObjectName { get; set; } + + /// + /// API Name of the lookup relationship field + /// + public String FieldName { get; set; } + + /// + /// IDs used by this lookup relationship in the form of key=object, value=referenced object, + /// saved during initial processing for reprocessing later + /// + public List> IdPairs { get; set; } + } +} diff --git a/SandboxberryLib/SalesforceTasks.cs b/SandboxberryLib/SalesforceTasks.cs index 0e480b1..0fd0544 100644 --- a/SandboxberryLib/SalesforceTasks.cs +++ b/SandboxberryLib/SalesforceTasks.cs @@ -166,11 +166,25 @@ public string BuildQuery(string sobjectName, List colList, string filter return sb.ToString(); } + /// + /// Gets data for all columns of an sobject + /// public List GetDataFromSObject(string sobjectName, string filter) { - LoginIfRequired(); List colNames = RemoveSystemColumns(GetObjectColumns(sobjectName)); string soql = BuildQuery(sobjectName, colNames, filter); + List allResults = GetDataFromSObject(sobjectName, colNames, filter); + return allResults; + } + + /// + /// Gets data for specified columns of an sobject + /// + public List GetDataFromSObject(string sobjectName, List colList, string filter) + { + LoginIfRequired(); + colList = RemoveSystemColumns(colList); + string soql = BuildQuery(sobjectName, colList, filter); bool allResultsReturned = false; List allResults = new List(); diff --git a/SandboxberryLib/SandboxberryLib.csproj b/SandboxberryLib/SandboxberryLib.csproj index 232bf67..aa42729 100644 --- a/SandboxberryLib/SandboxberryLib.csproj +++ b/SandboxberryLib/SandboxberryLib.csproj @@ -58,6 +58,7 @@ Settings.settings +