From f5d15ea12c1f062243ad81c790462cc49abb9eb4 Mon Sep 17 00:00:00 2001 From: Matt Wanchap Date: Tue, 7 May 2019 06:13:35 +1000 Subject: [PATCH 1/6] Unfinished attempt to update lookup fields that get missed during the initial import --- SandboxberryLib/PopulateSandbox.cs | 85 ++++++++++++++++++++++++++++++ SandboxberryLib/SalesforceTasks.cs | 16 +++++- 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/SandboxberryLib/PopulateSandbox.cs b/SandboxberryLib/PopulateSandbox.cs index 920a56f..9522547 100644 --- a/SandboxberryLib/PopulateSandbox.cs +++ b/SandboxberryLib/PopulateSandbox.cs @@ -86,6 +86,8 @@ public PopulateSandboxResult Start(IProgress progress) var targetUserIds = _targetTasks.GetAllUsers(); var missingUserIds = sourceUserIds.Except(targetUserIds).ToList(); logger.DebugFormat("Found {0} users in Source that are not in Target", missingUserIds.Count()); + var processedObjects = new List { "User" }; // user is copied already + var lookupsToRevisit = new List>(); foreach (SbbObject objLoop in _instructions.SbbObjects) { @@ -103,6 +105,17 @@ public PopulateSandboxResult Start(IProgress progress) transformer.ObjectRelationships = _sourceTasks.GetObjectRelationships(objLoop.ApiName); transformer.RecursiveRelationshipField = transformer.ObjectRelationships.FirstOrDefault(d => d.Value == objLoop.ApiName).Key; + + // find lookups on this object that can't be populated + var missingLookups = transformer.ObjectRelationships.Where( + d => d.Value != objLoop.ApiName // where it's not a recursive relationship + && !processedObjects.Contains(d.Value) // and we haven't already processed this object (i.e. it's not going to be mapped) + && !objLoop.SbbFieldOptions.Any( // and it's not one of the skipped fields + e => e.ApiName.Equals(d.Key) + && e.Skip)); + + lookupsToRevisit = lookupsToRevisit.Concat(missingLookups).ToList(); + if (transformer.RecursiveRelationshipField != null) logger.DebugFormat("Object {0} has a recurive relation to iteself in field {1}", objLoop.ApiName, transformer.RecursiveRelationshipField); @@ -179,7 +192,79 @@ public PopulateSandboxResult Start(IProgress progress) ProgressUpdate(string.Format("Summary for {0}: Success {1} Fail {2}", objLoop.ApiName, objres.SuccessCount, objres.FailCount)); + processedObjects.Add(objLoop.ApiName); + } + + // revisit lookups that will be missing and populate them now that all the records are + // loaded into the sandbox + foreach(var missingLookup in lookupsToRevisit) + { + // retrieve data from the source org just for this field + var objres = new PopulateObjectResult(); + List sourceData = null; + + try + { + // get all records where this lookup has a value + sourceData = _sourceTasks.GetDataFromSObject( + sobjectName: missingLookup.Key, + colList: new List { missingLookup.Key }, + filter: missingLookup.Value + " <> ''"); + } + catch (Exception e) + { + string errMess = string.Format("Error while fetching data for {0}: {1}", missingLookup.Value, e.Message); + throw new ApplicationException(errMess, e); + } + + objres.SourceRows = sourceData.Count(); + ProgressUpdate(string.Format("Received {0} {1} records from source", sourceData.Count, missingLookup.Value)); + + // switch the IDs with the new ones in the sandbox + // get working info and transform objects + var workingList = new List(); + foreach (sObject rowLoop in sourceData) + { + var wrap = new ObjectTransformer.sObjectWrapper(); + wrap.OriginalId = rowLoop.Id; + wrap.sObj = rowLoop; + + // TODO: transformer.ApplyTransformations(wrap); + + workingList.Add(wrap); + } + + // update records in batches + int batchSize = 100; + int done = 0; + bool allDone = false; + if (workingList.Count == 0) + allDone = true; + while (!allDone) + { + var workBatch = workingList.Skip(done).Take(batchSize).ToList(); + done += workBatch.Count; + if (done >= workingList.Count) + allDone = true; + var insertRes = _targetTasks.UpdateSObjects(missingLookup.Value, + workBatch.Select(w => w.sObj).ToArray()); + + for (int i = 0; i < insertRes.Length; i++) + { + if (insertRes[i].Success) + { + objres.SuccessCount += 1; + } + else + { + workBatch[i].ErrorMessage = insertRes[i].ErrorMessage; + logger.WarnFormat("Error when updating {0} {1} into target: {2}", + missingLookup.Value, workBatch[i].OriginalId, workBatch[i].ErrorMessage); + objres.FailCount += 1; + } + } + } } // log summary diff --git a/SandboxberryLib/SalesforceTasks.cs b/SandboxberryLib/SalesforceTasks.cs index 0e480b1..0fd0544 100644 --- a/SandboxberryLib/SalesforceTasks.cs +++ b/SandboxberryLib/SalesforceTasks.cs @@ -166,11 +166,25 @@ public string BuildQuery(string sobjectName, List colList, string filter return sb.ToString(); } + /// + /// Gets data for all columns of an sobject + /// public List GetDataFromSObject(string sobjectName, string filter) { - LoginIfRequired(); List colNames = RemoveSystemColumns(GetObjectColumns(sobjectName)); string soql = BuildQuery(sobjectName, colNames, filter); + List allResults = GetDataFromSObject(sobjectName, colNames, filter); + return allResults; + } + + /// + /// Gets data for specified columns of an sobject + /// + public List GetDataFromSObject(string sobjectName, List colList, string filter) + { + LoginIfRequired(); + colList = RemoveSystemColumns(colList); + string soql = BuildQuery(sobjectName, colList, filter); bool allResultsReturned = false; List allResults = new List(); From 9a0806f80e0b530a9ead01f38cfe1a50880f6672 Mon Sep 17 00:00:00 2001 From: Matt Wanchap Date: Wed, 8 May 2019 11:05:28 +1000 Subject: [PATCH 2/6] Got the update missing lookups functionality working (ugly, needs refactoring, but working) --- SandboxberryLib/PopulateSandbox.cs | 56 ++++++++++++++++++-------- SandboxberryLib/SandboxberryLib.csproj | 1 + 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/SandboxberryLib/PopulateSandbox.cs b/SandboxberryLib/PopulateSandbox.cs index 9522547..5c7925f 100644 --- a/SandboxberryLib/PopulateSandbox.cs +++ b/SandboxberryLib/PopulateSandbox.cs @@ -87,7 +87,7 @@ public PopulateSandboxResult Start(IProgress progress) var missingUserIds = sourceUserIds.Except(targetUserIds).ToList(); logger.DebugFormat("Found {0} users in Source that are not in Target", missingUserIds.Count()); var processedObjects = new List { "User" }; // user is copied already - var lookupsToRevisit = new List>(); + var lookupsToRevisit = new List(); foreach (SbbObject objLoop in _instructions.SbbObjects) { @@ -106,13 +106,20 @@ public PopulateSandboxResult Start(IProgress progress) transformer.ObjectRelationships = _sourceTasks.GetObjectRelationships(objLoop.ApiName); transformer.RecursiveRelationshipField = transformer.ObjectRelationships.FirstOrDefault(d => d.Value == objLoop.ApiName).Key; - // find lookups on this object that can't be populated - var missingLookups = transformer.ObjectRelationships.Where( - d => d.Value != objLoop.ApiName // where it's not a recursive relationship - && !processedObjects.Contains(d.Value) // and we haven't already processed this object (i.e. it's not going to be mapped) - && !objLoop.SbbFieldOptions.Any( // and it's not one of the skipped fields + // find lookups on this object that can't be populated, to revisit later + var missingLookups = transformer.ObjectRelationships + .Where(d => d.Value != objLoop.ApiName) // where it's not a recursive relationship + .Where(d => !processedObjects.Contains(d.Value)) // and this lookup can't be populated because we haven't already processed this object (i.e. the referenced record doesn't exist yet) + .Where(d => !objLoop.SbbFieldOptions.Any( // and it's not one of the skipped fields e => e.ApiName.Equals(d.Key) - && e.Skip)); + && e.Skip)) + // TODO: but is still one of the included object types (e.g. not Contact -> "rh2__PS_Describe__c") + .Select(d => new LookupInfo + { + FieldName = d.Key, + ObjectName = objLoop.ApiName, + RelatedObjectName = d.Value + }); lookupsToRevisit = lookupsToRevisit.Concat(missingLookups).ToList(); @@ -199,6 +206,9 @@ public PopulateSandboxResult Start(IProgress progress) // loaded into the sandbox foreach(var missingLookup in lookupsToRevisit) { + var transformer = new ObjectTransformer(); + transformer.RelationMapper = _relationMapper; + // retrieve data from the source org just for this field var objres = new PopulateObjectResult(); List sourceData = null; @@ -207,31 +217,43 @@ public PopulateSandboxResult Start(IProgress progress) { // get all records where this lookup has a value sourceData = _sourceTasks.GetDataFromSObject( - sobjectName: missingLookup.Key, - colList: new List { missingLookup.Key }, - filter: missingLookup.Value + " <> ''"); + sobjectName: missingLookup.ObjectName, //needs to be account, not contact + colList: new List { "Id", missingLookup.FieldName }, + filter: missingLookup.FieldName + " <> ''"); + // TODO: also include filters on this sobject from the instructions file } catch (Exception e) { - string errMess = string.Format("Error while fetching data for {0}: {1}", missingLookup.Value, e.Message); + string errMess = string.Format("Error while fetching data for {0}: {1}", missingLookup.ObjectName, e.Message); throw new ApplicationException(errMess, e); } objres.SourceRows = sourceData.Count(); - ProgressUpdate(string.Format("Received {0} {1} records from source", sourceData.Count, missingLookup.Value)); + ProgressUpdate(string.Format("Received {0} {1} records from source", sourceData.Count, missingLookup.ObjectName)); // switch the IDs with the new ones in the sandbox // get working info and transform objects var workingList = new List(); foreach (sObject rowLoop in sourceData) { + rowLoop.Any = rowLoop.Any.Where(e => e.LocalName != "Id").ToArray(); var wrap = new ObjectTransformer.sObjectWrapper(); wrap.OriginalId = rowLoop.Id; wrap.sObj = rowLoop; - // TODO: transformer.ApplyTransformations(wrap); - - workingList.Add(wrap); + transformer.FixRelatedIds(rowLoop, + new Dictionary + { + [missingLookup.FieldName] = missingLookup.RelatedObjectName + }); + + // also update the ID of the object itself + wrap.sObj.Id = _relationMapper.RecallNewId(missingLookup.ObjectName, wrap.sObj.Id); + + if (wrap.sObj.Id != null) + { + workingList.Add(wrap); + } } // update records in batches @@ -247,7 +269,7 @@ public PopulateSandboxResult Start(IProgress progress) if (done >= workingList.Count) allDone = true; - var insertRes = _targetTasks.UpdateSObjects(missingLookup.Value, + var insertRes = _targetTasks.UpdateSObjects(missingLookup.ObjectName, workBatch.Select(w => w.sObj).ToArray()); for (int i = 0; i < insertRes.Length; i++) @@ -260,7 +282,7 @@ public PopulateSandboxResult Start(IProgress progress) { workBatch[i].ErrorMessage = insertRes[i].ErrorMessage; logger.WarnFormat("Error when updating {0} {1} into target: {2}", - missingLookup.Value, workBatch[i].OriginalId, workBatch[i].ErrorMessage); + missingLookup.ObjectName, workBatch[i].OriginalId, workBatch[i].ErrorMessage); objres.FailCount += 1; } } diff --git a/SandboxberryLib/SandboxberryLib.csproj b/SandboxberryLib/SandboxberryLib.csproj index 232bf67..aa42729 100644 --- a/SandboxberryLib/SandboxberryLib.csproj +++ b/SandboxberryLib/SandboxberryLib.csproj @@ -58,6 +58,7 @@ Settings.settings + From a83e0c30ab022e98fb3f73fd9b12a931ea8f9a24 Mon Sep 17 00:00:00 2001 From: Matt Wanchap Date: Sat, 11 May 2019 16:38:23 +1000 Subject: [PATCH 3/6] Modified the update missing lookups feature so it doesn't require re-querying data before reprocessing, just stores the relevant fields and values during the import process. Also some minor refactoring in ObjectTransformer and PopulateSandbox --- SandboxberryLib/ObjectTransformer.cs | 24 +++ SandboxberryLib/PopulateSandbox.cs | 237 +++++++++++++++------------ 2 files changed, 157 insertions(+), 104 deletions(-) diff --git a/SandboxberryLib/ObjectTransformer.cs b/SandboxberryLib/ObjectTransformer.cs index 28fd6e3..890f1b6 100644 --- a/SandboxberryLib/ObjectTransformer.cs +++ b/SandboxberryLib/ObjectTransformer.cs @@ -38,6 +38,12 @@ public class ObjectTransformer public Dictionary ObjectRelationships { get; set; } + /// + /// Lookups that will need to be reprocessed later as the referenced objects didn't exist + /// yet when the wrapped object was created + /// + public List LookupsToReprocess { get; set; } + public List InactiveUserIds { get; set; } public List MissingUserIds { get; set; } @@ -51,6 +57,12 @@ public void ApplyTransformations(sObjectWrapper wrap) CorrectInactiveUser(wrap.sObj, this.InactiveUserIds, this.CurrentUserId); if (this.RecursiveRelationshipField != null) RememberRecursiveId(wrap, this.RecursiveRelationshipField); + + foreach(var lookup in this.LookupsToReprocess) + { + RemeberLookupIdsToReprocess(lookup, wrap.sObj); + } + FixRelatedIds(wrap.sObj, this.ObjectRelationships); RemoveIdFromSObject(wrap.sObj); @@ -169,7 +181,19 @@ private void RememberRecursiveId(ObjectTransformer.sObjectWrapper wrap, string r } + /// + /// Remember IDs to reprocess later for this lookup relationship + /// + private void RemeberLookupIdsToReprocess(LookupInfo lookup, sObject obj) + { + var lookupField = obj.Any.FirstOrDefault(e => e.LocalName == lookup.FieldName); + var relatedId = lookupField.InnerText; + if (!String.IsNullOrEmpty(relatedId)) + { + lookup.IdPairs.Add(new KeyValuePair(obj.Id, relatedId)); + } + } public class sObjectWrapper { diff --git a/SandboxberryLib/PopulateSandbox.cs b/SandboxberryLib/PopulateSandbox.cs index 5c7925f..45edf7f 100644 --- a/SandboxberryLib/PopulateSandbox.cs +++ b/SandboxberryLib/PopulateSandbox.cs @@ -87,7 +87,7 @@ public PopulateSandboxResult Start(IProgress progress) var missingUserIds = sourceUserIds.Except(targetUserIds).ToList(); logger.DebugFormat("Found {0} users in Source that are not in Target", missingUserIds.Count()); var processedObjects = new List { "User" }; // user is copied already - var lookupsToRevisit = new List(); + var objectsToReprocess = new List(); foreach (SbbObject objLoop in _instructions.SbbObjects) { @@ -106,10 +106,14 @@ public PopulateSandboxResult Start(IProgress progress) transformer.ObjectRelationships = _sourceTasks.GetObjectRelationships(objLoop.ApiName); transformer.RecursiveRelationshipField = transformer.ObjectRelationships.FirstOrDefault(d => d.Value == objLoop.ApiName).Key; - // find lookups on this object that can't be populated, to revisit later - var missingLookups = transformer.ObjectRelationships + if (transformer.RecursiveRelationshipField != null) + logger.DebugFormat("Object {0} has a recurive relation to iteself in field {1}", + objLoop.ApiName, transformer.RecursiveRelationshipField); + + // find lookups on this object that can't be populated, to reprocess later + transformer.LookupsToReprocess = transformer.ObjectRelationships .Where(d => d.Value != objLoop.ApiName) // where it's not a recursive relationship - .Where(d => !processedObjects.Contains(d.Value)) // and this lookup can't be populated because we haven't already processed this object (i.e. the referenced record doesn't exist yet) + .Where(d => !processedObjects.Contains(d.Value)) // and the referenced record doesn't exist yet .Where(d => !objLoop.SbbFieldOptions.Any( // and it's not one of the skipped fields e => e.ApiName.Equals(d.Key) && e.Skip)) @@ -119,14 +123,17 @@ public PopulateSandboxResult Start(IProgress progress) FieldName = d.Key, ObjectName = objLoop.ApiName, RelatedObjectName = d.Value - }); + }) + .ToList(); - lookupsToRevisit = lookupsToRevisit.Concat(missingLookups).ToList(); - - if (transformer.RecursiveRelationshipField != null) - logger.DebugFormat("Object {0} has a recurive relation to iteself in field {1}", - objLoop.ApiName, transformer.RecursiveRelationshipField); - + if(transformer.LookupsToReprocess.Count > 0) + { + objectsToReprocess.Add(transformer); + var fields = transformer.LookupsToReprocess.Select(lookup => lookup.FieldName); + logger.DebugFormat("Object {0} has lookups that will need to be reprocessed: {1}", + objLoop.ApiName, + String.Join(", ", fields)); + } List sourceData = null; try @@ -202,88 +209,95 @@ public PopulateSandboxResult Start(IProgress progress) processedObjects.Add(objLoop.ApiName); } - // revisit lookups that will be missing and populate them now that all the records are - // loaded into the sandbox - foreach(var missingLookup in lookupsToRevisit) + // reprocess lookups that were missed and populate them, now that all the records + // are loaded into the sandbox + foreach(var obj in objectsToReprocess) { - var transformer = new ObjectTransformer(); - transformer.RelationMapper = _relationMapper; - - // retrieve data from the source org just for this field - var objres = new PopulateObjectResult(); - List sourceData = null; - - try - { - // get all records where this lookup has a value - sourceData = _sourceTasks.GetDataFromSObject( - sobjectName: missingLookup.ObjectName, //needs to be account, not contact - colList: new List { "Id", missingLookup.FieldName }, - filter: missingLookup.FieldName + " <> ''"); - // TODO: also include filters on this sobject from the instructions file - } - catch (Exception e) + foreach (var field in obj.LookupsToReprocess) { - string errMess = string.Format("Error while fetching data for {0}: {1}", missingLookup.ObjectName, e.Message); - throw new ApplicationException(errMess, e); - } - - objres.SourceRows = sourceData.Count(); - ProgressUpdate(string.Format("Received {0} {1} records from source", sourceData.Count, missingLookup.ObjectName)); - - // switch the IDs with the new ones in the sandbox - // get working info and transform objects - var workingList = new List(); - foreach (sObject rowLoop in sourceData) - { - rowLoop.Any = rowLoop.Any.Where(e => e.LocalName != "Id").ToArray(); - var wrap = new ObjectTransformer.sObjectWrapper(); - wrap.OriginalId = rowLoop.Id; - wrap.sObj = rowLoop; + List updateList = new List(); + foreach(var idPair in field.IdPairs) + { + var updateObj = CreateSobjectWithLookup(field.ObjectName, + field.RelatedObjectName, field.FieldName, idPair); - transformer.FixRelatedIds(rowLoop, - new Dictionary + if (updateObj != null) { - [missingLookup.FieldName] = missingLookup.RelatedObjectName - }); - - // also update the ID of the object itself - wrap.sObj.Id = _relationMapper.RecallNewId(missingLookup.ObjectName, wrap.sObj.Id); + updateList.Add(updateObj); + } + } - if (wrap.sObj.Id != null) + // retrieve data from the source org just for this field + var objres = new PopulateObjectResult(); + //List sourceData = null; + + //try + //{ + // // get all records where this lookup has a value + // sourceData = _sourceTasks.GetDataFromSObject( + // sobjectName: field.ObjectName, //needs to be account, not contact + // colList: new List { "Id", field.FieldName }, + // filter: field.FieldName + " <> ''"); + // // TODO: also include filters on this sobject from the instructions file + //} + //catch (Exception e) + //{ + // string errMess = string.Format("Error while fetching data for {0}: {1}", + // field.ObjectName, e.Message); + // throw new ApplicationException(errMess, e); + //} + + objres.SourceRows = updateList.Count(); + ProgressUpdate(string.Format("Reprocessing {0} {1} records", + updateList.Count, field.ObjectName)); + + // switch the IDs with the new ones in the sandbox + // get working info and transform objects + var workingList = new List(); + foreach (sObject rowLoop in updateList) { - workingList.Add(wrap); + var wrap = new ObjectTransformer.sObjectWrapper(); + wrap.OriginalId = rowLoop.Id; + wrap.sObj = rowLoop; + + // update the ID of the object itself + wrap.sObj.Id = _relationMapper.RecallNewId(field.ObjectName, wrap.sObj.Id); + + if (wrap.sObj.Id != null) + { + workingList.Add(wrap); + } } - } - // update records in batches - int batchSize = 100; - int done = 0; - bool allDone = false; - if (workingList.Count == 0) - allDone = true; - while (!allDone) - { - var workBatch = workingList.Skip(done).Take(batchSize).ToList(); - done += workBatch.Count; - if (done >= workingList.Count) + // update records in batches + int batchSize = 100; + int done = 0; + bool allDone = false; + if (workingList.Count == 0) allDone = true; + while (!allDone) + { + var workBatch = workingList.Skip(done).Take(batchSize).ToList(); + done += workBatch.Count; + if (done >= workingList.Count) + allDone = true; - var insertRes = _targetTasks.UpdateSObjects(missingLookup.ObjectName, - workBatch.Select(w => w.sObj).ToArray()); + var insertRes = _targetTasks.UpdateSObjects(field.ObjectName, + workBatch.Select(w => w.sObj).ToArray()); - for (int i = 0; i < insertRes.Length; i++) - { - if (insertRes[i].Success) - { - objres.SuccessCount += 1; - } - else + for (int i = 0; i < insertRes.Length; i++) { - workBatch[i].ErrorMessage = insertRes[i].ErrorMessage; - logger.WarnFormat("Error when updating {0} {1} into target: {2}", - missingLookup.ObjectName, workBatch[i].OriginalId, workBatch[i].ErrorMessage); - objres.FailCount += 1; + if (insertRes[i].Success) + { + objres.SuccessCount += 1; + } + else + { + workBatch[i].ErrorMessage = insertRes[i].ErrorMessage; + logger.WarnFormat("Error when updating {0} {1} into target: {2}", + field.ObjectName, workBatch[i].OriginalId, workBatch[i].ErrorMessage); + objres.FailCount += 1; + } } } } @@ -302,9 +316,40 @@ public PopulateSandboxResult Start(IProgress progress) return res; } - + /// + /// Creates a new sObject record that only contains the specified lookup relationship field + /// and replaces IDs for the referenced objects + /// + /// The newly-constructed sObject with the correct referenced ID, otherwise null + private sObject CreateSobjectWithLookup(string objectName, string relatedObjectName, + string fieldName, KeyValuePair idPair) + { + var newObject = new sObject + { + type = objectName, + Id = idPair.Key + }; + + XmlDocument dummydoc = new XmlDocument(); + XmlElement recursiveEl = dummydoc.CreateElement(fieldName); + string replaceValue = _relationMapper.RecallNewId(relatedObjectName, idPair.Value); + + if (replaceValue == null) + { + logger.DebugFormat("Object {0} {1} relationship field {2} have value {3} could not translate - will ignore", + objectName, idPair.Key, fieldName, idPair.Value); + return null; + } + else + { + recursiveEl.InnerText = replaceValue; + newObject.Any = new XmlElement[] { recursiveEl }; + return newObject; + } + } + + - private void UpdateRecursiveField(string apiName, List workingList, string recursiveRelationshipField) @@ -318,36 +363,20 @@ private void UpdateRecursiveField(string apiName, List(wrapLoop.NewId, wrapLoop.RecursiveRelationshipOriginalId)); - if (replaceValue == null) - { - logger.DebugFormat("Object {0} {1} recursive field {2} have value {3} could not translate - will ignore", - apiName, wrapLoop.OriginalId, recursiveRelationshipField, wrapLoop.RecursiveRelationshipOriginalId); - } - else + if(updateObject != null) { - - recursiveEl.InnerText = replaceValue; - - upd.Any = new XmlElement[] { recursiveEl }; - - updateList.Add(upd); + updateList.Add(updateObject); } } - } logger.DebugFormat("{0} rows in Object {1} have recursive relation {2} to update ....", updateList.Count(), apiName, recursiveRelationshipField); + // TODO: refactor this stuff into a single method // update objects in batches int successCount = 0; int failCount = 0; From 791bb5666ebc8ab292c9d1535d7032c57ace1f2c Mon Sep 17 00:00:00 2001 From: Matt Wanchap Date: Sat, 11 May 2019 17:00:54 +1000 Subject: [PATCH 4/6] Whoops, forgot to add LookupInfo.cs --- SandboxberryLib/ResultsModel/LookupInfo.cs | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 SandboxberryLib/ResultsModel/LookupInfo.cs diff --git a/SandboxberryLib/ResultsModel/LookupInfo.cs b/SandboxberryLib/ResultsModel/LookupInfo.cs new file mode 100644 index 0000000..9202bd3 --- /dev/null +++ b/SandboxberryLib/ResultsModel/LookupInfo.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace SandboxberryLib.ResultsModel +{ + /// + /// Stores information about lookup relationships and the referenced IDs, sometimes they can't + /// be populated during the initial import (when the referenced object doesn't exist yet) and + /// need to be reprocessed after + /// + public class LookupInfo + { + public LookupInfo() + { + this.IdPairs = new List>(); + } + + /// + /// API Name of the object that owns the lookup relationship field + /// + public String ObjectName { get; set; } + + /// + /// API Name of the object that the lookup relationship field references + /// + public String RelatedObjectName { get; set; } + + /// + /// API Name of the lookup relationship field + /// + public String FieldName { get; set; } + + /// + /// IDs used by this lookup relationship in the form of key=object, value=referenced object, + /// saved during initial processing for reprocessing later + /// + public List> IdPairs { get; set; } + } +} From d035b2df2c6d1f462b4b75ce2359a9a0990f5606 Mon Sep 17 00:00:00 2001 From: Matt Wanchap Date: Thu, 16 May 2019 07:06:37 +1000 Subject: [PATCH 5/6] Refactored duplicated record-updating code into a single UpdateRecords method --- SandboxberryLib/PopulateSandbox.cs | 100 +++++++---------------------- 1 file changed, 22 insertions(+), 78 deletions(-) diff --git a/SandboxberryLib/PopulateSandbox.cs b/SandboxberryLib/PopulateSandbox.cs index 45edf7f..5bae4a1 100644 --- a/SandboxberryLib/PopulateSandbox.cs +++ b/SandboxberryLib/PopulateSandbox.cs @@ -227,79 +227,16 @@ public PopulateSandboxResult Start(IProgress progress) } } - // retrieve data from the source org just for this field - var objres = new PopulateObjectResult(); - //List sourceData = null; - - //try - //{ - // // get all records where this lookup has a value - // sourceData = _sourceTasks.GetDataFromSObject( - // sobjectName: field.ObjectName, //needs to be account, not contact - // colList: new List { "Id", field.FieldName }, - // filter: field.FieldName + " <> ''"); - // // TODO: also include filters on this sobject from the instructions file - //} - //catch (Exception e) - //{ - // string errMess = string.Format("Error while fetching data for {0}: {1}", - // field.ObjectName, e.Message); - // throw new ApplicationException(errMess, e); - //} - - objres.SourceRows = updateList.Count(); ProgressUpdate(string.Format("Reprocessing {0} {1} records", updateList.Count, field.ObjectName)); // switch the IDs with the new ones in the sandbox - // get working info and transform objects - var workingList = new List(); foreach (sObject rowLoop in updateList) { - var wrap = new ObjectTransformer.sObjectWrapper(); - wrap.OriginalId = rowLoop.Id; - wrap.sObj = rowLoop; - - // update the ID of the object itself - wrap.sObj.Id = _relationMapper.RecallNewId(field.ObjectName, wrap.sObj.Id); - - if (wrap.sObj.Id != null) - { - workingList.Add(wrap); - } + rowLoop.Id = _relationMapper.RecallNewId(field.ObjectName, rowLoop.Id); } - // update records in batches - int batchSize = 100; - int done = 0; - bool allDone = false; - if (workingList.Count == 0) - allDone = true; - while (!allDone) - { - var workBatch = workingList.Skip(done).Take(batchSize).ToList(); - done += workBatch.Count; - if (done >= workingList.Count) - allDone = true; - - var insertRes = _targetTasks.UpdateSObjects(field.ObjectName, - workBatch.Select(w => w.sObj).ToArray()); - - for (int i = 0; i < insertRes.Length; i++) - { - if (insertRes[i].Success) - { - objres.SuccessCount += 1; - } - else - { - workBatch[i].ErrorMessage = insertRes[i].ErrorMessage; - logger.WarnFormat("Error when updating {0} {1} into target: {2}", - field.ObjectName, workBatch[i].OriginalId, workBatch[i].ErrorMessage); - objres.FailCount += 1; - } - } - } + var result = UpdateRecords(field.ObjectName, updateList); } } @@ -366,7 +303,7 @@ private void UpdateRecursiveField(string apiName, List(wrapLoop.NewId, wrapLoop.RecursiveRelationshipOriginalId)); - if(updateObject != null) + if (updateObject != null) { updateList.Add(updateObject); } @@ -376,10 +313,15 @@ private void UpdateRecursiveField(string apiName, List + /// Updates a list of sobject records + /// + private PopulateObjectResult UpdateRecords(string objectName, List updateList) + { + var result = new PopulateObjectResult(); int done = 0; bool allDone = false; if (updateList.Count == 0) @@ -391,29 +333,31 @@ private void UpdateRecursiveField(string apiName, List= updateList.Count) allDone = true; - var updateRes = _targetTasks.UpdateSObjects(apiName, + var updateRes = _targetTasks.UpdateSObjects(objectName, batch.ToArray()); for (int i = 0; i < updateRes.Length; i++) { if (updateRes[i].Success) { - successCount+=1; + result.SuccessCount += 1; } else { - + logger.WarnFormat("Error when updating {0} {1} in target: {2}", - apiName, batch[i].Id, updateRes[i].ErrorMessage); - failCount += 1; + objectName, batch[i].Id, updateRes[i].ErrorMessage); + result.FailCount += 1; } } - + } - logger.DebugFormat("Object {0} recursive relation {1} updated. Attempted {2} Success {3} Failed {4}", - apiName, recursiveRelationshipField, updateList.Count, successCount, failCount); + logger.DebugFormat("Object {0} updated. Attempted {1} Success {2} Failed {3}", + objectName, updateList.Count, result.SuccessCount, result.FailCount); + + return result; } private void LoginToBoth() From 78473694fc45b88596a9a3b479e6e2c61c8a1e9f Mon Sep 17 00:00:00 2001 From: Matt Wanchap Date: Fri, 17 May 2019 06:35:01 +1000 Subject: [PATCH 6/6] Further refactored lookup reprocessing code into a separate method and added progress updates --- SandboxberryLib/PopulateSandbox.cs | 126 +++++++++++++++++------------ 1 file changed, 75 insertions(+), 51 deletions(-) diff --git a/SandboxberryLib/PopulateSandbox.cs b/SandboxberryLib/PopulateSandbox.cs index 5bae4a1..9d0f2c2 100644 --- a/SandboxberryLib/PopulateSandbox.cs +++ b/SandboxberryLib/PopulateSandbox.cs @@ -125,8 +125,8 @@ public PopulateSandboxResult Start(IProgress progress) RelatedObjectName = d.Value }) .ToList(); - - if(transformer.LookupsToReprocess.Count > 0) + + if (transformer.LookupsToReprocess.Count > 0) { objectsToReprocess.Add(transformer); var fields = transformer.LookupsToReprocess.Select(lookup => lookup.FieldName); @@ -209,14 +209,76 @@ public PopulateSandboxResult Start(IProgress progress) processedObjects.Add(objLoop.ApiName); } - // reprocess lookups that were missed and populate them, now that all the records - // are loaded into the sandbox - foreach(var obj in objectsToReprocess) + // reprocess lookup relationships that can be populated now that the inserts are done + var reprocessingResults = ReprocessObjects(objectsToReprocess); + + // log summary + ProgressUpdate("************************************************"); + foreach (var resLoop in res.ObjectResults) + { + ProgressUpdate(string.Format("Summary for {0}: Success {1} Fail {2}", + resLoop.ApiName, resLoop.SuccessCount, resLoop.FailCount)); + + } + + // log reprocssing summary + foreach (var resLoop in reprocessingResults.ObjectResults) + { + ProgressUpdate(string.Format("Reprocessing summary for {0}: Success {1} Fail {2}", + resLoop.ApiName, resLoop.SuccessCount, resLoop.FailCount)); + } + ProgressUpdate("************************************************"); + + return res; + } + + + + private void UpdateRecursiveField(string apiName, List workingList, string recursiveRelationshipField) + { + logger.DebugFormat("Object {0} has a recursive relation field {1}, now doing second pass to set it....", + apiName, recursiveRelationshipField); + + // make a new list of sObjects to do the update + List updateList = new List(); + foreach (var wrapLoop in workingList) + { + if (!string.IsNullOrEmpty(wrapLoop.RecursiveRelationshipOriginalId)) + { + var updateObject = CreateSobjectWithLookup(apiName, apiName, recursiveRelationshipField, + new KeyValuePair(wrapLoop.NewId, wrapLoop.RecursiveRelationshipOriginalId)); + + if (updateObject != null) + { + updateList.Add(updateObject); + } + } + } + + logger.DebugFormat("{0} rows in Object {1} have recursive relation {2} to update ....", + updateList.Count(), apiName, recursiveRelationshipField); + + var result = UpdateRecords(apiName, updateList); + } + + /// + /// Reprocesses lookup relationship fields that were missed during the initial import and + /// populates them, once all the records are loaded into the sandbox. Similar to + /// UpdateRecursiveField. + /// + private PopulateSandboxResult ReprocessObjects(List objectsToReprocess) + { + var results = new PopulateSandboxResult(); + + foreach (var obj in objectsToReprocess) { foreach (var field in obj.LookupsToReprocess) { + ProgressUpdate(string.Format("Reprocessing referenced objects for {0} field {1}", + field.ObjectName, field.FieldName)); + List updateList = new List(); - foreach(var idPair in field.IdPairs) + foreach (var idPair in field.IdPairs) { var updateObj = CreateSobjectWithLookup(field.ObjectName, field.RelatedObjectName, field.FieldName, idPair); @@ -227,30 +289,21 @@ public PopulateSandboxResult Start(IProgress progress) } } - ProgressUpdate(string.Format("Reprocessing {0} {1} records", - updateList.Count, field.ObjectName)); - // switch the IDs with the new ones in the sandbox foreach (sObject rowLoop in updateList) { rowLoop.Id = _relationMapper.RecallNewId(field.ObjectName, rowLoop.Id); } + ProgressUpdate(string.Format("Updating {0} {1} records", + updateList.Count, field.ObjectName)); + var result = UpdateRecords(field.ObjectName, updateList); + results.ObjectResults.Add(result); } } - // log summary - ProgressUpdate("************************************************"); - foreach (var resLoop in res.ObjectResults) - { - ProgressUpdate(string.Format("Summary for {0}: Success {1} Fail {2}", - resLoop.ApiName, resLoop.SuccessCount, resLoop.FailCount)); - - } - ProgressUpdate("************************************************"); - - return res; + return results; } /// @@ -285,43 +338,14 @@ private sObject CreateSobjectWithLookup(string objectName, string relatedObjectN } } - - - - - private void UpdateRecursiveField(string apiName, List workingList, string recursiveRelationshipField) - { - logger.DebugFormat("Object {0} has a recursive relation field {1}, now doing second pass to set it....", - apiName, recursiveRelationshipField); - - // make a new list of sObjects to do the update - List updateList = new List(); - foreach (var wrapLoop in workingList) - { - if (!string.IsNullOrEmpty(wrapLoop.RecursiveRelationshipOriginalId)) - { - var updateObject = CreateSobjectWithLookup(apiName, apiName, recursiveRelationshipField, - new KeyValuePair(wrapLoop.NewId, wrapLoop.RecursiveRelationshipOriginalId)); - - if (updateObject != null) - { - updateList.Add(updateObject); - } - } - } - - logger.DebugFormat("{0} rows in Object {1} have recursive relation {2} to update ....", - updateList.Count(), apiName, recursiveRelationshipField); - - var result = UpdateRecords(apiName, updateList); - } - /// /// Updates a list of sobject records /// private PopulateObjectResult UpdateRecords(string objectName, List updateList) { var result = new PopulateObjectResult(); + result.ApiName = objectName; + int done = 0; bool allDone = false; if (updateList.Count == 0)