From c3227fc12caa4c17e25176837ac48ad018e3ce23 Mon Sep 17 00:00:00 2001 From: petero-dk <2478689+petero-dk@users.noreply.github.com> Date: Fri, 23 Aug 2024 22:36:05 +0200 Subject: [PATCH 1/3] Purely to new Azure.Data clients, support for managed identities. Clean required packages. --- .../AzureTableUtilities.csproj | 10 +- src/AzureTableUtilities/BackupAzureTables.cs | 176 ++++++------------ src/AzureTableUtilities/CopyAzureTables.cs | 67 ++----- .../DynamicTableEntityJsonConverter.cs | 142 +++++++------- .../DynamicTableEntityJsonSerializer.cs | 41 ++-- src/AzureTableUtilities/Helper.cs | 5 +- src/AzureTableUtilities/RestoreAzureTables.cs | 165 ++++++---------- .../AzureTableUtilitiesBackupXUnitTest.cs | 20 +- .../AzureTableUtilitiesCopyXUnitTest.cs | 12 +- .../AzureTableUtilitiesDeleteXUnitTest.cs | 12 +- ...AzureTableUtilitiesFiltersBaseXUnitTest.cs | 14 +- .../AzureTableUtilitiesXUnitTest.cs | 15 +- .../AzureTableUtilitiesXUnitTest.csproj | 28 +-- 13 files changed, 264 insertions(+), 443 deletions(-) diff --git a/src/AzureTableUtilities/AzureTableUtilities.csproj b/src/AzureTableUtilities/AzureTableUtilities.csproj index 7a13edf..65eb900 100644 --- a/src/AzureTableUtilities/AzureTableUtilities.csproj +++ b/src/AzureTableUtilities/AzureTableUtilities.csproj @@ -50,16 +50,12 @@ v6.0.0 - ** Backup/Restore is NOT compatible with prior versions ** Upgrade fr - D:\github\TheByteStuff\AzureTableUtilities\src\AzureTableUtilities\AzureTableUtilities.xml + - - - - - - + + diff --git a/src/AzureTableUtilities/BackupAzureTables.cs b/src/AzureTableUtilities/BackupAzureTables.cs index 3fd84b3..fab05a0 100644 --- a/src/AzureTableUtilities/BackupAzureTables.cs +++ b/src/AzureTableUtilities/BackupAzureTables.cs @@ -1,28 +1,18 @@ -using System; +using Azure; +using Azure.Data.Tables; +using Azure.Storage; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Configuration; -using System.Diagnostics; using System.IO; using System.IO.Compression; -using System.Security; -using System.Net.Http; - -using AZStorage = Microsoft.Azure.Storage; -using Microsoft.Azure.Storage.Auth; -using AZBlob = Microsoft.Azure.Storage.Blob; -using Microsoft.Azure.Storage.Core; -using Microsoft.Azure.Storage.File; +using System.Linq; +using System.Text; +using TheByteStuff.AzureTableUtilities.Exceptions; +//using AZBlob = Microsoft.Azure.Storage.Blob; using AzureTables = Azure.Data.Tables; -using Azure.Data.Tables.Models; -using Azure; - -using Newtonsoft.Json; - -using TheByteStuff.AzureTableUtilities.Exceptions; namespace TheByteStuff.AzureTableUtilities { @@ -31,10 +21,9 @@ namespace TheByteStuff.AzureTableUtilities /// public class BackupAzureTables { - //private SecureString AzureTableConnectionSpec = new SecureString(); - //private SecureString AzureBlobConnectionSpec = new SecureString(); - private string AzureTableConnectionSpec = ""; - private string AzureBlobConnectionSpec = ""; + private TableServiceClient tableServiceClient; + private BlobServiceClient blobServiceClient; + /// /// Constructor, sets same connection spec for both the Azure Tables as well as the Azure Blob storage. @@ -42,19 +31,21 @@ public class BackupAzureTables /// Connection string for Azure Table and Blob Connections; ex "AccountName=devstoreaccount1;AccountKey={xxxxxxxxxxx};DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" public BackupAzureTables(string AzureConnection) : this(AzureConnection, AzureConnection) { - + tableServiceClient = new TableServiceClient(AzureConnection); + blobServiceClient = new BlobServiceClient(AzureConnection); } /// - /// Constructor, accepts SecureString and sets same connection spec for both the Azure Tables as well as the Azure Blob storage. + /// Directly set the Service Clients /// - /// Connection string for Azure Table and Blob Connections - /* - public BackupAzureTables(SecureString AzureConnection) : this(AzureConnection, AzureConnection) + /// + /// + public BackupAzureTables(TableServiceClient tableServiceClient, BlobServiceClient blobServiceClient) { - + this.tableServiceClient = tableServiceClient; + this.blobServiceClient = blobServiceClient; } - */ + /// /// Constructor, allows a different connection spec for Azure Table and Azure Blob storage. @@ -68,18 +59,8 @@ public BackupAzureTables(string AzureTableConnection, string AzureBlobConnection throw new ConnectionException(String.Format("Connection spec must be specified.")); } - AzureTableConnectionSpec = AzureTableConnection; - AzureBlobConnectionSpec = AzureBlobConnection; - /* - foreach (char c in AzureTableConnection.ToCharArray()) - { - AzureTableConnectionSpec.AppendChar(c); - } - foreach (char c in AzureBlobConnection.ToCharArray()) - { - AzureBlobConnectionSpec.AppendChar(c); - } - */ + tableServiceClient = new TableServiceClient(AzureTableConnection); + blobServiceClient = new BlobServiceClient(AzureBlobConnection); } @@ -115,9 +96,8 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl /// A string indicating the name of the blob file created as well as a count of how many files were aged. public string BackupTableToBlob(string TableName, string BlobRoot, string OutFileDirectory, bool Compress = false, bool Validate = false, int RetentionDays = 30, int TimeoutSeconds = 30, List filters = default(List)) { - string OutFileName = ""; + string OutFileName ; string OutFileNamePath = ""; - int BackupsAged = 0; if (String.IsNullOrWhiteSpace(TableName)) { @@ -144,42 +124,27 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl OutFileName = this.BackupTableToFile(TableName, OutFileDirectory, Compress, Validate, TimeoutSeconds, filters); OutFileNamePath = Path.Combine(OutFileDirectory, OutFileName); - if (!AZStorage.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureBlobConnectionSpec).Password, out AZStorage.CloudStorageAccount StorageAccountAZ)) - { - throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string."); - } - AZBlob.CloudBlobClient ClientBlob = AZBlob.BlobAccountExtensions.CreateCloudBlobClient(StorageAccountAZ); - var container = ClientBlob.GetContainerReference(BlobRoot); + var container = blobServiceClient.GetBlobContainerClient(BlobRoot); + container.CreateIfNotExists(); - AZBlob.CloudBlobDirectory directory = container.GetDirectoryReference(BlobRoot.ToLower() + "-table-" + TableName.ToLower()); + var directory = container.GetBlobClient(BlobRoot.ToLower() + "-table-" + TableName.ToLower() + "/" + OutFileName); - AZBlob.CloudBlockBlob BlobBlock = directory.GetBlockBlobReference(OutFileName); - BlobBlock.StreamWriteSizeInBytes = 1024 * 1024 * 32; //Set stream write size to 32MB - BlobBlock.UploadFromFile(OutFileNamePath); + var blobClient = container.GetBlobClient(OutFileName); + var blobUploadOptions = new BlobUploadOptions + { + TransferOptions = new StorageTransferOptions + { + MaximumTransferSize = 1024 * 1024 * 32 // Set stream write size to 32MB + } + }; + blobClient.Upload(OutFileNamePath, blobUploadOptions); DateTimeOffset OffsetTimeNow = System.DateTimeOffset.Now; DateTimeOffset OffsetTimeRetain = System.DateTimeOffset.Now.AddDays(-1 * RetentionDays); - //Cleanup old versions - var BlobList = directory.ListBlobs().OfType().ToList(); ; - foreach (var blob in BlobList) - { - if (blob.Properties.Created < OffsetTimeRetain) - { - try - { - blob.Delete(); - BackupsAged++; - } - catch (Exception ex) - { - throw new AgingException(String.Format("Error aging file '{0}'.", blob.Name), ex); - } - } - } - return String.Format("Table '{0}' backed up as '{2}' under blob '{3}\\{4}'; {1} files aged.", TableName, BackupsAged, OutFileName, BlobRoot , directory.ToString()); + return String.Format("Table '{0}' backed up as '{1}' under blob '{2}\\{3}';", TableName, OutFileName, BlobRoot , directory.ToString()); } catch (ConnectionException cex) { @@ -263,13 +228,11 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl TableSpec TableSpecStart = new TableSpec(TableName); - AzureTables.TableServiceClient clientSource = new AzureTables.TableServiceClient(AzureTableConnectionSpec.ToString()); - - Pageable queryResultsFilter = clientSource.GetTableClient(TableName).Query(filter: Filter.BuildFilterSpec(filters), maxPerPage: 100); + Pageable queryResultsFilter = tableServiceClient.GetTableClient(TableName).Query(filter: Filter.BuildFilterSpec(filters), maxPerPage: 100); using (StreamWriter OutFile = new StreamWriter(OutFileNamePath)) { - OutFile.WriteLine(JsonConvert.SerializeObject(TableSpecStart)); + OutFile.WriteLine(System.Text.Json.JsonSerializer.Serialize(TableSpecStart)); foreach (Page page in queryResultsFilter.AsPages()) { @@ -286,7 +249,7 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl } TableSpec TableSpecEnd = new TableSpec(TableName, RecordCount); - OutFile.WriteLine(JsonConvert.SerializeObject(TableSpecEnd)); + OutFile.WriteLine(System.Text.Json.JsonSerializer.Serialize(TableSpecEnd)); OutFile.Flush(); OutFile.Close(); @@ -316,7 +279,8 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl } while (DetailRec != null); InFile.Close(); - TableSpec footer= JsonConvert.DeserializeObject(FooterRec); + TableSpec footer = System.Text.Json.JsonSerializer.Deserialize(FooterRec); + if ((footer.RecordCount==InRecords) && (footer.TableName.Equals(TableName))) { //Do nothing, in count=out count @@ -412,36 +376,35 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl OutFileName = String.Format(TableName + "_Backup_" + System.DateTime.Now.ToString("yyyyMMddHHmmss") + ".txt"); } - if (!AZStorage.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureBlobConnectionSpec).Password, out AZStorage.CloudStorageAccount StorageAccountAZ)) - { - throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string."); - } - - AZBlob.CloudBlobClient ClientBlob = AZBlob.BlobAccountExtensions.CreateCloudBlobClient(StorageAccountAZ); - var container = ClientBlob.GetContainerReference(BlobRoot); + var container = blobServiceClient.GetBlobContainerClient(BlobRoot); container.CreateIfNotExists(); - AZBlob.CloudBlobDirectory directory = container.GetDirectoryReference(BlobRoot.ToLower() + "-table-" + TableName.ToLower()); + var directory = container.GetBlobClient(BlobRoot.ToLower() + "-table-" + TableName.ToLower() + "/" + OutFileName); - AZBlob.CloudBlockBlob BlobBlock = directory.GetBlockBlobReference(OutFileName); - BlobBlock.StreamWriteSizeInBytes = 1024 * 1024 * 32; //Set stream write size to 32MB + var blobClient = container.GetBlobClient(OutFileName); + var blobUploadOptions = new BlobUploadOptions + { + TransferOptions = new StorageTransferOptions + { + MaximumTransferSize = 1024 * 1024 * 32 // Set stream write size to 32MB + } + }; // start upload from stream, iterate through table, possible inline compress try { - AzureTables.TableServiceClient clientSource = new AzureTables.TableServiceClient(AzureTableConnectionSpec.ToString()); //TODO Timeout set? //table.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds); var entitiesSerialized = new List(); - DynamicTableEntityJsonSerializer serializer = new DynamicTableEntityJsonSerializer(); + var serializer = new DynamicTableEntityJsonSerializer(); - TableSpec TableSpecStart = new TableSpec(TableName); + var TableSpecStart = new TableSpec(TableName); var NewLineAsBytes = Encoding.UTF8.GetBytes("\n"); - var tempTableSpecStart = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(TableSpecStart)); - AZBlob.CloudBlobStream bs2 = BlobBlock.OpenWrite(); - Stream bs = BlobBlock.OpenWrite(); + var tempTableSpecStart = Encoding.UTF8.GetBytes(System.Text.Json.JsonSerializer.Serialize(TableSpecStart)); + var bs2 = blobClient.OpenWrite(true); + Stream bs = bs2; if (Compress) { @@ -457,7 +420,7 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl bs.Write(NewLineAsBytes, 0, NewLineAsBytes.Length); bs.Flush(); - Pageable queryResultsFilter = clientSource.GetTableClient(TableName).Query(filter: Filter.BuildFilterSpec(filters), maxPerPage: 100); + Pageable queryResultsFilter = tableServiceClient.GetTableClient(TableName).Query(filter: Filter.BuildFilterSpec(filters), maxPerPage: 100); foreach (Page page in queryResultsFilter.AsPages()) { foreach (AzureTables.TableEntity qEntity in page.Values) @@ -472,7 +435,7 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl } } TableSpec TableSpecEnd = new TableSpec(TableName, RecordCount); - var tempTableSpecEnd = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(TableSpecEnd)); + var tempTableSpecEnd = Encoding.UTF8.GetBytes(System.Text.Json.JsonSerializer.Serialize(TableSpecEnd)); bs.Write(tempTableSpecEnd, 0, tempTableSpecEnd.Length); bs.Flush(); bs.Write(NewLineAsBytes, 0, NewLineAsBytes.Length); @@ -487,23 +450,6 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl DateTimeOffset OffsetTimeNow = System.DateTimeOffset.Now; DateTimeOffset OffsetTimeRetain = System.DateTimeOffset.Now.AddDays(-1 * RetentionDays); - //Cleanup old versions - var BlobList = directory.ListBlobs().OfType().ToList(); ; - foreach (var blob in BlobList) - { - if (blob.Properties.Created < OffsetTimeRetain) - { - try - { - blob.Delete(); - BackupsAged++; - } - catch (Exception ex) - { - throw new AgingException(String.Format("Error aging file '{0}'.", blob.Name), ex); - } - } - } return String.Format("Table '{0}' backed up as '{2}' under blob '{3}\\{4}'; {1} files aged.", TableName, BackupsAged, OutFileName, BlobRoot, directory.ToString()); } @@ -540,7 +486,7 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl try { StringBuilder BackupResults = new StringBuilder(); - List TableNames = Helper.GetTableNames(AzureTableConnectionSpec); + List TableNames = Helper.GetTableNames(tableServiceClient); if (TableNames.Count() > 0) { foreach (string TableName in TableNames) diff --git a/src/AzureTableUtilities/CopyAzureTables.cs b/src/AzureTableUtilities/CopyAzureTables.cs index d0668c2..82d30eb 100644 --- a/src/AzureTableUtilities/CopyAzureTables.cs +++ b/src/AzureTableUtilities/CopyAzureTables.cs @@ -1,30 +1,13 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Configuration; -using System.Diagnostics; -using System.IO; -using System.IO.Compression; -using System.Security; -using System.Net.Http; - -using AZStorage = Microsoft.Azure.Storage; -using Microsoft.Azure.Storage.Auth; -using AZBlob = Microsoft.Azure.Storage.Blob; -using Microsoft.Azure.Storage.Core; -using Microsoft.Azure.Storage.File; - +using Azure; using Azure.Data.Tables; using Azure.Data.Tables.Models; -using Azure; - -using Newtonsoft.Json; - -using TheByteStuff.AzureTableUtilities.Exceptions; - +using Azure.Storage.Blobs; +using System; +using System.Collections.Generic; using System.Runtime.InteropServices; +using System.Security; +using System.Text; +using TheByteStuff.AzureTableUtilities.Exceptions; namespace TheByteStuff.AzureTableUtilities { @@ -33,10 +16,9 @@ namespace TheByteStuff.AzureTableUtilities /// public class CopyAzureTables { - //private SecureString AzureSourceTableConnection = new SecureString(); - //private SecureString AzureDestinationTableConnection = new SecureString(); - private StringBuilder AzureSourceTableConnection = new StringBuilder(); - private StringBuilder AzureDestinationTableConnection = new StringBuilder(); + + private TableServiceClient sourceTableServiceClient; + private TableServiceClient destinationTableServiceClient; /// /// Constructor, sets same connection spec for both the Source and Destination Azure Tables. /// @@ -67,16 +49,8 @@ public CopyAzureTables(string AzureSourceTableConnection, string AzureDestinatio throw new ConnectionException(String.Format("Connection spec must be specified.")); } - foreach (char c in AzureSourceTableConnection.ToCharArray()) - { - //this.AzureSourceTableConnection.AppendChar(c); - this.AzureSourceTableConnection.Append(c); - } - foreach (char c in AzureDestinationTableConnection.ToCharArray()) - { - //this.AzureDestinationTableConnection.AppendChar(c); - this.AzureDestinationTableConnection.Append(c); - } + sourceTableServiceClient = new TableServiceClient(AzureSourceTableConnection); + destinationTableServiceClient = new TableServiceClient(AzureDestinationTableConnection); } /// @@ -133,12 +107,9 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu try { - TableServiceClient clientSource = new TableServiceClient(AzureSourceTableConnection.ToString()); - - TableServiceClient clientDestination = new TableServiceClient(AzureDestinationTableConnection.ToString()); - TableItem TableDest = clientDestination.CreateTableIfNotExists(DestinationTableName); + TableItem TableDest = destinationTableServiceClient.CreateTableIfNotExists(DestinationTableName); - Pageable queryResultsFilter = clientSource.GetTableClient(SourceTableName).Query(filter: Filter.BuildFilterSpec(filters), maxPerPage: BatchSize); + Pageable queryResultsFilter = sourceTableServiceClient.GetTableClient(SourceTableName).Query(filter: Filter.BuildFilterSpec(filters), maxPerPage: BatchSize); // Set Timeout? BatchCount = 0; @@ -157,7 +128,7 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu } else { - Response> response = clientDestination.GetTableClient(DestinationTableName).SubmitTransaction(Batch); + Response> response = destinationTableServiceClient.GetTableClient(DestinationTableName).SubmitTransaction(Batch); TotalRecordCountOut = TotalRecordCountOut + Batch.Count; Batch = new List(); @@ -171,7 +142,7 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu { try { - Response> response = clientDestination.GetTableClient(DestinationTableName).SubmitTransaction(Batch); + Response> response = destinationTableServiceClient.GetTableClient(DestinationTableName).SubmitTransaction(Batch); TotalRecordCountOut = TotalRecordCountOut + Batch.Count; Batch = new List(); @@ -188,7 +159,7 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu } if (!BatchWritten) { - Response> response = clientDestination.GetTableClient(DestinationTableName).SubmitTransaction(Batch); + Response> response = destinationTableServiceClient.GetTableClient(DestinationTableName).SubmitTransaction(Batch); TotalRecordCountOut = TotalRecordCountOut + Batch.Count; Batch = new List(); PartitionKey = String.Empty; @@ -227,7 +198,7 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu public string CopyAllTables(int TimeoutSeconds = 30, List filters = default(List)) { //if (IsEqualTo(AzureSourceTableConnection.ToString(), AzureDestinationTableConnection.ToString())) - if (AzureSourceTableConnection.ToString().Equals(AzureDestinationTableConnection.ToString())) + if (sourceTableServiceClient.Uri.Equals(destinationTableServiceClient.Uri)) { throw new ParameterSpecException("Source and Destination Connection specs can not match for CopyAll."); } @@ -240,7 +211,7 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu StringBuilder BackupResults = new StringBuilder(); try { - List TableNames = Helper.GetTableNames(AzureSourceTableConnection.ToString()); + List TableNames = Helper.GetTableNames(sourceTableServiceClient); if (TableNames.Count > 0) { foreach (string TableName in TableNames) diff --git a/src/AzureTableUtilities/DynamicTableEntityJsonConverter.cs b/src/AzureTableUtilities/DynamicTableEntityJsonConverter.cs index b129220..7a60597 100644 --- a/src/AzureTableUtilities/DynamicTableEntityJsonConverter.cs +++ b/src/AzureTableUtilities/DynamicTableEntityJsonConverter.cs @@ -1,10 +1,9 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using System; -using System.Collections; +using System; using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; using Azure.Data.Tables; namespace TheByteStuff.AzureTableUtilities @@ -12,7 +11,7 @@ namespace TheByteStuff.AzureTableUtilities /// /// Based on classes from https://www.nuget.org/packages/DynamicTableEntityJsonSerializer/1.0.0 /// - class DynamicTableEntityJsonConverter : JsonConverter + class DynamicTableEntityJsonConverter : JsonConverter { private const int EntityPropertyIndex = 0; private const int EntityPropertyEdmTypeIndex = 1; @@ -25,44 +24,44 @@ public DynamicTableEntityJsonConverter(List excludedProperties = null) this.excludedProperties = excludedProperties; } - public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + public override void Write(Utf8JsonWriter writer, TableEntity value, JsonSerializerOptions options) { if (value == null) return; writer.WriteStartObject(); - DynamicTableEntityJsonConverter.WriteJsonProperties(writer, (TableEntity)value, this.excludedProperties); + WriteJsonProperties(writer, value, this.excludedProperties); writer.WriteEndObject(); } - public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + public override TableEntity Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - if (reader.TokenType == JsonToken.Null) - return (object)null; + if (reader.TokenType == JsonTokenType.Null) + return null; + TableEntity dynamicTableEntity = new TableEntity(); - using (List.Enumerator enumerator = JObject.Load(reader).Properties().ToList().GetEnumerator()) + using (JsonDocument document = JsonDocument.ParseValue(ref reader)) { - while (enumerator.MoveNext()) + foreach (JsonProperty property in document.RootElement.EnumerateObject()) { - JProperty current = enumerator.Current; - if (string.Equals(current.Name, "PartitionKey", StringComparison.Ordinal)) - dynamicTableEntity.PartitionKey = ((object)current.Value).ToString(); - else if (string.Equals(current.Name, "RowKey", StringComparison.Ordinal)) - dynamicTableEntity.RowKey = ((object)current.Value).ToString(); - else if (string.Equals(current.Name, "Timestamp", StringComparison.Ordinal)) - dynamicTableEntity.Timestamp = (DateTimeOffset)current.Value.ToObject(serializer); - else if (string.Equals(current.Name, "ETag", StringComparison.Ordinal)) + if (string.Equals(property.Name, "PartitionKey", StringComparison.Ordinal)) + dynamicTableEntity.PartitionKey = property.Value.GetString(); + else if (string.Equals(property.Name, "RowKey", StringComparison.Ordinal)) + dynamicTableEntity.RowKey = property.Value.GetString(); + else if (string.Equals(property.Name, "Timestamp", StringComparison.Ordinal)) + dynamicTableEntity.Timestamp = property.Value.GetDateTimeOffset(); + else if (string.Equals(property.Name, "ETag", StringComparison.Ordinal)) { - dynamicTableEntity.ETag = new Azure.ETag(current.Value.ToString()); + dynamicTableEntity.ETag = new Azure.ETag(property.Value.GetString()); } else { - KeyValuePair data = DynamicTableEntityJsonConverter.CreateKeyValue(serializer, current); + KeyValuePair data = CreateKeyValue(property); dynamicTableEntity.Add(data.Key, data.Value); } } } - return (object)dynamicTableEntity; + return dynamicTableEntity; } public override bool CanConvert(Type objectType) @@ -71,133 +70,124 @@ public override bool CanConvert(Type objectType) } private static void WriteJsonProperties( - JsonWriter writer, + Utf8JsonWriter writer, TableEntity entity, List excludedProperties = null) { if (entity == null) return; - writer.WritePropertyName("PartitionKey"); - writer.WriteValue(entity.PartitionKey); - writer.WritePropertyName("RowKey"); - writer.WriteValue(entity.RowKey); - writer.WritePropertyName("Timestamp"); - writer.WriteValue(entity.Timestamp); - //writer.WritePropertyName("ETag"); - //writer.WriteValue(entity.ETag); - //int i= 0; + writer.WriteString("PartitionKey", entity.PartitionKey); + writer.WriteString("RowKey", entity.RowKey); + writer.WriteString("Timestamp", entity.Timestamp?.ToString("o")); + for (int j = 0; j < entity.Count; j++) { - string ValueType = entity.ElementAt(j).Value.GetType().Name; - + string valueType = entity.ElementAt(j).Value.GetType().Name; if (excludedKeys.Contains(entity.ElementAt(j).Key)) { - + continue; } else { - EntityProperty ep = new EntityProperty(entity.ElementAt(j), EntityProperty.StringToType(ValueType)); // EntityProperty.EntityPropertyType.String); - DynamicTableEntityJsonConverter.WriteJsonProperty(writer, entity.ElementAt(j), EntityProperty.StringToType(ValueType)); + EntityProperty ep = new EntityProperty(entity.ElementAt(j), EntityProperty.StringToType(valueType)); + WriteJsonProperty(writer, entity.ElementAt(j), EntityProperty.StringToType(valueType)); } } - } private static void WriteJsonProperty( - JsonWriter writer, - KeyValuePair property, - EntityProperty.EntityPropertyType type) + Utf8JsonWriter writer, + KeyValuePair property, + EntityProperty.EntityPropertyType type) { - //https://www.newtonsoft.com/json/help/html/T_Newtonsoft_Json_JsonToken.htm if (string.IsNullOrWhiteSpace(property.Key) || property.Value == null) return; + switch ((int)type) { case 0: - DynamicTableEntityJsonConverter.WriteJsonPropertyWithEdmType(writer, property.Key, (object)property.Value, EntityProperty.EntityPropertyType.String); + WriteJsonPropertyWithEdmType(writer, property.Key, property.Value, EntityProperty.EntityPropertyType.String); break; case 1: - throw new NotSupportedException(string.Format((IFormatProvider)CultureInfo.InvariantCulture, "Unsupported EntityProperty.PropertyType:{0} detected during serialization.", type)); + throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, "Unsupported EntityProperty.PropertyType:{0} detected during serialization.", type)); case 2: - DynamicTableEntityJsonConverter.WriteJsonPropertyWithEdmType(writer, property.Key, (object)property.Value, EntityProperty.EntityPropertyType.Boolean); + WriteJsonPropertyWithEdmType(writer, property.Key, property.Value, EntityProperty.EntityPropertyType.Boolean); break; case 3: - DynamicTableEntityJsonConverter.WriteJsonPropertyWithEdmType(writer, property.Key, (object)property.Value, EntityProperty.EntityPropertyType.DateTime); + WriteJsonPropertyWithEdmType(writer, property.Key, property.Value, EntityProperty.EntityPropertyType.DateTime); break; case 4: - DynamicTableEntityJsonConverter.WriteJsonPropertyWithEdmType(writer, property.Key, (object)property.Value, EntityProperty.EntityPropertyType.Double); + WriteJsonPropertyWithEdmType(writer, property.Key, property.Value, EntityProperty.EntityPropertyType.Double); break; case 5: - DynamicTableEntityJsonConverter.WriteJsonPropertyWithEdmType(writer, property.Key, (object)property.Value, EntityProperty.EntityPropertyType.GUID); + WriteJsonPropertyWithEdmType(writer, property.Key, property.Value, EntityProperty.EntityPropertyType.GUID); break; case 6: - DynamicTableEntityJsonConverter.WriteJsonPropertyWithEdmType(writer, property.Key, (object)property.Value, EntityProperty.EntityPropertyType.Int32); + WriteJsonPropertyWithEdmType(writer, property.Key, property.Value, EntityProperty.EntityPropertyType.Int32); break; case 7: - DynamicTableEntityJsonConverter.WriteJsonPropertyWithEdmType(writer, property.Key, (object)property.Value, EntityProperty.EntityPropertyType.Int64); + WriteJsonPropertyWithEdmType(writer, property.Key, property.Value, EntityProperty.EntityPropertyType.Int64); break; default: - throw new NotSupportedException(string.Format((IFormatProvider)CultureInfo.InvariantCulture, "Unsupported EntityProperty.PropertyType:{0} detected during serialization.", type)); + throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, "Unsupported EntityProperty.PropertyType:{0} detected during serialization.", type)); } } private static void WriteJsonPropertyWithEdmType( - JsonWriter writer, + Utf8JsonWriter writer, string key, object value, - EntityProperty.EntityPropertyType TableEntityType) + EntityProperty.EntityPropertyType tableEntityType) { writer.WritePropertyName(key); writer.WriteStartObject(); writer.WritePropertyName(key); - writer.WriteValue(value); + writer.WriteStringValue(value.ToString()); writer.WritePropertyName("EdmType"); - writer.WriteValue(TableEntityType.ToString()); + writer.WriteStringValue(tableEntityType.ToString()); writer.WriteEndObject(); } - private static KeyValuePair CreateKeyValue( - JsonSerializer serializer, - JProperty property) + private static KeyValuePair CreateKeyValue(JsonProperty property) { - if (property == null) + if (property.Value.ValueKind == JsonValueKind.Null) return new KeyValuePair(); - List list = JObject.Parse(((object)property.Value).ToString()).Properties().ToList(); - EntityProperty.EntityPropertyType edmType = (EntityProperty.EntityPropertyType)Enum.Parse(typeof(EntityProperty.EntityPropertyType), ((object)list[1].Value).ToString(), true); - //EntityProperty entityProperty = new EntityProperty(new KeyValuePair("test", 123), edmType); - KeyValuePair KVP = new KeyValuePair(); + + JsonElement element = property.Value; + EntityProperty.EntityPropertyType edmType = (EntityProperty.EntityPropertyType)Enum.Parse(typeof(EntityProperty.EntityPropertyType), element.GetProperty("EdmType").GetString(), true); + + KeyValuePair kvp = new KeyValuePair(); switch ((int)edmType) { case 0: - KVP = new KeyValuePair(list[0].Name, (string)list[0].Value.ToObject(serializer)); + kvp = new KeyValuePair(property.Name, element.GetProperty(property.Name).GetString()); break; case 1: - KVP = new KeyValuePair(list[0].Name, (byte[])list[0].Value.ToObject(serializer)); - //entityProperty = EntityProperty.GeneratePropertyForByteArray((byte[])list[0].Value.ToObject(serializer)); + kvp = new KeyValuePair(property.Name, element.GetProperty(property.Name).GetBytesFromBase64()); break; case 2: - KVP = new KeyValuePair(list[0].Name, new bool?((bool)list[0].Value.ToObject(serializer))); + kvp = new KeyValuePair(property.Name, element.GetProperty(property.Name).GetBoolean()); break; case 3: - KVP = new KeyValuePair(list[0].Name, new DateTimeOffset?((DateTimeOffset)list[0].Value.ToObject(serializer))); + kvp = new KeyValuePair(property.Name, element.GetProperty(property.Name).GetDateTimeOffset()); break; case 4: - KVP = new KeyValuePair(list[0].Name, new double?((double)list[0].Value.ToObject(serializer))); + kvp = new KeyValuePair(property.Name, element.GetProperty(property.Name).GetDouble()); break; case 5: - KVP = new KeyValuePair(list[0].Name, new Guid?((Guid)list[0].Value.ToObject(serializer))); + kvp = new KeyValuePair(property.Name, element.GetProperty(property.Name).GetGuid()); break; case 6: - KVP = new KeyValuePair(list[0].Name, new int?((int)list[0].Value.ToObject(serializer))); + kvp = new KeyValuePair(property.Name, element.GetProperty(property.Name).GetInt32()); break; case 7: - KVP = new KeyValuePair(list[0].Name, new long?((long)list[0].Value.ToObject(serializer))); + kvp = new KeyValuePair(property.Name, element.GetProperty(property.Name).GetInt64()); break; default: - throw new NotSupportedException(string.Format((IFormatProvider)CultureInfo.InvariantCulture, "Unsupported EntityProperty.PropertyType:{0} detected during deserialization.", (object)edmType)); + throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, "Unsupported EntityProperty.PropertyType:{0} detected during deserialization.", edmType)); } - return KVP; + return kvp; } } -} \ No newline at end of file +} diff --git a/src/AzureTableUtilities/DynamicTableEntityJsonSerializer.cs b/src/AzureTableUtilities/DynamicTableEntityJsonSerializer.cs index 9324a4f..ce93738 100644 --- a/src/AzureTableUtilities/DynamicTableEntityJsonSerializer.cs +++ b/src/AzureTableUtilities/DynamicTableEntityJsonSerializer.cs @@ -1,9 +1,7 @@ using System; - -//using Microsoft.Azure.Cosmos.Table; -using Azure.Data.Tables; -using Newtonsoft.Json; using System.Collections.Generic; +using System.Text.Json; +using Azure.Data.Tables; namespace TheByteStuff.AzureTableUtilities { @@ -21,25 +19,28 @@ public DynamicTableEntityJsonSerializer(List excludedProperties = null) public string Serialize(TableEntity entity) { - string str; - if (entity != null) - str = JsonConvert.SerializeObject((object)entity, new JsonConverter[1] - { - (JsonConverter) this.jsonConverter - }); - else - str = (string)null; - return str; + if (entity == null) + return null; + + var options = new JsonSerializerOptions + { + Converters = { jsonConverter } + }; + + return JsonSerializer.Serialize(entity, options); } public TableEntity Deserialize(string serializedEntity) { - TableEntity local; - if (serializedEntity != null) - local = JsonConvert.DeserializeObject(serializedEntity, new JsonConverter[1] { (JsonConverter)this.jsonConverter }); - else - local = null; - return (TableEntity)local; + if (serializedEntity == null) + return null; + + var options = new JsonSerializerOptions + { + Converters = { jsonConverter } + }; + + return JsonSerializer.Deserialize(serializedEntity, options); } } -} \ No newline at end of file +} diff --git a/src/AzureTableUtilities/Helper.cs b/src/AzureTableUtilities/Helper.cs index 88de8aa..7cca605 100644 --- a/src/AzureTableUtilities/Helper.cs +++ b/src/AzureTableUtilities/Helper.cs @@ -48,13 +48,12 @@ public static bool IsStringNullOrEmpty(string value) /// /// Return a list of table names for the given azure connection. /// - /// + /// /// - public static List GetTableNames(String AzureTableConnection) + public static List GetTableNames(TableServiceClient client) { List TableNames = new List(); - TableServiceClient client = new TableServiceClient(AzureTableConnection.ToString()); Pageable result = client.Query(); foreach (TableItem Table in result) { diff --git a/src/AzureTableUtilities/RestoreAzureTables.cs b/src/AzureTableUtilities/RestoreAzureTables.cs index 250927d..14c7fbf 100644 --- a/src/AzureTableUtilities/RestoreAzureTables.cs +++ b/src/AzureTableUtilities/RestoreAzureTables.cs @@ -1,26 +1,11 @@ -using System; -using System.Configuration; +using Azure; +using Azure.Data.Tables; +using Azure.Data.Tables.Models; +using Azure.Storage.Blobs; +using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Diagnostics; using System.IO; using System.IO.Compression; -using System.Security; - -using AZStorage = Microsoft.Azure.Storage; -using Microsoft.Azure.Storage.Auth; -using AZBlob = Microsoft.Azure.Storage.Blob; -using Microsoft.Azure.Storage.Core; -using Microsoft.Azure.Storage.File; - -using Azure.Data.Tables; -using Azure.Data.Tables.Models; -using Azure; - -using Newtonsoft.Json; - using TheByteStuff.AzureTableUtilities.Exceptions; namespace TheByteStuff.AzureTableUtilities @@ -30,36 +15,37 @@ namespace TheByteStuff.AzureTableUtilities /// public class RestoreAzureTables { - //private SecureString AzureTableConnectionSpec = new SecureString(); - //private SecureString AzureBlobConnectionSpec = new SecureString(); - private string AzureTableConnectionSpec = ""; - private string AzureBlobConnectionSpec = ""; + private TableServiceClient tableServiceClient; + private BlobServiceClient blobServiceClient; + /// /// Constructor, sets same connection spec for both the Azure Tables as well as the Azure Blob storage. /// - /// Connection string for Azure Table and Blob Connections; ex "AccountName=devstoreaccount1;AccountKey={xxxxxxxxxxx};DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" + /// Connection string for Azure Table and Blob Connections; ex "AccountName=devstoreaccount1;AccountKey={xxxxxxxxxxx};DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" public RestoreAzureTables(string AzureConnection) : this(AzureConnection, AzureConnection) { - + tableServiceClient = new TableServiceClient(AzureConnection); + blobServiceClient = new BlobServiceClient(AzureConnection); } /// - /// Constructor, sets same connection spec for both the Azure Tables as well as the Azure Blob storage. + /// Directly set the Service Clients /// - /// - /* - public RestoreAzureTables(SecureString AzureConnection) : this(AzureConnection, AzureConnection) + /// + /// + public RestoreAzureTables(TableServiceClient tableServiceClient, BlobServiceClient blobServiceClient) { - + this.tableServiceClient = tableServiceClient; + this.blobServiceClient = blobServiceClient; } - */ + /// /// Constructor, allows a different connection spec for Azure Table and Azure Blob storage. /// - /// Connection string for Azure Table Connection; ex "AccountName=devstoreaccount1;AccountKey={xxxxxxxxxxx};DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" - /// Connection string for Azure Blob Connection; ex "AccountName=devstoreaccount1;AccountKey={xxxxxxxxxxx};DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" + /// Connection string for Azure Table Connection; ex "AccountName=devstoreaccount1;AccountKey={xxxxxxxxxxx};DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" + /// Connection string for Azure Blob Connection; ex "AccountName=devstoreaccount1;AccountKey={xxxxxxxxxxx};DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" public RestoreAzureTables(string AzureTableConnection, string AzureBlobConnection) { if (String.IsNullOrEmpty(AzureTableConnection) || String.IsNullOrEmpty(AzureBlobConnection)) @@ -67,18 +53,8 @@ public RestoreAzureTables(string AzureTableConnection, string AzureBlobConnectio throw new ConnectionException(String.Format("Connection spec must be specified.")); } - AzureTableConnectionSpec = AzureTableConnection; - AzureBlobConnectionSpec= AzureBlobConnection; - /* - foreach (char c in AzureTableConnection.ToCharArray()) - { - AzureTableConnectionSpec.AppendChar(c); - } - foreach (char c in AzureBlobConnection.ToCharArray()) - { - AzureBlobConnectionSpec.AppendChar(c); - } - */ + tableServiceClient = new TableServiceClient(AzureTableConnection); + blobServiceClient = new BlobServiceClient(AzureBlobConnection); } /// @@ -151,17 +127,12 @@ public string RestoreTableFromBlob(string DestinationTableName, string OriginalT throw new ParameterSpecException(String.Format("Invalid WorkingDirectory '{0}' specified.", WorkingDirectory)); } - if (!AZStorage.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureBlobConnectionSpec).Password, out AZStorage.CloudStorageAccount StorageAccountAZ)) - { - throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string."); - } try - { - AZBlob.CloudBlobClient ClientBlob = AZBlob.BlobAccountExtensions.CreateCloudBlobClient(StorageAccountAZ); - var container = ClientBlob.GetContainerReference(BlobRoot); + { + var container = blobServiceClient.GetBlobContainerClient(BlobRoot); container.CreateIfNotExists(); - AZBlob.CloudBlobDirectory directory = container.GetDirectoryReference(BlobRoot.ToLower() + "-table-" + OriginalTableName.ToLower()); + var directory = container.GetBlobClient(BlobRoot.ToLower() + "-table-" + OriginalTableName.ToLower() + "/" + BlobFileName); string WorkingFileNamePath = Path.Combine(WorkingDirectory, BlobFileName); string WorkingFileNamePathCompressed = Path.Combine(WorkingDirectory, BlobFileName); @@ -178,8 +149,9 @@ public string RestoreTableFromBlob(string DestinationTableName, string OriginalT //WorkingFileNamePathCompressed = WorkingFileNamePathCompressed.Replace(".txt", ".7z"); } - AZBlob.CloudBlockBlob BlobBlock = directory.GetBlockBlobReference(BlobFileName); - BlobBlock.DownloadToFile(WorkingFileNamePathCompressed, FileMode.Create); + var blobClient = container.GetBlobClient(BlobFileName); + blobClient.DownloadTo(WorkingFileNamePathCompressed); + //https://www.tutorialspoint.com/compressing-and-decompressing-files-using-gzip-format-in-chash if (Decompress) @@ -265,8 +237,7 @@ public string RestoreTableFromFile(string DestinationTableName, string InFilePat try { - TableServiceClient clientDestination = new TableServiceClient(AzureTableConnectionSpec.ToString()); - TableItem TableDest = clientDestination.CreateTableIfNotExists(DestinationTableName); + TableItem TableDest = tableServiceClient.CreateTableIfNotExists(DestinationTableName); DynamicTableEntityJsonSerializer serializer = new DynamicTableEntityJsonSerializer(); @@ -288,7 +259,7 @@ public string RestoreTableFromFile(string DestinationTableName, string InFilePat } else if (InFileLine.Contains("ProcessingMetaData") && InFileLine.Contains("Footer")) { - footer = JsonConvert.DeserializeObject(InFileLine); + footer = System.Text.Json.JsonSerializer.Deserialize(InFileLine); System.Console.WriteLine(String.Format("Footer {0}", InFileLine)); } else @@ -307,7 +278,7 @@ public string RestoreTableFromFile(string DestinationTableName, string InFilePat { try { - Response> response = clientDestination.GetTableClient(DestinationTableName).SubmitTransaction(Batch); + Response> response = tableServiceClient.GetTableClient(DestinationTableName).SubmitTransaction(Batch); Batch = new List(); PartitionKey = dte2.PartitionKey; Batch.Add(new TableTransactionAction(TableTransactionActionType.UpsertReplace, dte2)); @@ -324,7 +295,7 @@ public string RestoreTableFromFile(string DestinationTableName, string InFilePat { try { - Response> response = clientDestination.GetTableClient(DestinationTableName).SubmitTransaction(Batch); + Response> response = tableServiceClient.GetTableClient(DestinationTableName).SubmitTransaction(Batch); PartitionKey = String.Empty; Batch = new List(); BatchWritten = true; @@ -345,7 +316,7 @@ public string RestoreTableFromFile(string DestinationTableName, string InFilePat try { //TableDest.ExecuteBatch(Batch); - Response> response = clientDestination.GetTableClient(DestinationTableName).SubmitTransaction(Batch); + Response> response = tableServiceClient.GetTableClient(DestinationTableName).SubmitTransaction(Batch); PartitionKey = String.Empty; } catch (Exception ex) { @@ -404,20 +375,12 @@ private void DownloadFileFromBlob(string BlobRoot, string BlobDirectoryReference { try { - if (!AZStorage.CloudStorageAccount.TryParse(ConfigurationManager.ConnectionStrings["AzureBlobStorageConfigConnection"].ConnectionString, out AZStorage.CloudStorageAccount StorageAccountAZ)) - { - throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string."); - } - - AZBlob.CloudBlobClient ClientBlob = AZBlob.BlobAccountExtensions.CreateCloudBlobClient(StorageAccountAZ); - - var container = ClientBlob.GetContainerReference(BlobRoot); + BlobContainerClient container = blobServiceClient.GetBlobContainerClient(BlobRoot); container.CreateIfNotExists(); - AZBlob.CloudBlobDirectory directory = container.GetDirectoryReference(BlobDirectoryReference); - AZBlob.CloudBlockBlob BlobBlock = directory.GetBlockBlobReference(BlockBlobRef); + BlobClient blobClient = container.GetBlobClient($"{BlobDirectoryReference}/{BlockBlobRef}"); + blobClient.DownloadTo(LocalFileName); - BlobBlock.DownloadToFile(LocalFileName, FileMode.OpenOrCreate); } catch (Exception ex) @@ -466,53 +429,49 @@ public string RestoreTableFromBlobDirect(string DestinationTableName, string Ori try { - if (!AZStorage.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureBlobConnectionSpec).Password, out AZStorage.CloudStorageAccount StorageAccountAZ)) - { - throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string."); - } + + BlobContainerClient container = blobServiceClient.GetBlobContainerClient(BlobRoot); + container.CreateIfNotExists(); + BlobClient blobClient = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{BlobFileName}"); - AZBlob.CloudBlobClient ClientBlob = AZBlob.BlobAccountExtensions.CreateCloudBlobClient(StorageAccountAZ); - var container = ClientBlob.GetContainerReference(BlobRoot); - container.CreateIfNotExists(); - AZBlob.CloudBlobDirectory directory = container.GetDirectoryReference(BlobRoot.ToLower() + "-table-" + OriginalTableName.ToLower()); - - // If file is compressed, Decompress to a temp file in the blob - if (Decompress) - { - AZBlob.CloudBlockBlob BlobBlockTemp = directory.GetBlockBlobReference(TempFileName); - AZBlob.CloudBlockBlob BlobBlockRead = directory.GetBlockBlobReference(BlobFileName); - - using (AZBlob.CloudBlobStream decompressedStream = BlobBlockTemp.OpenWrite()) + // If file is compressed, Decompress to a temp file in the blob + if (Decompress) { - using (Stream readstream = BlobBlockRead.OpenRead()) + BlobClient blobClientTemp = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{TempFileName}"); + BlobClient blobClientRead = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{BlobFileName}"); + + using (var decompressedStream = blobClientTemp.OpenWrite(true)) { - using (var zip = new GZipStream(readstream, CompressionMode.Decompress, true)) + using (var readStream = blobClientRead.OpenRead()) { - zip.CopyTo(decompressedStream); + using (var zip = new GZipStream(readStream, CompressionMode.Decompress, true)) + { + zip.CopyTo(decompressedStream); + } } } + BlobFileName = TempFileName; } - BlobFileName = TempFileName; - } - AZBlob.CloudBlockBlob BlobBlock = directory.GetBlockBlobReference(BlobFileName); + blobClient = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{BlobFileName}"); - TableServiceClient clientDestination = new TableServiceClient(AzureTableConnectionSpec.ToString()); - TableItem TableDest = clientDestination.CreateTableIfNotExists(DestinationTableName); - using (Stream BlobStream = BlobBlock.OpenRead()) + TableItem TableDest = tableServiceClient.CreateTableIfNotExists(DestinationTableName); + + using (Stream blobStream = blobClient.OpenRead()) { - using (StreamReader InputFileStream = new StreamReader(BlobStream)) + using (StreamReader inputFileStream = new StreamReader(blobStream)) { - string result = RestoreFromStream(InputFileStream, clientDestination, DestinationTableName); + string result = RestoreFromStream(inputFileStream, tableServiceClient, DestinationTableName); if (Decompress) { - AZBlob.CloudBlockBlob BlobBlockTemp = directory.GetBlockBlobReference(TempFileName); - BlobBlockTemp.DeleteIfExists(); + BlobClient blobClientTemp = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{TempFileName}"); + blobClientTemp.DeleteIfExists(); } return result; } } + } catch (ConnectionException cex) { @@ -554,7 +513,7 @@ private string RestoreFromStream(StreamReader InputFileStream, TableServiceClien } else if (InFileLine.Contains("ProcessingMetaData") && InFileLine.Contains("Footer")) { - footer = JsonConvert.DeserializeObject(InFileLine); + footer = System.Text.Json.JsonSerializer.Deserialize(InFileLine); System.Console.WriteLine(String.Format("Footer {0}", InFileLine)); } else diff --git a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesBackupXUnitTest.cs b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesBackupXUnitTest.cs index 64e6ec3..c3100b2 100644 --- a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesBackupXUnitTest.cs +++ b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesBackupXUnitTest.cs @@ -1,20 +1,12 @@ -using System; -using System.IO; -using System.Reflection; -using System.Configuration; -using System.Security; +using Azure; +using Azure.Data.Tables; +using System; using System.Collections.Generic; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Configuration.Binder; -using Microsoft.Extensions.Configuration.FileExtensions; -using Microsoft.Extensions.Configuration.Json; -using Xunit; +using System.IO; using System.Linq; - -using Azure; -using Azure.Data.Tables; +using System.Reflection; using TheByteStuff.AzureTableUtilities; -using TheByteStuff.AzureTableUtilities.Exceptions; +using Xunit; namespace AzureTableUtilitiesXUnitTest diff --git a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesCopyXUnitTest.cs b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesCopyXUnitTest.cs index 66e4c7a..b6f6d7f 100644 --- a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesCopyXUnitTest.cs +++ b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesCopyXUnitTest.cs @@ -1,18 +1,10 @@ using System; +using System.Collections.Generic; using System.IO; using System.Reflection; -using System.Configuration; -using System.Security; -using System.Collections.Generic; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Configuration.Binder; -using Microsoft.Extensions.Configuration.FileExtensions; -using Microsoft.Extensions.Configuration.Json; -using Xunit; -using System.Linq; - using TheByteStuff.AzureTableUtilities; using TheByteStuff.AzureTableUtilities.Exceptions; +using Xunit; namespace AzureTableUtilitiesXUnitTest { diff --git a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesDeleteXUnitTest.cs b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesDeleteXUnitTest.cs index fd18d17..fd6f659 100644 --- a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesDeleteXUnitTest.cs +++ b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesDeleteXUnitTest.cs @@ -1,18 +1,10 @@ using System; +using System.Collections.Generic; using System.IO; using System.Reflection; -using System.Configuration; -using System.Security; -using System.Collections.Generic; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Configuration.Binder; -using Microsoft.Extensions.Configuration.FileExtensions; -using Microsoft.Extensions.Configuration.Json; -using Xunit; -using System.Linq; - using TheByteStuff.AzureTableUtilities; using TheByteStuff.AzureTableUtilities.Exceptions; +using Xunit; namespace AzureTableUtilitiesXUnitTest { diff --git a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesFiltersBaseXUnitTest.cs b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesFiltersBaseXUnitTest.cs index 08644d3..ccde893 100644 --- a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesFiltersBaseXUnitTest.cs +++ b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesFiltersBaseXUnitTest.cs @@ -1,16 +1,6 @@ -using System; -using System.Configuration; -using System.Security; -using System.Collections.Generic; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Configuration.Binder; -using Microsoft.Extensions.Configuration.FileExtensions; -using Microsoft.Extensions.Configuration.Json; -using Xunit; -using System.Linq; - +using System.Collections.Generic; using TheByteStuff.AzureTableUtilities; -using TheByteStuff.AzureTableUtilities.Exceptions; +using Xunit; namespace AzureTableUtilitiesXUnitTest { diff --git a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesXUnitTest.cs b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesXUnitTest.cs index 2dcc4fc..9054076 100644 --- a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesXUnitTest.cs +++ b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesXUnitTest.cs @@ -1,18 +1,11 @@ -using System; -using System.IO; -using System.Reflection; -using System.Configuration; -using System.Security; -using System.Collections.Generic; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Configuration.Binder; -using Microsoft.Extensions.Configuration.FileExtensions; using Microsoft.Extensions.Configuration.Json; -using Xunit; -using System.Linq; - +using System.Collections.Generic; +using System.IO; +using System.Reflection; using TheByteStuff.AzureTableUtilities; using TheByteStuff.AzureTableUtilities.Exceptions; +using Xunit; namespace AzureTableUtilitiesXUnitTest { diff --git a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesXUnitTest.csproj b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesXUnitTest.csproj index d940242..527c976 100644 --- a/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesXUnitTest.csproj +++ b/src/AzureTableUtilitiesXUnitTest/AzureTableUtilitiesXUnitTest.csproj @@ -5,24 +5,24 @@ --> - netcoreapp2.0 + net8.0 false - - - - - - - - - - - - - + + + + + + true + + + true + + + true + From d16685c6a3f480bc9f9ed9b02b43c3a318272794 Mon Sep 17 00:00:00 2001 From: petero-dk <2478689+petero-dk@users.noreply.github.com> Date: Sat, 24 Aug 2024 01:57:40 +0200 Subject: [PATCH 2/3] add commandline utility (simple) --- .gitignore | 2 + src/AzureTableUtilities.sln | 12 ++++-- .../AzureTableUtilitiesCLI.csproj | 18 +++++++++ src/AzureTableUtilitiesCLI/Program.cs | 38 +++++++++++++++++++ 4 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 src/AzureTableUtilitiesCLI/AzureTableUtilitiesCLI.csproj create mode 100644 src/AzureTableUtilitiesCLI/Program.cs diff --git a/.gitignore b/.gitignore index 589f6e8..80ab43d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ src/AzureTableUtilitiesXUnitTest/bin src/AzureTableUtilitiesXUnitTest/obj src/TestResults +/src/AzureTableUtilitiesCLI/bin +/src/AzureTableUtilitiesCLI/obj diff --git a/src/AzureTableUtilities.sln b/src/AzureTableUtilities.sln index a5bb078..b5c5b8a 100644 --- a/src/AzureTableUtilities.sln +++ b/src/AzureTableUtilities.sln @@ -1,11 +1,13 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.28307.1209 +# Visual Studio Version 17 +VisualStudioVersion = 17.8.34601.278 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureTableUtilities", "AzureTableUtilities\AzureTableUtilities.csproj", "{AB178BE5-5C5C-4EDD-B44F-FE175ECADF2B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureTableUtilitiesXUnitTest", "AzureTableUtilitiesXUnitTest\AzureTableUtilitiesXUnitTest.csproj", "{AFEE6E1E-EBC7-420B-ACF0-BCF9EC8A0852}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureTableUtilitiesXUnitTest", "AzureTableUtilitiesXUnitTest\AzureTableUtilitiesXUnitTest.csproj", "{AFEE6E1E-EBC7-420B-ACF0-BCF9EC8A0852}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureTableUtilitiesCLI", "AzureTableUtilitiesCLI\AzureTableUtilitiesCLI.csproj", "{67F0E345-53C2-4332-9B53-7B58BC4C3362}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -21,6 +23,10 @@ Global {AFEE6E1E-EBC7-420B-ACF0-BCF9EC8A0852}.Debug|Any CPU.Build.0 = Debug|Any CPU {AFEE6E1E-EBC7-420B-ACF0-BCF9EC8A0852}.Release|Any CPU.ActiveCfg = Release|Any CPU {AFEE6E1E-EBC7-420B-ACF0-BCF9EC8A0852}.Release|Any CPU.Build.0 = Release|Any CPU + {67F0E345-53C2-4332-9B53-7B58BC4C3362}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {67F0E345-53C2-4332-9B53-7B58BC4C3362}.Debug|Any CPU.Build.0 = Debug|Any CPU + {67F0E345-53C2-4332-9B53-7B58BC4C3362}.Release|Any CPU.ActiveCfg = Release|Any CPU + {67F0E345-53C2-4332-9B53-7B58BC4C3362}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/AzureTableUtilitiesCLI/AzureTableUtilitiesCLI.csproj b/src/AzureTableUtilitiesCLI/AzureTableUtilitiesCLI.csproj new file mode 100644 index 0000000..eba9dee --- /dev/null +++ b/src/AzureTableUtilitiesCLI/AzureTableUtilitiesCLI.csproj @@ -0,0 +1,18 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + diff --git a/src/AzureTableUtilitiesCLI/Program.cs b/src/AzureTableUtilitiesCLI/Program.cs new file mode 100644 index 0000000..c570ff4 --- /dev/null +++ b/src/AzureTableUtilitiesCLI/Program.cs @@ -0,0 +1,38 @@ +// See https://aka.ms/new-console-template for more information +using Azure.Data.Tables; +using Azure.Storage.Blobs; +using TheByteStuff.AzureTableUtilities; + + +string source; +string destination; +string name; + +if (args.Length > 0) +{ + if (args.Length == 3) + { + source = args[0]; + destination = args[1]; + name = args[2]; + } + else + { + Console.WriteLine("Invalid number of arguments"); + return; + } +} +else +{ + Console.WriteLine("No arguments"); + return; +} + +var credentials = new Azure.Identity.DefaultAzureCredential(); + +var tableClientService = new TableServiceClient(new Uri(source), credentials); +var blobClientService = new BlobServiceClient(new Uri(destination), credentials); + +var backup = new BackupAzureTables(tableClientService, blobClientService); + +backup.BackupAllTablesToBlob(name, true); \ No newline at end of file From 5ad72764d3a5e94a03912beb16668b08954c11f4 Mon Sep 17 00:00:00 2001 From: petero-dk <2478689+petero-dk@users.noreply.github.com> Date: Thu, 29 Aug 2024 20:41:28 +0200 Subject: [PATCH 3/3] added restore all to the command line, moved paramters for storage out --- src/AzureTableUtilities/BackupAzureTables.cs | 116 +++++++++++------- src/AzureTableUtilities/RestoreAzureTables.cs | 91 +++++++++++--- src/AzureTableUtilitiesCLI/Program.cs | 39 ++++-- 3 files changed, 177 insertions(+), 69 deletions(-) diff --git a/src/AzureTableUtilities/BackupAzureTables.cs b/src/AzureTableUtilities/BackupAzureTables.cs index fab05a0..f9b475e 100644 --- a/src/AzureTableUtilities/BackupAzureTables.cs +++ b/src/AzureTableUtilities/BackupAzureTables.cs @@ -8,6 +8,7 @@ using System.IO; using System.IO.Compression; using System.Linq; +using System.Net.NetworkInformation; using System.Text; using TheByteStuff.AzureTableUtilities.Exceptions; //using AZBlob = Microsoft.Azure.Storage.Blob; @@ -96,7 +97,7 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl /// A string indicating the name of the blob file created as well as a count of how many files were aged. public string BackupTableToBlob(string TableName, string BlobRoot, string OutFileDirectory, bool Compress = false, bool Validate = false, int RetentionDays = 30, int TimeoutSeconds = 30, List filters = default(List)) { - string OutFileName ; + string OutFileName; string OutFileNamePath = ""; if (String.IsNullOrWhiteSpace(TableName)) @@ -121,7 +122,7 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl try { - OutFileName = this.BackupTableToFile(TableName, OutFileDirectory, Compress, Validate, TimeoutSeconds, filters); + OutFileName = this.BackupTableToFile(TableName, OutFileDirectory, Compress, Validate, TimeoutSeconds, filters); OutFileNamePath = Path.Combine(OutFileDirectory, OutFileName); @@ -144,7 +145,7 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl DateTimeOffset OffsetTimeRetain = System.DateTimeOffset.Now.AddDays(-1 * RetentionDays); - return String.Format("Table '{0}' backed up as '{1}' under blob '{2}\\{3}';", TableName, OutFileName, BlobRoot , directory.ToString()); + return String.Format("Table '{0}' backed up as '{1}' under blob '{2}\\{3}';", TableName, OutFileName, BlobRoot, directory.ToString()); } catch (ConnectionException cex) { @@ -254,7 +255,7 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl OutFile.Flush(); OutFile.Close(); } - + if (Validate) { int InRecords = 0; @@ -281,7 +282,7 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl TableSpec footer = System.Text.Json.JsonSerializer.Deserialize(FooterRec); - if ((footer.RecordCount==InRecords) && (footer.TableName.Equals(TableName))) + if ((footer.RecordCount == InRecords) && (footer.TableName.Equals(TableName))) { //Do nothing, in count=out count } @@ -342,45 +343,47 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl /// /// Backup table directly to Blob. /// - /// Name of Azure Table to backup. - /// Name to use as blob root folder. - /// True to compress the file. + /// Name of Azure Table to backup. + /// Name to use as blob root folder. + /// True to compress the file. /// Process will age files in blob created more than x days ago. /// Set timeout for table client. /// A list of Filter objects to be applied to table values extracted. /// A string containing the name of the file created. - public string BackupTableToBlobDirect(string TableName, string BlobRoot, bool Compress = false, int RetentionDays = 30, int TimeoutSeconds = 30, List filters = default(List)) + public string BackupTableToBlobDirect(string tableName, string blobRoot, bool compress = false, int RetentionDays = 90, int TimeoutSeconds = 30, List filters = default) + { + + var fileName = tableName + "_Backup_" + System.DateTime.Now.ToString("yyyyMMddHHmmss"); //This is terrible because for multiple tables in a folder, the date will be different for each table + + var result = BackupTableToBlobDirect(tableName, blobRoot, blobRoot.ToLower() + "-table-" + tableName.ToLower(), fileName, compress, filters); + + DateTimeOffset OffsetTimeNow = System.DateTimeOffset.Now; + DateTimeOffset OffsetTimeRetain = System.DateTimeOffset.Now.AddDays(-1 * RetentionDays); + + // DO CLEANUP AS BEFORE + + return result; + } + public string BackupTableToBlobDirect(string tableName, string blobRoot, string folderName, string fileName, bool compress = false, List filters = default) { - string OutFileName = ""; int RecordCount = 0; - int BackupsAged = 0; - if (String.IsNullOrWhiteSpace(TableName)) - { + if (string.IsNullOrWhiteSpace(tableName)) throw new ParameterSpecException("TableName is missing."); - } - if (String.IsNullOrWhiteSpace(BlobRoot)) - { + if (string.IsNullOrWhiteSpace(blobRoot)) throw new ParameterSpecException("BlobRoot is missing."); - } try { - if (Compress) - { - OutFileName = String.Format(TableName + "_Backup_" + System.DateTime.Now.ToString("yyyyMMddHHmmss") + ".txt.7z"); - } - else - { - OutFileName = String.Format(TableName + "_Backup_" + System.DateTime.Now.ToString("yyyyMMddHHmmss") + ".txt"); - } - var container = blobServiceClient.GetBlobContainerClient(BlobRoot); + var container = blobServiceClient.GetBlobContainerClient(blobRoot); container.CreateIfNotExists(); - var directory = container.GetBlobClient(BlobRoot.ToLower() + "-table-" + TableName.ToLower() + "/" + OutFileName); - var blobClient = container.GetBlobClient(OutFileName); + + var outFileName = $"{(string.IsNullOrWhiteSpace(folderName) ? "" : folderName + "/")}{fileName}.txt{(compress ? ".7z" : "")}"; + + var blobClient = container.GetBlobClient(outFileName); var blobUploadOptions = new BlobUploadOptions { TransferOptions = new StorageTransferOptions @@ -399,14 +402,14 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl var entitiesSerialized = new List(); var serializer = new DynamicTableEntityJsonSerializer(); - var TableSpecStart = new TableSpec(TableName); + var TableSpecStart = new TableSpec(tableName); var NewLineAsBytes = Encoding.UTF8.GetBytes("\n"); var tempTableSpecStart = Encoding.UTF8.GetBytes(System.Text.Json.JsonSerializer.Serialize(TableSpecStart)); var bs2 = blobClient.OpenWrite(true); Stream bs = bs2; - if (Compress) + if (compress) { bs = new GZipStream(bs2, CompressionMode.Compress); } @@ -420,10 +423,10 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl bs.Write(NewLineAsBytes, 0, NewLineAsBytes.Length); bs.Flush(); - Pageable queryResultsFilter = tableServiceClient.GetTableClient(TableName).Query(filter: Filter.BuildFilterSpec(filters), maxPerPage: 100); - foreach (Page page in queryResultsFilter.AsPages()) + var queryResultsFilter = tableServiceClient.GetTableClient(tableName).Query(filter: Filter.BuildFilterSpec(filters), maxPerPage: 100); + foreach (var page in queryResultsFilter.AsPages()) { - foreach (AzureTables.TableEntity qEntity in page.Values) + foreach (var qEntity in page.Values) { // var tempDTE = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(qEntity)); // Int32 type gets lost in stock serializer var tempDTE = Encoding.UTF8.GetBytes(serializer.Serialize(qEntity)); @@ -433,25 +436,22 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl bs.Flush(); RecordCount++; } - } - TableSpec TableSpecEnd = new TableSpec(TableName, RecordCount); + } + var TableSpecEnd = new TableSpec(tableName, RecordCount); var tempTableSpecEnd = Encoding.UTF8.GetBytes(System.Text.Json.JsonSerializer.Serialize(TableSpecEnd)); bs.Write(tempTableSpecEnd, 0, tempTableSpecEnd.Length); bs.Flush(); bs.Write(NewLineAsBytes, 0, NewLineAsBytes.Length); bs.Flush(); - bs.Close(); + bs.Close(); } catch (Exception ex) { - throw new BackupFailedException(String.Format("Table '{0}' backup failed.", TableName), ex); + throw new BackupFailedException(String.Format("Table '{0}' backup failed.", tableName), ex); } - DateTimeOffset OffsetTimeNow = System.DateTimeOffset.Now; - DateTimeOffset OffsetTimeRetain = System.DateTimeOffset.Now.AddDays(-1 * RetentionDays); - - return String.Format("Table '{0}' backed up as '{2}' under blob '{3}\\{4}'; {1} files aged.", TableName, BackupsAged, OutFileName, BlobRoot, directory.ToString()); + return String.Format("Table '{0}' backed up as '{1}' under blob '{2}'.", tableName, outFileName, blobRoot); } catch (ConnectionException cex) { @@ -459,7 +459,7 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl } catch (Exception ex) { - throw new BackupFailedException(String.Format("Table '{0}' backup failed.", TableName), ex); + throw new BackupFailedException(String.Format("Table '{0}' backup failed.", tableName), ex); } finally { @@ -505,5 +505,37 @@ public BackupAzureTables(SecureString AzureTableConnection, SecureString AzureBl throw new BackupFailedException(String.Format("Backup of all tables to blob '{0}' failed.", BlobRoot), ex); } } // BackupAllTablesToBlob + + + + public string BackupAllTablesToBlob(string blobRoot, string folderName, bool compress = false, List filters = default(List)) + { + if (String.IsNullOrWhiteSpace(blobRoot)) + { + throw new ParameterSpecException("BlobRoot is missing."); + } + + try + { + StringBuilder BackupResults = new StringBuilder(); + List TableNames = Helper.GetTableNames(tableServiceClient); + if (TableNames.Count() > 0) + { + foreach (string tableName in TableNames) + { + BackupResults.Append(BackupTableToBlobDirect(tableName, blobRoot, folderName, tableName, compress, filters) + "|"); + } + return BackupResults.ToString(); + } + else + { + return "No Tables found."; + } + } + catch (Exception ex) + { + throw new BackupFailedException(string.Format("Backup of all tables to blob '{0}' failed.", blobRoot), ex); + } + } // BackupAllTablesToBlob } } diff --git a/src/AzureTableUtilities/RestoreAzureTables.cs b/src/AzureTableUtilities/RestoreAzureTables.cs index 14c7fbf..b4886da 100644 --- a/src/AzureTableUtilities/RestoreAzureTables.cs +++ b/src/AzureTableUtilities/RestoreAzureTables.cs @@ -2,10 +2,14 @@ using Azure.Data.Tables; using Azure.Data.Tables.Models; using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using System; using System.Collections.Generic; +using System.ComponentModel; using System.IO; using System.IO.Compression; +using System.Linq; +using System.Text; using TheByteStuff.AzureTableUtilities.Exceptions; namespace TheByteStuff.AzureTableUtilities @@ -319,13 +323,14 @@ public string RestoreTableFromFile(string DestinationTableName, string InFilePat Response> response = tableServiceClient.GetTableClient(DestinationTableName).SubmitTransaction(Batch); PartitionKey = String.Empty; } - catch (Exception ex) { + catch (Exception ex) + { throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex); } } } // using (StreamReader - if (null==footer) + if (null == footer) { throw new RestoreFailedException(String.Format("Table '{0}' restore failed, no footer record found.", DestinationTableName)); } @@ -429,31 +434,31 @@ public string RestoreTableFromBlobDirect(string DestinationTableName, string Ori try { - - BlobContainerClient container = blobServiceClient.GetBlobContainerClient(BlobRoot); - container.CreateIfNotExists(); - BlobClient blobClient = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{BlobFileName}"); - // If file is compressed, Decompress to a temp file in the blob - if (Decompress) - { - BlobClient blobClientTemp = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{TempFileName}"); - BlobClient blobClientRead = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{BlobFileName}"); + BlobContainerClient container = blobServiceClient.GetBlobContainerClient(BlobRoot); + container.CreateIfNotExists(); + BlobClient blobClient = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{BlobFileName}"); - using (var decompressedStream = blobClientTemp.OpenWrite(true)) + // If file is compressed, Decompress to a temp file in the blob + if (Decompress) + { + BlobClient blobClientTemp = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{TempFileName}"); + BlobClient blobClientRead = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{BlobFileName}"); + + using (var decompressedStream = blobClientTemp.OpenWrite(true)) + { + using (var readStream = blobClientRead.OpenRead()) { - using (var readStream = blobClientRead.OpenRead()) + using (var zip = new GZipStream(readStream, CompressionMode.Decompress, true)) { - using (var zip = new GZipStream(readStream, CompressionMode.Decompress, true)) - { - zip.CopyTo(decompressedStream); - } + zip.CopyTo(decompressedStream); } } - BlobFileName = TempFileName; } + BlobFileName = TempFileName; + } - blobClient = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{BlobFileName}"); + blobClient = container.GetBlobClient($"{BlobRoot.ToLower()}-table-{OriginalTableName.ToLower()}/{BlobFileName}"); TableItem TableDest = tableServiceClient.CreateTableIfNotExists(DestinationTableName); @@ -599,5 +604,53 @@ private string RestoreFromStream(StreamReader InputFileStream, TableServiceClien return String.Format("Restore to table '{0}' Successful; {1} entries.", DestinationTableName, TotalRecordCount); } + + public string RestoreAllTablesFromBlob(string blobRoot, string folder) + { + var container = blobServiceClient.GetBlobContainerClient(blobRoot); + if (!container.Exists()) + throw new RestoreFailedException(String.Format("Blob container '{0}' does not exist.", blobRoot)); + + var restoreResults = new StringBuilder(); + foreach (BlobHierarchyItem blobItem in container.GetBlobsByHierarchy(prefix: folder, delimiter: "/")) + { + if (blobItem.IsBlob) + { + var tableName = blobItem.Blob.Name.Split('.').First(); + var decompress = blobItem.Blob.Name.EndsWith(".7z"); + var tempFileName = String.Format("{0}.temp", blobItem.Blob.Name); + + var blobClient = container.GetBlobClient(blobItem.Blob.Name); + + // If file is compressed, Decompress to a temp file in the blob + if (decompress) + { + var blobClientTemp = container.GetBlobClient(tempFileName); + + using (var decompressedStream = blobClientTemp.OpenWrite(true)) + using (var readStream = blobClient.OpenRead()) + using (var zip = new GZipStream(readStream, CompressionMode.Decompress, true)) + { + zip.CopyTo(decompressedStream); + } + blobClient = container.GetBlobClient(tempFileName); + } + + var TableDest = tableServiceClient.CreateTableIfNotExists(tableName); + + using (Stream blobStream = blobClient.OpenRead()) + using (StreamReader inputFileStream = new StreamReader(blobStream)) + { + restoreResults.AppendLine(RestoreFromStream(inputFileStream, tableServiceClient, tableName)); + if (decompress) + { + var blobClientTemp = container.GetBlobClient(tempFileName); + blobClientTemp.DeleteIfExists(); + } + } + } + } + return restoreResults.ToString(); + } } } \ No newline at end of file diff --git a/src/AzureTableUtilitiesCLI/Program.cs b/src/AzureTableUtilitiesCLI/Program.cs index c570ff4..f363a58 100644 --- a/src/AzureTableUtilitiesCLI/Program.cs +++ b/src/AzureTableUtilitiesCLI/Program.cs @@ -7,20 +7,33 @@ string source; string destination; string name; +string folder; +string operation; if (args.Length > 0) { - if (args.Length == 3) + operation = args[0]; + if (operation == "restore" || operation == "backup") { - source = args[0]; - destination = args[1]; - name = args[2]; + if (args.Length == 5) + { + source = args[1]; + destination = args[2]; + name = args[3]; + folder = args[4]; + } + else + { + Console.WriteLine("Invalid number of arguments"); + return; + } } else { - Console.WriteLine("Invalid number of arguments"); + Console.WriteLine("Invalid operation"); return; - } + } + } else { @@ -33,6 +46,16 @@ var tableClientService = new TableServiceClient(new Uri(source), credentials); var blobClientService = new BlobServiceClient(new Uri(destination), credentials); -var backup = new BackupAzureTables(tableClientService, blobClientService); -backup.BackupAllTablesToBlob(name, true); \ No newline at end of file +if (operation == "backup") +{ + //var backupfolder = DateTime.Now.ToString("yyyyMMddTHHmm"); + + var backup = new BackupAzureTables(tableClientService, blobClientService); + backup.BackupAllTablesToBlob(name, folder, false); +} +else if (operation == "restore") +{ + var restore = new RestoreAzureTables(tableClientService, blobClientService); + restore.RestoreAllTablesFromBlob(name, folder); +} \ No newline at end of file