Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Purely to new Azure.Data clients, #11

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Purely to new Azure.Data clients, #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c3227fc
Purely to new Azure.Data clients, support for managed identities. Cle…
petero-dk Aug 23, 2024
d16685c
add commandline utility (simple)
petero-dk Aug 23, 2024
5ad7276
added restore all to the command line, moved paramters for storage out
petero-dk Aug 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,5 @@ src/AzureTableUtilitiesXUnitTest/bin
src/AzureTableUtilitiesXUnitTest/obj
src/TestResults

/src/AzureTableUtilitiesCLI/bin
/src/AzureTableUtilitiesCLI/obj
12 changes: 9 additions & 3 deletions src/AzureTableUtilities.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.28307.1209
# Visual Studio Version 17
VisualStudioVersion = 17.8.34601.278
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureTableUtilities", "AzureTableUtilities\AzureTableUtilities.csproj", "{AB178BE5-5C5C-4EDD-B44F-FE175ECADF2B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureTableUtilitiesXUnitTest", "AzureTableUtilitiesXUnitTest\AzureTableUtilitiesXUnitTest.csproj", "{AFEE6E1E-EBC7-420B-ACF0-BCF9EC8A0852}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureTableUtilitiesXUnitTest", "AzureTableUtilitiesXUnitTest\AzureTableUtilitiesXUnitTest.csproj", "{AFEE6E1E-EBC7-420B-ACF0-BCF9EC8A0852}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureTableUtilitiesCLI", "AzureTableUtilitiesCLI\AzureTableUtilitiesCLI.csproj", "{67F0E345-53C2-4332-9B53-7B58BC4C3362}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand All @@ -21,6 +23,10 @@ Global
{AFEE6E1E-EBC7-420B-ACF0-BCF9EC8A0852}.Debug|Any CPU.Build.0 = Debug|Any CPU
{AFEE6E1E-EBC7-420B-ACF0-BCF9EC8A0852}.Release|Any CPU.ActiveCfg = Release|Any CPU
{AFEE6E1E-EBC7-420B-ACF0-BCF9EC8A0852}.Release|Any CPU.Build.0 = Release|Any CPU
{67F0E345-53C2-4332-9B53-7B58BC4C3362}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{67F0E345-53C2-4332-9B53-7B58BC4C3362}.Debug|Any CPU.Build.0 = Debug|Any CPU
{67F0E345-53C2-4332-9B53-7B58BC4C3362}.Release|Any CPU.ActiveCfg = Release|Any CPU
{67F0E345-53C2-4332-9B53-7B58BC4C3362}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
10 changes: 3 additions & 7 deletions src/AzureTableUtilities/AzureTableUtilities.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,16 +50,12 @@ v6.0.0 - ** Backup/Restore is NOT compatible with prior versions ** Upgrade fr
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
<DocumentationFile>D:\github\TheByteStuff\AzureTableUtilities\src\AzureTableUtilities\AzureTableUtilities.xml</DocumentationFile>
<DocumentationFile></DocumentationFile>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Azure.Data.Tables" Version="12.8.0" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.13.0" />
<PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.1.7" />
<PackageReference Include="Microsoft.Azure.Storage.File" Version="11.1.7" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
<PackageReference Include="System.Configuration.ConfigurationManager" Version="7.0.0" />
<PackageReference Include="Azure.Data.Tables" Version="12.9.0" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.21.2" />
</ItemGroup>

</Project>
278 changes: 128 additions & 150 deletions src/AzureTableUtilities/BackupAzureTables.cs
View file
Open in desktop

Large diffs are not rendered by default.

67 changes: 19 additions & 48 deletions src/AzureTableUtilities/CopyAzureTables.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Configuration;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Security;
using System.Net.Http;

using AZStorage = Microsoft.Azure.Storage;
using Microsoft.Azure.Storage.Auth;
using AZBlob = Microsoft.Azure.Storage.Blob;
using Microsoft.Azure.Storage.Core;
using Microsoft.Azure.Storage.File;

using Azure;
using Azure.Data.Tables;
using Azure.Data.Tables.Models;
using Azure;

using Newtonsoft.Json;

using TheByteStuff.AzureTableUtilities.Exceptions;

using Azure.Storage.Blobs;
using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Security;
using System.Text;
using TheByteStuff.AzureTableUtilities.Exceptions;

namespace TheByteStuff.AzureTableUtilities
{
Expand All @@ -33,10 +16,9 @@ namespace TheByteStuff.AzureTableUtilities
/// </summary>
public class CopyAzureTables
{
//private SecureString AzureSourceTableConnection = new SecureString();
//private SecureString AzureDestinationTableConnection = new SecureString();
private StringBuilder AzureSourceTableConnection = new StringBuilder();
private StringBuilder AzureDestinationTableConnection = new StringBuilder();

private TableServiceClient sourceTableServiceClient;
private TableServiceClient destinationTableServiceClient;
/// <summary>
/// Constructor, sets same connection spec for both the Source and Destination Azure Tables.
/// </summary>
Expand Down Expand Up @@ -67,16 +49,8 @@ public CopyAzureTables(string AzureSourceTableConnection, string AzureDestinatio
throw new ConnectionException(String.Format("Connection spec must be specified."));
}

foreach (char c in AzureSourceTableConnection.ToCharArray())
{
//this.AzureSourceTableConnection.AppendChar(c);
this.AzureSourceTableConnection.Append(c);
}
foreach (char c in AzureDestinationTableConnection.ToCharArray())
{
//this.AzureDestinationTableConnection.AppendChar(c);
this.AzureDestinationTableConnection.Append(c);
}
sourceTableServiceClient = new TableServiceClient(AzureSourceTableConnection);
destinationTableServiceClient = new TableServiceClient(AzureDestinationTableConnection);
}

/// <summary>
Expand Down Expand Up @@ -133,12 +107,9 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu

try
{
TableServiceClient clientSource = new TableServiceClient(AzureSourceTableConnection.ToString());

TableServiceClient clientDestination = new TableServiceClient(AzureDestinationTableConnection.ToString());
TableItem TableDest = clientDestination.CreateTableIfNotExists(DestinationTableName);
TableItem TableDest = destinationTableServiceClient.CreateTableIfNotExists(DestinationTableName);

Pageable<TableEntity> queryResultsFilter = clientSource.GetTableClient(SourceTableName).Query<TableEntity>(filter: Filter.BuildFilterSpec(filters), maxPerPage: BatchSize);
Pageable<TableEntity> queryResultsFilter = sourceTableServiceClient.GetTableClient(SourceTableName).Query<TableEntity>(filter: Filter.BuildFilterSpec(filters), maxPerPage: BatchSize);
// Set Timeout?

BatchCount = 0;
Expand All @@ -157,7 +128,7 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu
}
else
{
Response<IReadOnlyList<Response>> response = clientDestination.GetTableClient(DestinationTableName).SubmitTransaction(Batch);
Response<IReadOnlyList<Response>> response = destinationTableServiceClient.GetTableClient(DestinationTableName).SubmitTransaction(Batch);
TotalRecordCountOut = TotalRecordCountOut + Batch.Count;
Batch = new List<TableTransactionAction>();

Expand All @@ -171,7 +142,7 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu
{
try
{
Response<IReadOnlyList<Response>> response = clientDestination.GetTableClient(DestinationTableName).SubmitTransaction(Batch);
Response<IReadOnlyList<Response>> response = destinationTableServiceClient.GetTableClient(DestinationTableName).SubmitTransaction(Batch);
TotalRecordCountOut = TotalRecordCountOut + Batch.Count;

Batch = new List<TableTransactionAction>();
Expand All @@ -188,7 +159,7 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu
}
if (!BatchWritten)
{
Response<IReadOnlyList<Response>> response = clientDestination.GetTableClient(DestinationTableName).SubmitTransaction(Batch);
Response<IReadOnlyList<Response>> response = destinationTableServiceClient.GetTableClient(DestinationTableName).SubmitTransaction(Batch);
TotalRecordCountOut = TotalRecordCountOut + Batch.Count;
Batch = new List<TableTransactionAction>();
PartitionKey = String.Empty;
Expand Down Expand Up @@ -227,7 +198,7 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu
public string CopyAllTables(int TimeoutSeconds = 30, List<Filter> filters = default(List<Filter>))
{
//if (IsEqualTo(AzureSourceTableConnection.ToString(), AzureDestinationTableConnection.ToString()))
if (AzureSourceTableConnection.ToString().Equals(AzureDestinationTableConnection.ToString()))
if (sourceTableServiceClient.Uri.Equals(destinationTableServiceClient.Uri))
{
throw new ParameterSpecException("Source and Destination Connection specs can not match for CopyAll.");
}
Expand All @@ -240,7 +211,7 @@ public CopyAzureTables(SecureString AzureSourceTableConnection, SecureString Azu
StringBuilder BackupResults = new StringBuilder();
try
{
List<string> TableNames = Helper.GetTableNames(AzureSourceTableConnection.ToString());
List<string> TableNames = Helper.GetTableNames(sourceTableServiceClient);
if (TableNames.Count > 0)
{
foreach (string TableName in TableNames)
Expand Down
Loading