Board Review: Introducing Storage STG 84 features

The Basics

Service team responsible for the client library: Storage
Link to documentation describing the service: Storage STG 84
Contact email (if service team, provide PM and Dev Lead): seanmcc@microsoft.com

About this client library

Name of the client library: Storage

Champion Scenarios

Tag Indexing for Versions

Users can now query for Blob Versions when calling BlobServiceClient.FindBlobsByTag(), .FindBlobsByTagAsync(), BlobContainerClient.FindBlobsByTag(), and .FindBlobsByTagAsync().

    [RecordedTest]
    [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2021_08_06)]
    public async Task FindBlobsByTagAsync_BlobVersions()
    {
        // Arrange
        BlobServiceClient service = GetServiceClient_SharedKey();
        await using DisposingContainer test = await GetTestContainerAsync();
        string blobName = GetNewBlobName();
        AppendBlobClient appendBlob = InstrumentClient(test.Container.GetAppendBlobClient(blobName));

        string tagKey = "myTagKey";
        string tagValue = "myTagValue";
        Dictionary<string, string> tags = new Dictionary<string, string>
        {
            { tagKey, tagValue }
        };
        AppendBlobCreateOptions options = new AppendBlobCreateOptions
        {
            Tags = tags
        };
        await appendBlob.CreateAsync(options);

        // Create blob again to trigger new version
        await appendBlob.CreateAsync(options);

        // It takes a few seconds for Filter Blobs to pick up new changes
        await Delay(2000);

        string expression = $"\"{tagKey}\"='{tagValue}'";

        // Act
        List<TaggedBlobItem> blobs = new List<TaggedBlobItem>();
        await foreach (TaggedBlobItem taggedBlobItem in service.FindBlobsByTagsAsync(
            tagFilterSqlExpression: expression,
            states: BlobStates.Version))
        {
            blobs.Add(taggedBlobItem);
        }
        blobs = blobs.Where(r => r.BlobName == appendBlob.Name).ToList();

        // Assert
        Assert.AreEqual(2, blobs.Count);
        Assert.IsNull(blobs[0].IsLatestVersion);
        Assert.IsNotNull(blobs[0].VersionId);
        Assert.IsTrue(blobs[1].IsLatestVersion);
        Assert.IsNotNull(blobs[1].VersionId);
    }

Append with Flush

Users can now specify that they would like to preform an Append and Flush on a Data Lake file in a single REST API call.

    [RecordedTest]
    [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)]
    public async Task AppendDataAsync_Flush()
    {
        await using DisposingFileSystem test = await GetNewFileSystem();

        // Arrange
        DataLakeFileClient file = InstrumentClient(test.FileSystem.GetFileClient(GetNewFileName()));
        await file.CreateIfNotExistsAsync();
        var data = GetRandomBuffer(Size);

        DataLakeFileAppendOptions options = new DataLakeFileAppendOptions
        {
            Flush = true
        };

        // Act
        using (var stream = new MemoryStream(data))
        {
            await file.AppendAsync(
                content: stream,
                offset: 0,
                options: options);
        }

        // Assert
        Response<FileDownloadInfo> response = await file.ReadAsync();

        Assert.AreEqual(data.Length, response.Value.ContentLength);
        var actual = new MemoryStream();
        await response.Value.Content.CopyToAsync(actual);
        TestHelper.AssertSequenceEqual(data, actual.ToArray());
    }

Data Lake Encryption Scopes

Users can now specify an Encryption Scope to encrypt an entire File System with in DataLakeFileSystemClient.Create()

    [RecordedTest]
    [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_10_02)]
    public async Task CreateAsync_EncryptionScopeOptions()
    {
        // Arrange
        DataLakeFileSystemEncryptionScopeOptions encryptionScopeOptions = new DataLakeFileSystemEncryptionScopeOptions
        {
            DefaultEncryptionScope = TestConfigHierarchicalNamespace.EncryptionScope
        };
        await using DisposingFileSystem test = await GetNewFileSystem(encryptionScopeOptions: encryptionScopeOptions);
    }

Users will now receive Encryption Scope as a per-File System response property on DataLakeServiceClient.GetFileSystem()

    [RecordedTest]
    [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_10_02)]
    public async Task GetFileSystemsAsync_EncryptionScope()
    {
        // Arrange
        DataLakeServiceClient service = GetServiceClient_SharedKey();
        string fileSystemName = GetNewFileSystemName();
        DataLakeFileSystemClient fileSystemClient = InstrumentClient(service.GetFileSystemClient(fileSystemName));
        DataLakeFileSystemEncryptionScopeOptions encryptionScopeOptions = new DataLakeFileSystemEncryptionScopeOptions
        {
            DefaultEncryptionScope = TestConfigHierarchicalNamespace.EncryptionScope
        };

        await fileSystemClient.CreateAsync(encryptionScopeOptions: encryptionScopeOptions);

        // Act
        IList<FileSystemItem> fileSystems = await service.GetFileSystemsAsync().ToListAsync();
        FileSystemItem fileSystemItem = fileSystems.Single(r => r.Name == fileSystemName);

        // Assert
        Assert.AreEqual(TestConfigHierarchicalNamespace.EncryptionScope, fileSystemItem.Properties.DefaultEncryptionScope);

        // Cleanup
        await fileSystemClient.DeleteIfExistsAsync();
    }

Users will now receive Encryption Scope as a response property on DataLakeFileSystemClient.GetProperties()

    [RecordedTest]
    [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_10_02)]
    public async Task CreateAsync_EncryptionScopeOptions()
    {
        // Arrange
        DataLakeFileSystemEncryptionScopeOptions encryptionScopeOptions = new DataLakeFileSystemEncryptionScopeOptions
        {
            DefaultEncryptionScope = TestConfigHierarchicalNamespace.EncryptionScope
        };
        await using DisposingFileSystem test = await GetNewFileSystem(encryptionScopeOptions: encryptionScopeOptions);

        // Assert - We are also testing GetPropertiesAsync() in this test.
        Response<FileSystemProperties> response = await test.FileSystem.GetPropertiesAsync();
        Assert.AreEqual(TestConfigHierarchicalNamespace.EncryptionScope, response.Value.DefaultEncryptionScope);
    }

Users will now receive Encryption Scope as a per-Path response property on DataLakeFileSystemClient.GetPaths()

    [RecordedTest]
    [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_10_02)]
    public async Task GetPathsAsync_EncryptionScopeOptions()
    {
        // Arrange
        DataLakeFileSystemEncryptionScopeOptions encryptionScopeOptions = new DataLakeFileSystemEncryptionScopeOptions
        {
            DefaultEncryptionScope = TestConfigHierarchicalNamespace.EncryptionScope
        };
        string directoryName = GetNewDirectoryName();
        await using DisposingFileSystem test = await GetNewFileSystem(encryptionScopeOptions: encryptionScopeOptions);
        DataLakeDirectoryClient directoryClient = InstrumentClient(test.FileSystem.GetDirectoryClient(directoryName));
        await directoryClient.CreateAsync();

        // Act
        AsyncPageable<PathItem> response = test.FileSystem.GetPathsAsync();
        IList<PathItem> paths = await response.ToListAsync();
        PathItem pathItem = paths.Single(r => r.Name == directoryName);

        // Assert
        Assert.AreEqual(TestConfigHierarchicalNamespace.EncryptionScope, pathItem.EncryptionScope);
    }

Users will now receive Encryption Scope as a response property on PathClient, DirectoryClient, and FileClient.GetProperties()

    [RecordedTest]
    [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_10_02)]
    public async Task GetPropertiesAsync_EncryptionScope()
    {
        // Arrange
        DataLakeFileSystemEncryptionScopeOptions encryptionScopeOptions = new DataLakeFileSystemEncryptionScopeOptions
        {
            DefaultEncryptionScope = TestConfigHierarchicalNamespace.EncryptionScope
        };
        await using DisposingFileSystem test = await GetNewFileSystem(encryptionScopeOptions: encryptionScopeOptions);
        DataLakeDirectoryClient directoryClient = InstrumentClient(test.FileSystem.GetDirectoryClient(GetNewDirectoryName()));
        await directoryClient.CreateAsync();

        // Act
        Response<PathProperties> response = await directoryClient.GetPropertiesAsync();

        // Assert
        Assert.AreEqual(TestConfigHierarchicalNamespace.EncryptionScope, response.Value.EncryptionScope);
    }

Data Lake Encryption Scope SAS

Users can now use a Service SAS URI to specify the Encryption Scope to encrypt a Data Lake Path with.

    [RecordedTest]
    [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_12_06)]
    public async Task CreateAsync_EncryptionScopeSas()
    {
        // Arrange
        await using DisposingFileSystem test = await GetNewFileSystem();
        string directoryName = GetNewDirectoryName();

        DataLakeSasBuilder sasBuilder = new DataLakeSasBuilder
        {
            FileSystemName = test.FileSystem.Name,
            Path = directoryName,
            EncryptionScope = TestConfigHierarchicalNamespace.EncryptionScope,
            ExpiresOn = Recording.UtcNow.AddDays(1)
        };
        sasBuilder.SetPermissions(DataLakeSasPermissions.All);

        DataLakeSasQueryParameters sasQueryParameters = sasBuilder.ToSasQueryParameters(
            new StorageSharedKeyCredential(
                TestConfigHierarchicalNamespace.AccountName,
                TestConfigHierarchicalNamespace.AccountKey));

        DataLakeUriBuilder uriBuilder = new DataLakeUriBuilder(test.FileSystem.Uri)
        {
            DirectoryOrFilePath = directoryName,
            Sas = sasQueryParameters
        };

        DataLakeDirectoryClient sasDirectoryClient = InstrumentClient(new DataLakeDirectoryClient(uriBuilder.ToUri(), GetOptions()));

        // Act
        Response<PathInfo> response = await sasDirectoryClient.CreateAsync();
    }

Users can now use an Identity SAS Uri to specify the Encryption Scope to encrypt a Data Lake Path with.

    [RecordedTest]
    [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_12_06)]
    public async Task CreateAsync_EncryptionScopeIdentitySas()
    {
        // Arrange
        await using DisposingFileSystem test = await GetNewFileSystem();
        Response<UserDelegationKey> userDelegationKey = await GetServiceClient_OAuth().GetUserDelegationKeyAsync(
            startsOn: null,
            expiresOn: Recording.UtcNow.AddHours(1));

        string directoryName = GetNewDirectoryName();

        DataLakeSasBuilder sasBuilder = new DataLakeSasBuilder
        {
            FileSystemName = test.FileSystem.Name,
            Path = directoryName,
            EncryptionScope = TestConfigHierarchicalNamespace.EncryptionScope,
            ExpiresOn = Recording.UtcNow.AddDays(1)
        };
        sasBuilder.SetPermissions(DataLakeSasPermissions.All);

        DataLakeUriBuilder uriBuilder = new DataLakeUriBuilder(test.FileSystem.Uri)
        {
            DirectoryOrFilePath = directoryName,
            Sas = sasBuilder.ToSasQueryParameters(userDelegationKey.Value, TestConfigHierarchicalNamespace.AccountName)
        };

        DataLakeDirectoryClient sasDirectoryClient = InstrumentClient(new DataLakeDirectoryClient(uriBuilder.ToUri(), GetOptions()));

        // Act
        await sasDirectoryClient.CreateAsync();
    }

Users can now use an Account SAS URI to specify the Encryption Scope to encrypt a Data Lake Path with.

Azure / azure-sdk