Support to multi delete Azure Threat Intelligence insights

[hazcod]

Feature Request

In order to delete threat intelligence indicators in Azure Sentinel, we currently need to create a Pager with a PageSize which will iterate over TI items. But we still need to delete them seperately, which is not performant since TI items are often very numerous.

Example current code:

package sentinel

import (
    "context"
    "fmt"
    "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
    "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
    insights "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/securityinsights/armsecurityinsights/v2"
    "github.com/sirupsen/logrus"
    "time"
)

func (s *Sentinel) CleanupThreatIntel(ctx context.Context, l *logrus.Logger, retentionDays uint32) error {
    logger := l.WithField("module", "sentinel_ti")

    cred, err := azidentity.NewClientSecretCredential(s.creds.TenantID, s.creds.ClientID, s.creds.ClientSecret, nil)
    if err != nil {
        return fmt.Errorf("could not authenticate to MS Sentinel: %v", err)
    }

    tiClient, err := insights.NewThreatIntelligenceIndicatorClient(s.creds.SubscriptionID, cred, nil)
    if err != nil {
        return fmt.Errorf("could not create TI client: %v", err)
    }

    yesterday := time.Now().AddDate(0, 0, -1)

    hasMorePages := true
    totalDeleted := 0
    pages := 0

    for hasMorePages {
        logger.WithField("page", pages).WithField("total_deleted", totalDeleted).Info("retrieving expired TI indicators")
        pager := tiClient.NewQueryIndicatorsPager(s.creds.ResourceGroup, s.creds.WorkspaceName, insights.ThreatIntelligenceFilteringCriteria{
            IncludeDisabled: to.Ptr(true),
            MaxValidUntil:   to.Ptr(yesterday.Format(time.RFC3339)),
            PageSize:        to.Ptr[int32](1000),
        }, nil)

        if !pager.More() {
            hasMorePages = false
            break
        }

        totalPageDeleted := 0

        for pager.More() {
            pages += 1

            items, err := pager.NextPage(ctx)
            if err != nil {
                logger.WithError(err).Debug("could not request more TI items")
                break
            }

            for _, value := range items.Value {
                ti := value.GetThreatIntelligenceInformation()

                logger.WithField("id", *ti.ID).
                    WithField("progress", fmt.Sprintf("%d/%d", totalPageDeleted, len(items.Value))).
                    WithField("total_deleted", totalDeleted).
                    Debug("deleting TI indicator")

                if _, err := tiClient.Delete(ctx, s.creds.ResourceGroup, s.creds.WorkspaceName, *ti.Name, nil); err != nil {
                    logger.WithError(err).WithField("id", *ti.ID).Error("could not delete TI indicator")
                    continue
                }

                totalDeleted += 1
                totalPageDeleted += 1
            }
        }
    }

    if totalDeleted > 0 {
        logger.WithField("num", totalDeleted).Info("deleted expired TI indicators")
    } else {
        logger.Info("no TI indicators to delete")
    }

    return nil
}

Proposal

Create, if supported by the Azure API, a method to mass-delete TI items by providing an array of UUIDs.

[github-actions[bot]]

Thank you for your feedback. This has been routed to the support team for assistance.

[hazcod]

Just peeking in here again, I sometimes have a pipeline that needs to run 1 hour because it can't mass-delete expired TI indicators.