[InstCombine] Infinite loop is happening with `-no-enable-noundef-analysis` option

[Saldivarcher]

After commit 8a156d1 this sample code (smallest I could get it), causes an infinte loop. The noundef-analysis flag has since been disabled, so by default the infinite loop doesn't happen.

I tracked down the issue to be that before the commit, this branch was getting hit because the previous check would always hit this return.

#define NULL 0

typedef struct ASN1_TEMPLATE_st ASN1_TEMPLATE;

struct ASN1_ITEM_st {
    char itype;                 /* The item type, primitive, SEQUENCE, CHOICE
                                 * or extern */
    long utype;                 /* underlying type */
    const ASN1_TEMPLATE *templates; /* If SEQUENCE or CHOICE this contains
                                     * the contents */
    long tcount;                /* Number of templates if SEQUENCE or CHOICE */
    const void *funcs;          /* functions that handle this type */
    long size;                  /* Structure size (usually) */
};

typedef struct ASN1_ITEM_st ASN1_ITEM;
typedef struct ASN1_VALUE_st ASN1_VALUE;

typedef const ASN1_ITEM ASN1_ITEM_EXP;

struct ASN1_TEMPLATE_st {
    unsigned long flags;        /* Various flags */
    long tag;                   /* tag, not used if no tagging */
    unsigned long offset;       /* Offset of this field in structure */
    ASN1_ITEM_EXP *item;        /* Relevant ASN1_ITEM or ASN1_ADB */
};

#define STACK_OF(type) struct stack_st_##type

const ASN1_TEMPLATE *asn1_do_adb(ASN1_VALUE **pval, const ASN1_TEMPLATE *tt,
                                 int nullerr);

static int asn1_template_ex_i2d(ASN1_VALUE **pval, unsigned char **out,
                                const ASN1_TEMPLATE *tt, int tag, int aclass);
int ASN1_object_size(int constructed, int length, int tag);
void ASN1_put_object(unsigned char **pp, int constructed, int length,
                     int tag, int xclass);
int ASN1_put_eoc(unsigned char **pp);

typedef struct stack_st {
    int num;
    char **data;
    int sorted;
    int num_alloc;
    int (*comp) (const void *, const void *);
} _STACK;                       /* Use STACK_OF(...) instead */

#define CHECKED_STACK_OF(type, p) \
    ((_STACK*) (1 ? p : (STACK_OF(type)*)0))

int sk_num(const _STACK *);
#define SKM_sk_num(type, st) \
        sk_num(CHECKED_STACK_OF(type, st))
#define sk_ASN1_VALUE_num(st) SKM_sk_num(ASN1_VALUE, (st))

void *sk_value(const _STACK *, int);
#define SKM_sk_value(type, st,i) \
        ((type *)sk_value(CHECKED_STACK_OF(type, st), i))
#define sk_ASN1_VALUE_value(st, i) SKM_sk_value(ASN1_VALUE, (st), (i))

#define ASN1_TFLG_NDEF          (0x1<<11)
#define ASN1_TFLG_SK_MASK       (0x3 << 1)
#define ASN1_TFLG_EXPTAG        (0x2 << 3)

#define ASN1_ITEM_ptr(iptr) (iptr)

int ASN1_item_ex_i2d(ASN1_VALUE **pval, unsigned char **out,
                     const ASN1_ITEM *it, int tag, int aclass)
{
    const ASN1_TEMPLATE *tt = NULL;
    int i;

    for (i = 0, tt = it->templates; i < it->tcount; tt++, i++) {
        const ASN1_TEMPLATE *seqtt;        ASN1_VALUE **pseqval;
        int tmplen;
        seqtt = asn1_do_adb(pval, tt, 1);
        tmplen = asn1_template_ex_i2d(pseqval, NULL, seqtt, -1, aclass);
    }

    for (i = 0, tt = it->templates; i < it->tcount; tt++, i++) {
        const ASN1_TEMPLATE *seqtt;
        ASN1_VALUE **pseqval;
        seqtt = asn1_do_adb(pval, tt, 1);
        asn1_template_ex_i2d(pseqval, out, seqtt, -1, aclass);
    }
    return 0;
}

static int asn1_template_ex_i2d(ASN1_VALUE **pval, unsigned char **out,
                                const ASN1_TEMPLATE *tt, int tag, int iclass)
{
    int i, ret, flags, ttag, tclass, ndef;
    ASN1_VALUE *tval;
    flags = tt->flags;

    if ((flags & ASN1_TFLG_NDEF) && (iclass & ASN1_TFLG_NDEF))
        ndef = 2;
    else
        ndef = 1;

    if (flags & ASN1_TFLG_SK_MASK) {
        STACK_OF(ASN1_VALUE) *sk = (STACK_OF(ASN1_VALUE) *)*pval;
        int sktag;
        int skcontlen, sklen;
        ASN1_VALUE *skitem;

        for (i = 0; i < sk_ASN1_VALUE_num(sk); i++) {
            skitem = sk_ASN1_VALUE_value(sk, i);
        }
        sklen = ASN1_object_size(ndef, skcontlen, sktag);

        if (flags & ASN1_TFLG_EXPTAG)
            ret = ASN1_object_size(ndef, sklen, ttag);

        if (!out || ret == -1)
            return ret;

        if (ndef == 2) {
            ASN1_put_eoc(out);
        }

        return ret;
    }

    return 0;
}

$ clang -c reproduce.c -march=znver2 -O3 -Xclang -no-enable-noundef-analysis

[llvmbot]

@llvm/issue-subscribers-clang-codegen

[nunoplopes]

cc @hyeongyukim