Cannot reuse ProjectionOperation inside $lookup pipeline [DATAMONGO-2615]

Daniel Theuke opened DATAMONGO-2615 and commented

If I try to use a ProjectionOperation with andExpression("$foo.bar") inside a custom $lookup document I get the following error:

java.lang.IllegalArgumentException: Invalid reference '$meta.name'!
    at org.springframework.data.mongodb.core.aggregation.ExposedFieldsAggregationOperationContext.getReference(ExposedFieldsAggregationOperationContext.java:114) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
    at org.springframework.data.mongodb.core.aggregation.ExposedFieldsAggregationOperationContext.getReference(ExposedFieldsAggregationOperationContext.java:86) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
    at org.springframework.data.mongodb.core.aggregation.AggregationExpressionTransformer$AggregationExpressionTransformationContext.getFieldReference(AggregationExpressionTransformer.java:82) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
    at org.springframework.data.mongodb.core.aggregation.SpelExpressionTransformer$CompoundExpressionNodeConversion.convert(SpelExpressionTransformer.java:541) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
    at org.springframework.data.mongodb.core.aggregation.SpelExpressionTransformer.transform(SpelExpressionTransformer.java:113) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
    at org.springframework.data.mongodb.core.aggregation.SpelExpressionTransformer.transform(SpelExpressionTransformer.java:105) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
    at org.springframework.data.mongodb.core.aggregation.ProjectionOperation$ExpressionProjectionOperationBuilder$ExpressionProjection.toMongoExpression(ProjectionOperation.java:438) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
    at org.springframework.data.mongodb.core.aggregation.ProjectionOperation$ExpressionProjectionOperationBuilder$ExpressionProjection.toDocument(ProjectionOperation.java:433) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
    at org.springframework.data.mongodb.core.aggregation.ProjectionOperation.toDocument(ProjectionOperation.java:261) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]

Code:

private static final AggregationOperation EXTRACT_ELEM = replaceRoot(ElemContainer._data);
private static final ProjectionOperation ELEM_AS_METADATA = Aggregation.project()
            .andInclude("id")
            .andInclude("type")
            .andExpression("$meta.name").as("name");
    private static final ProjectionOperation ELEM_AS_NESTED_METADATA = ELEM_AS_METADATA
            // Child ids
            .and(VariableOperators.mapItemsOf(
                    ObjectOperators.valueOf("childIds").toArray())
                    .as("this")
                    .andApply(ctx -> new Document()
                            .append("type", "$$this.k")
                            .append("id", "$$this.v")))
            .as("children"); // Named children to be overwritten later on

    private static final AggregationOperation LOOKUP_CHILDREN = ctx -> new Document()
        .append("$lookup", new Document()
            .append("from", "configuration")
            .append("let", new Document()
                    .append("childIds", "$children"))
            .append("pipeline", asList(
                    new Document() // Join condition
                            .append("$match", new Document()
                                    .append("$expr", new Document()
                                            .append("$in", asList("$_id", "$$childIds")))),
                    EXTRACT_ELEM,
                    ELEM_AS_METADATA.toDocument(ctx.continueOnMissingFieldReference()))) // <-- BOOM
            .append("as", "children"); // Overwrite "children" field

// --------------------------

final TypedAggregation<ElemContainer> aggregation = newAggregate(
                match(criteria),
                EXTRACT_ELEM,
                ELEM_AS_NESTED_METADATA,
                LOOKUP_CHILDREN);
        return this.mongoTemplate.aggregate(aggregation, NestedElementMetadata.class).getMappedResults();

In this case the continueOnMissingFieldReference() doesn't seem to work at all. (ExposedFieldsAggregationOperationContext doesn't implement that feature at all)

The default implementation should probably be rewritten to return a delegating variant that wraps the related call similar to RelaxedTypeBasedAggregationOperationContext.

Or is there another way to create a nested context?

Affects: 3.0.3 (Neumann SR3)

This feature is really necessary for the performance of query.

Its works for me. But it must be in this package.

org.springframework.data.mongodb.core.aggregation

public class ExcludeFieldsAggregation implements AggregationOperation
{
    private String[] fields;

    public ExcludeFieldsAggregation(String...fields)
    {
        this.fields = fields;
    }

    @Override
    public Document toDocument(AggregationOperationContext context)
    {
        Document result = new Document();
        for (String field: fields) {
            result.put(field,0);    
        }
        return new Document("$project", result);
    }

}

public class IncludeFieldsAggregation implements AggregationOperation
{
    private String[] fields;

    public IncludeFieldsAggregation(String...fields)
    {
        this.fields = fields;
    }

    @Override
    public Document toDocument(AggregationOperationContext context)
    {
        Document result = new Document();
        for (String field: fields) {
            result.put(field,1);    
        }
        return new Document("$project", result);
    }

}

package org.springframework.data.mongodb.core.aggregation;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

import org.bson.Document;
import org.springframework.data.mongodb.core.aggregation.ExposedFields.ExposedField;
import org.springframework.data.mongodb.core.aggregation.FieldsExposingAggregationOperation.InheritsFieldsAggregationOperation;
import org.springframework.data.mongodb.core.query.CriteriaDefinition;
import org.springframework.util.Assert;

import org.springframework.data.mongodb.core.aggregation.ExcludeFieldsAggregation;
import org.springframework.data.mongodb.core.aggregation.IncludeFieldsAggregation;

public class LookupPipelineOperation implements AggregationOperation, FieldsExposingAggregationOperation, InheritsFieldsAggregationOperation
{

    private Field from;
    private Field localField;
    private Field foreignField;
    private ExposedField as;

    private Document lets = null;
    private List<AggregationOperation> pipeline = new ArrayList<>();
    private String[] includeFields;
    private String[] excludeFields;

    /**
     * Creates a new {@link MatchOperation} for the given {@link CriteriaDefinition}.
     *
     * @param criteriaDefinition must not be {@literal null}.
     */
    public LookupPipelineOperation()
    {}

    public LookupPipelineOperation let(String fieldName, String expression)
    {
        Assert.notNull(fieldName, "Field name must not be null!");
        Assert.notNull(expression, "Expression must not be null!");
        if (this.lets==null) {
            this.lets = new Document();
        }
        this.lets.append(fieldName, expression);
        return this;
    }

    public LookupPipelineOperation from(String from)
    {
        Assert.notNull(from, "From must not be null!");
        this.from = Fields.field(from);
        return this;
    }

    public LookupPipelineOperation as(String as)
    {
        Assert.notNull(as, "As must not be null!");
        this.as = new ExposedField(Fields.field(as), true);
        return this;
    }

    public LookupPipelineOperation include(String... fields)
    {
        Assert.notNull(fields, "Fields must not be null!");
        this.includeFields = fields;
        return this;
    }

    public LookupPipelineOperation exclude(String... fields)
    {
        Assert.notNull(fields, "Fields must not be null!");
        this.excludeFields = fields;
        return this;
    }

    public LookupPipelineOperation pipeline(AggregationOperation pipeline)
    {
        Assert.notNull(pipeline, "Pipeline must not be null!");
        if (this.pipeline == null)
        {
            this.pipeline = new ArrayList<>();
        }
        this.pipeline.add(pipeline);
        return this;
    }

    public LookupPipelineOperation pipeline(List<AggregationOperation> pipeline)
    {
        Assert.notNull(pipeline, "Pipeline must not be null!");
        this.pipeline = pipeline;
        return this;
    }

    public LookupPipelineOperation localField(String localField)
    {
        Assert.notNull(localField, "localField must not be null!");
        this.localField = Fields.field(localField);
        return this;
    }

    public LookupPipelineOperation foreignField(String foreignField)
    {
        Assert.notNull(foreignField, "foreignField must not be null!");
        this.foreignField = Fields.field(foreignField);
        return this;
    }

    /*
     * (non-Javadoc)
     *
     * @see org.springframework.data.mongodb.core.aggregation.AggregationOperation#toDocument(org.
     * springframework.data.mongodb.core.aggregation.AggregationOperationContext)
     */
    @SuppressWarnings("deprecation")
    @Override
    public Document toDocument(final AggregationOperationContext context)
    {
        context.continueOnMissingFieldReference();
        if (this.excludeFields != null)
        {
            pipeline.add(0, new ExcludeFieldsAggregation(this.excludeFields));
        }
        if (this.includeFields != null)
        {
            pipeline.add(0, new IncludeFieldsAggregation(this.includeFields));
        }

        Document lookupDoc = new Document();

        lookupDoc.append("from", from.getTarget());

        if (this.localField!=null)
        {
            lookupDoc.append("localField", localField.getTarget());
        }

        if (this.foreignField!=null)
        {
            lookupDoc.append("foreignField", foreignField.getTarget());
        }

        if (lets != null && !lets.isEmpty())
        {
            lookupDoc.append("let", lets);
        }
        if (this.pipeline != null && this.pipeline.size() > 0)
        {
            lookupDoc.append("pipeline", pipeline.stream().map(op -> op.toDocument(context)).collect(Collectors.toList()));
        }
        if (this.as != null)
        {
            lookupDoc.append("as", as.getTarget());
        }
        return new Document(getOperator(), lookupDoc);
    }

    @Override
    public String getOperator()
    {
        return "$lookup";
    }

    @Override
    public ExposedFields getFields()
    {
        return ExposedFields.from(as);
    }
}

spring-projects / spring-data-mongodb

Cannot reuse ProjectionOperation inside $lookup pipeline [DATAMONGO-2615] #3469