THUDM / CodeGeeX

CodeGeeX: An Open Multilingual Code Generation Model (KDD 2023)
https://codegeex.cn
Apache License 2.0
8.18k stars 598 forks source link

jetbrain plugin Context length limit 代码超出上下文上限? #212

Open yhyu13 opened 1 year ago

yhyu13 commented 1 year ago

我输入的让codegeex解释的代码:

{
        // If we're going to stall the RHI thread for one query, we should stall it for all of them.
        // !(View.bIgnoreExistingQueries || bHZBOcclusion) is the code path that calls GetQueryForReading.
        const bool bShouldStallRHIThread = bSingleThreaded && GOcclusionSingleRHIThreadStall && !GSupportsParallelOcclusionQueries && IsInRenderingThread() && !(View.bIgnoreExistingQueries || bHZBOcclusion);
        FScopedRHIThreadStaller StallRHIThread(FRHICommandListExecutor::GetImmediateCommandList(), bShouldStallRHIThread);

        SCOPED_NAMED_EVENT_F(TEXT("forEach over %d entries"), FColor::Magenta, NumToProcess);

        //if we are load balanced then we iterate only the set bits, and the ranges have been pre-selected to evenly distribute set bits among the tasks with no overlaps.
        //if not, then the entire array is evenly divided by range.
#if BALANCE_LOAD
        for (FSceneSetBitIterator BitIt(View.PrimitiveVisibilityMap, StartIndex); BitIt && (NumProcessed < NumToProcess); ++BitIt, ++NumProcessed)
#else
        for (TBitArray<SceneRenderingBitArrayAllocator>::FIterator BitIt(View.PrimitiveVisibilityMap, StartIndex); BitIt && (NumProcessed < NumToProcess); ++BitIt, ++NumProcessed)
#endif
        {
#if !BALANCE_LOAD       
            if (!View.PrimitiveVisibilityMap.AccessCorrespondingBit(BitIt))
            {
                continue;
            }
#endif
            int32 Index = BitIt.GetIndex();

            const uint8 OcclusionFlags = Scene->PrimitiveOcclusionFlags[Index];

            if ((OcclusionFlags & EOcclusionFlags::CanBeOccluded) == 0)
            {
                View.PrimitiveDefinitelyUnoccludedMap.AccessCorrespondingBit(BitIt) = true;
                continue;
            }

            //we can't allow the prim history insertion array to realloc or it will invalidate pointers in the other output arrays.
            const bool bCanAllocPrimHistory = bSingleThreaded || InsertPrimitiveOcclusionHistory->Num() < InsertPrimitiveOcclusionHistory->Max();

#if WITH_EDITOR
            bool bCanBeOccluded = true;
            if (GIsEditor)
            {
                if (Scene->PrimitivesSelected[Index])
                {
                    // to render occluded outline for selected objects
                    bCanBeOccluded = false;
                }
            }
#else
            constexpr bool bCanBeOccluded = true;
#endif

            int32 NumSubQueries = 1;
            bool bSubQueries = false;
            const TArray<FBoxSphereBounds>* SubBounds = nullptr;

            check(Params.SubIsOccluded);
            TArray<bool>& SubIsOccluded = *Params.SubIsOccluded;
            int32 SubIsOccludedStart = SubIsOccluded.Num();
            if ((OcclusionFlags & EOcclusionFlags::HasSubprimitiveQueries) && GAllowSubPrimitiveQueries && !View.bDisableQuerySubmissions)
            {
                FPrimitiveSceneProxy* Proxy = Scene->Primitives[Index]->Proxy;
                SubBounds = Proxy->GetOcclusionQueries(&View);
                NumSubQueries = SubBounds->Num();
                bSubQueries = true;
                if (!NumSubQueries)
                {
                    View.PrimitiveVisibilityMap.AccessCorrespondingBit(BitIt) = false;
                    continue;
                }
                SubIsOccluded.Reserve(NumSubQueries);
            }

            bool bAllSubOcclusionStateIsDefinite = true;
            bool bAllSubOccluded = true;
            FPrimitiveComponentId PrimitiveId = Scene->PrimitiveComponentIds[Index];

            for (int32 SubQuery = 0; SubQuery < NumSubQueries; SubQuery++)
            {
                FPrimitiveOcclusionHistory* PrimitiveOcclusionHistory = ViewPrimitiveOcclusionHistory.Find(FPrimitiveOcclusionHistoryKey(PrimitiveId, SubQuery));

                bool bIsOccluded = false;
                bool bOcclusionStateIsDefinite = false;

                if (!PrimitiveOcclusionHistory)
                {
                    // If the primitive doesn't have an occlusion history yet, create it.
                    if (bSingleThreaded)
                    {
                        // In singlethreaded mode we can safely modify the view's history directly.
                        PrimitiveOcclusionHistory = &ViewPrimitiveOcclusionHistory[
                            ViewPrimitiveOcclusionHistory.Add(FPrimitiveOcclusionHistory(PrimitiveId, SubQuery))
                        ];
                    }
                    else if (bCanAllocPrimHistory)
                    {
                        // In multithreaded mode we have to buffer the new histories and add them to the view during a post-combine
                        PrimitiveOcclusionHistory = &(*InsertPrimitiveOcclusionHistory)[
                            InsertPrimitiveOcclusionHistory->Add(FPrimitiveOcclusionHistory(PrimitiveId, SubQuery))
                        ];
                    }

                    // If the primitive hasn't been visible recently enough to have a history, treat it as unoccluded this frame so it will be rendered as an occluder and its true occlusion state can be determined.
                    // already set bIsOccluded = false;

                    // Flag the primitive's occlusion state as indefinite, which will force it to be queried this frame.
                    // The exception is if the primitive isn't occludable, in which case we know that it's definitely unoccluded.
                    bOcclusionStateIsDefinite = !bCanBeOccluded;
                }
                else
                {
                    if (View.bIgnoreExistingQueries)
                    {
                        // If the view is ignoring occlusion queries, the primitive is definitely unoccluded.
                        // already set bIsOccluded = false;
                        bOcclusionStateIsDefinite = View.bDisableQuerySubmissions;
                    }
                    else if (bCanBeOccluded)
                    {
                        if (bUseOcclusionFeedback)
                        {
                            bIsOccluded = OcclusionFeedback.IsOccluded(PrimitiveId);
                            bOcclusionStateIsDefinite = true;
                        }
                        else if (bHZBOcclusion)
                        {
                            if (HZBOcclusionTests.IsValidFrame(PrimitiveOcclusionHistory->LastTestFrameNumber))
                            {
                                bIsOccluded = !HZBOcclusionTests.IsVisible(PrimitiveOcclusionHistory->HZBTestIndex);
                                bOcclusionStateIsDefinite = true;
                            }
                        }
                        else
                        {
                            // Read the occlusion query results.
                            uint64 NumSamples = 0;
                            bool bGrouped = false;
                            FRHIRenderQuery* PastQuery = PrimitiveOcclusionHistory->GetQueryForReading(OcclusionFrameCounter, NumBufferedFrames, ReadBackLagTolerance, bGrouped);
                            if (PastQuery)
                            {
                                //int32 RefCount = PastQuery.GetReference()->GetRefCount();
                                // NOTE: RHIGetOcclusionQueryResult should never fail when using a blocking call, rendering artifacts may show up.
                                if (RHIGetRenderQueryResult(PastQuery, NumSamples, true))
                                {
                                    // we render occlusion without MSAA
                                    uint32 NumPixels = (uint32)NumSamples;

                                    // The primitive is occluded if none of its bounding box's pixels were visible in the previous frame's occlusion query.
                                    bIsOccluded = (NumPixels == 0);

                                    if (!bIsOccluded)
                                    {
                                        checkSlow(View.OneOverNumPossiblePixels > 0.0f);
                                        PrimitiveOcclusionHistory->LastPixelsPercentage = NumPixels * View.OneOverNumPossiblePixels;
                                    }
                                    else
                                    {
                                        PrimitiveOcclusionHistory->LastPixelsPercentage = 0.0f;
                                    }

                                    // Flag the primitive's occlusion state as definite if it wasn't grouped.
                                    bOcclusionStateIsDefinite = !bGrouped;
                                }
                                else
                                {
                                    // If the occlusion query failed, treat the primitive as visible.  
                                    // already set bIsOccluded = false;
                                }
                            }
                            else
                            {
                                if (NumBufferedFrames > 1 || GRHIMaximumReccommendedOustandingOcclusionQueries < MAX_int32)
                                {
                                    // If there's no occlusion query for the primitive, assume it is whatever it was last frame
                                    bIsOccluded = PrimitiveOcclusionHistory->WasOccludedLastFrame;
                                    bOcclusionStateIsDefinite = PrimitiveOcclusionHistory->OcclusionStateWasDefiniteLastFrame;
                                }
                                else
                                {
                                    // If there's no occlusion query for the primitive, set it's visibility state to whether it has been unoccluded recently.
                                    bIsOccluded = (PrimitiveOcclusionHistory->LastProvenVisibleTime + GEngine->PrimitiveProbablyVisibleTime < CurrentRealTime);
                                    // the state was definite last frame, otherwise we would have ran a query
                                    bOcclusionStateIsDefinite = true;
                                }
                                if (bIsOccluded)
                                {
                                    PrimitiveOcclusionHistory->LastPixelsPercentage = 0.0f;
                                }
                                else
                                {
                                    PrimitiveOcclusionHistory->LastPixelsPercentage = GEngine->MaxOcclusionPixelsFraction;
                                }
                            }
                        }

                        if (GVisualizeOccludedPrimitives && OcclusionPDI && bIsOccluded)
                        {
                            const FBoxSphereBounds& Bounds = bSubQueries ? (*SubBounds)[SubQuery] : Scene->PrimitiveOcclusionBounds[Index];
                            DrawWireBox(OcclusionPDI, Bounds.GetBox(), FColor(50, 255, 50), SDPG_Foreground);
                        }
                    }
                    else
                    {
                        // Primitives that aren't occludable are considered definitely unoccluded.
                        // already set bIsOccluded = false;
                        bOcclusionStateIsDefinite = true;
                    }

                    if (bClearQueries)
                    {
                        if (bSingleThreaded)
                        {
                            PrimitiveOcclusionHistory->ReleaseQuery(OcclusionFrameCounter, NumBufferedFrames);
                        }
                        else
                        {
                            if (PrimitiveOcclusionHistory->GetQueryForEviction(OcclusionFrameCounter, NumBufferedFrames))
                            {
                                QueriesToRelease->Add(PrimitiveOcclusionHistory);
                            }
                        }
                    }
                }

                if (PrimitiveOcclusionHistory)
                {
                    if (bSubmitQueries && bCanBeOccluded)
                    {
                        bool bSkipNewlyConsidered = false;

                        if (bNewlyConsideredBBoxExpandActive)
                        {
                            if (!PrimitiveOcclusionHistory->BecameEligibleForQueryCooldown && OcclusionFrameCounter - PrimitiveOcclusionHistory->LastConsideredFrameNumber > uint32(GFramesNotOcclusionTestedToExpandBBoxes))
                            {
                                PrimitiveOcclusionHistory->BecameEligibleForQueryCooldown = GFramesToExpandNewlyOcclusionTestedBBoxes;
                            }

                            bSkipNewlyConsidered = !!PrimitiveOcclusionHistory->BecameEligibleForQueryCooldown;

                            if (bSkipNewlyConsidered)
                            {
                                PrimitiveOcclusionHistory->BecameEligibleForQueryCooldown--;
                            }
                        }

                        bool bAllowBoundsTest;
                        const FBoxSphereBounds OcclusionBounds = (bSubQueries ? (*SubBounds)[SubQuery] : Scene->PrimitiveOcclusionBounds[Index]).ExpandBy(GExpandAllTestedBBoxesAmount + (bSkipNewlyConsidered ? GExpandNewlyOcclusionTestedBBoxesAmount : 0.0));
                        if (FVector::DistSquared(ViewOrigin, OcclusionBounds.Origin) < NeverOcclusionTestDistanceSquared)
                        {
                            bAllowBoundsTest = false;
                        }
                        else if (View.bHasNearClippingPlane)
                        {
                            bAllowBoundsTest = View.NearClippingPlane.PlaneDot(OcclusionBounds.Origin) <
                                -(FVector::BoxPushOut(View.NearClippingPlane, OcclusionBounds.BoxExtent));

                        }
                        else if (!View.IsPerspectiveProjection())
                        {
                            // Transform parallel near plane
                            static_assert((int32)ERHIZBuffer::IsInverted != 0, "Check equation for culling!");
                            bAllowBoundsTest = View.WorldToScreen(OcclusionBounds.Origin).Z - View.ViewMatrices.GetProjectionMatrix().M[2][2] * OcclusionBounds.SphereRadius < 1;
                        }
                        else
                        {
                            bAllowBoundsTest = OcclusionBounds.SphereRadius < HALF_WORLD_MAX;
                        }

                        if (bAllowBoundsTest)
                        {
                            PrimitiveOcclusionHistory->LastTestFrameNumber = OcclusionFrameCounter;

                            if (bUseOcclusionFeedback)
                            {
                                const FVector BoundOrigin = OcclusionBounds.Origin + View.ViewMatrices.GetPreViewTranslation();
                                const FVector BoundExtent = OcclusionBounds.BoxExtent;
                                OcclusionFeedback.AddPrimitive(PrimitiveId, BoundOrigin, BoundExtent, *DynamicVertexBufferIfSingleThreaded);
                            }
                            else if (bHZBOcclusion)
                            {
                                // Always run
                                if (bSingleThreaded)
                                {
                                    PrimitiveOcclusionHistory->HZBTestIndex = HZBOcclusionTests.AddBounds(OcclusionBounds.Origin, OcclusionBounds.BoxExtent);
                                }
                                else
                                {
                                    HZBBoundsToAdd->Emplace(PrimitiveOcclusionHistory, OcclusionBounds.Origin, OcclusionBounds.BoxExtent);
                                }
                            }
                            else
                            {
                                // decide if a query should be run this frame
                                bool bRunQuery, bGroupedQuery;

                                if (!bSubQueries && // sub queries are never grouped, we assume the custom code knows what it is doing and will group internally if it wants
                                    (OcclusionFlags & EOcclusionFlags::AllowApproximateOcclusion))
                                {
                                    if (bIsOccluded)
                                    {
                                        // Primitives that were occluded the previous frame use grouped queries.
                                        bGroupedQuery = true;
                                        bRunQuery = true;
                                    }
                                    else if (bOcclusionStateIsDefinite)
                                    {
                                        bGroupedQuery = false;
                                        float Rnd = GOcclusionRandomStream.GetFraction();
                                        if (GRHISupportsExactOcclusionQueries)
                                        {
                                            float FractionMultiplier = FMath::Max(PrimitiveOcclusionHistory->LastPixelsPercentage / GEngine->MaxOcclusionPixelsFraction, 1.0f);
                                            bRunQuery = (FractionMultiplier * Rnd) < GEngine->MaxOcclusionPixelsFraction;
                                        }
                                        else
                                        {
                                            bRunQuery = CurrentRealTime - PrimitiveOcclusionHistory->LastProvenVisibleTime > PrimitiveProbablyVisibleTime * (0.5f * 0.25f * Rnd);
                                        }
                                    }
                                    else
                                    {
                                        bGroupedQuery = false;
                                        bRunQuery = true;
                                    }
                                }
                                else
                                {
                                    // Primitives that need precise occlusion results use individual queries.
                                    bGroupedQuery = false;
                                    bRunQuery = true;
                                }

                                if (bRunQuery)
                                {
                                    const FVector BoundOrigin = OcclusionBounds.Origin + View.ViewMatrices.GetPreViewTranslation();
                                    const FVector BoundExtent = OcclusionBounds.BoxExtent;

                                    if (bSingleThreaded)
                                    {
                                        checkSlow(DynamicVertexBufferIfSingleThreaded);

                                        if (GRHIMaximumReccommendedOustandingOcclusionQueries < MAX_int32 && !bGroupedQuery)
                                        {
                                            QueriesToAdd->Emplace(FPrimitiveOcclusionHistoryKey(PrimitiveId, SubQuery), BoundOrigin, BoundExtent, PrimitiveOcclusionHistory->LastQuerySubmitFrame());
                                        }
                                        else
                                        {
                                            PrimitiveOcclusionHistory->SetCurrentQuery(OcclusionFrameCounter,
                                                bGroupedQuery ?
                                                View.GroupedOcclusionQueries.BatchPrimitive(BoundOrigin, BoundExtent, *DynamicVertexBufferIfSingleThreaded) :
                                                View.IndividualOcclusionQueries.BatchPrimitive(BoundOrigin, BoundExtent, *DynamicVertexBufferIfSingleThreaded),
                                                NumBufferedFrames,
                                                bGroupedQuery,
                                                Params.bNeedsScanOnRead
                                            );
                                        }
                                    }
                                    else
                                    {
                                        QueriesToAdd->Emplace(PrimitiveOcclusionHistory, BoundOrigin, BoundExtent, bGroupedQuery);
                                    }
                                }
                            }
                        }
                        else
                        {
                            // If the primitive's bounding box intersects the near clipping plane, treat it as definitely unoccluded.
                            bIsOccluded = false;
                            bOcclusionStateIsDefinite = true;
                        }
                    }
                    // Set the primitive's considered time to keep its occlusion history from being trimmed.
                    PrimitiveOcclusionHistory->LastConsideredTime = CurrentRealTime;
                    if (!bIsOccluded && bOcclusionStateIsDefinite)
                    {
                        PrimitiveOcclusionHistory->LastProvenVisibleTime = CurrentRealTime;
                    }
                    PrimitiveOcclusionHistory->LastConsideredFrameNumber = OcclusionFrameCounter;
                    PrimitiveOcclusionHistory->WasOccludedLastFrame = bIsOccluded;
                    PrimitiveOcclusionHistory->OcclusionStateWasDefiniteLastFrame = bOcclusionStateIsDefinite;
                }

                if (bSubQueries)
                {
                    SubIsOccluded.Add(bIsOccluded);
                    if (!bIsOccluded)
                    {
                        bAllSubOccluded = false;
                    }
                    if (bIsOccluded || !bOcclusionStateIsDefinite)
                    {
                        bAllSubOcclusionStateIsDefinite = false;
                    }
                }
                else
                {

                    if (bIsOccluded)
                    {
                        View.PrimitiveVisibilityMap.AccessCorrespondingBit(BitIt) = false;
                        STAT(NumOccludedPrimitives++);
                    }
                    else if (bOcclusionStateIsDefinite)
                    {
                        View.PrimitiveDefinitelyUnoccludedMap.AccessCorrespondingBit(BitIt) = true;
                    }
                }
            }

            if (bSubQueries)
            {
                if (SubIsOccluded.Num() > 0)
                {
                    FPrimitiveSceneProxy* Proxy = Scene->Primitives[Index]->Proxy;
                    Proxy->AcceptOcclusionResults(&View, &SubIsOccluded, SubIsOccludedStart, SubIsOccluded.Num() - SubIsOccludedStart);
                }

                if (bAllSubOccluded)
                {
                    View.PrimitiveVisibilityMap.AccessCorrespondingBit(BitIt) = false;
                    STAT(NumOccludedPrimitives++);
                }
                else if (bAllSubOcclusionStateIsDefinite)
                {
                    View.PrimitiveDefinitelyUnoccludedMap.AccessCorrespondingBit(BitIt) = true;
                }
            }
        }
    }

Submit之后,模型实际收到的的代码:

>SetCurrentQuery(OcclusionFrameCounter,
                                                bGroupedQuery ?
                                                View.GroupedOcclusionQueries.BatchPrimitive(BoundOrigin, BoundExtent, *DynamicVertexBufferIfSingleThreaded) :
                                                View.IndividualOcclusionQueries.BatchPrimitive(BoundOrigin, BoundExtent, *DynamicVertexBufferIfSingleThreaded),
                                                NumBufferedFrames,
                                                bGroupedQuery,
                                                Params.bNeedsScanOnRead
                                            );
                                        }
                                    }
                                    else
                                    {
                                        QueriesToAdd->Emplace(PrimitiveOcclusionHistory, BoundOrigin, BoundExtent, bGroupedQuery);
                                    }
                                }
                            }
                        }
                        else
                        {
                            // If the primitive's bounding box intersects the near clipping plane, treat it as definitely unoccluded.
                            bIsOccluded = false;
                            bOcclusionStateIsDefinite = true;
                        }
                    }
                    // Set the primitive's considered time to keep its occlusion history from being trimmed.
                    PrimitiveOcclusionHistory->LastConsideredTime = CurrentRealTime;
                    if (!bIsOccluded && bOcclusionStateIsDefinite)
                    {
                        PrimitiveOcclusionHistory->LastProvenVisibleTime = CurrentRealTime;
                    }
                    PrimitiveOcclusionHistory->LastConsideredFrameNumber = OcclusionFrameCounter;
                    PrimitiveOcclusionHistory->WasOccludedLastFrame = bIsOccluded;
                    PrimitiveOcclusionHistory->OcclusionStateWasDefiniteLastFrame = bOcclusionStateIsDefinite;
                }

                if (bSubQueries)
                {
                    SubIsOccluded.Add(bIsOccluded);
                    if (!bIsOccluded)
                    {
                        bAllSubOccluded = false;
                    }
                    if (bIsOccluded || !bOcclusionStateIsDefinite)
                    {
                        bAllSubOcclusionStateIsDefinite = false;
                    }
                }
                else
                {

                    if (bIsOccluded)
                    {
                        View.PrimitiveVisibilityMap.AccessCorrespondingBit(BitIt) = false;
                        STAT(NumOccludedPrimitives++);
                    }
                    else if (bOcclusionStateIsDefinite)
                    {
                        View.PrimitiveDefinitelyUnoccludedMap.AccessCorrespondingBit(BitIt) = true;
                    }
                }
            }

            if (bSubQueries)
            {
                if (SubIsOccluded.Num() > 0)
                {
                    FPrimitiveSceneProxy* Proxy = Scene->Primitives[Index]->Proxy;
                    Proxy->AcceptOcclusionResults(&View, &SubIsOccluded, SubIsOccludedStart, SubIsOccluded.Num() - SubIsOccludedStart);
                }

                if (bAllSubOccluded)
                {
                    View.PrimitiveVisibilityMap.AccessCorrespondingBit(BitIt) = false;
                    STAT(NumOccludedPrimitives++);
                }
                else if (bAllSubOcclusionStateIsDefinite)
                {
                    View.PrimitiveDefinitelyUnoccludedMap.AccessCorrespondingBit(BitIt) = true;
                }
            }
        }
    }

有很大一部分输入都被截断了。模型的输出也是基于被截断了的输入解释的。 谢谢