search

carla-simulator / carla

Open-source simulator for autonomous driving research.
http://carla.org
MIT License
11.4k stars 3.7k forks source link

[Instance Segmentation] make an instance segmentation fisheye camera #6368

Open JosenJiang opened 1 year ago

JosenJiang commented 1 year ago

Based on annaornatskaya's PR, fisheye RGB camera can work on Carla.

39

For getting some ground truth, I try to make an instance segmentation fisheye camera. I imitated the code of InstanceSegmentationCamera to make an InstanceSegmentationFisheye. But it looks weird.

163

I still haven't figured out how InstanceSegmentationCamera knows the labels of the actors in the image. Any Suggestions are welcome!

CARLA version: 0.9.14 Platform/OS: ubuntu 18.04

FActorDefinition AInstanceSegmentationFisheye::GetSensorDefinition()
{
  auto Definition = UActorBlueprintFunctionLibrary::MakeGenericSensorDefinition(
      TEXT("camera"),
      TEXT("instance_segmentation_fisheye"));

  FActorVariation XSize;
  XSize.Id = TEXT("x_size");
  XSize.Type = EActorAttributeType::Float;
  XSize.RecommendedValues = { TEXT("1000.0") };
  XSize.bRestrictToRecommended = false;

  FActorVariation YSize;
  YSize.Id = TEXT("y_size");
  YSize.Type = EActorAttributeType::Float;
  YSize.RecommendedValues = { TEXT("900.0") };
  YSize.bRestrictToRecommended = false;

  FActorVariation MaxAngle;
  MaxAngle.Id = TEXT("max_angle");
  MaxAngle.Type = EActorAttributeType::Float;
  MaxAngle.RecommendedValues = { TEXT("200.0") };
  MaxAngle.bRestrictToRecommended = false;

  FActorVariation Fx;
  Fx.Id = TEXT("f_x");
  Fx.Type = EActorAttributeType::Float;
  Fx.RecommendedValues = { TEXT("300.0") };
  Fx.bRestrictToRecommended = false;

  FActorVariation Fy;
  Fy.Id = TEXT("f_y");
  Fy.Type = EActorAttributeType::Float;
  Fy.RecommendedValues = { TEXT("300.0") };
  Fy.bRestrictToRecommended = false;

  FActorVariation Cx;
  Cx.Id = TEXT("c_x");
  Cx.Type = EActorAttributeType::Float;
  Cx.RecommendedValues = { TEXT("600.0") };
  Cx.bRestrictToRecommended = false;

  FActorVariation Cy;
  Cy.Id = TEXT("c_y");
  Cy.Type = EActorAttributeType::Float;
  Cy.RecommendedValues = { TEXT("400.0") };
  Cy.bRestrictToRecommended = false;

  FActorVariation D1;
  D1.Id = TEXT("d_1");
  D1.Type = EActorAttributeType::Float;
  D1.RecommendedValues = { TEXT("0.0") };
  D1.bRestrictToRecommended = false;

  FActorVariation D2;
  D2.Id = TEXT("d_2");
  D2.Type = EActorAttributeType::Float;
  D2.RecommendedValues = { TEXT("0.0") };
  D2.bRestrictToRecommended = false;

  FActorVariation D3;
  D3.Id = TEXT("d_3");
  D3.Type = EActorAttributeType::Float;
  D3.RecommendedValues = { TEXT("0.0") };
  D3.bRestrictToRecommended = false;

  FActorVariation D4;
  D4.Id = TEXT("d_4");
  D4.Type = EActorAttributeType::Float;
  D4.RecommendedValues = { TEXT("0.0") };
  D4.bRestrictToRecommended = false;

  Definition.Variations.Append({ XSize, YSize, MaxAngle, Fx, Fy, Cx, Cy, D1, D2, D3, D4});

  return Definition;
}

AInstanceSegmentationFisheye::AInstanceSegmentationFisheye(const FObjectInitializer &ObjectInitializer)
  : Super(ObjectInitializer)
{
//   AddPostProcessingMaterial(TEXT("Material'/Carla/PostProcessingMaterials/PhysicLensDistortion.PhysicLensDistortion'"));
}

void AInstanceSegmentationFisheye::SetUpSceneCaptureComponentCube(USceneCaptureComponentCube &SceneCapture)
{
    Super::SetUpSceneCaptureComponentCube(SceneCapture);

    ApplyViewMode(VMI_Unlit, true, SceneCapture.ShowFlags);

    SceneCapture.ShowFlags.SetNotDrawTaggedComponents(false); // TaggedComponent detects this and sets view relevance for proxy material

    SceneCapture.ShowFlags.SetAtmosphere(false);

    SceneCapture.PrimitiveRenderMode = ESceneCapturePrimitiveRenderMode::PRM_UseShowOnlyList;

    TArray<UObject *> TaggedComponents;
    GetObjectsOfClass(UTaggedComponent::StaticClass(), TaggedComponents, false, EObjectFlags::RF_ClassDefaultObject, EInternalObjectFlags::AllFlags);

    TArray<UPrimitiveComponent *> ShowOnlyComponents;
    for (UObject *Object : TaggedComponents)
    {
        UPrimitiveComponent *Component = Cast<UPrimitiveComponent>(Object);
        SceneCapture.ShowOnlyComponents.Emplace(Component);
    }
}

void AInstanceSegmentationFisheye::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
// Add the view information every tick. It's only used for one tick and then removed by the streamer.
{
  TRACE_CPUPROFILER_EVENT_SCOPE(AInstanceSegmentationFisheye::PostPhysTick);

  USceneCaptureComponentCube* SceneCapture = GetCaptureComponentCube();
  TArray<UObject *> TaggedComponents;
  GetObjectsOfClass(UTaggedComponent::StaticClass(), TaggedComponents, false, EObjectFlags::RF_ClassDefaultObject, EInternalObjectFlags::AllFlags);

  SceneCapture->ClearShowOnlyComponents();
  for (UObject *Object : TaggedComponents) {
    UPrimitiveComponent *Component = Cast<UPrimitiveComponent>(Object);
    SceneCapture->ShowOnlyComponents.Emplace(Component);
  }

    SendPixelsInRenderThread(*this, MaxAngle, XSize, YSize, Fx, Fy, Cx, Cy, D1, D2, D3, D4);
}
AHAHA124 commented 1 year ago

Hello, As a beginner in learning Linux and CARLA, i have read annaornatskaya's PR https://github.com/carla-simulator/carla/pull/3755 and downloaded .zip document, but i don't know how to use the .patch document. Can you give me some advice? Thanks very much.

JosenJiang commented 1 year ago

Hello, As a beginner in learning Linux and CARLA, i have read annaornatskaya's PR #3755 and downloaded .zip document, but i don't know how to use the .patch document. Can you give me some advice? Thanks very much.

The command is git apply <.patch file>. You can search how to apply git patch for detail on Google.

lucalazzaroni commented 1 year ago

Does the patch still work on the newest Carla version?

AHAHA124 commented 1 year ago

Does the patch still work on the newest Carla version?

I think it can work in newest Carla version, if your Unreal Engine version is 4.24. This patch is based on the Unreal Engine version 4.24. But my teamwork should work on 4.26 and it can't be used directly used, so i haven't used it yet. If you have the same version problem you can get more information in https://github.com/carla-simulator/carla/issues/5678. I haven't resolved this version issue yet. If you can adapt this patch to the new UE version, please reply to me. Thanks very much.

dongkeyan commented 1 year ago

I use carla 0.9.13 and UE 4.26.2 ,and add fisheye sensor model to carla. You can follow issue No.3755 and No.5678 to solve problem.

But I have a problem when using fisheye sensor model. I deploy one fisheye on ego car ,the resolution is 3840x2160. The FPS of carla is 14, but I use RGB sensor model, the fps is 28. So how to solve the low fps problem?

GPU:RTX 4090, CPU: AMD 7950

gouchaoer commented 1 year ago

I have RTX 4090 and have 15fps too. Have you sovled the problem?

TurtleZhong commented 1 year ago

@JosenJiang @dongkeyan @AHAHA124 I follow this PR, and I use carla 0.9.13 and UE 4.26.2, when I use git apply *.patch, It turns out crashed ad line99, so I change the file mannually. I complied suscess, but when I run mannual_control.py, It crashed with IndexError: blueprint 'sensor.camera.fisheye' not found So I wanna konw how you solved the problem!

baituhuangyu commented 1 year ago

Based on annaornatskaya's PR, fisheye RGB camera can work on Carla.

39

For getting some ground truth, I try to make an instance segmentation fisheye camera. I imitated the code of InstanceSegmentationCamera to make an InstanceSegmentationFisheye. But it looks weird.

163

I still haven't figured out how InstanceSegmentationCamera knows the labels of the actors in the image. Any Suggestions are welcome!

CARLA version: 0.9.14 Platform/OS: ubuntu 18.04

FActorDefinition AInstanceSegmentationFisheye::GetSensorDefinition()
{
  auto Definition = UActorBlueprintFunctionLibrary::MakeGenericSensorDefinition(
      TEXT("camera"),
      TEXT("instance_segmentation_fisheye"));

  FActorVariation XSize;
  XSize.Id = TEXT("x_size");
  XSize.Type = EActorAttributeType::Float;
  XSize.RecommendedValues = { TEXT("1000.0") };
  XSize.bRestrictToRecommended = false;

  FActorVariation YSize;
  YSize.Id = TEXT("y_size");
  YSize.Type = EActorAttributeType::Float;
  YSize.RecommendedValues = { TEXT("900.0") };
  YSize.bRestrictToRecommended = false;

  FActorVariation MaxAngle;
  MaxAngle.Id = TEXT("max_angle");
  MaxAngle.Type = EActorAttributeType::Float;
  MaxAngle.RecommendedValues = { TEXT("200.0") };
  MaxAngle.bRestrictToRecommended = false;

  FActorVariation Fx;
  Fx.Id = TEXT("f_x");
  Fx.Type = EActorAttributeType::Float;
  Fx.RecommendedValues = { TEXT("300.0") };
  Fx.bRestrictToRecommended = false;

  FActorVariation Fy;
  Fy.Id = TEXT("f_y");
  Fy.Type = EActorAttributeType::Float;
  Fy.RecommendedValues = { TEXT("300.0") };
  Fy.bRestrictToRecommended = false;

  FActorVariation Cx;
  Cx.Id = TEXT("c_x");
  Cx.Type = EActorAttributeType::Float;
  Cx.RecommendedValues = { TEXT("600.0") };
  Cx.bRestrictToRecommended = false;

  FActorVariation Cy;
  Cy.Id = TEXT("c_y");
  Cy.Type = EActorAttributeType::Float;
  Cy.RecommendedValues = { TEXT("400.0") };
  Cy.bRestrictToRecommended = false;

  FActorVariation D1;
  D1.Id = TEXT("d_1");
  D1.Type = EActorAttributeType::Float;
  D1.RecommendedValues = { TEXT("0.0") };
  D1.bRestrictToRecommended = false;

  FActorVariation D2;
  D2.Id = TEXT("d_2");
  D2.Type = EActorAttributeType::Float;
  D2.RecommendedValues = { TEXT("0.0") };
  D2.bRestrictToRecommended = false;

  FActorVariation D3;
  D3.Id = TEXT("d_3");
  D3.Type = EActorAttributeType::Float;
  D3.RecommendedValues = { TEXT("0.0") };
  D3.bRestrictToRecommended = false;

  FActorVariation D4;
  D4.Id = TEXT("d_4");
  D4.Type = EActorAttributeType::Float;
  D4.RecommendedValues = { TEXT("0.0") };
  D4.bRestrictToRecommended = false;

  Definition.Variations.Append({ XSize, YSize, MaxAngle, Fx, Fy, Cx, Cy, D1, D2, D3, D4});

  return Definition;
}

AInstanceSegmentationFisheye::AInstanceSegmentationFisheye(const FObjectInitializer &ObjectInitializer)
  : Super(ObjectInitializer)
{
//   AddPostProcessingMaterial(TEXT("Material'/Carla/PostProcessingMaterials/PhysicLensDistortion.PhysicLensDistortion'"));
}

void AInstanceSegmentationFisheye::SetUpSceneCaptureComponentCube(USceneCaptureComponentCube &SceneCapture)
{
    Super::SetUpSceneCaptureComponentCube(SceneCapture);

    ApplyViewMode(VMI_Unlit, true, SceneCapture.ShowFlags);

    SceneCapture.ShowFlags.SetNotDrawTaggedComponents(false); // TaggedComponent detects this and sets view relevance for proxy material

    SceneCapture.ShowFlags.SetAtmosphere(false);

    SceneCapture.PrimitiveRenderMode = ESceneCapturePrimitiveRenderMode::PRM_UseShowOnlyList;

    TArray<UObject *> TaggedComponents;
    GetObjectsOfClass(UTaggedComponent::StaticClass(), TaggedComponents, false, EObjectFlags::RF_ClassDefaultObject, EInternalObjectFlags::AllFlags);

    TArray<UPrimitiveComponent *> ShowOnlyComponents;
    for (UObject *Object : TaggedComponents)
    {
        UPrimitiveComponent *Component = Cast<UPrimitiveComponent>(Object);
        SceneCapture.ShowOnlyComponents.Emplace(Component);
    }
}

void AInstanceSegmentationFisheye::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
// Add the view information every tick. It's only used for one tick and then removed by the streamer.
{
  TRACE_CPUPROFILER_EVENT_SCOPE(AInstanceSegmentationFisheye::PostPhysTick);

  USceneCaptureComponentCube* SceneCapture = GetCaptureComponentCube();
  TArray<UObject *> TaggedComponents;
  GetObjectsOfClass(UTaggedComponent::StaticClass(), TaggedComponents, false, EObjectFlags::RF_ClassDefaultObject, EInternalObjectFlags::AllFlags);

  SceneCapture->ClearShowOnlyComponents();
  for (UObject *Object : TaggedComponents) {
    UPrimitiveComponent *Component = Cast<UPrimitiveComponent>(Object);
    SceneCapture->ShowOnlyComponents.Emplace(Component);
  }

    SendPixelsInRenderThread(*this, MaxAngle, XSize, YSize, Fx, Fy, Cx, Cy, D1, D2, D3, D4);
}

Dear Josen, Have you sovled this problem?