Skip to content

Revert "Keep instance segmentation refreshes fast with many annotated components" #171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
WouterJansen merged 1 commit into from
Oct 5, 2026
Merged

Revert "Keep instance segmentation refreshes fast with many annotated components" #171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 2 additions & 12 deletions Unreal/Plugins/AirSim/Source/Annotation/ObjectAnnotator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1049,7 +1049,6 @@ bool FObjectAnnotator::PaintRGBComponent(UMeshComponent* component, const FColor
AnnotationComponent->MarkRenderStateDirty();
UPrimitiveComponent* PrimitiveComponent = Cast<UPrimitiveComponent>(AnnotationComponent);
annotation_component_list_.Add(PrimitiveComponent);
new_annotation_components_.Add(PrimitiveComponent);
return true;
}

Expand Down Expand Up @@ -1094,8 +1093,7 @@ bool FObjectAnnotator::PaintTextureComponent(UMeshComponent* component, const FS
AnnotationComponent->bVisibleInRealTimeSkyCaptures = false;
AnnotationComponent->MarkRenderStateDirty();
UPrimitiveComponent* PrimitiveComponent = Cast<UPrimitiveComponent>(AnnotationComponent);
annotation_component_list_.Add(PrimitiveComponent);
new_annotation_components_.Add(PrimitiveComponent);
annotation_component_list_.Add(PrimitiveComponent);
return true;
}

Expand Down Expand Up @@ -1185,7 +1183,6 @@ void FObjectAnnotator::UpdateAnnotationComponents(UWorld* World)
EInternalObjectFlags ExclusionInternalFlags = EInternalObjectFlags::None;
GetObjectsOfClass(UAnnotationComponent::StaticClass(), UObjectList, bIncludeDerivedClasses, ExclusionFlags, ExclusionInternalFlags);

TSet<TWeakObjectPtr<UPrimitiveComponent>> listed(annotation_component_list_); // Contains on the list was quadratic
for (UObject* Object : UObjectList)
{
if (!IsValid(Object)) continue;
Expand All @@ -1194,13 +1191,11 @@ void FObjectAnnotator::UpdateAnnotationComponents(UWorld* World)
FName componentFName = *Component->GetName();
FString componentName = componentFName.ToString();
if (Component->GetWorld() == World
&& !listed.Contains(Component)
&& !annotation_component_list_.Contains(Component)
&& componentName.Contains(name_)
&& !componentName.Contains("annotation_sphere"))
{
annotation_component_list_.Add(Component);
new_annotation_components_.Add(Component);
listed.Add(Component);
}
}

Expand All @@ -1218,10 +1213,6 @@ TArray<TWeakObjectPtr<UPrimitiveComponent>> FObjectAnnotator::GetAnnotationComp
return annotation_component_list_;
}

TArray<TWeakObjectPtr<UPrimitiveComponent>> FObjectAnnotator::TakeNewAnnotationComponents() {
return MoveTemp(new_annotation_components_);
}

std::vector<std::string> FObjectAnnotator::GetAllComponentNames() {
std::vector<std::string> retval;
TMap<FString, uint32> nameToColorIndexMapTemp = name_to_color_index_map_;
Expand Down Expand Up @@ -1273,7 +1264,6 @@ void FObjectAnnotator::EndPlay() {
name_to_color_index_map_.Empty();
name_to_component_map_.Empty();
annotation_component_list_.Empty();
new_annotation_components_.Empty();
name_to_gammacorrected_color_map_.Empty();
name_to_value_map_.Empty();
name_to_texture_path_map_.Empty();
Expand Down
3 changes: 0 additions & 3 deletions Unreal/Plugins/AirSim/Source/Annotation/ObjectAnnotator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,6 @@ class AIRSIM_API FObjectAnnotator

void UpdateAnnotationComponents(UWorld* World);
TArray<TWeakObjectPtr<UPrimitiveComponent>> GetAnnotationComponents();
// The annotation components added since the previous call, so cameras can hide them without going over all of them
TArray<TWeakObjectPtr<UPrimitiveComponent>> TakeNewAnnotationComponents();

static void SetViewForAnnotationRender(FEngineShowFlags& show_flags);

Expand Down Expand Up @@ -124,6 +122,5 @@ class AIRSIM_API FObjectAnnotator
TMap<FString, UMeshComponent*> name_to_component_map_;
TMap<UMeshComponent*, FString> component_to_name_map_;
TArray<TWeakObjectPtr<UPrimitiveComponent>> annotation_component_list_;
TArray<TWeakObjectPtr<UPrimitiveComponent>> new_annotation_components_; // added since TakeNewAnnotationComponents
};

53 changes: 10 additions & 43 deletions Unreal/Plugins/AirSim/Source/PIPCamera.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,24 +11,6 @@
#include <exception>
#include "AirBlueprintLib.h"

//adds the annotation components missing from a list of hidden components, and drops the entries of
//components that no longer exist; a set keeps this linear in the number of components (AddUnique for
//each one was quadratic, which stalls the game thread once thousands are annotated, and the stale
//entries made the lists grow without bound)
static void hideComponents(TArray<TWeakObjectPtr<UPrimitiveComponent>>& hidden, const TArray<TWeakObjectPtr<UPrimitiveComponent>>& components)
{
hidden.RemoveAll([](const TWeakObjectPtr<UPrimitiveComponent>& component) { return !component.IsValid(); });
TSet<TWeakObjectPtr<UPrimitiveComponent>> present;
present.Reserve(hidden.Num() + components.Num());
present.Append(hidden);
for (const TWeakObjectPtr<UPrimitiveComponent>& component : components) {
bool already_hidden;
present.Add(component, &already_hidden);
if (!already_hidden)
hidden.Add(component);
}
}

//CinemAirSim
APIPCamera::APIPCamera(const FObjectInitializer& ObjectInitializer)
: Super(ObjectInitializer
Expand Down Expand Up @@ -547,28 +529,11 @@ void APIPCamera::updateInstanceSegmentationAnnotation(TArray<TWeakObjectPtr<UPri
if(!only_hide)
captures_[Utils::toNumeric(ImageType::Segmentation)]->ShowOnlyComponents = ComponentList;
APlayerController* controller = this->GetWorld()->GetFirstPlayerController();
hideComponents(captures_[Utils::toNumeric(ImageType::Scene)]->HiddenComponents, ComponentList);
hideComponents(captures_[Utils::toNumeric(ImageType::Lighting)]->HiddenComponents, ComponentList);
hideComponents(controller->HiddenPrimitiveComponents, ComponentList);
}

void APIPCamera::updateInstanceSegmentationAnnotation(TArray<TWeakObjectPtr<UPrimitiveComponent> >& ComponentList,
const TArray<TWeakObjectPtr<UPrimitiveComponent> >& NewComponents, uint64 RefreshSerial, bool only_hide) {
if(!only_hide)
captures_[Utils::toNumeric(ImageType::Segmentation)]->ShowOnlyComponents = ComponentList;
if (instance_segmentation_serial_ == RefreshSerial)
return;
const bool has_previous = instance_segmentation_serial_ != 0 && instance_segmentation_serial_ + 1 == RefreshSerial;
for (ImageType type : {ImageType::Scene, ImageType::Lighting}) {
TArray<TWeakObjectPtr<UPrimitiveComponent>>& hidden = captures_[Utils::toNumeric(type)]->HiddenComponents;
if (has_previous)
hidden.Append(NewComponents);
else
hideComponents(hidden, ComponentList);
if (hidden.Num() > 2 * ComponentList.Num() + 1024)
hidden.RemoveAll([](const TWeakObjectPtr<UPrimitiveComponent>& component) { return !component.IsValid(); });
}
instance_segmentation_serial_ = RefreshSerial;
for(TWeakObjectPtr<UPrimitiveComponent> component : ComponentList) {
captures_[Utils::toNumeric(ImageType::Scene)]->HiddenComponents.AddUnique(component);
captures_[Utils::toNumeric(ImageType::Lighting)]->HiddenComponents.AddUnique(component);
controller->HiddenPrimitiveComponents.AddUnique(component);
}
}

void APIPCamera::updateAnnotation(TArray<TWeakObjectPtr<UPrimitiveComponent> >& ComponentList, FString annotation_name, bool only_hide) {
Expand All @@ -582,9 +547,11 @@ void APIPCamera::updateAnnotation(TArray<TWeakObjectPtr<UPrimitiveComponent> >&
captures_[annotator_name_to_index_map_[annotation_name]]->ShowOnlyComponents.Add(sphere_annotation_component_map_[annotation_name]);
}
APlayerController* controller = this->GetWorld()->GetFirstPlayerController();
hideComponents(captures_[Utils::toNumeric(ImageType::Scene)]->HiddenComponents, ComponentList);
hideComponents(captures_[Utils::toNumeric(ImageType::Lighting)]->HiddenComponents, ComponentList);
hideComponents(controller->HiddenPrimitiveComponents, ComponentList);
for (TWeakObjectPtr<UPrimitiveComponent> component : ComponentList) {
captures_[Utils::toNumeric(ImageType::Scene)]->HiddenComponents.AddUnique(component);
captures_[Utils::toNumeric(ImageType::Lighting)]->HiddenComponents.AddUnique(component);
controller->HiddenPrimitiveComponents.AddUnique(component);
}
}

void APIPCamera::addAnnotationCamera(FString name, FObjectAnnotator::AnnotatorType type, float max_view_distance)
Expand Down
6 changes: 0 additions & 6 deletions Unreal/Plugins/AirSim/Source/PIPCamera.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,6 @@ class AIRSIM_API APIPCamera : public ACineCameraActor //CinemAirSim
void setCameraTypeUpdate(ImageType type, bool nodisplay, std::string annotation_name = "");
void setCameraOrientation(const FRotator& rotator);
void updateInstanceSegmentationAnnotation(TArray<TWeakObjectPtr<UPrimitiveComponent> >& ComponentList, bool only_hide=false);
// The same, given the components added since the previous refresh: refresh serial numbers tell
// whether this camera has hidden all earlier ones, otherwise all are hidden. The player's view is
// left to the caller.
void updateInstanceSegmentationAnnotation(TArray<TWeakObjectPtr<UPrimitiveComponent> >& ComponentList,
const TArray<TWeakObjectPtr<UPrimitiveComponent> >& NewComponents, uint64 RefreshSerial, bool only_hide=false);
bool GetAnnotationNameExist(std::string annotation_name);
void updateAnnotation(TArray<TWeakObjectPtr<UPrimitiveComponent> >& ComponentList, FString annotation_name, bool only_hide = false);
void updateAnnotationComponentsFromObjectAnnotator(FObjectAnnotator& annotator, const FString& annotator_name = TEXT("InstanceSegmentation"));
Expand All @@ -96,7 +91,6 @@ class AIRSIM_API APIPCamera : public ACineCameraActor //CinemAirSim
msr::airlib::Pose getPose() const;

private: //members
uint64 instance_segmentation_serial_ = 0; // the last segmentation refresh whose components the captures hide
UPROPERTY()
UMaterialParameterCollection* distortion_param_collection_;
UPROPERTY()
Expand Down
33 changes: 5 additions & 28 deletions Unreal/Plugins/AirSim/Source/SimMode/SimModeBase.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
#include "Runtime/Launch/Resources/Version.h"
#include "UObject/ConstructorHelpers.h"
#include "Kismet/GameplayStatics.h"
#include "GameFramework/PlayerController.h"
#include "Misc/OutputDeviceNull.h"
#include "Engine/World.h"
#include "Kismet/BlueprintFunctionLibrary.h"
Expand Down Expand Up @@ -1250,28 +1249,6 @@ void ASimModeBase::ForceUpdateAnnotation(FString annotation_name) {
void ASimModeBase::updateInstanceSegmentationAnnotation() {
instance_segmentation_annotator_.UpdateAnnotationComponents(this->GetWorld());
TArray<TWeakObjectPtr<UPrimitiveComponent>> current_segmentation_components = instance_segmentation_annotator_.GetAnnotationComponents();
// Cameras hide only the components added since the previous refresh: hiding all of them in every
// camera at each refresh stalls the game once thousands are annotated
const TArray<TWeakObjectPtr<UPrimitiveComponent>> new_segmentation_components = instance_segmentation_annotator_.TakeNewAnnotationComponents();
const uint64 refresh_serial = ++instance_segmentation_refresh_serial_;
if (APlayerController* controller = GetWorld()->GetFirstPlayerController()) {
if (instance_segmentation_hidden_controller_ != controller) {
instance_segmentation_hidden_controller_ = controller;
instance_segmentation_controller_serial_ = 0;
}
TArray<TWeakObjectPtr<UPrimitiveComponent>>& hidden = controller->HiddenPrimitiveComponents;
if (instance_segmentation_controller_serial_ != 0 && instance_segmentation_controller_serial_ + 1 == refresh_serial)
hidden.Append(new_segmentation_components);
else {
TSet<TWeakObjectPtr<UPrimitiveComponent>> present(hidden);
for (const TWeakObjectPtr<UPrimitiveComponent>& component : current_segmentation_components)
if (!present.Contains(component))
hidden.Add(component);
}
instance_segmentation_controller_serial_ = refresh_serial;
if (hidden.Num() > 2 * current_segmentation_components.Num() + 1024)
hidden.RemoveAll([](const TWeakObjectPtr<UPrimitiveComponent>& component) { return !component.IsValid(); });
}

TArray<AActor*> cameras_found;
UAirBlueprintLib::RunCommandOnGameThread([this, &cameras_found]() {
Expand All @@ -1280,7 +1257,7 @@ void ASimModeBase::updateInstanceSegmentationAnnotation() {
if (cameras_found.Num() >= 0) {
for (auto camera_actor : cameras_found) {
APIPCamera* cur_camera = static_cast<APIPCamera*>(camera_actor);
cur_camera->updateInstanceSegmentationAnnotation(current_segmentation_components, new_segmentation_components, refresh_serial);
cur_camera->updateInstanceSegmentationAnnotation(current_segmentation_components);
}
}
TArray<AActor*> lidar_cameras_found;
Expand All @@ -1296,13 +1273,13 @@ void ASimModeBase::updateInstanceSegmentationAnnotation() {
}
if (CameraDirector != nullptr) {
if(CameraDirector->getFpvCamera() != nullptr)
CameraDirector->getFpvCamera()->updateInstanceSegmentationAnnotation(current_segmentation_components, new_segmentation_components, refresh_serial, true);
CameraDirector->getFpvCamera()->updateInstanceSegmentationAnnotation(current_segmentation_components, true);
if (CameraDirector->getExternalCamera() != nullptr)
CameraDirector->getExternalCamera()->updateInstanceSegmentationAnnotation(current_segmentation_components, new_segmentation_components, refresh_serial, true);
CameraDirector->getExternalCamera()->updateInstanceSegmentationAnnotation(current_segmentation_components, true);
if (CameraDirector->getBackupCamera() != nullptr)
CameraDirector->getBackupCamera()->updateInstanceSegmentationAnnotation(current_segmentation_components, new_segmentation_components, refresh_serial, true);
CameraDirector->getBackupCamera()->updateInstanceSegmentationAnnotation(current_segmentation_components, true);
if (CameraDirector->getFrontCamera() != nullptr)
CameraDirector->getFrontCamera()->updateInstanceSegmentationAnnotation(current_segmentation_components, new_segmentation_components, refresh_serial, true);
CameraDirector->getFrontCamera()->updateInstanceSegmentationAnnotation(current_segmentation_components, true);
}
}

Expand Down
4 changes: 0 additions & 4 deletions Unreal/Plugins/AirSim/Source/SimMode/SimModeBase.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,10 +334,6 @@ class AIRSIM_API ASimModeBase : public AActor
TMap<FString, ALight*> world_lights_;

private:
// Instance segmentation refreshes (see updateInstanceSegmentationAnnotation)
uint64 instance_segmentation_refresh_serial_ = 0;
TWeakObjectPtr<APlayerController> instance_segmentation_hidden_controller_;
uint64 instance_segmentation_controller_serial_ = 0;
void InitializeInstanceSegmentation();
void InitializeAnnotation();
void AddAnnotatorCamera(FString name, FObjectAnnotator::AnnotatorType type, float max_view_distance = -1.0f);
Expand Down
Loading