IndexOutOfBoundsException error in SemanticTokensInspectorManager

[CppCXY]

java.lang.IndexOutOfBoundsException: Wrong offset: 10104; documentLength: 277
    at com.intellij.openapi.editor.impl.DocumentImpl.insertString(DocumentImpl.java:535)
    at com.redhat.devtools.lsp4ij.LSPIJUtils.applyEdit(LSPIJUtils.java:1002)
    at com.redhat.devtools.lsp4ij.features.semanticTokens.inspector.SemanticTokensInspectorManager.doFormat(SemanticTokensInspectorManager.java:92)
    at com.redhat.devtools.lsp4ij.features.semanticTokens.inspector.SemanticTokensInspectorManager.lambda$format$1(SemanticTokensInspectorManager.java:77)
    at com.intellij.openapi.command.WriteCommandAction.lambda$runWriteCommandAction$5(WriteCommandAction.java:342)
    at com.intellij.openapi.command.WriteCommandAction$BuilderImpl.lambda$compute$3(WriteCommandAction.java:161)
    at com.intellij.openapi.command.WriteCommandAction$BuilderImpl.lambda$doRunWriteCommandAction$1(WriteCommandAction.java:144)
    at com.intellij.openapi.application.impl.RwLockHolder.runWriteAction(RwLockHolder.kt:344)
    at com.intellij.openapi.application.impl.ApplicationImpl.runWriteAction(ApplicationImpl.java:883)
    at com.intellij.openapi.command.WriteCommandAction$BuilderImpl.lambda$doRunWriteCommandAction$2(WriteCommandAction.java:142)
    at com.intellij.openapi.command.impl.CoreCommandProcessor.executeCommand(CoreCommandProcessor.java:225)
    at com.intellij.openapi.command.impl.CoreCommandProcessor.executeCommand(CoreCommandProcessor.java:187)
    at com.intellij.openapi.command.WriteCommandAction$BuilderImpl.doRunWriteCommandAction(WriteCommandAction.java:151)
    at com.intellij.openapi.command.WriteCommandAction$BuilderImpl.lambda$run$0(WriteCommandAction.java:122)
    at com.intellij.openapi.application.TransactionGuardImpl.runWithWritingAllowed(TransactionGuardImpl.java:209)
    at com.intellij.openapi.application.TransactionGuardImpl.access$100(TransactionGuardImpl.java:22)
    at com.intellij.openapi.application.TransactionGuardImpl$1.run(TransactionGuardImpl.java:191)
    at com.intellij.openapi.application.impl.RwLockHolder.runIntendedWriteActionOnCurrentThread(RwLockHolder.kt:204)
    at com.intellij.openapi.application.impl.ApplicationImpl.runIntendedWriteActionOnCurrentThread(ApplicationImpl.java:830)
    at com.intellij.openapi.application.impl.ApplicationImpl$2.run(ApplicationImpl.java:419)
    at com.intellij.openapi.application.impl.LaterInvocator$1.run(LaterInvocator.java:101)
    at com.intellij.openapi.application.impl.RwLockHolder.runWithEnabledImplicitRead(RwLockHolder.kt:138)
    at com.intellij.openapi.application.impl.RwLockHolder.runWithImplicitRead(RwLockHolder.kt:129)
    at com.intellij.openapi.application.impl.ApplicationImpl.runWithImplicitRead(ApplicationImpl.java:1152)
    at com.intellij.openapi.application.impl.FlushQueue.doRun(FlushQueue.java:81)
    at com.intellij.openapi.application.impl.FlushQueue.runNextEvent(FlushQueue.java:123)
    at com.intellij.openapi.application.impl.FlushQueue.flushNow(FlushQueue.java:43)
    at java.desktop/java.awt.event.InvocationEvent.dispatch(InvocationEvent.java:318)
    at java.desktop/java.awt.EventQueue.dispatchEventImpl(EventQueue.java:792)
    at java.desktop/java.awt.EventQueue$3.run(EventQueue.java:739)
    at java.desktop/java.awt.EventQueue$3.run(EventQueue.java:733)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/java.security.ProtectionDomain$JavaSecurityAccessImpl.doIntersectionPrivilege(ProtectionDomain.java:86)
    at java.desktop/java.awt.EventQueue.dispatchEvent(EventQueue.java:761)
    at com.intellij.ide.IdeEventQueue.defaultDispatchEvent(IdeEventQueue.kt:698)
    at com.intellij.ide.IdeEventQueue._dispatchEvent$lambda$12(IdeEventQueue.kt:593)
    at com.intellij.openapi.application.impl.RwLockHolder.runWithoutImplicitRead(RwLockHolder.kt:105)
    at com.intellij.ide.IdeEventQueue._dispatchEvent(IdeEventQueue.kt:593)
    at com.intellij.ide.IdeEventQueue.access$_dispatchEvent(IdeEventQueue.kt:77)
    at com.intellij.ide.IdeEventQueue$dispatchEvent$processEventRunnable$1$1$1.compute(IdeEventQueue.kt:362)
    at com.intellij.ide.IdeEventQueue$dispatchEvent$processEventRunnable$1$1$1.compute(IdeEventQueue.kt:361)
    at com.intellij.openapi.progress.impl.CoreProgressManager.computePrioritized(CoreProgressManager.java:843)
    at com.intellij.ide.IdeEventQueue$dispatchEvent$processEventRunnable$1$1.invoke(IdeEventQueue.kt:361)
    at com.intellij.ide.IdeEventQueue$dispatchEvent$processEventRunnable$1$1.invoke(IdeEventQueue.kt:356)
    at com.intellij.ide.IdeEventQueueKt.performActivity$lambda$1(IdeEventQueue.kt:1021)
    at com.intellij.openapi.application.TransactionGuardImpl.performActivity(TransactionGuardImpl.java:106)
    at com.intellij.ide.IdeEventQueueKt.performActivity(IdeEventQueue.kt:1021)
    at com.intellij.ide.IdeEventQueue.dispatchEvent$lambda$7(IdeEventQueue.kt:356)
    at com.intellij.openapi.application.impl.RwLockHolder.runIntendedWriteActionOnCurrentThread(RwLockHolder.kt:209)
    at com.intellij.openapi.application.impl.ApplicationImpl.runIntendedWriteActionOnCurrentThread(ApplicationImpl.java:830)
    at com.intellij.ide.IdeEventQueue.dispatchEvent(IdeEventQueue.kt:398)
    at java.desktop/java.awt.EventDispatchThread.pumpOneEventForFilters(EventDispatchThread.java:207)
    at java.desktop/java.awt.EventDispatchThread.pumpEventsForFilter(EventDispatchThread.java:128)
    at java.desktop/java.awt.EventDispatchThread.pumpEventsForHierarchy(EventDispatchThread.java:117)
    at java.desktop/java.awt.EventDispatchThread.pumpEvents(EventDispatchThread.java:113)
    at java.desktop/java.awt.EventDispatchThread.pumpEvents(EventDispatchThread.java:105)
    at java.desktop/java.awt.EventDispatchThread.run(EventDispatchThread.java:92)

[fbricon]

@CppCXY Have you managed to reproduce this after multiline token support was added?

[angelozerr]

I think we could have this problem again when user will type quickly and the document text could be unsynchronized with the semantic tokens.

Another use cases is when semantic tokens will return large value like 99999.

We need to take care of this 2 usecase

[fbricon]

Reproduced it after several edits undone with Ctrl-Z: