java.lang.IllegalAccessError

[githubcheng2978]

1. i redefine org.apache.http.impl.client.CloseableHttpClient in java agent
2. the agent has follow code: ClassInjector.UsingUnsafe.Factory factory = ClassInjector.UsingUnsafe.Factory.resolve(instrumentation); factory.make(null, null).injectRaw(classesTypeMap); agentBuilder = agentBuilder.with(new AgentBuilder.InjectionStrategy.UsingUnsafe.OfFactory(factory));
3. the application will load class org.apache.http.impl.client.CloseableHttpClient with customer classloader. code: ` import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URL; import java.net.URLClassLoader; import java.util.Enumeration; import java.util.Properties;

public class IClassLoader extends URLClassLoader { private static final String version = "1.0.0.5"; private static IClassLoader instance; private OpenClassLoader parent; private static final boolean isForceUpdate = false; private static final String standardClasspathConfigFile = "/sar/sar.properties"; private static final String standardClasspathDir = "/sar/jars/"; private static String workDir = System.getProperty("user.home");

private IClassLoader(URL[] urls) {
    this(urls, IClassLoader.class.getClassLoader());
}

private IClassLoader(URL[] urls, ClassLoader parent) {
    super(urls, (ClassLoader)null);
    this.parent = new OpenClassLoader(parent);
}

public static synchronized IClassLoader getInstance() throws IOException {
    if (instance != null) {
        return instance;
    } else {
        instance = getInstance("/sar/sar.properties");
        return instance;
    }
}

private static IClassLoader getInstance(String classpathConfigFile) throws IOException {
    Class clazz = IClassLoader.class;
    InputStream in = clazz.getResourceAsStream(classpathConfigFile);
    Properties config = new Properties();
    String sarDir = workDir + "/sar/jars/".replace("/", File.separator) + "1.0.0.5" + File.separator;
    File sarDirFile = new File(sarDir);
    if (!sarDirFile.exists() && !sarDirFile.mkdirs()) {
        throw new RuntimeException("create sar dir fail:" + sarDir);
    } else {
        Logger.error("init sar ：" + sarDirFile);

        IClassLoader var34;
        try {
            config.load(in);
            String content = config.getProperty("jars");
            Logger.error("sar中jar包列表如下：" + content);
            String[] jars = content.split(";");
            URL[] urls = new URL[jars.length];

            for(int i = 0; i < jars.length; ++i) {
                String jarName = jars[i];
                jarName = jarName.trim();
                String resource = "/sar/jars/" + jarName;
                URL url = clazz.getResource(resource);
                if (url == null) {
                    throw new RuntimeException("can't found resources:" + resource);
                }

                String urlStr = url.toString();
                String jarFileName;
                if (urlStr.startsWith("wsjar:")) {
                    jarFileName = urlStr.substring(2);
                    url = new URL(jarFileName);
                }

                if (url.getProtocol().equals("jar")) {
                    jarFileName = sarDir + jarName;
                    File jarFile = new File(jarFileName);
                    if (jarFile.exists()) {
                        Logger.error("jar包已经存在：" + jarFileName);
                    } else {
                        jarFile.createNewFile();
                        InputStream jarIn = null;
                        OutputStream jarOut = null;

                        try {
                            jarIn = clazz.getResourceAsStream(resource);
                            jarOut = new FileOutputStream(jarFile);
                            transfer(jarIn, jarOut);
                        } catch (Exception var29) {
                            Exception e = var29;
                            throw new RuntimeException(e);
                        } finally {
                            if (jarIn != null) {
                                jarIn.close();
                            }

                            if (jarOut != null) {
                                jarOut.flush();
                                jarOut.close();
                            }

                        }

                        Logger.error("jar init successful：" + jarFileName);
                    }

                    url = jarFile.toURI().toURL();
                } else {
                    Logger.error(：" + resource);
                }

                Logger.error("load url：" + url);
                urls[i] = url;
            }

            var34 = new IClassLoader(urls);
        } catch (IOException var31) {
            IOException e = var31;
            Logger.error("loading error", e);
            throw e;
        } finally {
            if (in != null) {
                in.close();
            }

        }

        return var34;
    }
}

private static void transfer(InputStream is, OutputStream os) throws IOException {
    byte[] buf = new byte[4096];

    while(true) {
        int count = is.read(buf);
        if (count < 0) {
            return;
        }

        os.write(buf, 0, count);
    }
}

public Class<?> loadClass(String name) throws ClassNotFoundException {
    try {
        return super.loadClass(name);
    } catch (ClassNotFoundException var3) {
        return this.parent.loadClass(name);
    }
}

public Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
    try {
        return super.loadClass(name, resolve);
    } catch (ClassNotFoundException var4) {
        return this.parent.loadClass(name, resolve);
    }
}

public URL getResource(String name) {
    URL url = super.getResource(name);
    if (url == null) {
        url = this.parent.getResource(name);
    }

    return url;
}

public Enumeration<URL> getResources(String name) throws IOException {
    Enumeration[] tmp = new Enumeration[2];
    Enumeration<URL> resources = super.getResources(name);
    tmp[0] = resources;
    if (this.parent != null) {
        Enumeration<URL> parentResources = this.parent.getResources(name);
        tmp[1] = parentResources;
    }

    return new CompoundEnumeration(tmp);
}

public InputStream getResourceAsStream(String name) {
    InputStream in = super.getResourceAsStream(name);
    return in != null ? in : this.parent.getResourceAsStream(name);
}

public static void setWorkDir(String workDir) {
    if (workDir != null && workDir.trim().length() >= 1) {
        if (workDir.endsWith(File.separator)) {
            workDir = workDir.substring(0, workDir.length() - 1);
        }

        IClassLoader.workDir = workDir;
    } else {
        throw new RuntimeException("illege");
    }
}

interface ConfigKey {
    String jars = "jars";
}

} ` the code 'org.apache.http.impl.client.CloseableHttpClient' in "/sar/jars/" + jarName。 springboot project has also 'org.apache.http.impl.client.CloseableHttpClient'

when i delete code InjectionStrategy.UsingUnsafe.OfFactor, my applicaiotn works well. but, throw an exception。

help me~~ this is production incident

[raphw]

What is the exception?

[githubcheng2978]

java.lang.IllegalAccessError: tried to access class org.apache.http.impl.client.InternalHttpClient$$sw$auxiliary$8shium3 from class org.apache.http.impl.client.InternalHttpClient

[raphw]

Are you using the module system? It's hard to tell from your code why this is. Are the two classes on different class loaders?

[githubcheng2978]

1. InternalHttpClient class is loaded by springboot classLoader and then it is loader by customer classloader(IClassLoader)，to prevent class conflicts by it is throw java.lang.IllegalAccessError.
2. If I prioritize using a custom classloader(IClassLoader) to load, did not throw any exceptions
3. if i remove AgentBuilder.InjectionStrategy.UsingUnsafe.OfFactory(factory)) code, did not throw any exceptions。

[raphw]

I assume that your intercepted class is not visible to your intercepted code. The exception suggests that you are loading code on different class loaders. You should avoid auxiliary classes on agents for this reason. Maybe have a look at Advice to avoid it,

[githubcheng2978]

I use the Skywalking agent, and the key code is as follows: newClassBuilder = newClassBuilder.method(junction) .intercept(MethodDelegation.withDefaultConfiguration() .to(new InstMethodsInter(interceptor, classLoader), delegateNamingResolver.resolve(instanceMethodsInterceptPoint)));

What is the class of generated auxiliary： `class InternalHttpClient$$sw$auxiliary$urqm873 implements Runnable, Callable { private InternalHttpClient argument0; private HttpHost argument1; private HttpRequest argument2; private HttpContext argument3;

public Object call() throws Exception {
    return this.argument0.$sw$original$doExecute$7gb7dq1$accessor$$sw$l56slt2(this.argument1, this.argument2, this.argument3);
}

public void run() {
    this.argument0.$sw$original$doExecute$7gb7dq1$accessor$$sw$l56slt2(this.argument1, this.argument2, this.argument3);
}

InternalHttpClient$$sw$auxiliary$urqm873(InternalHttpClient var1, HttpHost var2, HttpRequest var3, HttpContext var4) {
    this.argument0 = var1;
    this.argument1 = var2;
    this.argument2 = var3;
    this.argument3 = var4;
}

}`

[raphw]

it's a proxy to invoke the super method of the intercepted code. It needs to be injected into the right class loader. Did you configure a proper injection strategy?

[githubcheng2978]

https://github.com/apache/skywalking-java/blob/3a9645849529736ca3f67170d673daaee0ed0cda/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/plugin/bootstrap/BootstrapInstrumentBoost.java#L117

the injection strategy is:
ClassInjector.UsingUnsafe.Factory factory = ClassInjector.UsingUnsafe.Factory.resolve(instrumentation); factory.make(null, null).injectRaw(classesTypeMap); agentBuilder = agentBuilder.with(new AgentBuilder.InjectionStrategy.UsingUnsafe.OfFactory(factory));

[raphw]

Not sure why you encounter this. Might it be that injected classes end up on two class loaders?

[githubcheng2978]

My project uses Skywalking and Apache HTTP client. At the same time, a third-party SDK was used in the project, which also introduced the Apache HTTP client through a jar package. The SDK customized a classloader to load its own jar. If I prioritize initializing the SDK, the project is normal, but if I prior itize initializing the HTTP client in the project, an exception will be thrown I wrote a simple demo: Demo.zip

download site: https://dlcdn.apache.org/skywalking/java-agent/9.2.0/apache-skywalking-java-agent-9.2.0.tgz and add jvm options with agent. -javaagent:/home/blue/workspace/skywalking-agent/skywalking-agent.jar

Plugins that require bootstrap to be enabled. change config/agent.config plugin.mount=plugins,bootstrap-plugins

[raphw]

I would assume that this is related to the Skywalking agent somehow, I would suggest you reach out to the maintainers there.

[githubcheng2978]

I also tried to get help in Skywalking, and the answer I got was the problem with the class loader. I don't know exactly where the problem occurred, so I came here to seek help

https://github.com/apache/skywalking/discussions/12252

[raphw]

Not sure if I can help. This seems to be related to how the instrumentation is implemented. In general, if you create a class on two class loaders on the same hirarchy, this will often lead to hanging.

[dsc6636926]

when i use two agentBuilder(use InjectionStrategy.UsingUnsafe to enhance jdk classes,use InjectionStrategy.UsingReflection.INSTANCE to enhance other classes), the error disappeared

[raphw]

Then you might have it some module boundary before. Handles are not as powerful as injection, but use standard API.