search

spring-cloud / spring-cloud-sleuth

Distributed tracing for spring cloud
https://spring.io/projects/spring-cloud-sleuth
Apache License 2.0
1.76k stars 781 forks source link

Always get traces from channel hystrixStreamOutput and broker in zipkin. #986

Closed hanrw closed 6 years ago

hanrw commented 6 years ago

Bug report: From zipkin you can see account-service x3 4ms broker x2 4ms account-service x3 4ms broker x2 4ms account-service x3 4ms broker x2 4ms ....

screen shot 2018-05-22 at 11 43 55 am screen shot 2018-05-22 at 12 22 37 pm screen shot 2018-05-22 at 1 32 33 pm

Which is not useful

So i tried config below which are not working

    scheduled:
      skipPattern: .*HystrixStreamTask.*

and 

    scheduled:
      skipPattern: org.springframework.cloud.netflix.hystrix.stream.HystrixStreamTask

DEBUG INFO screen shot 2018-05-22 at 3 53 16 pm screen shot 2018-05-22 at 3 54 06 pm screen shot 2018-05-22 at 3 54 27 pm screen shot 2018-05-22 at 4 23 38 pm

screen shot 2018-05-22 at 5 13 44 pm screen shot 2018-05-22 at 5 13 25 pm screen shot 2018-05-22 at 5 12 12 pm

Traces json from http://localhost:9411/api/v2/traces?serviceName=account-service&limit=10

[
    [
        {
            "traceId":"9c28f66e3155b5e8",
            "parentId":"9c28f66e3155b5e8",
            "id":"c4af518d9bb1f6ae",
            "kind":"CONSUMER",
            "timestamp":1526981310683110,
            "duration":17,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        },
        {
            "traceId":"9c28f66e3155b5e8",
            "parentId":"c4af518d9bb1f6ae",
            "id":"5c7fb622e82e837f",
            "name":"handle",
            "timestamp":1526981310683006,
            "duration":292,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            }
        },
        {
            "traceId":"9c28f66e3155b5e8",
            "id":"9c28f66e3155b5e8",
            "kind":"PRODUCER",
            "name":"send",
            "timestamp":1526981310683011,
            "duration":549,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        }
    ],
    [
        {
            "traceId":"830db45ee353fd8e",
            "parentId":"830db45ee353fd8e",
            "id":"3842164619522807",
            "kind":"CONSUMER",
            "timestamp":1526981310682173,
            "duration":17,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        },
        {
            "traceId":"830db45ee353fd8e",
            "parentId":"3842164619522807",
            "id":"2f4a80308eb63990",
            "name":"handle",
            "timestamp":1526981310682006,
            "duration":486,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            }
        },
        {
            "traceId":"830db45ee353fd8e",
            "id":"830db45ee353fd8e",
            "kind":"PRODUCER",
            "name":"send",
            "timestamp":1526981310682010,
            "duration":868,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        }
    ],
    [
        {
            "traceId":"76dc792a02621b94",
            "parentId":"76dc792a02621b94",
            "id":"40aa739988cd7bc2",
            "kind":"CONSUMER",
            "timestamp":1526981310185141,
            "duration":19,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        },
        {
            "traceId":"76dc792a02621b94",
            "parentId":"40aa739988cd7bc2",
            "id":"b3573af4fdeaa36e",
            "name":"handle",
            "timestamp":1526981310185006,
            "duration":398,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            }
        },
        {
            "traceId":"76dc792a02621b94",
            "id":"76dc792a02621b94",
            "kind":"PRODUCER",
            "name":"send",
            "timestamp":1526981310185011,
            "duration":669,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        }
    ],
    [
        {
            "traceId":"741166e2b87db1c8",
            "parentId":"741166e2b87db1c8",
            "id":"cd77bc705b7afb68",
            "kind":"CONSUMER",
            "timestamp":1526981310182191,
            "duration":21,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        },
        {
            "traceId":"741166e2b87db1c8",
            "parentId":"cd77bc705b7afb68",
            "id":"06e2a9d36c5b8603",
            "name":"handle",
            "timestamp":1526981310182008,
            "duration":1286,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            }
        },
        {
            "traceId":"741166e2b87db1c8",
            "id":"741166e2b87db1c8",
            "kind":"PRODUCER",
            "name":"send",
            "timestamp":1526981310182007,
            "duration":2847,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        }
    ],
    [
        {
            "traceId":"d7029f67302b74c3",
            "parentId":"d7029f67302b74c3",
            "id":"b75bb7a9dafd969b",
            "kind":"CONSUMER",
            "timestamp":1526981309685078,
            "duration":13,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        },
        {
            "traceId":"d7029f67302b74c3",
            "parentId":"b75bb7a9dafd969b",
            "id":"3bf00b04459eea21",
            "name":"handle",
            "timestamp":1526981309685006,
            "duration":260,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            }
        },
        {
            "traceId":"d7029f67302b74c3",
            "id":"d7029f67302b74c3",
            "kind":"PRODUCER",
            "name":"send",
            "timestamp":1526981309685017,
            "duration":387,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        }
    ],
    [
        {
            "traceId":"a4a0ce4cef0d785a",
            "parentId":"a4a0ce4cef0d785a",
            "id":"126a6f04a26f7189",
            "kind":"CONSUMER",
            "timestamp":1526981309683266,
            "duration":20,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        },
        {
            "traceId":"a4a0ce4cef0d785a",
            "parentId":"126a6f04a26f7189",
            "id":"b6a4b76c1779425d",
            "name":"handle",
            "timestamp":1526981309684199,
            "duration":648,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            }
        },
        {
            "traceId":"a4a0ce4cef0d785a",
            "id":"a4a0ce4cef0d785a",
            "kind":"PRODUCER",
            "name":"send",
            "timestamp":1526981309683037,
            "duration":1221,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        }
    ],
    [
        {
            "traceId":"1c678c42421fa5ca",
            "parentId":"1c678c42421fa5ca",
            "id":"172e5c610c04384f",
            "kind":"CONSUMER",
            "timestamp":1526981309183078,
            "duration":15,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        },
        {
            "traceId":"1c678c42421fa5ca",
            "parentId":"172e5c610c04384f",
            "id":"8f73b1f605e438e4",
            "name":"handle",
            "timestamp":1526981309183007,
            "duration":406,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            }
        },
        {
            "traceId":"1c678c42421fa5ca",
            "id":"1c678c42421fa5ca",
            "kind":"PRODUCER",
            "name":"send",
            "timestamp":1526981309183008,
            "duration":611,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        }
    ],
    [
        {
            "traceId":"fdd495d3a98eb86b",
            "parentId":"fdd495d3a98eb86b",
            "id":"963be7bbe30b81e2",
            "kind":"CONSUMER",
            "timestamp":1526981309182195,
            "duration":39,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        },
        {
            "traceId":"fdd495d3a98eb86b",
            "parentId":"963be7bbe30b81e2",
            "id":"b936147da1655324",
            "name":"handle",
            "timestamp":1526981309182006,
            "duration":673,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            }
        },
        {
            "traceId":"fdd495d3a98eb86b",
            "id":"fdd495d3a98eb86b",
            "kind":"PRODUCER",
            "name":"send",
            "timestamp":1526981309182010,
            "duration":1034,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        }
    ],
    [
        {
            "traceId":"8bafcb7984a95364",
            "parentId":"8bafcb7984a95364",
            "id":"c88ad9ed9d36c821",
            "kind":"CONSUMER",
            "timestamp":1526981308683111,
            "duration":13,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        },
        {
            "traceId":"8bafcb7984a95364",
            "parentId":"c88ad9ed9d36c821",
            "id":"46679e7a989f1b25",
            "name":"handle",
            "timestamp":1526981308684007,
            "duration":461,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            }
        },
        {
            "traceId":"8bafcb7984a95364",
            "id":"8bafcb7984a95364",
            "kind":"PRODUCER",
            "name":"send",
            "timestamp":1526981308683006,
            "duration":771,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        }
    ],
    [
        {
            "traceId":"cab82e2e0b2af828",
            "parentId":"cab82e2e0b2af828",
            "id":"efc9c4da019100a3",
            "kind":"CONSUMER",
            "timestamp":1526981308682204,
            "duration":19,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        },
        {
            "traceId":"cab82e2e0b2af828",
            "parentId":"efc9c4da019100a3",
            "id":"68f81b4066047fd5",
            "name":"handle",
            "timestamp":1526981308682006,
            "duration":710,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            }
        },
        {
            "traceId":"cab82e2e0b2af828",
            "id":"cab82e2e0b2af828",
            "kind":"PRODUCER",
            "name":"send",
            "timestamp":1526981308682027,
            "duration":1300,
            "localEndpoint":{
                "serviceName":"account-service",
                "ipv4":"172.30.0.11"
            },
            "remoteEndpoint":{
                "serviceName":"broker"
            },
            "tags":{
                "channel":"hystrixStreamOutput"
            }
        }
    ]
]

Pom dependencies

<dependencyManagement>
    <dependencies>
      <dependency>
        <!-- Import dependency management from Spring Boot -->
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-dependencies</artifactId>
        <version>2.0.1.RELEASE</version>
        <type>pom</type>
        <scope>import</scope>
      </dependency>

      <dependency>
        <groupId>org.springframework.cloud</groupId>
        <artifactId>spring-cloud-dependencies</artifactId>
        <version>Finchley.RC1</version>
        <type>pom</type>
        <scope>import</scope>
      </dependency>
    </dependencies>
  </dependencyManagement>
<dependencies>
    <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-starter-zipkin</artifactId>
    </dependency>

    <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
    </dependency>
    <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-starter-openfeign</artifactId>
    </dependency>
    <dependency>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-starter-data-mongodb</artifactId>
    </dependency>
    <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-starter-config</artifactId>
    </dependency>
    <dependency>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-starter-security</artifactId>
    </dependency>
    <dependency>
      <groupId>org.springframework.security.oauth.boot</groupId>
      <artifactId>spring-security-oauth2-autoconfigure</artifactId>
    </dependency>
    <dependency>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-starter-web</artifactId>
    </dependency>

    <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-starter-oauth2</artifactId>
    </dependency>

    <dependency>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-starter-actuator</artifactId>
    </dependency>
    <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-starter-bus-amqp</artifactId>
    </dependency>
    <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-netflix-hystrix-stream</artifactId>
    </dependency>

    <dependency>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-starter-test</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>de.flapdoodle.embed</groupId>
      <artifactId>de.flapdoodle.embed.mongo</artifactId>
      <version>1.50.3</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>com.jayway.jsonpath</groupId>
      <artifactId>json-path</artifactId>
      <version>2.2.0</version>
      <scope>test</scope>
    </dependency>
  </dependencies>
hanrw commented 6 years ago

How to fix that:

  sleuth:
    integration:
      enabled: false
    scheduled:
      skip-pattern: org.springframework.cloud.netflix.hystrix.stream.HystrixStreamTask
hanrw commented 6 years ago

My issue related two parts 1.for HystrixStreamTask, spans came into zipkin to let zipkin search not working. pls see issue openzipkin/docker-zipkin#177 This is fixed by skip-pattern: org.springframework.cloud.netflix.hystrix.stream.HystrixStreamTask

2.for broker why it’s came into zipkin This is fixed by integration: enabled: false

hanrw commented 6 years ago

i suggest that

skip-pattern: org.springframework.cloud.netflix.hystrix.stream.HystrixStreamTask set as default.

The reason is that I want traceing method invoke only when using sleuth and zipkin at first time. So for me org.springframework.cloud.netflix.hystrix.stream.HystrixStreamTask tracing is like advance feature.

hanrw commented 6 years ago

Working fine with snapshots version Thanks @marcingrzejszczak