mirror of
https://github.com/hanzoai/krakend-otel.git
synced 2026-09-30 22:49:38 +00:00
initial implementation of otel metrics
This commit is contained in:
David Hontecillas
@@ -0,0 +1,46 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to creating a positive environment include:
|
||||
|
||||
* Using welcoming and inclusive language
|
||||
* Being respectful of differing viewpoints and experiences
|
||||
* Gracefully accepting constructive criticism
|
||||
* Focusing on what is best for the community
|
||||
* Showing empathy towards other community members
|
||||
|
||||
Examples of unacceptable behavior by participants include:
|
||||
|
||||
* The use of sexualized language or imagery and unwelcome sexual attention or advances
|
||||
* Trolling, insulting/derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or electronic address, without explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a professional setting
|
||||
|
||||
## Our Responsibilities
|
||||
|
||||
Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
|
||||
|
||||
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at hello@krakend.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
|
||||
|
||||
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
|
||||
|
||||
[homepage]: http://contributor-covenant.org
|
||||
[version]: http://contributor-covenant.org/version/1/4/
|
||||
@@ -0,0 +1,60 @@
|
||||
# Contributing
|
||||
|
||||
Thank you for your interest in contributing to KrakenD-OTEL, there are several ways
|
||||
you can contribute and make this project more awesome, please see below:
|
||||
|
||||
## Reporting an Issue
|
||||
|
||||
If you believe you have found an issue with the code please do not hesitate to file an issue in [Github](https://github.com/krakend/krakend-otel/issues). When
|
||||
filing the issue please describe the problem with the maximum level of detail
|
||||
and the steps to reproduce the problem, including information about your
|
||||
environment.
|
||||
|
||||
You can also open an issue requesting for help or doing a question and it's
|
||||
also a good way of contributing since other users might be in a similar
|
||||
position.
|
||||
|
||||
Please note we have a [code of conduct](./CODE_OF_CONDUCT.md), please follow it in all your interactions with the project.
|
||||
|
||||
## Code Contributions
|
||||
|
||||
When contributing to this repository, it is generally a good idea to discuss
|
||||
the change with the owners before investing a lot of time coding. The process
|
||||
could be:
|
||||
|
||||
1. Open an issue explaining the improvment or fix you want to add
|
||||
2. [Fork the project](https://github.com/krakend/krakend-otel/fork)
|
||||
3. Code it in your fork
|
||||
4. Submit a [pull request](https://help.github.com/articles/creating-a-pull-request) referencing the issue
|
||||
|
||||
|
||||
Your work will then be reviewed as soon as possible (suggestions about some
|
||||
changes, improvements or alternatives may be given).
|
||||
|
||||
**Don't forget to add tests**, make sure that they all pass!
|
||||
|
||||
# Help with Git
|
||||
|
||||
Once the repository is forked, you should track the upstream (original) one
|
||||
using the following command:
|
||||
|
||||
git remote add upstream https://github.com/krakend/krakend-otel.git
|
||||
|
||||
Then you should create your own branch:
|
||||
|
||||
git checkout -b <prefix>/<micro-title>-<issue-number>
|
||||
|
||||
Once your changes are done (`git commit -am '<descriptive-message>'`), get the
|
||||
upstream changes:
|
||||
|
||||
git checkout master
|
||||
git pull --rebase origin master
|
||||
git pull --rebase upstream master
|
||||
git checkout <your-branch>
|
||||
git rebase master
|
||||
|
||||
Finally, publish your changes:
|
||||
|
||||
git push -f origin <your-branch>
|
||||
|
||||
You should be now ready to make a pull request.
|
||||
@@ -0,0 +1,9 @@
|
||||
test:
|
||||
go test ./client/...
|
||||
go test ./config/...
|
||||
go test ./exporter/...
|
||||
go test ./io/...
|
||||
go test ./lura/...
|
||||
go test ./state/...
|
||||
go test ./router/...
|
||||
.PHONY: test
|
||||
@@ -1,72 +1,444 @@
|
||||
# krakend-otel
|
||||
KrakenD component for OpenTelemetry
|
||||
|
||||
**While we still need to start to implement OpenTelemetry natively in this repo**, doing a bridge is **already possible** today.
|
||||
OpenTelemetry building blocks to instrument [Lura](https://github.com/luraproject/lura) / [KrakenD](https://krakend.io)
|
||||
based API Gateways.
|
||||
|
||||
For instance:
|
||||
[Apache Licnese 2.0](.LICENSE)
|
||||
|
||||
`otel-collector.yaml`:
|
||||
```yaml
|
||||
receivers:
|
||||
opencensus:
|
||||
[Check the official KrakenD Open Telemetry documentation](https://www.krakend.io/docs/telemetry/opentelemetry/)
|
||||
|
||||
exporters:
|
||||
logging:
|
||||
verbosity: detailed
|
||||
|
||||
azuremonitor:
|
||||
maxbatchsize: 100
|
||||
maxbatchinterval: 10s
|
||||
# REPLACE WITH YOUR KEY:
|
||||
instrumentation_key: YOUR-KEY-HERE
|
||||
## Example
|
||||
|
||||
For a quick look at the observability the library can provide, check the
|
||||
[example documentation](./example/README.md).
|
||||
|
||||
## Configuration from Lura's [ServiceConfig](https://github.com/luraproject/lura/blob/master/config/config.go)
|
||||
|
||||
In order to configure the open telemetry stack to instrument the API Gateway, a
|
||||
new entry must be added to the `ExtraConfig` root propertry of `ServiceConfig`
|
||||
using the `telemetry/opentelemetry` key, with the `krakend-otel`'s configuration.
|
||||
|
||||
## `krakend-otel` Configuration
|
||||
|
||||
In the configuration we find the following root entries:
|
||||
|
||||
- `service_name`: to provide a custom name for this service. However if there is
|
||||
a `ServiceConfig.Name` already set, that will be the one used.
|
||||
- `[exporters](exporters)`: in this section we define our different exporter
|
||||
configurations, giving them our own "custom name" to be referenced later
|
||||
when we deine the instance. This allows us to use the same `kind` of exporter,
|
||||
with different port / host configurations, and report the same metric / traces
|
||||
to different systems (imagine migrating from a self hosted grafana stack to the
|
||||
cloud version, and that for a while you want to report to both places before
|
||||
making the switch), of report to different `kind` of exporters (migrating from one cloud
|
||||
observability system to a different one).
|
||||
- `layers`: in this section we can fine tune the amount of metrics / traces that
|
||||
we want to report at each "stage" of the processing pipeline: `router`, `pipe` or `backend`.
|
||||
- `skip_paths`: an option to define endpoints that we do not want to instrument. By default,
|
||||
if it is not provided, it will skip the following "internal" paths:
|
||||
- `/healthz`
|
||||
- `/_ah/health`
|
||||
- `/__debug`
|
||||
- `/__echo`
|
||||
If we want to instrument those paths too, just provide an empty string as endpoint to
|
||||
skip.
|
||||
|
||||
In a visual way, this is the realation between the `exporters` configuration, and
|
||||
how we select as `metric_providers` or `trace_providers`:
|
||||
|
||||
|
||||
|
||||
|
||||
However, for providers that can export both metric and traces (like in the OTLP case), you
|
||||
can disable any of those.
|
||||
|
||||
### Exporters
|
||||
|
||||
Example:
|
||||
|
||||
service:
|
||||
telemetry:
|
||||
logs:
|
||||
level: "warn"
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [opencensus]
|
||||
exporters: [azuremonitor]
|
||||
```
|
||||
KrakenD configuration:
|
||||
```json
|
||||
{
|
||||
"telemetry/opencensus": {
|
||||
"sample_rate": 100,
|
||||
"reporting_period": 60,
|
||||
"exporters": {
|
||||
"ocagent": {
|
||||
"address": "otel-collector:55678",
|
||||
"service_name": "myKrakenD",
|
||||
"insecure": true
|
||||
"exporters": [
|
||||
{
|
||||
"name": "local_prometheus",
|
||||
"kind": "prometheus",
|
||||
"config": {
|
||||
"port": 9092,
|
||||
"process_metrics": true,
|
||||
"go_metrics": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "local_tempo",
|
||||
"kind": "opentelemetry",
|
||||
"config": {
|
||||
"port": 4317,
|
||||
"use_http": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "local_jaeger",
|
||||
"kind": "opentelemetry",
|
||||
"config": {
|
||||
"port": 5317,
|
||||
"use_http": false
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### Layers
|
||||
|
||||
We can differentiate the processing of a request in KrakenD in 3 main stages (each one
|
||||
including or wrapping the inner stage):
|
||||
|
||||
- `global`: this part that comes before the `Lura`'s framework starts working with
|
||||
the request. In the case of [KrakenD CE](https://github.com/krakend/krakend-ce),
|
||||
this stage is implemented usin [gin](https://github.com/gin-gonic/gin)
|
||||
|
||||
- `proxy`: this is the `Lura`'s framework part where it deals with one of the
|
||||
API Gateway exposed endpoints, and includes spawning the required
|
||||
requests to the backends, as well as the manipulation at the endpoint
|
||||
level before and after the requests are performed.
|
||||
|
||||
- `backend`: this is the `Lura`'s framework part where it deals with each
|
||||
single backend request (including the manipulation at that request level).
|
||||
|
||||
|
||||
For each of those layers it can be selected the deatail of metrics and traces
|
||||
that we want to report.
|
||||
|
||||
#### global
|
||||
|
||||
At the router level we have 3 main options:
|
||||
|
||||
- `disable_metrics`: boolean to enable / disable if we want to report metrics for this layer
|
||||
- `disable_traces`: boolean to enable / disable if we want to report traces for this layer
|
||||
- `disable_propagation`: boolean to disable the consumption of a propagation header for
|
||||
traces (so spans from a previous layer are linked to the KrakenD trace).
|
||||
|
||||
```json
|
||||
"global": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false,
|
||||
"disable_propagation": false
|
||||
}
|
||||
```
|
||||
|
||||
##### Metrics
|
||||
|
||||
- `http.server.duration`: histogram of the time it takes to produce the response.
|
||||
Attributes:
|
||||
- `http.request.method`: the HTTP method (`GET`, `POST`, `HEAD`, ...)
|
||||
- `http.response.status_code`: status code of the produced response
|
||||
- `url.path`: the matched endpoint path
|
||||
- `http.server.response.size`: histogram of the size of the body produced for the response.
|
||||
Attributes:
|
||||
- `http.request.method`: the HTTP method (`GET`, `POST`, `HEAD`, ...)
|
||||
- `http.response.status_code`: status code of the produced response
|
||||
- `url.path`: the matched endpoint path
|
||||
|
||||
|
||||
##### Traces
|
||||
|
||||
A trace is created with the received request's **path** (not the matched endpoint, because
|
||||
the trace is started before any matching is performed).
|
||||
|
||||
The attributes of the global trace are:
|
||||
|
||||
- `krakend.stage`: always with `global` value (to easily find all global traces)
|
||||
- `url.path`: the matched endpoint (in case no matching is made, for example in
|
||||
`404` results, it would be empty)
|
||||
- `http.response.status_code`: the response status code
|
||||
- `http.response.body.size`: the returned body size
|
||||
|
||||
In case an error happens, the span will record the error and set the status to
|
||||
error (value = `1`), in case of success the status is set to ok (value = `2`).
|
||||
|
||||
#### pipe
|
||||
|
||||
At the pipe level we only have 2 options:
|
||||
|
||||
- `metrics`: boolean to enable / disable if we want to report metrics for this layer
|
||||
- `traces`: boolean to enable / disable if we want to report traces for this layer
|
||||
|
||||
```json
|
||||
"proxy": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": true
|
||||
}
|
||||
```
|
||||
|
||||
##### Metrics
|
||||
|
||||
- `krakend.pipe.duration`: histogram of the time it takes to produce the response.
|
||||
Attributes:
|
||||
- `url.path`: the matched endpoint path that **krakend is serving** (is different
|
||||
than in `backend`, krakend stage, when this property is the path
|
||||
for the backend we are targetting).
|
||||
|
||||
Attributes:
|
||||
|
||||
- `krakend.stage`: always with value `pipe`
|
||||
- `complete`: a `true` / `false` value to know when a response is complete (all
|
||||
backends returned a successful response).
|
||||
- `canceled`: if appears, will always be `true`, and indicates a request
|
||||
that has been canceled (usually when parallel requests are used).
|
||||
- `error`: in case an error happened, the description of the error.
|
||||
|
||||
#### backend
|
||||
|
||||
At the backend level is where we have more granularity selecting the information
|
||||
that we want to obtain.
|
||||
|
||||
There are three entries:
|
||||
|
||||
- `metrics`: to define the amount of info we want to report in backend metrics
|
||||
- `traces`: to define the amount of info we want to report in backend traces
|
||||
|
||||
For both, the `metrics` and `traces` part, we can select the same options:
|
||||
|
||||
- `disable_stage`: to enable metrics / traces for the full backend processing part
|
||||
- `round_trip`: to enable metrics /traces for the actual http request for the backend
|
||||
(not taking into account the manipulation part at the backend level).
|
||||
- `read_payload`: to enable metrics / traces only for the response reading payload
|
||||
(not taking into account the http connection part of the request).
|
||||
- `detailed_connection`: to enable metrics / traces for the connection details, like
|
||||
time to query the DNS, the time spent in TLS, and so one.
|
||||
- `static_attributes`: an array of `key: value` pairs to be used as tags / labels in
|
||||
the reported metric / traces.
|
||||
|
||||
```json
|
||||
"backend": {
|
||||
"metrics": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_metric_attr",
|
||||
"value": "my_middle_metric"
|
||||
}
|
||||
]
|
||||
},
|
||||
"traces": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_trace_attr",
|
||||
"value": "my_middle_trace"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
##### Metrics
|
||||
|
||||
- `krakend.backend.duration`: histogram of the time it takes to produce the response. Controlled
|
||||
by the `disable_stage` flag (if set to `true` this metric will not appear).
|
||||
Attributes:
|
||||
- `http.request.method`: the method used to make the request
|
||||
- `url.path`: the matched endpoint path that **krakend is serving** (is different
|
||||
than in `backend`, krakend stage, when this property is the path
|
||||
- `krakend.endpoint`: this attribute is set to the krakend exposed endpoint that
|
||||
is the "parent" of the backend request.
|
||||
- `server.address`: the target host (in case more than one are provided, those
|
||||
are joined with `_`).
|
||||
|
||||
###### RoundTrip metrics
|
||||
|
||||
The following metrics are enabled if `round_trip` is set to true, and share the same attributes:
|
||||
- `http.request.method_original`: the methos used for the request to the backend
|
||||
(GET, POST,...)
|
||||
- `url.path`: the requested path
|
||||
- `server.address`: the target host (just the first in the list of provided hosts).
|
||||
(**TODO**: check if we want the concat list of hosts like in the `stage-duration`
|
||||
metric, or if we should change that attribute there).
|
||||
- `krakend.stage`: always with value `backend-request`
|
||||
|
||||
- `http.client.duration`: histogram with the time taken since starting a request, until
|
||||
until having the first byte of the body ready to read.
|
||||
|
||||
- `http.client.request.started.count`: number of requests started.
|
||||
- `http.client.request.failed.count`: number of requests [failed](failed).
|
||||
- `http.client.request.canceled.count`: number of canceled request.
|
||||
- `http.client.request.timedout.count`: number of timed out requests.
|
||||
- `http.client.request.size`: counter with the sum of `Content-Length` header for the
|
||||
sent payload for the request.
|
||||
|
||||
- `http.client.response.size`: histogram with the size of response bodies
|
||||
as read from the `Content-Length` header.
|
||||
|
||||
|
||||
###### Read Payload metrics
|
||||
|
||||
- `read-size`: counted with the read bytes (**TODO** should we remove this onw as is redundant with
|
||||
the histogram).
|
||||
- `read-size-hist`: histogram with the read bytes
|
||||
- `read-time`: counter with seconds spent reading the body payload of the response.(**TODO** remove
|
||||
if this is redundant wiht the `read-time-hist`).
|
||||
- `read-time-hist`: histogram with the seconds spent reading the bdy.
|
||||
- `read-errors`: counter of number of errors that happened reading the response body.
|
||||
|
||||
###### Detailed connection metrics
|
||||
|
||||
The following metrics are enabled if `detailed_connection` is set to **_true_**, and share the same attributes:
|
||||
- `http.request.method_original`: the methos used for the request to the backend
|
||||
(GET, POST,...)
|
||||
- `url.path`: the requested path
|
||||
- `server.address`: the target host (just the first in the list of provided hosts).
|
||||
(**TODO**: check if we want the concat list of hosts like in the `stage-duration`
|
||||
metric, or if we should change that attribute there).
|
||||
- `krakend.stage`: always with value `backend-request`
|
||||
|
||||
- `request-get-conn-latency`: time to get a connection from the connection pool
|
||||
- `request-dns-latency`: time spen5 resolving a DNS name
|
||||
- `request-tls-latency`: time spent on TLS Handshake
|
||||
|
||||
##### Traces
|
||||
|
||||
**Stage Span** attributes (controlled by the `disable_stage` flag: will not appear if set to `true`):
|
||||
|
||||
- `krakend.stage`: always with value `backend`
|
||||
- `complete`: a `true` / `false` value to know when a response is complete (all
|
||||
backends returned a successful response).
|
||||
- `canceled`: if appears, will always be `true`, and indicates a request
|
||||
that has been canceled (usually when parallel requests are used).
|
||||
- `error`: in case an error happened, the description of the error.
|
||||
|
||||
**read-tracer** span with when `read_paylaod` option is set to true.
|
||||
|
||||
### Instance
|
||||
|
||||
Given that any Exporter could implement both traces and metrics, we might want
|
||||
to select to use it only for one of those roles: that's why we have this section
|
||||
to specify what to use. Also, it allows to easily enable / disable exporters
|
||||
without needing to delete the exporters configuration.
|
||||
|
||||
The fields of the `instance` section are:
|
||||
|
||||
- `metric_providers`: this is the list of the exporter names that we want to use to
|
||||
report metrics. Provides an easy way to just select a few of the configured exporters
|
||||
for metrics (the exporters might have support for metrics **and / or** traces, so
|
||||
that should be taken into account because if we try to use an exporter that
|
||||
does not have support for metrics, it will fail).
|
||||
- `metric_reporting_period`: how often we want to flush the metrics in seconds.
|
||||
- `trace_providers`: the list of the exporter names that we want to use to
|
||||
report traces. Provides an easy way to just select a few of the configured exporters
|
||||
for traces (the exporters might have support for metrics **and / or** traces, so
|
||||
that should be taken into account because if we try to use an exporter that
|
||||
does not have support for traces, it will fail).
|
||||
- `trace_sample_rate`: a number between `0` and `1.0` to define the sample rate for
|
||||
traces: if we set it for example to `0.25` only one in four requests will be
|
||||
reporting traces (usefull to reduce the amount of data generated while keeping
|
||||
some traces to identify / debug issues).
|
||||
|
||||
Example:
|
||||
|
||||
```json
|
||||
"instance": {
|
||||
"metric_providers": [
|
||||
"local_prometheus"
|
||||
],
|
||||
"metric_reporting_period": 1,
|
||||
"trace_providers": [
|
||||
"local_tempo",
|
||||
"local_jaeger"
|
||||
],
|
||||
"trace_sample_rate": 1
|
||||
}
|
||||
```
|
||||
|
||||
### Example configuration:
|
||||
|
||||
Putting it all together, here we have an example of a configuration:
|
||||
|
||||
```json
|
||||
"telemetry/opentelemetry": {
|
||||
"service_name": "krakend_middle_service",
|
||||
"exporters": [
|
||||
{
|
||||
"name": "local_prometheus",
|
||||
"kind": "prometheus",
|
||||
"config": {
|
||||
"port": 9092,
|
||||
"process_metrics": true,
|
||||
"go_metrics": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "local_tempo",
|
||||
"kind": "opentelemetry",
|
||||
"config": {
|
||||
"port": 4317,
|
||||
"use_http": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "local_jaeger",
|
||||
"kind": "opentelemetry",
|
||||
"config": {
|
||||
"port": 5317,
|
||||
"use_http": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"layers": {
|
||||
"global": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false,
|
||||
"disable_propagation": false
|
||||
},
|
||||
"proxy": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false
|
||||
},
|
||||
"backend": {
|
||||
"metrics": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": {
|
||||
"my_metric_attr": "my_middle_metric"
|
||||
}
|
||||
},
|
||||
"traces": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": {
|
||||
"my_trace_attr": "my_middle_trace"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"instance": {
|
||||
"metric_providers": [
|
||||
"local_prometheus"
|
||||
],
|
||||
"metric_reporting_period": 1,
|
||||
"trace_providers": [
|
||||
"local_tempo",
|
||||
"local_jaeger"
|
||||
],
|
||||
"trace_sample_rate": 1
|
||||
},
|
||||
"skip_paths": [""],
|
||||
"extra": {
|
||||
"custom": "extra",
|
||||
"future_expansion": true
|
||||
}
|
||||
}
|
||||
```
|
||||
Docker-compose sample:
|
||||
```yaml
|
||||
version: "3"
|
||||
services:
|
||||
krakend:
|
||||
image: devopsfaith/krakend:2
|
||||
command: [ "krakend", "run", "-d", "-c", "/etc/krakend/krakend.json"]
|
||||
volumes:
|
||||
- ./:/etc/krakend
|
||||
ports:
|
||||
- 8080:8080
|
||||
depends_on:
|
||||
- otel-collector
|
||||
# Collector
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib
|
||||
command: ["--config=/etc/otel-collector.yaml"]
|
||||
environment:
|
||||
- GRPC_GO_LOG_SEVERITY_LEVEL=warn
|
||||
- GRPC_GO_LOG_VERBOSITY_LEVEL=warn
|
||||
volumes:
|
||||
- ./otel-collector.yaml:/etc/otel-collector.yaml
|
||||
ports:
|
||||
- 55678:55678
|
||||
```
|
||||
|
||||
# Other Info
|
||||
|
||||
- Check the [example documentation](./example/README.md)
|
||||
- Some [notes about current implementation](./doc/implementation_details.md)
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
// Package config defines the configuration to be used to setup
|
||||
// the metrics and traces for each stage of a KrakenD instances
|
||||
// as well as the level of detail we want for each stage.
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Config is the root configuration for the OTEL observability stack
|
||||
type Config struct {
|
||||
ServiceName string `json:"service_name"`
|
||||
Layers *LayersOpts `json:"layers"`
|
||||
Exporters Exporters `json:"exporters"`
|
||||
SkipPaths []string `json:"skip_paths"`
|
||||
MetricReportingPeriod *int `json:"metric_reporting_period"`
|
||||
TraceSampleRate *float64 `json:"trace_sample_rate"`
|
||||
}
|
||||
|
||||
func (c *Config) Validate() error {
|
||||
return c.Exporters.Validate()
|
||||
}
|
||||
|
||||
func (c *Config) UnsetFieldsToDefaults() {
|
||||
if c.MetricReportingPeriod == nil {
|
||||
reportingPeriod := 30
|
||||
c.MetricReportingPeriod = &reportingPeriod
|
||||
}
|
||||
if c.TraceSampleRate == nil {
|
||||
sampleRate := float64(1.0)
|
||||
c.TraceSampleRate = &sampleRate
|
||||
}
|
||||
}
|
||||
|
||||
type Exporters struct {
|
||||
OTLP []OTLPExporter `json:"otlp"`
|
||||
Prometheus []PrometheusExporter `json:"prometheus"`
|
||||
}
|
||||
|
||||
func (e *Exporters) Validate() error {
|
||||
uniqueNames := make(map[string]bool, len(e.OTLP)+len(e.Prometheus))
|
||||
for idx, ecfg := range e.OTLP {
|
||||
if uniqueNames[ecfg.Name] {
|
||||
return fmt.Errorf("OTLP exporter with duplicate name: %s (at idx %d)", ecfg.Name, idx)
|
||||
}
|
||||
uniqueNames[ecfg.Name] = true
|
||||
}
|
||||
for idx, ecfg := range e.Prometheus {
|
||||
if uniqueNames[ecfg.Name] {
|
||||
return fmt.Errorf("prometheus with duplicate name: %s (at idx %d)", ecfg.Name, idx)
|
||||
}
|
||||
uniqueNames[ecfg.Name] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type OTLPExporter struct {
|
||||
Name string `json:"name"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
UseHTTP bool `json:"use_http"`
|
||||
DisableMetrics bool `json:"disable_metrics"`
|
||||
DisableTraces bool `json:"disable_traces"`
|
||||
}
|
||||
|
||||
type PrometheusExporter struct {
|
||||
Name string `json:"name"`
|
||||
Port int `json:"port"`
|
||||
Host string `json:"host"`
|
||||
ProcessMetrics bool `json:"process_metrics"`
|
||||
GoMetrics bool `json:"go_metrics"`
|
||||
DisableMetrics bool `json:"disable_metrics"`
|
||||
}
|
||||
|
||||
// LayersOpts contains the level of telemetry detail
|
||||
// that we want for each KrakenD stage
|
||||
type LayersOpts struct {
|
||||
Global *GlobalOpts `json:"global"`
|
||||
Pipe *PipeOpts `json:"proxy"`
|
||||
Backend *BackendOpts `json:"backend"`
|
||||
}
|
||||
|
||||
// GlobalOpts has the options for the KrakenD
|
||||
// http handler stage.
|
||||
// We can select if we want to disable the metrics,
|
||||
// the traces, and / or the trace propagation.
|
||||
type GlobalOpts struct {
|
||||
DisableMetrics bool `json:"disable_metrics"`
|
||||
DisableTraces bool `json:"disable_traces"`
|
||||
DisablePropagation bool `json:"disable_propagation"`
|
||||
ReportHeaders bool `json:"report_headers"`
|
||||
}
|
||||
|
||||
// PipeOpts has the options for the KrakenD pipe stage
|
||||
// to disable metrics and traces.
|
||||
type PipeOpts struct {
|
||||
DisableMetrics bool `json:"disable_metrics"`
|
||||
DisableTraces bool `json:"disable_traces"`
|
||||
ReportHeaders bool `json:"report_headers"`
|
||||
}
|
||||
|
||||
// Enabled returns if either metrics or traces are enabled
|
||||
// for the pipe stage.
|
||||
func (o *PipeOpts) Enabled() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return !o.DisableMetrics || !o.DisableTraces
|
||||
}
|
||||
|
||||
// BackendOpts defines the instrumentation detail level for
|
||||
// backend requests.
|
||||
// SkipInstrumentationPaths allows us to provide a list of path
|
||||
// that we do not want to have instrumentation for: those could
|
||||
// be the __debug , __health, or __echo endpoint, for example.
|
||||
type BackendOpts struct {
|
||||
Metrics *BackendMetricOpts `json:"metrics"`
|
||||
Traces *BackendTraceOpts `json:"traces"`
|
||||
}
|
||||
|
||||
// Enabled returns if either metrics or traces enabled
|
||||
// for the backend stage.
|
||||
func (o *BackendOpts) Enabled() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.Metrics.Enabled() || o.Traces.Enabled()
|
||||
}
|
||||
|
||||
type KeyValue struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type Attributes []KeyValue
|
||||
|
||||
func (a Attributes) ToMap() (map[string]string, error) {
|
||||
var err error
|
||||
m := make(map[string]string, len(a))
|
||||
for _, attr := range a {
|
||||
if _, ok := m[attr.Key]; ok {
|
||||
err = fmt.Errorf("duplicate attribute keys")
|
||||
}
|
||||
m[attr.Key] = attr.Value
|
||||
}
|
||||
return m, err
|
||||
}
|
||||
|
||||
// BackendMetricOpts provides the options for the metrics
|
||||
// to be reported at the backend level.
|
||||
//
|
||||
// DisableStage option means it will perevent to report metrics
|
||||
// for ALLL the full backend part (request + manipulations at the backend
|
||||
// level), so other fields will have no effect.
|
||||
//
|
||||
// RoundTrip options will report metrics on the actual request
|
||||
// made for this backend: latency, body size, response code...
|
||||
//
|
||||
// ReadPayload will report the metrics about the reading the
|
||||
// body content of the request (from first time to read, until
|
||||
// all the body has been read). This last options gives extra
|
||||
// fined grained times, that might not be always useful.
|
||||
type BackendMetricOpts struct {
|
||||
DisableStage bool `json:"disable_stage"`
|
||||
RoundTrip bool `json:"round_trip"`
|
||||
ReadPayload bool `json:"read_payload"`
|
||||
DetailedConnection bool `json:"detailed_connection"`
|
||||
StaticAttributes Attributes `json:"static_attributes"`
|
||||
}
|
||||
|
||||
// Enabled tells if there are any metrics to be reported.
|
||||
func (o *BackendMetricOpts) Enabled() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return !o.DisableStage || o.RoundTrip || o.ReadPayload || o.DetailedConnection
|
||||
}
|
||||
|
||||
// BackendTraceOpts provides the options for the tracing
|
||||
// to be reported at the backend level.
|
||||
//
|
||||
// DisableStage means it will avoid creating a Span for ALL
|
||||
// the full backend part (request + manipulations at the backend
|
||||
// level), so other fields will have no effect.
|
||||
//
|
||||
// RoundTrip options will create an span for the actual request
|
||||
// made for this backend.
|
||||
//
|
||||
// ReadPayload will create an additional span just for the reading
|
||||
// the response body part.
|
||||
type BackendTraceOpts struct {
|
||||
DisableStage bool `json:"disable_stage"`
|
||||
RoundTrip bool `json:"round_trip"`
|
||||
ReadPayload bool `json:"read_payload"`
|
||||
DetailedConnection bool `json:"detailed_connection"`
|
||||
StaticAttributes Attributes `json:"static_attributes"`
|
||||
ReportHeaders bool `json:"report_headers"`
|
||||
}
|
||||
|
||||
// Enabled tells if there are any traces to be reported.
|
||||
func (o *BackendTraceOpts) Enabled() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return !o.DisableStage || o.RoundTrip || o.ReadPayload || o.DetailedConnection
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
)
|
||||
|
||||
var (
|
||||
TimeBucketsOpt = metric.WithExplicitBucketBoundaries(
|
||||
0.010, 0.020, 0.050, 0.075,
|
||||
0.100, 0.125, 0.150, 0.175,
|
||||
0.200, 0.250, 0.300, 0.350,
|
||||
0.500, 0.750, 1.000, 1.500,
|
||||
2.000, 3.500, 5.000, 10.000)
|
||||
|
||||
SizeBucketsOpt = metric.WithExplicitBucketBoundaries(
|
||||
128, 256, 512, 1024, // <- reasonable response sizes
|
||||
4*1024, 8*1024, 16*1024, 32*1024, // <- these starts to be big
|
||||
64*1024, 4*64*1024, 8*64*1024, 16*64*1024, // <- those are huge for an api response 64k to 1 Meg
|
||||
4*1024*1024, 16*1024*1024, 64*1024*1024, // <- what are you sending here !? 64 Megs max detail ?
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,98 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
"github.com/luraproject/lura/v2/config"
|
||||
)
|
||||
|
||||
const (
|
||||
// Namespace is the key under the Lura's "extra_config" root
|
||||
// section, for a valid config. See [config] documentation for
|
||||
// details.
|
||||
Namespace = "telemetry/opentelemetry"
|
||||
)
|
||||
|
||||
// ErrNoConfig is used to signal no config was found
|
||||
var ErrNoConfig = errors.New("no config found for opentelemetry")
|
||||
|
||||
// FromLura extracts the configuration from the Lura's ServiceConfig
|
||||
// "extra_config" field.
|
||||
//
|
||||
// In case no "Layers" config is provided, a set of defaults with
|
||||
// everything enabled will be used.
|
||||
func FromLura(srvCfg config.ServiceConfig) (*Config, error) {
|
||||
cfg := new(Config)
|
||||
tmp, ok := srvCfg.ExtraConfig[Namespace]
|
||||
if !ok {
|
||||
return nil, ErrNoConfig
|
||||
}
|
||||
buf := new(bytes.Buffer)
|
||||
json.NewEncoder(buf).Encode(tmp)
|
||||
if err := json.NewDecoder(buf).Decode(cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if cfg.Layers == nil {
|
||||
cfg.Layers = &LayersOpts{}
|
||||
}
|
||||
|
||||
if cfg.Layers.Global == nil {
|
||||
cfg.Layers.Global = &GlobalOpts{
|
||||
DisableMetrics: false,
|
||||
DisableTraces: false,
|
||||
DisablePropagation: false,
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.Layers.Pipe == nil {
|
||||
cfg.Layers.Pipe = &PipeOpts{
|
||||
DisableMetrics: false,
|
||||
DisableTraces: false,
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.Layers.Backend == nil {
|
||||
cfg.Layers.Backend = &BackendOpts{}
|
||||
}
|
||||
|
||||
if cfg.Layers.Backend.Metrics == nil {
|
||||
cfg.Layers.Backend.Metrics = &BackendMetricOpts{
|
||||
DisableStage: false,
|
||||
RoundTrip: true,
|
||||
ReadPayload: true,
|
||||
DetailedConnection: true,
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.Layers.Backend.Traces == nil {
|
||||
cfg.Layers.Backend.Traces = &BackendTraceOpts{
|
||||
DisableStage: false,
|
||||
RoundTrip: true,
|
||||
ReadPayload: true,
|
||||
DetailedConnection: true,
|
||||
}
|
||||
}
|
||||
|
||||
if len(cfg.SkipPaths) == 0 {
|
||||
// if there are no defined skip paths, we use the default ones:
|
||||
// to avoid using defaultSkipPaths, provide a list with an empty string
|
||||
cfg.SkipPaths = []string{
|
||||
"/healthz",
|
||||
"/_ah/health",
|
||||
"/__debug",
|
||||
"/__echo",
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.ServiceName == "" {
|
||||
if srvCfg.Name != "" {
|
||||
cfg.ServiceName = srvCfg.Name
|
||||
} else {
|
||||
cfg.ServiceName = "KrakenD"
|
||||
}
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var templatedURLPattern = regexp.MustCompile(`{{\.(.*?)}}`)
|
||||
|
||||
func NormalizeURLPattern(u string) string {
|
||||
s := templatedURLPattern.ReplaceAllStringFunc(u,
|
||||
func(x string) string {
|
||||
return ":" + strings.ToLower(x[3:len(x)-2])
|
||||
})
|
||||
return s
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
# TODO
|
||||
|
||||
- **we are reporting the size and time in `io.instruments.go` as both a counter and a histogram**
|
||||
(count can be extracted from histogram in all "backends": prometheus, datadog, new relic, etc..?)
|
||||
|
||||
- allow different formats for the propagation of the trace (the `TextMapPropagator`)
|
||||
in [endpoint.go](../router/gin/endpoint.go) and in [state.go](../state/state.go).
|
||||
|
||||
- review how we pass the state.
|
||||
|
||||
- we cannot use `skipPaths` with the same value that we have to define the endpoints
|
||||
at the global layer, because we cannot know the matched pattern
|
||||
|
||||
- clean shutdown
|
||||
|
||||
- add configuration options to allow to use grpc credentials
|
||||
|
||||
- allow to tweak the `bucket` limits for different histograms (like
|
||||
latency and size)
|
||||
- `http/client/transport_metrics.go`: `timeBucketsOpt`, `sizeBucketsOpt`
|
||||
- `http/server/metrics.go`: `timeBucketsOpt`, `sizeBucketsOpt`
|
||||
- `io/instrumens.go`: `timeBucketsOpt`, `sizeBucketsOpt`
|
||||
- `lura/proxy.go`: timeBucketsOpt, sizeBucketsOpt
|
||||
|
||||
- review all metric instruments and procive the correct `metric.WithUnit` value.
|
||||
|
||||
# TO DECIDE
|
||||
|
||||
- do we want to have the `service_name` to override the global ServiceConfig name for
|
||||
the KrakenD gateway ?
|
||||
|
||||
- in `exporter/prometheus/prometheus.go` we could add a config option that will
|
||||
add a prefix to all reported metrics (using the `WithNamespace` option).
|
||||
|
||||
- in `lura/backend.go` and `lura/proxy.go` we are setting the static attrs,
|
||||
and using `semconv.ServerAddress`, to set the concatenated list of hosts.
|
||||
Is that something that we want ?
|
||||
|
||||
# TO CHECK
|
||||
|
||||
- in `exporter/otelcollector/otelcollector.go`, the `WithEndpoint` states that
|
||||
**no http** schema should be in cluded (nor path). To use `http` reporting methods,
|
||||
the `WithInsecure` option should be used (and to user a path `WithURLPath`).
|
||||
Check that current implementation works as expected, or fix it.
|
||||
|
||||
- in `http/client/transport.go` we have commented out the `StartOptions` for
|
||||
trace: review if would be useful to expose that in the config.
|
||||
|
||||
# Known Issues
|
||||
|
||||
There is an issue, that we might have already started an span at
|
||||
the global layer, because we do not know if that path had to be ignored.
|
||||
So, at the global layer, there is no way to skip the paths
|
||||
@@ -0,0 +1,35 @@
|
||||
# Implementation Details
|
||||
|
||||
How the code is organized:
|
||||
|
||||
- [io package](../io): has the wrappers for basic io classes to instrument the
|
||||
bytes read / written with an `io.Reader` / `io.Writer` (event the
|
||||
writer is not currently used in other packages)
|
||||
|
||||
- [http package](../http): has the common functions used to instrument an http
|
||||
request or response (currently, it only has a way to extract
|
||||
trace attributes).
|
||||
|
||||
- [http/client package](../http/client): this package provides the wrapper
|
||||
for an instrumented http client. It has all the
|
||||
|
||||
- [config package](../config): this package contains the configuration
|
||||
definitions for all the krakend-otel library
|
||||
|
||||
- [state package](../state): contains exporter instances
|
||||
and the meter and tracer instances created from a configuration
|
||||
(also a global shared state that can be used from anywhere).
|
||||
|
||||
- [exporter package](../exporter): here are the interfaces to implement
|
||||
the metric / traces exporters, as well as specific implementations
|
||||
for some "providers", like prometheus, otel collector, ...
|
||||
|
||||
- [lura package](../lura): specific Lura middlewares to instruments
|
||||
the `pipe` and `backend` stages.
|
||||
|
||||
- [router/gin](../router/gin): contains specifig gin middleware to
|
||||
instrument a gin router.
|
||||
|
||||
- [example](../example): an example of how to use the krakend-otel
|
||||
library: check the [example documentation](../example/README.md) for
|
||||
more info.
|
||||
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 20 KiB |
@@ -0,0 +1,45 @@
|
||||
KRAKEND_LOCAL_IP ?= 192.168.1.12
|
||||
|
||||
build:
|
||||
go build -o srv ./server
|
||||
.PHONY: build
|
||||
|
||||
image:
|
||||
docker build -f ./server/Dockerfile -t krakend-otel-example:latest ..
|
||||
|
||||
conf:
|
||||
cat ./docker_compose/conf.local/prometheus/prometheus.env_tmpl.yml | envsubst > ./docker_compose/conf.local/prometheus/prometheus.yml
|
||||
|
||||
docker_up:
|
||||
cd ./docker_compose && docker-compose -p krakend_otel -f compose.yaml up -d
|
||||
|
||||
docker_down:
|
||||
cd ./docker_compose && docker-compose -p krakend_otel -f compose.yaml down --remove-orphans -v
|
||||
|
||||
docker_local_up:
|
||||
cd ./docker_compose && docker-compose -p krakend_otel_local -f compose.local.yaml up -d
|
||||
|
||||
docker_local_down:
|
||||
cd ./docker_compose && docker-compose -p krakend_otel_local -f compose.local.yaml down --remove-orphans -v
|
||||
|
||||
run_local_up: srv
|
||||
cd ./docker_compose && docker-compose -p krakend_otel_local -f compose.local.yaml up -d
|
||||
./srv -c ./docker_compose/conf.local/krakend_front/configuration.json > log_frontend.log &
|
||||
./srv -c ./docker_compose/conf.local/krakend_middle/configuration.json > log_middle.log &
|
||||
./srv -c ./docker_compose/conf.local/krakend_back/configuration.json > log_back.log
|
||||
|
||||
run_local: srv
|
||||
./srv -c ./docker_compose/conf.local/krakend_front/configuration.json > log_frontend.log &
|
||||
./srv -c ./docker_compose/conf.local/krakend_middle/configuration.json > log_middle.log &
|
||||
./srv -c ./docker_compose/conf.local/krakend_back/configuration.json > log_back.log
|
||||
|
||||
kill:
|
||||
cd ./docker_compose && docker-compose -p krakend_otel_local -f compose.local.yaml down
|
||||
@ps -aux | grep 'srv.*krakend' | grep -v 'grep' | sed "s/[[:space:]]\+/ /g" | cut -d ' ' -f 2 | xargs kill
|
||||
|
||||
kill_srv:
|
||||
@ps -aux | grep 'srv.*krakend' | grep -v 'grep' | sed "s/[[:space:]]\+/ /g" | cut -d ' ' -f 2 | xargs kill
|
||||
|
||||
dbg:
|
||||
dlv debug ./server --wd . -- -p 7979 -c ./krakend.json
|
||||
.PHONY: dbg
|
||||
@@ -0,0 +1,65 @@
|
||||
# How to run the example
|
||||
|
||||
In the `docker_compose` folder there is the environment to run the observability software.
|
||||
|
||||
## Run the dockerized example:
|
||||
|
||||
```
|
||||
make image
|
||||
cd docker-compose && make up
|
||||
```
|
||||
|
||||
## Run local krakend instances and other services in docker compose
|
||||
|
||||
Since the docker-compose environment runs in its own network, in order to reach
|
||||
your local running services, you need to provide it with your network visible ip:
|
||||
you might have for example `localhost` that maps to `127.0.0.1` and an IP
|
||||
assigned to your ethernet interface, that could be `192.168.1.12` for example. The
|
||||
localhost one will not when used from a container, however, the other one can
|
||||
be used to gather data from your running services (that is needed to scrape
|
||||
prometheus metrics).
|
||||
|
||||
So, you should run:
|
||||
|
||||
```
|
||||
export KRAKEND_LOCAL_IP="{your non localhost machine ip}"
|
||||
make conf
|
||||
make run
|
||||
```
|
||||
|
||||
# What do you get
|
||||
|
||||
By running the example, you spawn 3 servers based on the Lura framework (we could
|
||||
say the example is a super simplified KrakenD CE).
|
||||
|
||||
- `krakend_front`: the one to receive the requests
|
||||
- `krakend_middle`: a middle krakend to check how traces are propagated
|
||||
- `krakend_back`: a back krakend that makes request to `https://jsonplaceholder.typicode.com` to
|
||||
get some dummy data.
|
||||
|
||||
The config files can be found in:
|
||||
|
||||
- [./docker_compose/conf/krakend_*](./docker_compose/conf/) directories for the dockerized version of the example.
|
||||
- [./docker_compose/conf.local/krakend_*](./docker_compose/conf.local/) directories for running
|
||||
the krakends in the local environment.
|
||||
|
||||
|
||||
# Execute some requests
|
||||
|
||||
There is a bash script to make some requests, that you can run with:
|
||||
|
||||
```bash
|
||||
bash ./make_request.sh
|
||||
```
|
||||
|
||||
(You might want to edit the script to point to the dockerized version of the krakend frontend
|
||||
server or the local one).
|
||||
|
||||
# Access the dashboards
|
||||
|
||||
In both, the dockerized and the local options, you will end with some containers running some
|
||||
services that you can use to check the metrics / traces:
|
||||
|
||||
- Grafana: at http://localhost:3000 , with the username: `krakend` and the password: `krakend` as
|
||||
admin users
|
||||
- Jaeger: at http://localhost:16686.
|
||||
@@ -0,0 +1,26 @@
|
||||
import http from 'k6/http';
|
||||
import { sleep } from 'k6';
|
||||
|
||||
export const options = {
|
||||
// A number specifying the number of VUs to run concurrently.
|
||||
vus: 2,
|
||||
// A string specifying the total duration of the test run.
|
||||
duration: '150m',
|
||||
};
|
||||
|
||||
// The function that defines VU logic.
|
||||
//
|
||||
// See https://grafana.com/docs/k6/latest/examples/get-started-with-k6/ to learn more
|
||||
// about authoring k6 scripts.
|
||||
//
|
||||
export default function() {
|
||||
// let address = 'http://192.168.1.12:54444';
|
||||
let address = 'http://192.168.1.12:44444';
|
||||
http.get(address + '/fake/fsf');
|
||||
http.get(address + '/combination/1');
|
||||
http.get(address + '/direct/slow');
|
||||
http.get(address + '/direct/delayed');
|
||||
http.get(address + '/direct/drop');
|
||||
http.get(address + '/does_not_exist');
|
||||
sleep(1);
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
|
||||
# to generate a new base script:
|
||||
#
|
||||
# docker run --rm -i -v ${PWD}:/app \
|
||||
# --user $(id -u):$(id -g) \
|
||||
# -w /app grafana/k6 \
|
||||
# new client.js
|
||||
|
||||
docker run --rm -i -v ${PWD}:/app \
|
||||
--user $(id -u):$(id -g) \
|
||||
-w /app grafana/k6 \
|
||||
run k6client/client.js
|
||||
|
||||
## for i in {1..1}
|
||||
## # for i in {1..1000}
|
||||
## do
|
||||
## # curl localhost:54444/fake/fsf
|
||||
## # curl localhost:54444/combination/2
|
||||
## curl localhost:54444/combination/1
|
||||
## # curl localhost:54444/direct/slow
|
||||
## sleep 0.1
|
||||
## # curl localhost:54444/direct/delayed
|
||||
## # curl localhost:54444/direct/drop
|
||||
## done
|
||||
##
|
||||
## # curl localhost:44444/fake/fsf | jq
|
||||
## # echo -e "\n"
|
||||
## # curl localhost:44444/combination/23 | jq
|
||||
@@ -0,0 +1,23 @@
|
||||
up:
|
||||
docker-compose -p krakend_otel up -d
|
||||
.PHONY: up
|
||||
|
||||
down:
|
||||
docker-compose -p krakend_otel down --volumes --remove-orphans
|
||||
.PHONY: down
|
||||
|
||||
stop:
|
||||
docker-compose -p krakend_otel stop
|
||||
.PHONY: stop
|
||||
|
||||
local_up:
|
||||
docker-compose -p krakend_otel_local -f compose.local.yaml up -d
|
||||
.PHONY: up
|
||||
|
||||
local_down:
|
||||
docker-compose -p krakend_otel_local -f compose.local.yaml down --volumes --remove-orphans
|
||||
.PHONY: down
|
||||
|
||||
local_stop:
|
||||
docker-compose -p krakend_otel_local -f compose.local.yaml stop
|
||||
.PHONY: stop
|
||||
@@ -0,0 +1,28 @@
|
||||
# Start the environment
|
||||
|
||||
|
||||
# Grafana
|
||||
|
||||
Go to `http://localhost:3000` and login with `krakend` :: `krakend`.
|
||||
|
||||
|
||||
# Generate traces
|
||||
|
||||
In the [opentelemetry.io collector getting started page](https://opentelemetry.io/docs/collector/getting-started/)'s
|
||||
step `2` we can see that we can download a tool to generate traces:
|
||||
|
||||
```
|
||||
go install github.com/open-telemetry/opentelemetry-collector-contrib/cmd/telemetrygen@latest
|
||||
```
|
||||
|
||||
The `gRPC` endpoint is on port `4317` by default.
|
||||
|
||||
```
|
||||
telemetrygen traces --otlp-insecure --duration 5s
|
||||
```
|
||||
|
||||
|
||||
## References
|
||||
|
||||
- [Grafana's Tempo Documentation](https://grafana.com/docs/tempo/latest/)
|
||||
- [Grafana's tempo docker-compose example](https://github.com/grafana/tempo/tree/main/example/docker-compose)
|
||||
@@ -0,0 +1,83 @@
|
||||
volumes:
|
||||
vol_local_tempo:
|
||||
vol_local_grafana:
|
||||
vol_local_prometheus:
|
||||
services:
|
||||
tracelogger:
|
||||
image: otel/opentelemetry-collector:0.94.0
|
||||
ports:
|
||||
- "24317:4317"
|
||||
- "25679:55679"
|
||||
volumes:
|
||||
- "./conf.local/tracelogger/config.yaml:/etc/otelcol/config.yaml"
|
||||
- "./tmp:/var/log/example"
|
||||
user: "1000"
|
||||
loki:
|
||||
image: grafana/loki:latest
|
||||
domainname: loki
|
||||
healthcheck:
|
||||
test: [CMD, wget, --spicer, -S, http://localhost:3100/ready]
|
||||
ports:
|
||||
- "53100:3100"
|
||||
tempo:
|
||||
image: grafana/tempo:latest
|
||||
domainname: tempo
|
||||
command: [ "-config.file=/etc/tempo.yaml" ]
|
||||
volumes:
|
||||
- "./conf.local/tempo/tempo.yaml:/etc/tempo.yaml"
|
||||
- "vol_local_tempo:/tmp/tempo"
|
||||
ports:
|
||||
- "53200:3200" # tempo
|
||||
- "59095:9095" # tempo grpc
|
||||
- "53317:4317" # otlp grpc
|
||||
- "53318:4318" # otlp http
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
domainname: prometheus
|
||||
ports:
|
||||
- "59590:9090"
|
||||
volumes:
|
||||
- "./conf.local/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml"
|
||||
- "vol_local_prometheus:/prometheus"
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
domainname: grafana
|
||||
ports:
|
||||
- "53000:3000"
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: krakend
|
||||
GF_SECURITY_ADMIN_PASSWORD: krakend
|
||||
GF_AUT_ANONYMOUS_ENABLED: "true"
|
||||
volumes:
|
||||
- "vol_local_grafana:/var/lib/grafana"
|
||||
- "./conf.local/grafana/conf/provisioning/datasources:/etc/grafana/provisioning/datasources"
|
||||
- "./conf.local/grafana/conf/provisioning/dashboards:/etc/grafana/provisioning/dashboards"
|
||||
- "./conf.local/grafana/conf/data/dashboards:/var/lib/grafana/dashboards"
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
jaeger:
|
||||
image: jaegertracing/all-in-one:1.54
|
||||
domainname: jaeger
|
||||
environment:
|
||||
COLLECTOR_ZIPKIN_HOST_PORT: ":9411"
|
||||
ports:
|
||||
- "55778:5778" # serve configs
|
||||
- "56686:16686" # serve frontend UI
|
||||
- "55317:4317" # otlp grpc: we remap this to be able to run other envs
|
||||
- "55318:4318" # otlp http: we reamp this to be able to run other envs
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4096M
|
||||
fakeapi:
|
||||
image: dhontecillas/reqstatsrv:latest
|
||||
domainname: fakeapi
|
||||
ports:
|
||||
- "9876:9876"
|
||||
command:
|
||||
- "./reqstatsrv"
|
||||
- "/etc/reqstatsrv/config/example.dockerized.json"
|
||||
volumes:
|
||||
- "./conf.local/fakeapi:/etc/reqstatsrv"
|
||||
@@ -0,0 +1,102 @@
|
||||
volumes:
|
||||
vol_tempo:
|
||||
vol_grafana:
|
||||
vol_prometheus:
|
||||
services:
|
||||
krakend_front:
|
||||
domainname: krakend_front
|
||||
image: krakend-otel-example:latest
|
||||
volumes:
|
||||
- "./conf/krakend_front:/etc/krakend"
|
||||
ports:
|
||||
- "44444:44444"
|
||||
- "9091:9090"
|
||||
krakend_middle:
|
||||
domainname: krakend_middle
|
||||
image: krakend-otel-example:latest
|
||||
volumes:
|
||||
- "./conf/krakend_middle:/etc/krakend"
|
||||
ports:
|
||||
- "44445:44445"
|
||||
- "9092:9090"
|
||||
krakend_back:
|
||||
domainname: krakend_back
|
||||
image: krakend-otel-example:latest
|
||||
volumes:
|
||||
- "./conf/krakend_back:/etc/krakend"
|
||||
ports:
|
||||
- "44446:44446"
|
||||
- "9093:9090"
|
||||
loki:
|
||||
image: grafana/loki:latest
|
||||
domainname: loki
|
||||
healthcheck:
|
||||
test: [CMD, wget, --spicer, -S, http://localhost:3100/ready]
|
||||
ports:
|
||||
- "3100:3100"
|
||||
tempo:
|
||||
image: grafana/tempo:latest
|
||||
domainname: tempo
|
||||
command: [ "-config.file=/etc/tempo.yaml" ]
|
||||
volumes:
|
||||
- "./conf/tempo/tempo.yaml:/etc/tempo.yaml"
|
||||
- "vol_tempo:/tmp/tempo"
|
||||
ports:
|
||||
- "3200:3200" # tempo
|
||||
- "9095:9095" # tempo grpc
|
||||
- "4317:4317" # otlp grpc
|
||||
- "4318:4318" # otlp http
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
domainname: prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- "./conf/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml"
|
||||
- "vol_prometheus:/prometheus"
|
||||
depends_on:
|
||||
- krakend_front
|
||||
- krakend_middle
|
||||
- krakend_back
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
domainname: grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: krakend
|
||||
GF_SECURITY_ADMIN_PASSWORD: krakend
|
||||
GF_AUT_ANONYMOUS_ENABLED: "true"
|
||||
volumes:
|
||||
- "vol_grafana:/var/lib/grafana"
|
||||
- "./conf/grafana/conf/provisioning/datasources:/etc/grafana/provisioning/datasources"
|
||||
- "./conf/grafana/conf/provisioning/dashboards:/etc/grafana/provisioning/dashboards"
|
||||
- "./conf/grafana/conf/data/dashboards:/var/lib/grafana/dashboards"
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
jaeger:
|
||||
image: jaegertracing/all-in-one:1.54
|
||||
domainname: jaeger
|
||||
environment:
|
||||
COLLECTOR_ZIPKIN_HOST_PORT: ":9411"
|
||||
ports:
|
||||
- "5778:5778" # serve configs
|
||||
- "16686:16686" # serve frontend UI
|
||||
- "5317:4317" # otlp grpc: we remap this to be able to run other envs
|
||||
- "5318:4318" # otlp http: we reamp this to be able to run other envs
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4096M
|
||||
fakeapi:
|
||||
image: dhontecillas/reqstatsrv:latest
|
||||
domainname: fakeapi
|
||||
ports:
|
||||
- "9876:9876"
|
||||
command:
|
||||
- "./reqstatsrv"
|
||||
- "/etc/reqstatsrv/config/example.dockerized.json"
|
||||
volumes:
|
||||
- "./conf.local/fakeapi:/etc/reqstatsrv"
|
||||
@@ -0,0 +1,173 @@
|
||||
{
|
||||
"port": 9876,
|
||||
"host": "0.0.0.0",
|
||||
"endpoints": [
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/",
|
||||
"behaviour": [
|
||||
{
|
||||
"name": "slower",
|
||||
"config": {
|
||||
"max_bytes_per_second": 20,
|
||||
"flush_bytes": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "status_distributor",
|
||||
"config": {
|
||||
"code_distribution": [
|
||||
{"key": 200, "val": 0.5},
|
||||
{"key": 201, "val": 0.3}
|
||||
],
|
||||
"seed": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "delayer",
|
||||
"config": {
|
||||
"delay_millis_distribution": [
|
||||
{"key": 0, "val": 0.05},
|
||||
{"key": 10, "val": 0.2},
|
||||
{"key": 50, "val": 0.0},
|
||||
{"key": 100, "val": 0.75}
|
||||
],
|
||||
"seed": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"content": {
|
||||
"source": "directory",
|
||||
"config": {
|
||||
"dir": "/etc/reqstatsrv/data"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/drop/",
|
||||
"behaviour": [
|
||||
{
|
||||
"name": "connection_closer",
|
||||
"config": {
|
||||
"freq": 0.5,
|
||||
"seed": 3
|
||||
}
|
||||
}
|
||||
],
|
||||
"content" : {
|
||||
"source": "file",
|
||||
"config": {
|
||||
"path": "/etc/reqstatsrv/data/bar.json"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/delayed/",
|
||||
"behaviour": [
|
||||
{
|
||||
"name": "delayer",
|
||||
"config": {
|
||||
"delay_millis_distribution": [
|
||||
{"key": 0, "val": 0.0},
|
||||
{"key": 10, "val": 0.0},
|
||||
{"key": 50, "val": 0.0},
|
||||
{"key": 100, "val": 0.25},
|
||||
{"key": 200, "val": 0.25},
|
||||
{"key": 700, "val": 0.5}
|
||||
],
|
||||
"seed": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "status_distributor",
|
||||
"config": {
|
||||
"code_distribution": [
|
||||
{"key": 200, "val": 0.7},
|
||||
{"key": 201, "val": 0.3}
|
||||
],
|
||||
"seed": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"content": {
|
||||
"source": "directory",
|
||||
"config": {
|
||||
"dir": "/etc/reqstatsrv/data"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/file",
|
||||
"behaviour": [],
|
||||
"content": {
|
||||
"source": "file",
|
||||
"config": {
|
||||
"path": "/etc/reqstatsrv/data/bar.json"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/stats",
|
||||
"behaviour": [
|
||||
|
||||
],
|
||||
"content": {
|
||||
"source": "stats",
|
||||
"config": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/suslik",
|
||||
"behaviour": [
|
||||
{
|
||||
"name": "status_distributor",
|
||||
"config": {
|
||||
"code_distribution": [
|
||||
{"key": 200, "val": 0.6},
|
||||
{"key": 201, "val": 0.2},
|
||||
{"key": 202, "val": 0.2}
|
||||
],
|
||||
"seed": 1
|
||||
}
|
||||
}
|
||||
|
||||
],
|
||||
"content": {
|
||||
"source": "status_content_selector",
|
||||
"config": {
|
||||
"default_content": {
|
||||
"source": "stats",
|
||||
"config": {}
|
||||
},
|
||||
"status_contents": [
|
||||
{
|
||||
"from": 201,
|
||||
"to": 202,
|
||||
"content": {
|
||||
"source": "file",
|
||||
"config": {
|
||||
"path": "/etc/reqstatsrv/data/201_created.json"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"from": 202,
|
||||
"to": 203,
|
||||
"content": {
|
||||
"source": "file",
|
||||
"config": {
|
||||
"path": "/etc/reqstatsrv/data/202_accepted.json"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"result": 201,
|
||||
"desc": "something created"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"batch_process_id": "A629S99IAKSRtT"
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"source": "bar.json",
|
||||
"dummy_string": "something here",
|
||||
"dummy_number": 42.0
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"foo": "bar"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"foo": "bar"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'default'
|
||||
orgId: 1
|
||||
folder: ''
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 60
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards/KrakenDOTEL.json
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"apiVersion": 1,
|
||||
"datasources": [
|
||||
{
|
||||
"id": 3,
|
||||
"uid": "b2a29c2f-ef77-4ba5-b934-b42aaa6ae3b4",
|
||||
"orgId": 1,
|
||||
"name": "Loki",
|
||||
"type": "loki",
|
||||
"typeName": "Loki",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/loki/img/loki_icon.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://loki:3100",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": false,
|
||||
"jsonData": {},
|
||||
"readOnly": false
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e",
|
||||
"orgId": 1,
|
||||
"name": "Prometheus",
|
||||
"type": "prometheus",
|
||||
"typeName": "Prometheus",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/prometheus/img/prometheus_logo.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://prometheus:9090",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": true,
|
||||
"jsonData": {
|
||||
"httpMethod": "POST"
|
||||
},
|
||||
"readOnly": false
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"uid": "e7eea071-7df9-44ff-8a6a-155248d05f42",
|
||||
"orgId": 1,
|
||||
"name": "Tempo",
|
||||
"type": "tempo",
|
||||
"typeName": "Tempo",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/tempo/img/tempo_logo.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://tempo:3200",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": false,
|
||||
"jsonData": {},
|
||||
"readOnly": false
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
[
|
||||
{
|
||||
"id": 3,
|
||||
"uid": "b2a29c2f-ef77-4ba5-b934-b42aaa6ae3b4",
|
||||
"orgId": 1,
|
||||
"name": "Loki",
|
||||
"type": "loki",
|
||||
"typeName": "Loki",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/loki/img/loki_icon.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://loki:3100",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": false,
|
||||
"jsonData": {},
|
||||
"readOnly": false
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e",
|
||||
"orgId": 1,
|
||||
"name": "Prometheus",
|
||||
"type": "prometheus",
|
||||
"typeName": "Prometheus",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/prometheus/img/prometheus_logo.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://prometheus:9090",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": true,
|
||||
"jsonData": {
|
||||
"httpMethod": "POST"
|
||||
},
|
||||
"readOnly": false
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"uid": "e7eea071-7df9-44ff-8a6a-155248d05f42",
|
||||
"orgId": 1,
|
||||
"name": "Tempo",
|
||||
"type": "tempo",
|
||||
"typeName": "Tempo",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/tempo/img/tempo_logo.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://tempo:3200",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": false,
|
||||
"jsonData": {},
|
||||
"readOnly": false
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/bin/bash
|
||||
curl http://krakend:krakend@localhost:53000/api/datasources
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"id": 3,
|
||||
"uid": "b2a29c2f-ef77-4ba5-b934-b42aaa6ae3b4",
|
||||
"orgId": 1,
|
||||
"name": "Loki",
|
||||
"type": "loki",
|
||||
"typeName": "Loki",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/loki/img/loki_icon.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://loki:3100",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": false,
|
||||
"jsonData": {},
|
||||
"readOnly": false
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"id": 1,
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e",
|
||||
"orgId": 1,
|
||||
"name": "Prometheus",
|
||||
"type": "prometheus",
|
||||
"typeName": "Prometheus",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/prometheus/img/prometheus_logo.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://prometheus:9090",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": true,
|
||||
"jsonData": {
|
||||
"httpMethod": "POST"
|
||||
},
|
||||
"readOnly": false
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
|
||||
# echo '{ "dashboard": ' > tmp.json
|
||||
# cat KrakenD_OTEL_Dashboard.json >> tmp.json
|
||||
# echo '}' >> tmp.json
|
||||
|
||||
curl -X POST --insecure --header "Content-Type: application/json" \
|
||||
http://krakend:krakend@localhost:53000/api/dashboards/db \
|
||||
-d @tmp.json
|
||||
|
||||
# rm tmp.json
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
|
||||
curl -X POST --header "Content-Type: application/json" \
|
||||
http://krakend:krakend@localhost:53000/api/datasources \
|
||||
-d @prometheus_datasource.json
|
||||
|
||||
curl -X POST --header "Content-Type: application/json" \
|
||||
http://krakend:krakend@localhost:53000/api/datasources \
|
||||
-d @tempo_datasource.json
|
||||
|
||||
curl -X POST --header "Content-Type: application/json" \
|
||||
http://krakend:krakend@localhost:53000/api/datasources \
|
||||
-d @loki_datasource.json
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"id": 2,
|
||||
"uid": "e7eea071-7df9-44ff-8a6a-155248d05f42",
|
||||
"orgId": 1,
|
||||
"name": "Tempo",
|
||||
"type": "tempo",
|
||||
"typeName": "Tempo",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/tempo/img/tempo_logo.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://tempo:3200",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": false,
|
||||
"jsonData": {},
|
||||
"readOnly": false
|
||||
}
|
||||
@@ -0,0 +1,784 @@
|
||||
{ "dashboard":
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Example of collected OTEL metrics",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": 3,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 8,
|
||||
"panels": [],
|
||||
"title": "Row title",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"description": "Router Requests Per Minute",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"fixedColor": "blue",
|
||||
"mode": "fixed"
|
||||
},
|
||||
"fieldMinMax": false,
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 3,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"pluginVersion": "10.2.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(router_response_latency_count{instance=\"$instance\"}[$__rate_interval]) * 60)",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": false,
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "Router Requests",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"fixedColor": "blue",
|
||||
"mode": "fixed"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 3,
|
||||
"x": 3,
|
||||
"y": 1
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "10.2.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "code",
|
||||
"expr": "avg(histogram_quantile(0.99, sum by(le) (rate(router_response_latency_bucket{instance=\"$instance\"}[$__rate_interval]))))",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": false,
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "Router Latency p99",
|
||||
"transparent": true,
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 3,
|
||||
"x": 6,
|
||||
"y": 1
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "10.2.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "builder",
|
||||
"expr": "histogram_quantile(0.99, sum by(le) (rate(stage_duration_bucket{krakend_stage=\"pipe\"}[$__rate_interval])))",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": false,
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "Pipe Stage Duration 99p",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 3,
|
||||
"x": 9,
|
||||
"y": 1
|
||||
},
|
||||
"id": 12,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "10.2.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "builder",
|
||||
"expr": "histogram_quantile(0.99, sum by(le) (rate(stage_duration_bucket{krakend_stage=\"backend\"}[$__rate_interval])))",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": false,
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "Backend Stage Duration 99p",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 6
|
||||
},
|
||||
"id": 4,
|
||||
"panels": [],
|
||||
"title": "Router",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 7
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "builder",
|
||||
"expr": "histogram_quantile(0.95, sum by(le, url_path) (rate(router_response_latency_bucket{instance=\"$instance\"}[$__rate_interval])))",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": false,
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "95p Router Endpoint Latencies",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 7
|
||||
},
|
||||
"id": 13,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"title": "Panel Title",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"id": 3,
|
||||
"panels": [],
|
||||
"title": "Pipeline",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"id": 2,
|
||||
"panels": [],
|
||||
"title": "Backends",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 15,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 17
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "builder",
|
||||
"expr": "histogram_quantile(0.95, sum by(le, url_path, server_address, otel_scope_name) (rate(response_latency_bucket{instance=\"$instance\"}[$__rate_interval])))",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": false,
|
||||
"instant": true,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "95p backends latency",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 32
|
||||
},
|
||||
"id": 9,
|
||||
"panels": [],
|
||||
"title": "Tracing",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "e7eea071-7df9-44ff-8a6a-155248d05f42"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"cellOptions": {
|
||||
"type": "auto"
|
||||
},
|
||||
"inspect": false
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 19,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 33
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"cellHeight": "sm",
|
||||
"footer": {
|
||||
"countRows": false,
|
||||
"fields": "",
|
||||
"reducer": [
|
||||
"sum"
|
||||
],
|
||||
"show": false
|
||||
},
|
||||
"showHeader": true
|
||||
},
|
||||
"pluginVersion": "10.2.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "af13a7d6-b296-4f6e-978a-942e087b2c9e"
|
||||
},
|
||||
"filters": [
|
||||
{
|
||||
"id": "2adf4ac6",
|
||||
"operator": "=",
|
||||
"scope": "span"
|
||||
},
|
||||
{
|
||||
"id": "service-name",
|
||||
"operator": "=",
|
||||
"scope": "resource",
|
||||
"tag": "service.name",
|
||||
"value": [],
|
||||
"valueType": "string"
|
||||
}
|
||||
],
|
||||
"key": "Q-7e494413-0c5c-41b3-bf83-a0c8dc0b4205-0",
|
||||
"limit": 10,
|
||||
"queryType": "traceqlSearch",
|
||||
"refId": "A",
|
||||
"spss": 100,
|
||||
"tableType": "traces"
|
||||
}
|
||||
],
|
||||
"title": "New Panel",
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
"schemaVersion": 38,
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "192.168.1.12:59091",
|
||||
"value": "192.168.1.12:59091"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e"
|
||||
},
|
||||
"definition": "label_values(instance)",
|
||||
"description": "The Instance that we want to monitor",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Instance",
|
||||
"multi": false,
|
||||
"name": "instance",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(instance)",
|
||||
"refId": "PrometheusVariableQueryEditor-VariableQuery"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-5m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "KrakenD OTEL",
|
||||
"version": 9,
|
||||
"weekStart": ""
|
||||
}}
|
||||
@@ -0,0 +1,122 @@
|
||||
{
|
||||
"version": 3,
|
||||
"name": "Back KrakenD",
|
||||
"port": 54446,
|
||||
"cache_ttl": "3600s",
|
||||
"timeout": "2500ms",
|
||||
"host": [ "http://127.0.0.1:54446/" ],
|
||||
"debug_endpoint": true,
|
||||
"endpoints": [
|
||||
{
|
||||
"endpoint": "/back_fake/{something}",
|
||||
"method": "GET",
|
||||
"concurrent_calls": 3,
|
||||
"backend": [
|
||||
{
|
||||
"url_pattern": "/__debug/{something}",
|
||||
"mapping": {
|
||||
"message": "backend1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"url_pattern": "/__debug/{something}/{something}",
|
||||
"mapping": {
|
||||
"message": "backend2"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/back_combination/{id}",
|
||||
"backend": [
|
||||
{
|
||||
"host": [
|
||||
"https://jsonplaceholder.typicode.com"
|
||||
],
|
||||
"url_pattern": "/posts?userId={id}",
|
||||
"is_collection": true,
|
||||
"mapping": {
|
||||
"collection": "posts"
|
||||
}
|
||||
},
|
||||
{
|
||||
"host": [
|
||||
"https://jsonplaceholder.typicode.com"
|
||||
],
|
||||
"url_pattern": "/users/{id}",
|
||||
"mapping": {
|
||||
"email": "personal_email"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"extra_config": {
|
||||
"telemetry/opentelemetry": {
|
||||
"service_name": "krakend_back_service",
|
||||
"layers": {
|
||||
"global": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false,
|
||||
"disable_propagation": false
|
||||
},
|
||||
"proxy": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false
|
||||
},
|
||||
"backend": {
|
||||
"metrics": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_metric_attr",
|
||||
"value": "my_metric_val"
|
||||
}
|
||||
]
|
||||
},
|
||||
"traces": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_trace_attr",
|
||||
"value": "my_trace_val"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"exporters": {
|
||||
"prometheus": [
|
||||
{
|
||||
"name": "local_prometheus",
|
||||
"port": 59093,
|
||||
"process_metrics": true,
|
||||
"go_metrics": true
|
||||
}
|
||||
],
|
||||
"otlp": [
|
||||
{
|
||||
"name": "local_tempo",
|
||||
"host": "127.0.0.1",
|
||||
"port": 53317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
},
|
||||
{
|
||||
"name": "local_jaeger",
|
||||
"host": "127.0.0.1",
|
||||
"port": 55317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
{
|
||||
"version": 3,
|
||||
"name": "Front KrakenD",
|
||||
"port": 54444,
|
||||
"cache_ttl": "3600s",
|
||||
"timeout": "2500ms",
|
||||
"debug": true,
|
||||
"host": [ "http://127.0.0.1:54445/" ],
|
||||
"endpoints": [
|
||||
{
|
||||
"endpoint": "/fake/{something}",
|
||||
"concurrent_calls": 3,
|
||||
"backend": [
|
||||
{
|
||||
"url_pattern": "/middle_fake/{something}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/combination/{id}",
|
||||
"backend": [
|
||||
{
|
||||
"url_pattern": "/middle_combination/{id}",
|
||||
"group": "front_1"
|
||||
},
|
||||
{
|
||||
"url_pattern": "/middle_combination/{id}",
|
||||
"group": "front_2"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/direct/slow",
|
||||
"backend": [
|
||||
{
|
||||
"host": ["http://127.0.0.1:9876"],
|
||||
"url_pattern": "/foo.json",
|
||||
"group": "data"
|
||||
},
|
||||
{
|
||||
"host": ["http://127.0.0.1:9876"],
|
||||
"url_pattern": "/bar.json",
|
||||
"group": "data"
|
||||
},
|
||||
{
|
||||
"host": ["http://127.0.0.1:9876"],
|
||||
"url_pattern": "/stats"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/direct/drop",
|
||||
"backend": [
|
||||
{
|
||||
"host": ["http://127.0.0.1:9876"],
|
||||
"url_pattern": "/drop/"
|
||||
},
|
||||
{
|
||||
"host": ["http://127.0.0.1:9876"],
|
||||
"url_pattern": "/stats"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/direct/delayed",
|
||||
"backend": [
|
||||
{
|
||||
"host": ["http://127.0.0.1:9876"],
|
||||
"url_pattern": "/delayed/"
|
||||
},
|
||||
{
|
||||
"host": ["http://127.0.0.1:9876"],
|
||||
"url_pattern": "/file"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"extra_config": {
|
||||
"telemetry/opentelemetry": {
|
||||
"service_name": "krakend_frontend_service",
|
||||
"metric_reporting_period": 1,
|
||||
"trace_sample_rate": 1,
|
||||
"layers": {
|
||||
"global": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false,
|
||||
"disable_propagation": false,
|
||||
"report_headers": true
|
||||
},
|
||||
"proxy": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false,
|
||||
"report_headers": true
|
||||
},
|
||||
"backend": {
|
||||
"metrics": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes":[
|
||||
{
|
||||
"key": "my_metric_attr",
|
||||
"value": "my_metric_val"
|
||||
}
|
||||
]
|
||||
},
|
||||
"traces": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_trace_attr",
|
||||
"value": "my_trace_val"
|
||||
}
|
||||
],
|
||||
"report_headers": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"exporters": {
|
||||
"prometheus": [
|
||||
{
|
||||
"name": "local_prometheus",
|
||||
"port": 59091,
|
||||
"process_metrics": true,
|
||||
"go_metrics": true
|
||||
}
|
||||
],
|
||||
"otlp": [
|
||||
{
|
||||
"name": "local_tempo",
|
||||
"host": "127.0.0.1",
|
||||
"port": 53317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
},
|
||||
{
|
||||
"name": "local_jaeger",
|
||||
"host": "127.0.0.1",
|
||||
"port": 55317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
},
|
||||
{
|
||||
"name": "local_tracelogger",
|
||||
"host": "127.0.0.1",
|
||||
"port": 24317,
|
||||
"use_http": false,
|
||||
"disable_traces": true,
|
||||
"disable_metrics": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"version": 3,
|
||||
"name": "Middle KrakenD",
|
||||
"port": 54445,
|
||||
"cache_ttl": "3600s",
|
||||
"timeout": "2500ms",
|
||||
"host": [ "http://127.0.0.1:54446/" ],
|
||||
"endpoints": [
|
||||
{
|
||||
"endpoint": "/middle_fake/{something}",
|
||||
"concurrent_calls": 3,
|
||||
"backend": [
|
||||
{
|
||||
"url_pattern": "/back_fake/{something}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/middle_combination/{id}",
|
||||
"backend": [
|
||||
{
|
||||
"url_pattern": "/back_combination/{id}",
|
||||
"group": "middle_1"
|
||||
},
|
||||
{
|
||||
"url_pattern": "/back_combination/{id}",
|
||||
"group": "middle_2"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"extra_config": {
|
||||
"telemetry/opentelemetry": {
|
||||
"service_name": "krakend_middle_service",
|
||||
"metric_reporting_period": 1,
|
||||
"trace_sample_rate": 1,
|
||||
"layers": {
|
||||
"global": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false,
|
||||
"disable_propagation": false
|
||||
},
|
||||
"proxy": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false
|
||||
},
|
||||
"backend": {
|
||||
"metrics": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_metric_attr",
|
||||
"value": "my_middle_metric"
|
||||
}
|
||||
]
|
||||
},
|
||||
"traces": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_trace_attr",
|
||||
"value": "my_middle_trace"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"exporters": {
|
||||
"prometheus": [
|
||||
{
|
||||
"name": "local_prometheus",
|
||||
"port": 59092,
|
||||
"process_metrics": true,
|
||||
"go_metrics": true
|
||||
}
|
||||
],
|
||||
"otlp": [
|
||||
{
|
||||
"name": "local_tempo",
|
||||
"host": "127.0.0.1",
|
||||
"port": 53317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
},
|
||||
{
|
||||
"name": "local_jaeger",
|
||||
"host": "127.0.0.1",
|
||||
"port": 55317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
external_labels:
|
||||
monitor: kotel_monitor
|
||||
|
||||
scrape_configs:
|
||||
- job_name: kotel_example
|
||||
scrape_interval: 5s
|
||||
metrics_path: '/metrics'
|
||||
static_configs:
|
||||
- targets:
|
||||
- '${KRAKEND_LOCAL_IP}:59091'
|
||||
- '${KRAKEND_LOCAL_IP}:59092'
|
||||
- '${KRAKEND_LOCAL_IP}:59093'
|
||||
labels:
|
||||
app: kotel_example
|
||||
@@ -0,0 +1,16 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
external_labels:
|
||||
monitor: kotel_monitor
|
||||
|
||||
scrape_configs:
|
||||
- job_name: kotel_example
|
||||
scrape_interval: 5s
|
||||
metrics_path: '/metrics'
|
||||
static_configs:
|
||||
- targets:
|
||||
- '192.168.1.12:59091'
|
||||
- '192.168.1.12:59092'
|
||||
- '192.168.1.12:59093'
|
||||
labels:
|
||||
app: kotel_example
|
||||
@@ -0,0 +1,31 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
grpc_listen_port: 0
|
||||
|
||||
clients:
|
||||
- url: http://kotel_loki:3100/loki/api/v1/push
|
||||
|
||||
scrape_configs:
|
||||
- job_name: "kotel_example"
|
||||
pipeline_stages:
|
||||
- match:
|
||||
selector: '{app="kotel_app"}'
|
||||
stages:
|
||||
- json:
|
||||
expressions:
|
||||
level: level
|
||||
method: method
|
||||
path: path
|
||||
file: file
|
||||
msg: msg
|
||||
timestamp: time
|
||||
- labels:
|
||||
level:
|
||||
path:
|
||||
method:
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
app: "kotel_ap"
|
||||
__path__: "/var/log/tmp/*.log"
|
||||
@@ -0,0 +1,60 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
query_frontend:
|
||||
search:
|
||||
duration_slo: 5s
|
||||
throughput_bytes_slo: 1.073741824e+09
|
||||
trace_by_id:
|
||||
duration_slo: 5s
|
||||
|
||||
# this configuration will listen on all ports and protocols that tempo is capable of.
|
||||
# the receives all come from the OpenTelemetry collector. more configuration information can
|
||||
# be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
|
||||
# for a production deployment you should only enable the receivers you need!
|
||||
distributor:
|
||||
receivers:
|
||||
jaeger:
|
||||
protocols:
|
||||
thrift_http:
|
||||
grpc:
|
||||
thrift_binary:
|
||||
thrift_compact:
|
||||
zipkin:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
grpc:
|
||||
|
||||
# cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
# overall Tempo trace retention. set for demo purposes
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 1h
|
||||
|
||||
metrics_generator:
|
||||
registry:
|
||||
external_labels:
|
||||
source: tempo
|
||||
cluster: docker-compose
|
||||
storage:
|
||||
path: /tmp/tempo/generator/wal
|
||||
remote_write:
|
||||
- url: http://prometheus:9090/api/v1/write
|
||||
send_exemplars: true
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local # backend configuration to use
|
||||
wal:
|
||||
path: /tmp/tempo/wal # where to store the the wal locally
|
||||
local:
|
||||
path: /tmp/tempo/blocks
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
metrics_generator:
|
||||
processors: [service-graphs, span-metrics] # enables metrics generator
|
||||
@@ -0,0 +1,15 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
exporters:
|
||||
file:
|
||||
path: /var/log/example/output.json
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: []
|
||||
exporters: [file]
|
||||
@@ -0,0 +1,173 @@
|
||||
{
|
||||
"port": 9876,
|
||||
"host": "0.0.0.0",
|
||||
"endpoints": [
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/",
|
||||
"behaviour": [
|
||||
{
|
||||
"name": "slower",
|
||||
"config": {
|
||||
"max_bytes_per_second": 20,
|
||||
"flush_bytes": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "status_distributor",
|
||||
"config": {
|
||||
"code_distribution": [
|
||||
{"key": 200, "val": 0.5},
|
||||
{"key": 201, "val": 0.3}
|
||||
],
|
||||
"seed": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "delayer",
|
||||
"config": {
|
||||
"delay_millis_distribution": [
|
||||
{"key": 0, "val": 0.05},
|
||||
{"key": 10, "val": 0.2},
|
||||
{"key": 50, "val": 0.0},
|
||||
{"key": 100, "val": 0.75}
|
||||
],
|
||||
"seed": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"content": {
|
||||
"source": "directory",
|
||||
"config": {
|
||||
"dir": "/etc/reqstatsrv/data"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/drop/",
|
||||
"behaviour": [
|
||||
{
|
||||
"name": "connection_closer",
|
||||
"config": {
|
||||
"freq": 0.5,
|
||||
"seed": 3
|
||||
}
|
||||
}
|
||||
],
|
||||
"content" : {
|
||||
"source": "file",
|
||||
"config": {
|
||||
"path": "/etc/reqstatsrv/data/bar.json"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/delayed/",
|
||||
"behaviour": [
|
||||
{
|
||||
"name": "delayer",
|
||||
"config": {
|
||||
"delay_millis_distribution": [
|
||||
{"key": 0, "val": 0.0},
|
||||
{"key": 10, "val": 0.0},
|
||||
{"key": 50, "val": 0.0},
|
||||
{"key": 100, "val": 0.25},
|
||||
{"key": 200, "val": 0.25},
|
||||
{"key": 700, "val": 0.5}
|
||||
],
|
||||
"seed": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "status_distributor",
|
||||
"config": {
|
||||
"code_distribution": [
|
||||
{"key": 200, "val": 0.7},
|
||||
{"key": 201, "val": 0.3}
|
||||
],
|
||||
"seed": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"content": {
|
||||
"source": "directory",
|
||||
"config": {
|
||||
"dir": "/etc/reqstatsrv/data"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/file",
|
||||
"behaviour": [],
|
||||
"content": {
|
||||
"source": "file",
|
||||
"config": {
|
||||
"path": "/etc/reqstatsrv/data/bar.json"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/stats",
|
||||
"behaviour": [
|
||||
|
||||
],
|
||||
"content": {
|
||||
"source": "stats",
|
||||
"config": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path_pattern": "/suslik",
|
||||
"behaviour": [
|
||||
{
|
||||
"name": "status_distributor",
|
||||
"config": {
|
||||
"code_distribution": [
|
||||
{"key": 200, "val": 0.6},
|
||||
{"key": 201, "val": 0.2},
|
||||
{"key": 202, "val": 0.2}
|
||||
],
|
||||
"seed": 1
|
||||
}
|
||||
}
|
||||
|
||||
],
|
||||
"content": {
|
||||
"source": "status_content_selector",
|
||||
"config": {
|
||||
"default_content": {
|
||||
"source": "stats",
|
||||
"config": {}
|
||||
},
|
||||
"status_contents": [
|
||||
{
|
||||
"from": 201,
|
||||
"to": 202,
|
||||
"content": {
|
||||
"source": "file",
|
||||
"config": {
|
||||
"path": "/etc/reqstatsrv/data/201_created.json"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"from": 202,
|
||||
"to": 203,
|
||||
"content": {
|
||||
"source": "file",
|
||||
"config": {
|
||||
"path": "/etc/reqstatsrv/data/202_accepted.json"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"result": 201,
|
||||
"desc": "something created"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"batch_process_id": "A629S99IAKSRtT"
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"source": "bar.json",
|
||||
"dummy_string": "something here",
|
||||
"dummy_number": 42.0
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"foo": "bar"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"foo": "bar"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'default'
|
||||
orgId: 1
|
||||
folder: ''
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 60
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards/KrakenDOTEL.json
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"apiVersion": 1,
|
||||
"datasources": [
|
||||
{
|
||||
"id": 3,
|
||||
"uid": "b2a29c2f-ef77-4ba5-b934-b42aaa6ae3b4",
|
||||
"orgId": 1,
|
||||
"name": "Loki",
|
||||
"type": "loki",
|
||||
"typeName": "Loki",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/loki/img/loki_icon.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://loki:3100",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": false,
|
||||
"jsonData": {},
|
||||
"readOnly": false
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"uid": "b9564da7-2fba-447a-934e-e0217745974e",
|
||||
"orgId": 1,
|
||||
"name": "Prometheus",
|
||||
"type": "prometheus",
|
||||
"typeName": "Prometheus",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/prometheus/img/prometheus_logo.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://prometheus:9090",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": true,
|
||||
"jsonData": {
|
||||
"httpMethod": "POST"
|
||||
},
|
||||
"readOnly": false
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"uid": "e7eea071-7df9-44ff-8a6a-155248d05f42",
|
||||
"orgId": 1,
|
||||
"name": "Tempo",
|
||||
"type": "tempo",
|
||||
"typeName": "Tempo",
|
||||
"typeLogoUrl": "/public/app/plugins/datasource/tempo/img/tempo_logo.svg",
|
||||
"access": "proxy",
|
||||
"url": "http://tempo:3200",
|
||||
"user": "",
|
||||
"database": "",
|
||||
"basicAuth": false,
|
||||
"isDefault": false,
|
||||
"jsonData": {},
|
||||
"readOnly": false
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"version": 3,
|
||||
"name": "Back KrakenD",
|
||||
"port": 44446,
|
||||
"cache_ttl": "3600s",
|
||||
"timeout": "2500ms",
|
||||
"host": [ "http://krakend_back:44446/" ],
|
||||
"debug_endpoint": true,
|
||||
"endpoints": [
|
||||
{
|
||||
"endpoint": "/back_fake/{something}",
|
||||
"method": "GET",
|
||||
"concurrent_calls": 3,
|
||||
"backend": [
|
||||
{
|
||||
"url_pattern": "/__debug/{something}",
|
||||
"mapping": {
|
||||
"message": "backend1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"url_pattern": "/__debug/{something}/{something}",
|
||||
"mapping": {
|
||||
"message": "backend2"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/back_combination/{id}",
|
||||
"backend": [
|
||||
{
|
||||
"host": [
|
||||
"https://jsonplaceholder.typicode.com"
|
||||
],
|
||||
"url_pattern": "/posts?userId={id}",
|
||||
"is_collection": true,
|
||||
"mapping": {
|
||||
"collection": "posts"
|
||||
}
|
||||
},
|
||||
{
|
||||
"host": [
|
||||
"https://jsonplaceholder.typicode.com"
|
||||
],
|
||||
"url_pattern": "/users/{id}",
|
||||
"mapping": {
|
||||
"email": "personal_email"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"extra_config": {
|
||||
"telemetry/opentelemetry": {
|
||||
"service_name": "krakend_backend_service",
|
||||
"metric_reporting_period": 1,
|
||||
"trace_sample_rate": 1,
|
||||
"layers": {
|
||||
"global": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false,
|
||||
"disable_propagation": false
|
||||
},
|
||||
"proxy": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false
|
||||
},
|
||||
"backend": {
|
||||
"metrics": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_metric_attr",
|
||||
"value": "my_metric_val"
|
||||
}
|
||||
]
|
||||
},
|
||||
"traces": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_trace_attr",
|
||||
"value": "my_trace_val"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"exporters": {
|
||||
"prometheus": [
|
||||
{
|
||||
"name": "local_prometheus",
|
||||
"port": 9090,
|
||||
"process_metrics": true,
|
||||
"go_metrics": true
|
||||
}
|
||||
],
|
||||
"otlp": [
|
||||
{
|
||||
"name": "local_tempo",
|
||||
"host": "tempo",
|
||||
"port": 4317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
},
|
||||
{
|
||||
"name": "local_jaeger",
|
||||
"host": "jaeger",
|
||||
"port": 4317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
{
|
||||
"version": 3,
|
||||
"name": "Front KrakenD",
|
||||
"port": 44444,
|
||||
"cache_ttl": "3600s",
|
||||
"timeout": "2500ms",
|
||||
"host": [ "http://krakend_middle:44445/" ],
|
||||
"endpoints": [
|
||||
{
|
||||
"endpoint": "/fake/{something}",
|
||||
"concurrent_calls": 3,
|
||||
"backend": [
|
||||
{
|
||||
"url_pattern": "/middle_fake/{something}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/combination/{id}",
|
||||
"backend": [
|
||||
{
|
||||
"url_pattern": "/middle_combination/{id}",
|
||||
"group": "front_1"
|
||||
},
|
||||
{
|
||||
"url_pattern": "/middle_combination/{id}",
|
||||
"group": "front_2"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/direct/slow",
|
||||
"backend": [
|
||||
{
|
||||
"host": ["http://fakeapi:9876"],
|
||||
"url_pattern": "/foo.json",
|
||||
"group": "data"
|
||||
},
|
||||
{
|
||||
"host": ["http://fakeapi:9876"],
|
||||
"url_pattern": "/bar.json",
|
||||
"group": "data"
|
||||
},
|
||||
{
|
||||
"host": ["http://fakeapi:9876"],
|
||||
"url_pattern": "/stats"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/direct/drop",
|
||||
"backend": [
|
||||
{
|
||||
"host": ["http://fakeapi:9876"],
|
||||
"url_pattern": "/drop/"
|
||||
},
|
||||
{
|
||||
"host": ["http://fakeapi:9876"],
|
||||
"url_pattern": "/stats"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/direct/delayed",
|
||||
"backend": [
|
||||
{
|
||||
"host": ["http://fakeapi:9876"],
|
||||
"url_pattern": "/delayed/"
|
||||
},
|
||||
{
|
||||
"host": ["http://fakeapi:9876"],
|
||||
"url_pattern": "/file"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"extra_config": {
|
||||
"telemetry/opentelemetry": {
|
||||
"service_name": "krakend_frontend_service",
|
||||
"metric_reporting_period": 1,
|
||||
"trace_sample_rate": 1,
|
||||
"layers": {
|
||||
"global": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false,
|
||||
"disable_propagation": false,
|
||||
"report_headers": true
|
||||
},
|
||||
"proxy": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false,
|
||||
"report_headers": true
|
||||
},
|
||||
"backend": {
|
||||
"metrics": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_metric_attr",
|
||||
"value": "my_metric_val"
|
||||
}
|
||||
]
|
||||
},
|
||||
"traces": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_trace_attr",
|
||||
"value": "my_trace_val"
|
||||
}
|
||||
],
|
||||
"report_headers": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"exporters": {
|
||||
"prometheus": [
|
||||
{
|
||||
"name": "local_prometheus",
|
||||
"port": 9090,
|
||||
"process_metrics": true,
|
||||
"go_metrics": true
|
||||
}
|
||||
],
|
||||
"otlp": [
|
||||
{
|
||||
"name": "local_tempo",
|
||||
"host": "tempo",
|
||||
"port": 4317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
},
|
||||
{
|
||||
"name": "local_jaeger",
|
||||
"host": "jaeger",
|
||||
"port": 4317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"version": 3,
|
||||
"name": "Middle KrakenD",
|
||||
"port": 44445,
|
||||
"cache_ttl": "3600s",
|
||||
"timeout": "2500ms",
|
||||
"host": [ "http://krakend_back:44446/" ],
|
||||
"endpoints": [
|
||||
{
|
||||
"endpoint": "/middle_fake/{something}",
|
||||
"concurrent_calls": 3,
|
||||
"backend": [
|
||||
{
|
||||
"url_pattern": "/back_fake/{something}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endpoint": "/middle_combination/{id}",
|
||||
"backend": [
|
||||
{
|
||||
"url_pattern": "/back_combination/{id}",
|
||||
"group": "middle_1"
|
||||
},
|
||||
{
|
||||
"url_pattern": "/back_combination/{id}",
|
||||
"group": "middle_2"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"extra_config": {
|
||||
"telemetry/opentelemetry": {
|
||||
"service_name": "krakend_middle_service",
|
||||
"metric_reporting_period": 1,
|
||||
"trace_sample_rate": 1,
|
||||
"layers": {
|
||||
"global": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false,
|
||||
"disable_propagation": false
|
||||
},
|
||||
"proxy": {
|
||||
"disable_metrics": false,
|
||||
"disable_traces": false
|
||||
},
|
||||
"backend": {
|
||||
"metrics": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_metric_attr",
|
||||
"value": "my_middle_metric"
|
||||
}
|
||||
]
|
||||
},
|
||||
"traces": {
|
||||
"disable_stage": false,
|
||||
"round_trip": true,
|
||||
"read_payload": true,
|
||||
"detailed_connection": true,
|
||||
"static_attributes": [
|
||||
{
|
||||
"key": "my_trace_attr",
|
||||
"value": "my_middle_trace"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"exporters": {
|
||||
"prometheus": [
|
||||
{
|
||||
"name": "local_prometheus",
|
||||
"port": 9090,
|
||||
"process_metrics": true,
|
||||
"go_metrics": true
|
||||
}
|
||||
],
|
||||
"otlp": [
|
||||
{
|
||||
"name": "local_tempo",
|
||||
"host": "tempo",
|
||||
"port": 4317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
},
|
||||
{
|
||||
"name": "local_jaeger",
|
||||
"host": "jaeger",
|
||||
"port": 4317,
|
||||
"use_http": false,
|
||||
"disable_metrics": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
external_labels:
|
||||
monitor: kotel_monitor
|
||||
|
||||
scrape_configs:
|
||||
- job_name: kotel_example
|
||||
scrape_interval: 5s
|
||||
metrics_path: '/metrics'
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'krakend_front:9090'
|
||||
- 'krakend_middle:9090'
|
||||
- 'krakend_back:9090'
|
||||
labels:
|
||||
app: kotel_example
|
||||
@@ -0,0 +1,31 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
grpc_listen_port: 0
|
||||
|
||||
clients:
|
||||
- url: http://kotel_loki:3100/loki/api/v1/push
|
||||
|
||||
scrape_configs:
|
||||
- job_name: "kotel_example"
|
||||
pipeline_stages:
|
||||
- match:
|
||||
selector: '{app="kotel_app"}'
|
||||
stages:
|
||||
- json:
|
||||
expressions:
|
||||
level: level
|
||||
method: method
|
||||
path: path
|
||||
file: file
|
||||
msg: msg
|
||||
timestamp: time
|
||||
- labels:
|
||||
level:
|
||||
path:
|
||||
method:
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
app: "kotel_ap"
|
||||
__path__: "/var/log/tmp/*.log"
|
||||
@@ -0,0 +1,60 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
query_frontend:
|
||||
search:
|
||||
duration_slo: 5s
|
||||
throughput_bytes_slo: 1.073741824e+09
|
||||
trace_by_id:
|
||||
duration_slo: 5s
|
||||
|
||||
# this configuration will listen on all ports and protocols that tempo is capable of.
|
||||
# the receives all come from the OpenTelemetry collector. more configuration information can
|
||||
# be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
|
||||
# for a production deployment you should only enable the receivers you need!
|
||||
distributor:
|
||||
receivers:
|
||||
jaeger:
|
||||
protocols:
|
||||
thrift_http:
|
||||
grpc:
|
||||
thrift_binary:
|
||||
thrift_compact:
|
||||
zipkin:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
grpc:
|
||||
|
||||
# cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
# overall Tempo trace retention. set for demo purposes
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 1h
|
||||
|
||||
metrics_generator:
|
||||
registry:
|
||||
external_labels:
|
||||
source: tempo
|
||||
cluster: docker-compose
|
||||
storage:
|
||||
path: /tmp/tempo/generator/wal
|
||||
remote_write:
|
||||
- url: http://prometheus:9090/api/v1/write
|
||||
send_exemplars: true
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local # backend configuration to use
|
||||
wal:
|
||||
path: /tmp/tempo/wal # where to store the the wal locally
|
||||
local:
|
||||
path: /tmp/tempo/blocks
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
metrics_generator:
|
||||
processors: [service-graphs, span-metrics] # enables metrics generator
|
||||
@@ -0,0 +1,21 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
# - "first.rules"
|
||||
# - "second.rules"
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
- job_name: 'one'
|
||||
static_configs:
|
||||
- targets: ['localhost:9091']
|
||||
- job_name: 'two'
|
||||
static_configs:
|
||||
- targets: ['localhost:9092']
|
||||
- job_name: 'three'
|
||||
static_configs:
|
||||
- targets: ['localhost:9093']
|
||||
@@ -0,0 +1,10 @@
|
||||
FROM golang:1.20-alpine AS builder
|
||||
|
||||
COPY ./ /app
|
||||
WORKDIR /app
|
||||
RUN go get ./...
|
||||
RUN go build -o ./server ./example/server/main.go
|
||||
|
||||
FROM alpine
|
||||
COPY --from=builder /app/server /server
|
||||
CMD ["/server"]
|
||||
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
Srv creates a few basic routes to test the configured instrumentation.
|
||||
|
||||
Usage:
|
||||
|
||||
srv [flags]
|
||||
|
||||
The flags are:
|
||||
|
||||
-p [port_number]
|
||||
To select the port number where we want to run the server
|
||||
|
||||
-d
|
||||
To enable debug logs
|
||||
|
||||
-c [config_file]
|
||||
To select the config file to use.
|
||||
*/
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/luraproject/lura/v2/config"
|
||||
"github.com/luraproject/lura/v2/logging"
|
||||
"github.com/luraproject/lura/v2/proxy"
|
||||
krakendgin "github.com/luraproject/lura/v2/router/gin"
|
||||
"github.com/luraproject/lura/v2/transport/http/client"
|
||||
"github.com/luraproject/lura/v2/transport/http/server"
|
||||
|
||||
kotel "github.com/krakend/krakend-otel"
|
||||
kotelconfig "github.com/krakend/krakend-otel/config"
|
||||
otellura "github.com/krakend/krakend-otel/lura"
|
||||
otelgin "github.com/krakend/krakend-otel/router/gin"
|
||||
"github.com/krakend/krakend-otel/state"
|
||||
)
|
||||
|
||||
func main() {
|
||||
port := flag.Int("p", 0, "Port of the service")
|
||||
logLevel := flag.String("l", "ERROR", "Logging level")
|
||||
debug := flag.Bool("d", false, "Enable the debug")
|
||||
configFile := flag.String("c", "/etc/krakend/configuration.json", "Path to the configuration filename")
|
||||
flag.Parse()
|
||||
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
go func() {
|
||||
select {
|
||||
case sig := <-sigs:
|
||||
log.Println("Signal intercepted:", sig)
|
||||
cancel()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
parser := config.NewParser()
|
||||
serviceConfig, err := parser.Parse(*configFile)
|
||||
if err != nil {
|
||||
fmt.Printf("ERROR: %s\n", err.Error())
|
||||
return
|
||||
}
|
||||
serviceConfig.Debug = serviceConfig.Debug || *debug
|
||||
if *port != 0 {
|
||||
serviceConfig.Port = *port
|
||||
}
|
||||
|
||||
logger, _ := logging.NewLogger(*logLevel, os.Stdout, "[KRAKEND]")
|
||||
|
||||
obsConfig, err := kotelconfig.FromLura(serviceConfig)
|
||||
if err != nil {
|
||||
fmt.Printf("ERROR: no config found for open telemetry: %s\n", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
shutdownFn, err := kotel.Register(ctx, serviceConfig)
|
||||
if err != nil {
|
||||
fmt.Printf("--- failed to register: %s\n", err.Error())
|
||||
return
|
||||
}
|
||||
defer shutdownFn()
|
||||
|
||||
otelStateFn := state.GlobalState
|
||||
|
||||
bf := func(backendConfig *config.Backend) proxy.Proxy {
|
||||
reqExec := otellura.HTTPRequestExecutorFromConfig(client.NewHTTPClient,
|
||||
backendConfig, obsConfig.Layers.Backend, obsConfig.SkipPaths, otelStateFn)
|
||||
return proxy.NewHTTPProxyWithHTTPExecutor(backendConfig, reqExec, backendConfig.Decoder)
|
||||
}
|
||||
bf = otellura.BackendFactory(bf, otelStateFn, obsConfig.Layers.Backend, obsConfig.SkipPaths)
|
||||
|
||||
defaultPF := proxy.NewDefaultFactory(bf, logger)
|
||||
pf := otellura.ProxyFactory(defaultPF, otelStateFn, obsConfig.Layers.Pipe, obsConfig.SkipPaths)
|
||||
|
||||
handlerF := otelgin.New(krakendgin.EndpointHandler, obsConfig.SkipPaths)
|
||||
|
||||
runserverChain := krakendgin.RunServerFunc(
|
||||
otellura.GlobalRunServer(logger, obsConfig, otelStateFn, server.RunServer))
|
||||
|
||||
engine := gin.Default()
|
||||
engine.RedirectTrailingSlash = true
|
||||
engine.RedirectFixedPath = true
|
||||
engine.HandleMethodNotAllowed = true
|
||||
engine.ContextWithFallback = true // <- this is important for trace span propagation
|
||||
|
||||
// setup the krakend router
|
||||
routerFactory := krakendgin.NewFactory(krakendgin.Config{
|
||||
Engine: engine,
|
||||
ProxyFactory: pf,
|
||||
Middlewares: []gin.HandlerFunc{},
|
||||
Logger: logger,
|
||||
HandlerFactory: handlerF,
|
||||
RunServer: runserverChain,
|
||||
})
|
||||
|
||||
// start the engine
|
||||
routerFactory.NewWithContext(ctx).Run(serviceConfig)
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
// Package exporter defines the interfaces required to implement in
|
||||
// order to add additional exporters.
|
||||
package exporter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
|
||||
"github.com/krakend/krakend-otel/config"
|
||||
"github.com/krakend/krakend-otel/exporter/otelcollector"
|
||||
"github.com/krakend/krakend-otel/exporter/prometheus"
|
||||
)
|
||||
|
||||
// MetricReader is the interface required in order to
|
||||
// export metrics.
|
||||
type MetricReader interface {
|
||||
MetricReader(reportingPeriod time.Duration) sdkmetric.Reader
|
||||
MetricDefaultReporting() bool
|
||||
}
|
||||
|
||||
// SpanExporter is the interface required in order to
|
||||
// export traces.
|
||||
type SpanExporter interface {
|
||||
SpanExporter() sdktrace.SpanExporter
|
||||
TraceDefaultReporting() bool
|
||||
}
|
||||
|
||||
var (
|
||||
metricsInstances map[string]MetricReader
|
||||
tracesInstances map[string]SpanExporter
|
||||
mu = new(sync.RWMutex)
|
||||
)
|
||||
|
||||
func CreateOTLPExporters(ctx context.Context, otlpConfs []config.OTLPExporter) (map[string]MetricReader, map[string]SpanExporter, error) {
|
||||
m := make(map[string]MetricReader, len(otlpConfs))
|
||||
s := make(map[string]SpanExporter, len(otlpConfs))
|
||||
for idx, ecfg := range otlpConfs {
|
||||
c, err := otelcollector.Exporter(ctx, ecfg)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("OTLP Exporter %s (at idx %d) failed: %s", ecfg.Name, idx, err.Error())
|
||||
}
|
||||
s[ecfg.Name] = c
|
||||
m[ecfg.Name] = c
|
||||
}
|
||||
return m, s, nil
|
||||
}
|
||||
|
||||
func CreatePrometheusExporters(ctx context.Context, promConfs []config.PrometheusExporter) (map[string]MetricReader, error) {
|
||||
m := make(map[string]MetricReader, len(promConfs))
|
||||
for idx, ecfg := range promConfs {
|
||||
c, err := prometheus.Exporter(ctx, ecfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prometheus exporter %s (at idx %d) failed: %s", ecfg.Name, idx, err.Error())
|
||||
}
|
||||
m[ecfg.Name] = c
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// Instances create instances for a given configuration.
|
||||
func Instances(ctx context.Context, cfg *config.Config) (map[string]MetricReader, map[string]SpanExporter, error) {
|
||||
// Create OTLP (OpenTelemetry Line Protocol) exporters
|
||||
m, s, err := CreateOTLPExporters(ctx, cfg.Exporters.OTLP)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// Create Prometheus exporters
|
||||
pm, err := CreatePrometheusExporters(ctx, cfg.Exporters.Prometheus)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for k, v := range pm {
|
||||
m[k] = v
|
||||
}
|
||||
return m, s, nil
|
||||
}
|
||||
|
||||
// SetGlobalExporterInstances sets the provided metric and traces
|
||||
// as global defaults.
|
||||
func SetGlobalExporterInstances(m map[string]MetricReader, t map[string]SpanExporter) {
|
||||
mu.Lock()
|
||||
metricsInstances = make(map[string]MetricReader, len(m))
|
||||
tracesInstances = make(map[string]SpanExporter, len(t))
|
||||
for k, v := range m {
|
||||
metricsInstances[k] = v
|
||||
}
|
||||
for k, v := range t {
|
||||
tracesInstances[k] = v
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
// GetGlobalExporterInstances gets the global metrics and traces exporters
|
||||
func GetGlobalExporterInstances() (map[string]MetricReader, map[string]SpanExporter) {
|
||||
mu.RLock()
|
||||
m := make(map[string]MetricReader, len(metricsInstances))
|
||||
t := make(map[string]SpanExporter, len(tracesInstances))
|
||||
for k, v := range metricsInstances {
|
||||
m[k] = v
|
||||
}
|
||||
for k, v := range tracesInstances {
|
||||
t[k] = v
|
||||
}
|
||||
mu.RUnlock()
|
||||
return m, t
|
||||
}
|
||||
|
||||
// GlobalTraceInstance gets a global trace exporter by name
|
||||
func GlobalTraceInstance(name string) (SpanExporter, error) {
|
||||
mu.RLock()
|
||||
i, ok := tracesInstances[name]
|
||||
mu.RUnlock()
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("not found")
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
||||
// GlobalMetricInstance get a global metrics exporter by name
|
||||
func GlobalMetricInstance(name string) (MetricReader, error) {
|
||||
mu.RLock()
|
||||
i, ok := metricsInstances[name]
|
||||
mu.RUnlock()
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("not found")
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
// Package otelcollector implements the Open Telemetry exporter.
|
||||
package otelcollector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
|
||||
"github.com/krakend/krakend-otel/config"
|
||||
)
|
||||
|
||||
// OtelCollector implements the traces exporter.
|
||||
type OtelCollector struct {
|
||||
exporter sdktrace.SpanExporter
|
||||
metricExporter sdkmetric.Exporter
|
||||
metricsDisabledByDefault bool
|
||||
tracesDisabledByDefault bool
|
||||
}
|
||||
|
||||
// SpanExporter implements the interface to export traces.
|
||||
func (c *OtelCollector) SpanExporter() sdktrace.SpanExporter {
|
||||
return c.exporter
|
||||
}
|
||||
|
||||
func (c *OtelCollector) MetricReader(reportingPeriod time.Duration) sdkmetric.Reader {
|
||||
return sdkmetric.NewPeriodicReader(c.metricExporter,
|
||||
sdkmetric.WithInterval(reportingPeriod))
|
||||
}
|
||||
|
||||
func (c *OtelCollector) MetricDefaultReporting() bool {
|
||||
return !c.metricsDisabledByDefault
|
||||
}
|
||||
|
||||
func (c *OtelCollector) TraceDefaultReporting() bool {
|
||||
return !c.tracesDisabledByDefault
|
||||
}
|
||||
|
||||
func httpExporterWithOptions(ctx context.Context, cfg config.OTLPExporter,
|
||||
options []interface{},
|
||||
) (*OtelCollector, error) {
|
||||
tOpts := make([]otlptracehttp.Option, 0, len(options)+1)
|
||||
mOpts := make([]otlpmetrichttp.Option, 0, len(options)+1)
|
||||
for _, iopt := range options {
|
||||
if to, ok := iopt.(otlptracehttp.Option); ok {
|
||||
tOpts = append(tOpts, to)
|
||||
}
|
||||
if mo, ok := iopt.(otlpmetrichttp.Option); ok {
|
||||
mOpts = append(mOpts, mo)
|
||||
}
|
||||
}
|
||||
|
||||
endpoint := fmt.Sprintf("%s:%d", cfg.Host, cfg.Port)
|
||||
tOpts = append(tOpts, otlptracehttp.WithEndpoint(endpoint))
|
||||
exporter, err := otlptracehttp.New(ctx, tOpts...)
|
||||
if err != nil {
|
||||
return nil, errors.New("cannot create http trace exporter")
|
||||
}
|
||||
|
||||
mOpts = append(mOpts, otlpmetrichttp.WithEndpoint(endpoint))
|
||||
metricExporter, err := otlpmetrichttp.New(ctx, mOpts...)
|
||||
if err != nil {
|
||||
return nil, errors.New("cannot create http metric exporter")
|
||||
}
|
||||
|
||||
return &OtelCollector{
|
||||
exporter: exporter,
|
||||
metricExporter: metricExporter,
|
||||
metricsDisabledByDefault: cfg.DisableMetrics,
|
||||
tracesDisabledByDefault: cfg.DisableTraces,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func grpcExporterWithOptions(ctx context.Context, cfg config.OTLPExporter,
|
||||
options []interface{},
|
||||
) (*OtelCollector, error) {
|
||||
tOpts := make([]otlptracegrpc.Option, 0, len(options)+1)
|
||||
mOpts := make([]otlpmetricgrpc.Option, 0, len(options)+1)
|
||||
for _, iopt := range options {
|
||||
if to, ok := iopt.(otlptracegrpc.Option); ok {
|
||||
tOpts = append(tOpts, to)
|
||||
}
|
||||
if mo, ok := iopt.(otlpmetricgrpc.Option); ok {
|
||||
mOpts = append(mOpts, mo)
|
||||
}
|
||||
}
|
||||
|
||||
endpoint := fmt.Sprintf("%s:%d", cfg.Host, cfg.Port)
|
||||
tOpts = append(tOpts, otlptracegrpc.WithEndpoint(endpoint))
|
||||
exporter, err := otlptracegrpc.New(ctx, tOpts...)
|
||||
if err != nil {
|
||||
return nil, errors.New("cannot create grpc traces exporter")
|
||||
}
|
||||
mOpts = append(mOpts, otlpmetricgrpc.WithEndpoint(endpoint))
|
||||
metricExporter, err := otlpmetricgrpc.New(ctx, mOpts...)
|
||||
if err != nil {
|
||||
return nil, errors.New("cannot create grpc metric exporter")
|
||||
}
|
||||
|
||||
return &OtelCollector{
|
||||
exporter: exporter,
|
||||
metricExporter: metricExporter,
|
||||
metricsDisabledByDefault: cfg.DisableMetrics,
|
||||
tracesDisabledByDefault: cfg.DisableTraces,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ExporterWithOptions(ctx context.Context, cfg config.OTLPExporter, options []interface{}) (*OtelCollector, error) {
|
||||
if cfg.Port == 0 {
|
||||
cfg.Port = 4317
|
||||
}
|
||||
if cfg.Host == "" {
|
||||
cfg.Host = "localhost"
|
||||
}
|
||||
|
||||
if cfg.UseHTTP {
|
||||
return httpExporterWithOptions(ctx, cfg, options)
|
||||
}
|
||||
return grpcExporterWithOptions(ctx, cfg, options)
|
||||
}
|
||||
|
||||
// Exporter creates an Open Telemetry exporter instance.
|
||||
func Exporter(ctx context.Context, cfg config.OTLPExporter) (*OtelCollector, error) {
|
||||
options := make([]interface{}, 0, 2)
|
||||
// by default, grpc conections have TLS enabled:
|
||||
options = append(options, otlptracegrpc.WithInsecure())
|
||||
options = append(options, otlpmetricgrpc.WithInsecure())
|
||||
return ExporterWithOptions(ctx, cfg, options)
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
// Package prometheus implements a Prometheus metrics exporter.
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
prom "github.com/prometheus/client_golang/prometheus"
|
||||
promcollectors "github.com/prometheus/client_golang/prometheus/collectors"
|
||||
promhttp "github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.opentelemetry.io/otel/exporters/prometheus"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
|
||||
"github.com/krakend/krakend-otel/config"
|
||||
)
|
||||
|
||||
// PrometheusCollector implemnts the metrics exporter
|
||||
type PrometheusCollector struct {
|
||||
registry *prom.Registry
|
||||
exporter *prometheus.Exporter
|
||||
disabledByDefault bool
|
||||
}
|
||||
|
||||
// MetricReader implements the interface to exporte metrics.
|
||||
func (c *PrometheusCollector) MetricReader(_ time.Duration) sdkmetric.Reader {
|
||||
return c.exporter
|
||||
}
|
||||
|
||||
func (c *PrometheusCollector) MetricDefaultReporting() bool {
|
||||
return !c.disabledByDefault
|
||||
}
|
||||
|
||||
// Exporter creates a Prometheus exporter instance.
|
||||
func Exporter(ctx context.Context, cfg config.PrometheusExporter) (*PrometheusCollector, error) {
|
||||
if cfg.Port == 0 {
|
||||
cfg.Port = 9090
|
||||
}
|
||||
prometheusRegistry := prom.NewRegistry()
|
||||
|
||||
if cfg.ProcessMetrics {
|
||||
err := prometheusRegistry.Register(promcollectors.NewProcessCollector(promcollectors.ProcessCollectorOpts{}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.GoMetrics {
|
||||
err := prometheusRegistry.Register(promcollectors.NewGoCollector())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
exporter, err := prometheus.New(prometheus.WithRegisterer(prometheusRegistry))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
router := http.NewServeMux()
|
||||
router.Handle("/metrics", promhttp.HandlerFor(prometheusRegistry,
|
||||
promhttp.HandlerOpts{}))
|
||||
server := http.Server{
|
||||
Handler: router,
|
||||
Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
|
||||
ReadHeaderTimeout: 3 * time.Second,
|
||||
}
|
||||
|
||||
go func() {
|
||||
if serverErr := server.ListenAndServe(); serverErr != http.ErrServerClosed {
|
||||
log.Printf("[SERVICE: kotel] The Prometheus exporter failed to listen and serve: %v", serverErr)
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
server.Shutdown(ctx)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
return &PrometheusCollector{
|
||||
registry: prometheusRegistry,
|
||||
exporter: exporter,
|
||||
disabledByDefault: cfg.DisableMetrics,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
module github.com/krakend/krakend-otel
|
||||
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/gin-gonic/gin v1.9.1
|
||||
github.com/luraproject/lura/v2 v2.5.1-0.20240216134027-7bca41332dd6
|
||||
github.com/prometheus/client_golang v1.17.0
|
||||
go.opentelemetry.io/otel v1.22.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.45.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.45.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.44.0
|
||||
go.opentelemetry.io/otel/metric v1.22.0
|
||||
go.opentelemetry.io/otel/sdk v1.22.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.22.0
|
||||
go.opentelemetry.io/otel/trace v1.22.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bytedance/sonic v1.9.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-logr/logr v1.4.1 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.14.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/krakendio/flatmap v1.1.1 // indirect
|
||||
github.com/leodido/go-urn v1.2.4 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
|
||||
github.com/prometheus/client_model v0.5.0 // indirect
|
||||
github.com/prometheus/common v0.44.0 // indirect
|
||||
github.com/prometheus/procfs v0.11.1 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.11 // indirect
|
||||
github.com/valyala/fastrand v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/crypto v0.17.0 // indirect
|
||||
golang.org/x/net v0.17.0 // indirect
|
||||
golang.org/x/sys v0.16.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
|
||||
google.golang.org/grpc v1.60.1 // indirect
|
||||
google.golang.org/protobuf v1.32.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -0,0 +1,45 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/semconv/v1.21.0"
|
||||
)
|
||||
|
||||
// TraceRequestAttrs returns a list of attributes to be set
|
||||
// for a given http.Request (only useful for traces, as
|
||||
// it reports the url string with any variable parameter
|
||||
// that it might contain).
|
||||
func TraceRequestAttrs(r *http.Request) []attribute.KeyValue {
|
||||
attrs := make([]attribute.KeyValue, 0, 5)
|
||||
attrs = append(attrs,
|
||||
semconv.URLFull(r.URL.String()),
|
||||
semconv.ServerAddress(r.Host),
|
||||
semconv.ClientAddress(r.RemoteAddr),
|
||||
semconv.HTTPRequestMethodKey.String(r.Method),
|
||||
)
|
||||
|
||||
if r.ContentLength >= 0 {
|
||||
attrs = append(attrs, semconv.HTTPRequestBodySize(int(r.ContentLength)))
|
||||
}
|
||||
|
||||
userAgent := r.UserAgent()
|
||||
if userAgent != "" {
|
||||
attrs = append(attrs, semconv.UserAgentOriginal(userAgent))
|
||||
}
|
||||
|
||||
return attrs
|
||||
}
|
||||
|
||||
// TraceResponseAttrs returns a list of attributes to be set
|
||||
// for a given http.Response.
|
||||
func TraceResponseAttrs(resp *http.Response) []attribute.KeyValue {
|
||||
if resp == nil {
|
||||
return []attribute.KeyValue{}
|
||||
}
|
||||
return []attribute.KeyValue{
|
||||
semconv.HTTPResponseStatusCode(int(resp.StatusCode)),
|
||||
semconv.HTTPResponseBodySize(int(resp.ContentLength)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// Package client provides the instrumentation for an http client.
|
||||
//
|
||||
// The detail that metrics and traces provides can be configured
|
||||
// with the [TransportOptions] struct.
|
||||
//
|
||||
// Metrics:
|
||||
//
|
||||
// Tags for all metrics:
|
||||
// - Host
|
||||
// - URLPattern
|
||||
// - Name (the name might be the combination of the endpoint + url pattern)
|
||||
//
|
||||
// Requests:
|
||||
//
|
||||
// - num cancelled requests:
|
||||
// - per context timeout
|
||||
// - per caller cancellation (for example when sending to multiple hosts)
|
||||
// - content-lenght: the size of the payload to be sent
|
||||
//
|
||||
// Responses:
|
||||
// Tags for all responses:
|
||||
// - Status Code
|
||||
//
|
||||
// - content length: the server side provided content length
|
||||
// - latency: the time since we send the request, until we have the
|
||||
// response available (might or might not included the time for
|
||||
// buffereing part or all the payload).
|
||||
//
|
||||
// - response bytes read: using the body tracker, account for the actual
|
||||
// number of bytes read by the client.
|
||||
// - response reading time: measures the time spent
|
||||
//
|
||||
// Traces:
|
||||
package client
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// InstrumentedHTTPClient creates a new instrumented http client with the options provided.
|
||||
// If the provided options are nil, the default options (with everything enabled, except
|
||||
// the detailed connection data: DNS, TLS, time to get a connection...) will be used.
|
||||
// Check the [TransportOptions] struct for details..
|
||||
func InstrumentedHTTPClient(c *http.Client, t *TransportOptions, clientName string) *http.Client {
|
||||
if t == nil {
|
||||
return c
|
||||
}
|
||||
|
||||
transport := c.Transport
|
||||
if transport == nil {
|
||||
transport = http.DefaultTransport
|
||||
}
|
||||
|
||||
rtt := NewRoundTripper(transport, t.MetricsOpts, t.TracesOpts, clientName, t.OTELInstance)
|
||||
if rtt == nil {
|
||||
return c
|
||||
}
|
||||
wc := &http.Client{
|
||||
Transport: rtt,
|
||||
CheckRedirect: c.CheckRedirect,
|
||||
Jar: c.Jar,
|
||||
Timeout: c.Timeout,
|
||||
}
|
||||
return wc
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
// "go.opentelemetry.io/otel/codes"
|
||||
"github.com/krakend/krakend-otel/state"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/metric/metricdata"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
sdktracetest "go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type fakeService struct{}
|
||||
|
||||
func (s *fakeService) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("foo bar"))
|
||||
}
|
||||
|
||||
type testOTEL struct {
|
||||
tracer trace.Tracer
|
||||
metricProvider metric.MeterProvider
|
||||
|
||||
metricReader *sdkmetric.ManualReader
|
||||
spanRecorder *sdktracetest.SpanRecorder
|
||||
}
|
||||
|
||||
func newTestOTEL() *testOTEL {
|
||||
spanRecorder := sdktracetest.NewSpanRecorder()
|
||||
tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spanRecorder))
|
||||
tracer := tracerProvider.Tracer("testotel-tracer")
|
||||
|
||||
metricReader := sdkmetric.NewManualReader()
|
||||
metricProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(metricReader))
|
||||
return &testOTEL{
|
||||
tracer: tracer,
|
||||
metricProvider: metricProvider,
|
||||
metricReader: metricReader,
|
||||
spanRecorder: spanRecorder,
|
||||
}
|
||||
}
|
||||
|
||||
func (o *testOTEL) Tracer() trace.Tracer {
|
||||
return o.tracer
|
||||
}
|
||||
|
||||
func (o *testOTEL) MeterProvider() metric.MeterProvider {
|
||||
return o.metricProvider
|
||||
}
|
||||
|
||||
func (o *testOTEL) Propagator() propagation.TextMapPropagator {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *testOTEL) Shutdown(ctx context.Context) {
|
||||
}
|
||||
|
||||
func TestInstrumentedHTTPClient(t *testing.T) {
|
||||
svc := &fakeService{}
|
||||
server := httptest.NewServer(svc)
|
||||
|
||||
innerClient := &http.Client{}
|
||||
otelInstance := newTestOTEL()
|
||||
transportOptions := &TransportOptions{
|
||||
OTELInstance: func() state.OTEL {
|
||||
return otelInstance
|
||||
},
|
||||
TracesOpts: TransportTracesOptions{
|
||||
RoundTrip: true,
|
||||
ReadPayload: true,
|
||||
FixedAttributes: []attribute.KeyValue{
|
||||
attribute.String("test-trace-attr", "a_trace"),
|
||||
},
|
||||
},
|
||||
MetricsOpts: TransportMetricsOptions{
|
||||
RoundTrip: true,
|
||||
ReadPayload: true,
|
||||
FixedAttributes: []attribute.KeyValue{
|
||||
attribute.String("test-metric-attr", "a_metric"),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
c, err := InstrumentedHTTPClient(innerClient, transportOptions, "test-http-client")
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", server.URL, nil)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error creating request: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected client error: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if resp == nil {
|
||||
t.Errorf("nil response")
|
||||
return
|
||||
}
|
||||
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if len(b) == 0 {
|
||||
t.Errorf("no bytes read")
|
||||
return
|
||||
}
|
||||
|
||||
// check that we have two traces, and one is the parent of the other
|
||||
endedSpans := otelInstance.spanRecorder.Ended()
|
||||
if len(endedSpans) != 2 {
|
||||
t.Errorf("traces, want: 2 got: %d", len(endedSpans))
|
||||
return
|
||||
}
|
||||
parentSpan := endedSpans[0]
|
||||
childSpan := endedSpans[1]
|
||||
fromChildTraceID := childSpan.Parent().TraceID()
|
||||
fromParentTraceID := parentSpan.SpanContext().TraceID()
|
||||
for idx, c := range fromChildTraceID {
|
||||
p := fromParentTraceID[idx]
|
||||
if p != c {
|
||||
t.Errorf("trace id mismatch at idx: %d (%#v != %#v)", idx,
|
||||
fromChildTraceID, fromParentTraceID)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// check the metrics
|
||||
mdata := metricdata.ResourceMetrics{}
|
||||
err = otelInstance.metricReader.Collect(context.Background(), &mdata)
|
||||
if err != nil {
|
||||
t.Errorf("cannot collect the recorded metrics")
|
||||
return
|
||||
}
|
||||
|
||||
if len(mdata.ScopeMetrics) != 1 {
|
||||
t.Errorf("wrong amount of metrics, want: 1, got: %d", len(mdata.ScopeMetrics))
|
||||
for idx, sm := range mdata.ScopeMetrics {
|
||||
t.Errorf("%d -> %#v", idx, sm)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// --> check that we have all the metrics we want to report
|
||||
sm := mdata.ScopeMetrics[0]
|
||||
wantedMetrics := map[string]bool{
|
||||
// "requests-failed-count": false // <- we do not have requests that failed
|
||||
// "requests-canceled-count": false // <- we do not have requests cancelled
|
||||
// "requests-timedout-count": false // <- we do not have requests timed out
|
||||
"http.client.request.started.count": false,
|
||||
"http.client.request.size": false,
|
||||
"http.client.duration": false,
|
||||
"http.client.response.size": false,
|
||||
"http.client.response.read.size": false,
|
||||
"http.client.response.read.size.hist": false,
|
||||
"http.client.response.read.duration": false,
|
||||
"http.client.response.read.duration.hist": false,
|
||||
// "reader-errors": false,
|
||||
}
|
||||
numWantedMetrics := len(wantedMetrics)
|
||||
gotMetrics := make(map[string]metricdata.Metrics, numWantedMetrics)
|
||||
for _, m := range sm.Metrics {
|
||||
gotMetrics[m.Name] = m
|
||||
}
|
||||
for k := range wantedMetrics {
|
||||
if _, ok := gotMetrics[k]; !ok {
|
||||
t.Errorf("missing metric %s", k)
|
||||
return
|
||||
}
|
||||
}
|
||||
// check that we do not have not expected metrics:
|
||||
for k := range gotMetrics {
|
||||
if _, ok := wantedMetrics[k]; !ok {
|
||||
t.Errorf("got unexpected metric %s", k)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// --> check that the metrics have the expected attributes set
|
||||
readSize := gotMetrics["read-size"]
|
||||
readSizeSum, ok := readSize.Data.(metricdata.Sum[int64])
|
||||
if !ok {
|
||||
t.Errorf("cannot access read size aggregation: %#v", readSize.Data)
|
||||
return
|
||||
}
|
||||
if len(readSizeSum.DataPoints) != 1 {
|
||||
t.Errorf("read sum data points, want: 1, got: %d", len(readSizeSum.DataPoints))
|
||||
return
|
||||
}
|
||||
dp := readSizeSum.DataPoints[0]
|
||||
|
||||
if dp.Attributes.Len() != 1 {
|
||||
t.Errorf("missing attributes, want 1, got: %d\n%#v", dp.Attributes.Len(), dp.Attributes)
|
||||
return
|
||||
}
|
||||
payload := "foo bar"
|
||||
if dp.Value != int64(len(payload)) {
|
||||
t.Errorf("metric size, want: %d, got: %d", len(payload), dp.Value)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
otelio "github.com/krakend/krakend-otel/io"
|
||||
"github.com/krakend/krakend-otel/state"
|
||||
)
|
||||
|
||||
// TransportOptions defines the detail we want for
|
||||
// the metrics and traces.
|
||||
// See [TrasnportMetricsOptions] and [TransportTracesOptions] for
|
||||
// more details.
|
||||
// The OTELInstance member defines a function to obtain a State to
|
||||
// use with this transport. The state getter is used at configuration
|
||||
// time, not at runtime.
|
||||
type TransportOptions struct {
|
||||
MetricsOpts TransportMetricsOptions
|
||||
TracesOpts TransportTracesOptions
|
||||
|
||||
OTELInstance state.GetterFn
|
||||
}
|
||||
|
||||
// readerWrapper defines a function to wrap a reader
|
||||
type readerWrapperFn func(r io.Reader, ctx context.Context) io.ReadCloser
|
||||
|
||||
// Transport is an http.RoundTripper that instruments all outgoing requests with
|
||||
// OpenTelemetry metrics and tracing.
|
||||
//
|
||||
// The zero value is intended to be a useful default, but for
|
||||
// now it's recommended that you explicitly set Propagation, since the default
|
||||
// for this may change.
|
||||
type Transport struct {
|
||||
// Base may be set to wrap another http.RoundTripper that does the actual
|
||||
// requests. By default http.DefaultTransport is used.
|
||||
//
|
||||
// If base HTTP roundtripper implements CancelRequest,
|
||||
// the returned round tripper will be cancelable.
|
||||
base http.RoundTripper
|
||||
|
||||
// Propagation defines how traces are propagated. If unspecified, a default
|
||||
// (currently B3 format can be configured outside) will be used.
|
||||
propagator propagation.TextMapPropagator
|
||||
|
||||
// StartOptions are applied to the span started by this Transport around each
|
||||
// request.
|
||||
//
|
||||
// StartOptions.SpanKind will always be set to trace.SpanKindClient
|
||||
// for spans started by this transport.
|
||||
// StartOptions trace.StartOptions
|
||||
|
||||
tracesOpts TransportTracesOptions
|
||||
metricsOpts TransportMetricsOptions
|
||||
getOTELInstance state.GetterFn
|
||||
|
||||
metrics *transportMetrics
|
||||
traces *transportTraces
|
||||
|
||||
readerWrapper readerWrapperFn
|
||||
}
|
||||
|
||||
func readWrapperBuilder(metricsOpts *TransportMetricsOptions, tracesOpts *TransportTracesOptions,
|
||||
meter metric.Meter, tracer trace.Tracer,
|
||||
) readerWrapperFn {
|
||||
if !metricsOpts.ReadPayload && !tracesOpts.ReadPayload {
|
||||
// no metrics or traces for the payload reading
|
||||
return func(r io.Reader, ctx context.Context) io.ReadCloser {
|
||||
rc, ok := r.(io.ReadCloser)
|
||||
if !ok {
|
||||
rc = io.NopCloser(r)
|
||||
}
|
||||
return rc
|
||||
}
|
||||
}
|
||||
|
||||
var attrM []attribute.KeyValue
|
||||
var attrT []attribute.KeyValue
|
||||
var t trace.Tracer
|
||||
var m metric.Meter
|
||||
|
||||
if metricsOpts.ReadPayload {
|
||||
attrM = metricsOpts.FixedAttributes
|
||||
m = meter
|
||||
}
|
||||
|
||||
if tracesOpts.ReadPayload {
|
||||
attrT = tracesOpts.FixedAttributes
|
||||
t = tracer
|
||||
}
|
||||
|
||||
irf := otelio.NewInstrumentedReaderFactory("http.client.request.read.", attrT, attrM, t, m)
|
||||
return func(r io.Reader, ctx context.Context) io.ReadCloser {
|
||||
return irf(r, ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// NewRoundTripper creates an instrumented round tripper.
|
||||
func NewRoundTripper(base http.RoundTripper, metricsOpts TransportMetricsOptions,
|
||||
tracesOpts TransportTracesOptions, clientName string, stateGetter state.GetterFn,
|
||||
) http.RoundTripper {
|
||||
rt := newTransport(base, metricsOpts, tracesOpts, clientName, stateGetter)
|
||||
if rt == nil {
|
||||
return base
|
||||
}
|
||||
return rt
|
||||
}
|
||||
|
||||
func newTransport(base http.RoundTripper, metricsOpts TransportMetricsOptions,
|
||||
tracesOpts TransportTracesOptions, clientName string, stateGetter state.GetterFn,
|
||||
) *Transport {
|
||||
if !tracesOpts.Enabled() && !metricsOpts.Enabled() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// This could be using the call to `otel.GetGlobalTracer` directly,
|
||||
// but for testability purposes, we have a way to obtain the tracer
|
||||
// from a "state provider" function call
|
||||
if stateGetter == nil {
|
||||
stateGetter = state.GlobalState
|
||||
}
|
||||
inst := stateGetter()
|
||||
|
||||
if inst == nil {
|
||||
// nothing to do here: the global observability state has not been configured,
|
||||
// or we do not have traces nor metrics enabled.
|
||||
return nil
|
||||
}
|
||||
|
||||
meter := inst.Meter()
|
||||
tracer := inst.Tracer()
|
||||
|
||||
return &Transport{
|
||||
base: base,
|
||||
propagator: otel.GetTextMapPropagator(),
|
||||
getOTELInstance: stateGetter,
|
||||
tracesOpts: tracesOpts,
|
||||
metricsOpts: metricsOpts,
|
||||
metrics: newTransportMetrics(&metricsOpts, meter, clientName),
|
||||
traces: newTransportTraces(&tracesOpts, tracer, clientName),
|
||||
readerWrapper: readWrapperBuilder(&metricsOpts, &tracesOpts, meter, tracer),
|
||||
}
|
||||
}
|
||||
|
||||
// RoundTrip implements http.RoundTripper, delegating to Base and recording
|
||||
// metrics and traces for the request.
|
||||
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
rtt := roundTripTracking{
|
||||
req: req,
|
||||
}
|
||||
if t.tracesOpts.DetailedConnection || t.metricsOpts.DetailedConnection {
|
||||
rtt.withClientTrace()
|
||||
}
|
||||
t.traces.start(&rtt, t.propagator)
|
||||
|
||||
requestSentAt := time.Now()
|
||||
rtt.resp, rtt.err = t.base.RoundTrip(rtt.req)
|
||||
rtt.latencyInSecs = float64(time.Since(requestSentAt)) / float64(time.Second)
|
||||
|
||||
t.metrics.report(&rtt, t.metricsOpts.FixedAttributes)
|
||||
|
||||
if rtt.resp != nil && rtt.resp.Body != nil {
|
||||
rtt.resp.Body = t.readerWrapper(rtt.resp.Body, rtt.req.Context())
|
||||
}
|
||||
t.traces.end(&rtt)
|
||||
return rtt.resp, rtt.err
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/semconv/v1.21.0"
|
||||
|
||||
kotelconfig "github.com/krakend/krakend-otel/config"
|
||||
)
|
||||
|
||||
// TransportMetricsOptions contains the options to enable / disable
|
||||
// for reporting metrics, and a set of fixed attributes to add
|
||||
// to all metrics.
|
||||
type TransportMetricsOptions struct {
|
||||
RoundTrip bool // provide the round trip metrics
|
||||
ReadPayload bool // provide metrics for the reading the full body
|
||||
DetailedConnection bool // provide detailed metrics about the connection: dns lookup, tls ...
|
||||
FixedAttributes []attribute.KeyValue // "static" attributes set at config time.
|
||||
}
|
||||
|
||||
// Enabled tells if metrics should be reported for the transport.
|
||||
func (o *TransportMetricsOptions) Enabled() bool {
|
||||
return o.RoundTrip || o.ReadPayload
|
||||
}
|
||||
|
||||
// transportMetrics holds the metric instruments for the round trip
|
||||
type transportMetrics struct {
|
||||
// total of initiated requests (sucessful, failed and cancelled)
|
||||
requestsStarted metric.Int64Counter
|
||||
requestsFailed metric.Int64Counter
|
||||
requestsCanceled metric.Int64Counter
|
||||
requestsTimedOut metric.Int64Counter
|
||||
|
||||
// the value of the Content-Length header for the request (not the
|
||||
// actual written bytes of the request, that might be cancelled
|
||||
// when it is already on flight.
|
||||
requestContentLength metric.Int64Counter
|
||||
|
||||
responseLatency metric.Float64Histogram
|
||||
|
||||
// the response content lenght comes from the server provided header
|
||||
// and might differ from the actual number of bytes read from the body
|
||||
responseContentLength metric.Int64Histogram
|
||||
responseNoContentLength metric.Int64Counter
|
||||
|
||||
// from the httptrace details
|
||||
detailsEnabled bool
|
||||
getConnLatency metric.Float64Histogram
|
||||
dnsLatency metric.Float64Histogram
|
||||
tlsLatency metric.Float64Histogram
|
||||
|
||||
// to identify the source of the request (in KrakenD the front facing endpoint)
|
||||
clientName string
|
||||
}
|
||||
|
||||
func newTransportMetrics(metricsOpts *TransportMetricsOptions, meter metric.Meter, clientName string) *transportMetrics {
|
||||
if meter == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var tm transportMetrics
|
||||
tm.requestsStarted, _ = meter.Int64Counter("http.client.request.started.count") // number of reqs started
|
||||
tm.requestsFailed, _ = meter.Int64Counter("http.client.request.failed.count") // number of reqs failed
|
||||
tm.requestsCanceled, _ = meter.Int64Counter("http.client.request.canceled.count") // number of canceled requests
|
||||
tm.requestsTimedOut, _ = meter.Int64Counter("http.client.request.timedout.count") // numer of timedout request (inclued in failed)
|
||||
|
||||
tm.requestContentLength, _ = meter.Int64Counter("http.client.request.size") // the value of the Content-Length header for the request
|
||||
|
||||
tm.responseLatency, _ = meter.Float64Histogram("http.client.duration", kotelconfig.TimeBucketsOpt)
|
||||
tm.responseContentLength, _ = meter.Int64Histogram("http.client.response.size", kotelconfig.SizeBucketsOpt)
|
||||
tm.responseNoContentLength, _ = meter.Int64Counter("http.client.response.no_content_length")
|
||||
|
||||
tm.detailsEnabled = metricsOpts.DetailedConnection
|
||||
tm.getConnLatency, _ = meter.Float64Histogram("http.client.request.get-conn.duration", kotelconfig.TimeBucketsOpt)
|
||||
tm.dnsLatency, _ = meter.Float64Histogram("http.client.request.dns.duration", kotelconfig.TimeBucketsOpt)
|
||||
tm.tlsLatency, _ = meter.Float64Histogram("http.client.request.tls.duration", kotelconfig.TimeBucketsOpt)
|
||||
return &tm
|
||||
}
|
||||
|
||||
func (m *transportMetrics) report(rtt *roundTripTracking, attrs []attribute.KeyValue) {
|
||||
if m == nil || m.requestsStarted == nil {
|
||||
// if metrics are nil or not initialized, we just return
|
||||
return
|
||||
}
|
||||
|
||||
attrM := make([]attribute.KeyValue, len(attrs), len(attrs)+2)
|
||||
copy(attrM, attrs)
|
||||
if len(m.clientName) > 0 {
|
||||
attrM = append(attrM, attribute.Key("clientname").String(m.clientName))
|
||||
}
|
||||
attrM = append(attrM, semconv.HTTPRequestMethodKey.String(rtt.req.Method))
|
||||
statusCode := 0
|
||||
if rtt.err == nil {
|
||||
// if we fail on the client side, we do not have a status code, but we
|
||||
// want it set to 0 to be displayed on the dashboard
|
||||
statusCode = int(rtt.resp.StatusCode)
|
||||
}
|
||||
attrM = append(attrM, semconv.HTTPResponseStatusCode(statusCode))
|
||||
attrOpt := metric.WithAttributeSet(attribute.NewSet(attrM...))
|
||||
|
||||
ctx := rtt.req.Context()
|
||||
|
||||
m.requestsStarted.Add(ctx, 1, attrOpt)
|
||||
if rtt.req.ContentLength >= 0 {
|
||||
// TOOD: should we check the http verb / method to report this ?
|
||||
m.requestContentLength.Add(ctx, rtt.req.ContentLength, attrOpt)
|
||||
}
|
||||
|
||||
if rtt.err != nil {
|
||||
reqCtx := rtt.req.Context()
|
||||
var ctxErr error
|
||||
if reqCtx != nil {
|
||||
ctxErr = rtt.req.Context().Err()
|
||||
}
|
||||
if errors.Is(ctxErr, context.Canceled) {
|
||||
// ATTENTION: a Cancelled requests is not considered failed
|
||||
m.requestsCanceled.Add(ctx, 1, attrOpt)
|
||||
} else if errors.Is(ctxErr, context.DeadlineExceeded) {
|
||||
m.requestsTimedOut.Add(ctx, 1, attrOpt)
|
||||
m.requestsFailed.Add(ctx, 1, attrOpt)
|
||||
} else {
|
||||
m.requestsFailed.Add(ctx, 1, attrOpt)
|
||||
}
|
||||
}
|
||||
|
||||
m.responseLatency.Record(ctx, rtt.latencyInSecs, attrOpt)
|
||||
if rtt.req.Method != "HEAD" && rtt.resp != nil {
|
||||
if rtt.resp.ContentLength >= 0 {
|
||||
// it might be the case were we receive a chunked response, and then
|
||||
// we will not record a metric for it.
|
||||
m.responseContentLength.Record(ctx, rtt.resp.ContentLength, attrOpt)
|
||||
} else {
|
||||
m.responseNoContentLength.Add(ctx, 1, attrOpt)
|
||||
}
|
||||
}
|
||||
|
||||
if m.detailsEnabled {
|
||||
m.getConnLatency.Record(ctx, rtt.getConnLatency, attrOpt)
|
||||
m.dnsLatency.Record(ctx, rtt.dnsLatency, attrOpt)
|
||||
m.tlsLatency.Record(ctx, rtt.tlsLatency, attrOpt)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
otelhttp "github.com/krakend/krakend-otel/http"
|
||||
)
|
||||
|
||||
// TransportTracesOptions defines what information
|
||||
// is enabled, and extra fixed attributes to add to
|
||||
// the trace.
|
||||
type TransportTracesOptions struct {
|
||||
RoundTrip bool // use a span for the round trip
|
||||
ReadPayload bool // use a span for the process of reading the full body
|
||||
DetailedConnection bool // add extra detail about the connection to the server: dns lookup, tls...
|
||||
FixedAttributes []attribute.KeyValue // "static" attributes set at config time.
|
||||
ReportHeaders bool
|
||||
}
|
||||
|
||||
// Enabled returns if the transport should create a trace.
|
||||
func (o *TransportTracesOptions) Enabled() bool {
|
||||
return o.RoundTrip || o.ReadPayload
|
||||
}
|
||||
|
||||
type transportTraces struct {
|
||||
tracer trace.Tracer
|
||||
spanName string
|
||||
fixedAttrs []attribute.KeyValue
|
||||
detailedConnection bool
|
||||
reportHeaders bool
|
||||
}
|
||||
|
||||
func newTransportTraces(tracesOpts *TransportTracesOptions, tracer trace.Tracer, spanName string) *transportTraces {
|
||||
return &transportTraces{
|
||||
tracer: tracer,
|
||||
spanName: spanName,
|
||||
fixedAttrs: tracesOpts.FixedAttributes,
|
||||
detailedConnection: tracesOpts.DetailedConnection,
|
||||
reportHeaders: tracesOpts.ReportHeaders,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *transportTraces) start(rtt *roundTripTracking,
|
||||
propagator propagation.TextMapPropagator,
|
||||
) {
|
||||
if t == nil || rtt.req == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ctx, span := t.tracer.Start(rtt.req.Context(), t.spanName)
|
||||
if span == nil || !span.IsRecording() {
|
||||
// we might not be recording because of sampling
|
||||
return
|
||||
}
|
||||
rtt.span = span
|
||||
rtt.req = rtt.req.WithContext(ctx)
|
||||
|
||||
reqAttrs := otelhttp.TraceRequestAttrs(rtt.req)
|
||||
// propagate the context, see `example/passthrough/main.go` in OTEL repo
|
||||
// SpanContextToRequest will modify its Request argument, which is
|
||||
// contrary to the contract for http.RoundTripper, so we need to
|
||||
// pass it a copy of the Request.
|
||||
// However, the Request struct itself was already copied by
|
||||
// the WithContext calls above and so we just need to copy the header.
|
||||
header := make(http.Header, len(rtt.req.Header)+1)
|
||||
for k, v := range rtt.req.Header {
|
||||
header[k] = v
|
||||
if t.reportHeaders {
|
||||
reqAttrs = append(reqAttrs,
|
||||
attribute.StringSlice("http.request.header."+strings.ToLower(k), v))
|
||||
}
|
||||
}
|
||||
rtt.req.Header = header
|
||||
propagator.Inject(rtt.req.Context(), propagation.HeaderCarrier(rtt.req.Header))
|
||||
|
||||
rtt.span.SetAttributes(t.fixedAttrs...)
|
||||
rtt.span.SetAttributes(reqAttrs...)
|
||||
}
|
||||
|
||||
func (t *transportTraces) end(rtt *roundTripTracking) {
|
||||
if t == nil || rtt.span == nil || !rtt.span.IsRecording() {
|
||||
return
|
||||
}
|
||||
|
||||
if rtt.err != nil {
|
||||
rtt.span.RecordError(rtt.err)
|
||||
rtt.span.SetStatus(codes.Error, rtt.err.Error())
|
||||
} else {
|
||||
respAttrs := otelhttp.TraceResponseAttrs(rtt.resp)
|
||||
if t.reportHeaders {
|
||||
for k, v := range rtt.resp.Header {
|
||||
respAttrs = append(respAttrs,
|
||||
attribute.StringSlice("http.response.header."+strings.ToLower(k), v))
|
||||
}
|
||||
}
|
||||
rtt.span.SetAttributes(respAttrs...)
|
||||
rtt.span.SetAttributes(attribute.Float64("response-latency", rtt.latencyInSecs))
|
||||
if t.detailedConnection {
|
||||
rtt.span.SetAttributes(
|
||||
attribute.Float64("get-conn-latency", rtt.getConnLatency),
|
||||
attribute.Float64("dns-latency", rtt.dnsLatency),
|
||||
attribute.Float64("tls-latency", rtt.tlsLatency),
|
||||
)
|
||||
rtt.span.AddEvent("first-byte-time", trace.WithTimestamp(rtt.firstByteTime))
|
||||
}
|
||||
rtt.span.SetStatus(codes.Ok, "")
|
||||
}
|
||||
|
||||
rtt.span.End()
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net/http"
|
||||
"net/http/httptrace"
|
||||
"net/textproto"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// roundTripTracking holds all the data to report
|
||||
// for a round trip: some data will be used only
|
||||
// for metrics, some for traces, and some for both.
|
||||
type roundTripTracking struct {
|
||||
req *http.Request
|
||||
resp *http.Response
|
||||
|
||||
span trace.Span
|
||||
|
||||
latencyInSecs float64
|
||||
err error
|
||||
|
||||
getConnStart time.Time
|
||||
getConnLatency float64
|
||||
|
||||
firstByteTime time.Time // reported as an span event
|
||||
|
||||
dnsStart time.Time
|
||||
dnsLatency float64
|
||||
|
||||
tlsStart time.Time
|
||||
tlsLatency float64
|
||||
}
|
||||
|
||||
// withClientTrace may be set to a function allowing the current *trace.Span
|
||||
// to be annotated with HTTP request event information emitted by the
|
||||
// httptrace package.
|
||||
func (t *roundTripTracking) withClientTrace() {
|
||||
// Commented out are the callback calls that we are not interested in:
|
||||
httpTrace := &httptrace.ClientTrace{
|
||||
GetConn: t.GetConn,
|
||||
GotConn: t.GotConn,
|
||||
// PutIdleConn: t.PutIdleConn,
|
||||
GotFirstResponseByte: t.GotFirstResponseByte,
|
||||
// Got100Continue: t.Got100Continue,
|
||||
// Got1xxResponse: t.Got1xxResponse,
|
||||
DNSStart: t.DNSStart,
|
||||
DNSDone: t.DNSDone,
|
||||
// ConnectStart: t.ConnectStart,
|
||||
// ConnectDone: t.ConnectDone,
|
||||
TLSHandshakeStart: t.TLSHandshakeStart,
|
||||
TLSHandshakeDone: t.TLSHandshakeDone,
|
||||
// WroteHeaderField: t.WroteHeaderField,
|
||||
// WroteHeaders: t.WroteHeaders,
|
||||
// Wait100Continue: t.Wait100Continue,
|
||||
// WroteRequest: t.WroteRequest,
|
||||
}
|
||||
t.req = t.req.WithContext(httptrace.WithClientTrace(t.req.Context(), httpTrace))
|
||||
}
|
||||
|
||||
// GetConn is called before a connection is created or
|
||||
// retrieved from an idle pool. The hostPort is the
|
||||
// "host:port" of the target or proxy. GetConn is called even
|
||||
// if there's already an idle cached connection available.
|
||||
func (t *roundTripTracking) GetConn(hostPort string) {
|
||||
t.getConnStart = time.Now()
|
||||
}
|
||||
|
||||
// GotConn is called after a successful connection is
|
||||
// obtained. There is no hook for failure to obtain a
|
||||
// connection; instead, use the error from
|
||||
// Transport.RoundTrip.
|
||||
func (t *roundTripTracking) GotConn(info httptrace.GotConnInfo) {
|
||||
t.getConnLatency = float64(time.Since(t.getConnStart)) / float64(time.Second)
|
||||
}
|
||||
|
||||
// PutIdleConn is called when the connection is returned to
|
||||
// the idle pool. If err is nil, the connection was
|
||||
// successfully returned to the idle pool. If err is non-nil,
|
||||
// it describes why not. PutIdleConn is not called if
|
||||
// connection reuse is disabled via Transport.DisableKeepAlives.
|
||||
// PutIdleConn is called before the caller's Response.Body.Close
|
||||
// call returns.
|
||||
// For HTTP/2, this hook is not currently used.
|
||||
func (t *roundTripTracking) PutIdleConn(err error) {
|
||||
// we are not interested in this metric, but we leave the function
|
||||
// definition to make clear that leaving this out has been
|
||||
// a decision
|
||||
}
|
||||
|
||||
// GotFirstResponseByte is called when the first byte of the response
|
||||
// headers is available.
|
||||
func (t *roundTripTracking) GotFirstResponseByte() {
|
||||
t.firstByteTime = time.Now()
|
||||
}
|
||||
|
||||
// Got100Continue is called if the server replies with a "100
|
||||
// Continue" response.
|
||||
func (t *roundTripTracking) Got100Continue() {
|
||||
// we are not interested in this metric, but we leave the function
|
||||
// definition to make clear that leaving this out has been
|
||||
// a decision
|
||||
}
|
||||
|
||||
// Got1xxResponse is called for each 1xx informational response header
|
||||
// returned before the final non-1xx response. Got1xxResponse is called
|
||||
// for "100 Continue" responses, even if Got100Continue is also defined.
|
||||
// If it returns an error, the client request is aborted with that error value.
|
||||
func (t *roundTripTracking) Got1xxResponse(code int, header textproto.MIMEHeader) error {
|
||||
// we are not interested in this metric, but we leave the function
|
||||
// definition to make clear that leaving this out has been
|
||||
// a decision
|
||||
return nil
|
||||
}
|
||||
|
||||
// DNSStart is called when a DNS lookup begins.
|
||||
func (t *roundTripTracking) DNSStart(httptrace.DNSStartInfo) {
|
||||
t.dnsStart = time.Now()
|
||||
}
|
||||
|
||||
// DNSDone is called when a DNS lookup ends.
|
||||
func (t *roundTripTracking) DNSDone(httptrace.DNSDoneInfo) {
|
||||
t.dnsLatency = float64(time.Since(t.dnsStart)) / float64(time.Second)
|
||||
}
|
||||
|
||||
// ConnectStart is called when a new connection's Dial begins.
|
||||
// If net.Dialer.DualStack (IPv6 "Happy Eyeballs") support is
|
||||
// enabled, this may be called multiple times.
|
||||
func (t *roundTripTracking) ConnectStart(network, addr string) {
|
||||
// we are not interested in this metric, but we leave the function
|
||||
// definition to make clear that leaving this out has been
|
||||
// a decision
|
||||
}
|
||||
|
||||
// ConnectDone is called when a new connection's Dial
|
||||
// completes. The provided err indicates whether the
|
||||
// connection completed successfully.
|
||||
// If net.Dialer.DualStack ("Happy Eyeballs") support is
|
||||
// enabled, this may be called multiple times.
|
||||
func (t *roundTripTracking) ConnectDone(network, addr string, err error) {
|
||||
// we are not interested in this metric, but we leave the function
|
||||
// definition to make clear that leaving this out has been
|
||||
// a decision
|
||||
}
|
||||
|
||||
// TLSHandshakeStart is called when the TLS handshake is started. When
|
||||
// connecting to an HTTPS site via an HTTP proxy, the handshake happens
|
||||
// after the CONNECT request is processed by the proxy.
|
||||
func (t *roundTripTracking) TLSHandshakeStart() {
|
||||
t.tlsStart = time.Now()
|
||||
}
|
||||
|
||||
// TLSHandshakeDone is called after the TLS handshake with either the
|
||||
// successful handshake's connection state, or a non-nil error on handshake
|
||||
// failure.
|
||||
func (t *roundTripTracking) TLSHandshakeDone(tls.ConnectionState, error) {
|
||||
t.tlsLatency = float64(time.Since(t.tlsStart)) / float64(time.Second)
|
||||
}
|
||||
|
||||
// WroteHeaderField is called after the Transport has written
|
||||
// each request header. At the time of this call the values
|
||||
// might be buffered and not yet written to the network.
|
||||
func (t *roundTripTracking) WroteHeaderField(key string, value []string) {
|
||||
// we are not interested in this metric, but we leave the function
|
||||
// definition to make clear that leaving this out has been
|
||||
// a decision
|
||||
}
|
||||
|
||||
// WroteHeaders is called after the Transport has written
|
||||
// all request headers.
|
||||
func (t *roundTripTracking) WroteHeaders() {
|
||||
// we are not interested in this metric, but we leave the function
|
||||
// definition to make clear that leaving this out has been
|
||||
// a decision
|
||||
}
|
||||
|
||||
// Wait100Continue is called if the Request specified
|
||||
// "Expect: 100-continue" and the Transport has written the
|
||||
// request headers but is waiting for "100 Continue" from the
|
||||
// server before writing the request body.
|
||||
func (t *roundTripTracking) Wait100Continue() {
|
||||
// we are not interested in this metric, but we leave the function
|
||||
// definition to make clear that leaving this out has been
|
||||
// a decision
|
||||
}
|
||||
|
||||
// WroteRequest is called with the result of writing the
|
||||
// request and any body. It may be called multiple times
|
||||
// in the case of retried requests.
|
||||
func (t *roundTripTracking) WroteRequest(httptrace.WroteRequestInfo) {
|
||||
// we are not interested in this metric, but we leave the function
|
||||
// definition to make clear that leaving this out has been
|
||||
// a decision
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/semconv/v1.21.0"
|
||||
|
||||
kotelconfig "github.com/krakend/krakend-otel/config"
|
||||
)
|
||||
|
||||
type metricsHTTP struct {
|
||||
fixedAttrs []attribute.KeyValue
|
||||
fixedAttrsOpts metric.MeasurementOption
|
||||
|
||||
latency metric.Float64Histogram
|
||||
size metric.Int64Histogram
|
||||
}
|
||||
|
||||
func newMetricsHTTP(meter metric.Meter, attrs []attribute.KeyValue) *metricsHTTP {
|
||||
var m metricsHTTP
|
||||
m.latency, _ = meter.Float64Histogram("http.server.duration", kotelconfig.TimeBucketsOpt)
|
||||
m.size, _ = meter.Int64Histogram("http.server.response.size", kotelconfig.SizeBucketsOpt)
|
||||
if len(attrs) > 0 {
|
||||
m.fixedAttrs = make([]attribute.KeyValue, len(attrs))
|
||||
copy(m.fixedAttrs, attrs)
|
||||
m.fixedAttrsOpts = metric.WithAttributeSet(attribute.NewSet(m.fixedAttrs...))
|
||||
} else {
|
||||
m.fixedAttrsOpts = metric.WithAttributeSet(attribute.NewSet())
|
||||
}
|
||||
return &m
|
||||
}
|
||||
|
||||
func (m *metricsHTTP) report(t *tracking, r *http.Request) {
|
||||
if m == nil || m.latency == nil {
|
||||
return
|
||||
}
|
||||
dynAttrsOpts := metric.WithAttributes(
|
||||
semconv.HTTPRoute(t.EndpointPattern()),
|
||||
semconv.HTTPRequestMethodKey.String(r.Method),
|
||||
semconv.HTTPResponseStatusCode(t.responseStatus),
|
||||
)
|
||||
m.latency.Record(t.ctx, t.latencyInSecs, m.fixedAttrsOpts, dynAttrsOpts)
|
||||
m.size.Record(t.ctx, int64(t.responseSize), m.fixedAttrsOpts, dynAttrsOpts)
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type TrackingResponseWriter struct {
|
||||
track *tracking
|
||||
recordHeaders bool
|
||||
rw http.ResponseWriter
|
||||
flusher http.Flusher
|
||||
hijacker http.Hijacker
|
||||
}
|
||||
|
||||
func (w *TrackingResponseWriter) gatherHeaders() {
|
||||
if !w.recordHeaders {
|
||||
return
|
||||
}
|
||||
if w.track.responseHeaders != nil {
|
||||
// we already recorded the response headers
|
||||
return
|
||||
}
|
||||
h := w.rw.Header()
|
||||
w.track.responseHeaders = make(map[string][]string, len(h))
|
||||
for k, v := range h {
|
||||
w.track.responseHeaders[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
func (w *TrackingResponseWriter) Header() http.Header {
|
||||
return w.rw.Header()
|
||||
}
|
||||
|
||||
func (w *TrackingResponseWriter) Write(b []byte) (int, error) {
|
||||
w.gatherHeaders()
|
||||
if w.recordHeaders && w.track.responseHeaders == nil {
|
||||
}
|
||||
nBytes, e := w.rw.Write(b)
|
||||
if e != nil {
|
||||
w.track.writeErrs = append(w.track.writeErrs, e)
|
||||
}
|
||||
w.track.responseSize += nBytes
|
||||
return nBytes, e
|
||||
}
|
||||
|
||||
func (w *TrackingResponseWriter) WriteHeader(statusCode int) {
|
||||
w.gatherHeaders()
|
||||
w.track.responseStatus = statusCode
|
||||
w.rw.WriteHeader(statusCode)
|
||||
}
|
||||
|
||||
func (w *TrackingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
||||
if w.hijacker != nil {
|
||||
return w.hijacker.Hijack()
|
||||
}
|
||||
return nil, nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (w *TrackingResponseWriter) Flush() {
|
||||
if w.flusher != nil {
|
||||
w.flusher.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func newTrackingResponseWriter(rw http.ResponseWriter, t *tracking, recordHeaders bool) *TrackingResponseWriter {
|
||||
return &TrackingResponseWriter{
|
||||
track: t,
|
||||
recordHeaders: recordHeaders,
|
||||
rw: rw,
|
||||
flusher: rw.(http.Flusher),
|
||||
hijacker: rw.(http.Hijacker),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
kotelconfig "github.com/krakend/krakend-otel/config"
|
||||
"github.com/krakend/krakend-otel/state"
|
||||
)
|
||||
|
||||
type trackingHandler struct {
|
||||
next http.Handler
|
||||
|
||||
prop propagation.TextMapPropagator
|
||||
metrics *metricsHTTP
|
||||
traces *tracesHTTP
|
||||
reportHeaders bool
|
||||
}
|
||||
|
||||
func (h *trackingHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
|
||||
t := newTracking()
|
||||
t.ctx = r.Context()
|
||||
if h.prop != nil {
|
||||
t.ctx = h.prop.Extract(t.ctx, propagation.HeaderCarrier(r.Header))
|
||||
if t.ctx != r.Context() {
|
||||
t.span = trace.SpanFromContext(t.ctx)
|
||||
}
|
||||
}
|
||||
t.ctx = context.WithValue(t.ctx, krakenDContextTrackingStrKey, t)
|
||||
r = r.WithContext(t.ctx)
|
||||
|
||||
if h.metrics != nil || h.traces != nil {
|
||||
rw = newTrackingResponseWriter(rw, t, h.reportHeaders)
|
||||
}
|
||||
|
||||
t.Start()
|
||||
r = h.traces.start(r, t)
|
||||
h.next.ServeHTTP(rw, r)
|
||||
t.Finish()
|
||||
h.traces.end(t)
|
||||
h.metrics.report(t, r)
|
||||
}
|
||||
|
||||
func NewTrackingHandler(next http.Handler, obsCfg *kotelconfig.Config, stateFn state.GetterFn) http.Handler {
|
||||
s := stateFn()
|
||||
|
||||
gCfg := &kotelconfig.GlobalOpts{}
|
||||
if obsCfg.Layers != nil && obsCfg.Layers.Global != nil {
|
||||
gCfg = obsCfg.Layers.Global
|
||||
}
|
||||
|
||||
if gCfg.DisablePropagation && gCfg.DisableMetrics && gCfg.DisableTraces {
|
||||
// nothing to do if everything is disabled
|
||||
return next
|
||||
}
|
||||
|
||||
var prop propagation.TextMapPropagator
|
||||
if !gCfg.DisablePropagation {
|
||||
prop = s.Propagator()
|
||||
}
|
||||
|
||||
var m *metricsHTTP
|
||||
if !gCfg.DisableMetrics {
|
||||
m = newMetricsHTTP(s.Meter(), []attribute.KeyValue{})
|
||||
}
|
||||
|
||||
var t *tracesHTTP
|
||||
if !gCfg.DisableTraces {
|
||||
t = newTracesHTTP(s.Tracer(), []attribute.KeyValue{
|
||||
attribute.String("krakend.stage", "global"),
|
||||
}, gCfg.ReportHeaders)
|
||||
}
|
||||
|
||||
return &trackingHandler{
|
||||
next: next,
|
||||
prop: prop,
|
||||
metrics: m,
|
||||
traces: t,
|
||||
reportHeaders: gCfg.ReportHeaders,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/semconv/v1.21.0"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
otelhttp "github.com/krakend/krakend-otel/http"
|
||||
)
|
||||
|
||||
type tracesHTTP struct {
|
||||
tracer trace.Tracer
|
||||
fixedAttrs []attribute.KeyValue
|
||||
reportHeaders bool
|
||||
}
|
||||
|
||||
func newTracesHTTP(tracer trace.Tracer, attrs []attribute.KeyValue, reportHeaders bool) *tracesHTTP {
|
||||
var fa []attribute.KeyValue
|
||||
if len(attrs) > 0 {
|
||||
fa = make([]attribute.KeyValue, len(attrs))
|
||||
copy(fa, attrs)
|
||||
}
|
||||
return &tracesHTTP{
|
||||
tracer: tracer,
|
||||
fixedAttrs: fa,
|
||||
reportHeaders: reportHeaders,
|
||||
}
|
||||
}
|
||||
|
||||
// start starts a trace from using the handlerTracking information provided.
|
||||
//
|
||||
// When traces are disabled, the tracer will be nil.
|
||||
func (t *tracesHTTP) start(r *http.Request, tr *tracking) *http.Request {
|
||||
if t == nil || t.tracer == nil || r.URL == nil {
|
||||
return r
|
||||
}
|
||||
tr.ctx, tr.span = t.tracer.Start(r.Context(), r.URL.Path)
|
||||
r = r.WithContext(tr.ctx)
|
||||
attrs := otelhttp.TraceRequestAttrs(r)
|
||||
tr.span.SetAttributes(attrs...)
|
||||
if len(t.fixedAttrs) > 0 {
|
||||
tr.span.SetAttributes(t.fixedAttrs...)
|
||||
}
|
||||
if t.reportHeaders {
|
||||
// report all incoming headers
|
||||
for hk, hv := range r.Header {
|
||||
tr.span.SetAttributes(attribute.StringSlice("http.request.header."+strings.ToLower(hk), hv))
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// end finishes the span started and tracked using the [handlerTracking] info.
|
||||
func (t *tracesHTTP) end(tr *tracking) {
|
||||
if t == nil || tr.span == nil || !tr.span.IsRecording() {
|
||||
return
|
||||
}
|
||||
|
||||
tr.span.SetAttributes(
|
||||
semconv.HTTPRoute(tr.EndpointPattern()),
|
||||
semconv.HTTPResponseStatusCode(tr.responseStatus),
|
||||
semconv.HTTPResponseBodySize(tr.responseSize))
|
||||
|
||||
if tr.responseHeaders != nil {
|
||||
// report all incoming headers
|
||||
for hk, hv := range tr.responseHeaders {
|
||||
tr.span.SetAttributes(attribute.StringSlice("http.response.header."+strings.ToLower(hk), hv))
|
||||
}
|
||||
}
|
||||
if len(tr.writeErrs) > 0 {
|
||||
e := tr.writeErrs[0]
|
||||
tr.span.RecordError(e)
|
||||
tr.span.SetStatus(codes.Error, e.Error())
|
||||
} else {
|
||||
tr.span.SetStatus(codes.Ok, "")
|
||||
}
|
||||
tr.span.End()
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type KrakenDContextTrackingTypeKey string
|
||||
|
||||
const (
|
||||
// KrakenDContextOTELStrKey is a special key to be used when there
|
||||
// is no way to obtain the span context from an inner context
|
||||
// (like when gin has not the fallback option enabled in the engine).
|
||||
krakenDContextTrackingStrKey KrakenDContextTrackingTypeKey = "KrakendD-Context-OTEL"
|
||||
)
|
||||
|
||||
type tracking struct {
|
||||
startTime time.Time
|
||||
ctx context.Context
|
||||
span trace.Span
|
||||
|
||||
latencyInSecs float64
|
||||
responseSize int
|
||||
responseStatus int
|
||||
responseHeaders map[string][]string
|
||||
writeErrs []error
|
||||
endpointPattern string
|
||||
}
|
||||
|
||||
func (t *tracking) EndpointPattern() string {
|
||||
if len(t.endpointPattern) == 0 {
|
||||
return "404 Not Found"
|
||||
}
|
||||
return t.endpointPattern
|
||||
}
|
||||
|
||||
func newTracking() *tracking {
|
||||
return &tracking{
|
||||
responseStatus: 200,
|
||||
}
|
||||
}
|
||||
|
||||
func fromContext(ctx context.Context) *tracking {
|
||||
v := ctx.Value(krakenDContextTrackingStrKey)
|
||||
if v != nil {
|
||||
return v.(*tracking)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetEndpointPattern allows to set the endpoint attribute once it
|
||||
// has been matched down the http handling pipeline.
|
||||
func SetEndpointPattern(ctx context.Context, endpointPattern string) {
|
||||
if t := fromContext(ctx); t != nil {
|
||||
t.endpointPattern = endpointPattern
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tracking) Start() {
|
||||
t.startTime = time.Now()
|
||||
}
|
||||
|
||||
func (t *tracking) Finish() {
|
||||
t.latencyInSecs = float64(time.Since(t.startTime)) / float64(time.Second)
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
// Package otelio implements the instrumentation around the
|
||||
// io.Reader and io.Writer.
|
||||
package otelio
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
noopmetric "go.opentelemetry.io/otel/metric/noop"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
nooptrace "go.opentelemetry.io/otel/trace/noop"
|
||||
|
||||
kotelconfig "github.com/krakend/krakend-otel/config"
|
||||
)
|
||||
|
||||
// instruments holds the instruments for the transfer of
|
||||
// information (either through a Reader or a Writer):
|
||||
// - size (in bytes)
|
||||
// - time (in seconds)
|
||||
// - error (if an error happened).
|
||||
type instruments struct {
|
||||
metricFixedAttrs []attribute.KeyValue
|
||||
traceFixedAttrs []attribute.KeyValue
|
||||
|
||||
sizeCount metric.Int64Counter
|
||||
sizeHistogram metric.Int64Histogram
|
||||
timeCount metric.Float64Counter
|
||||
timeHistogram metric.Float64Histogram
|
||||
errorsMeter metric.Int64Counter // error rate when doing i/o
|
||||
|
||||
tracer trace.Tracer
|
||||
traceSizeAttrName string
|
||||
traceTimeAttrName string
|
||||
traceName string
|
||||
|
||||
// attribute sets to be reused, and not computed in the
|
||||
// critical path
|
||||
metricAttributeSetOpt metric.MeasurementOption
|
||||
metricAttributeSetWithErrorOpt metric.MeasurementOption
|
||||
}
|
||||
|
||||
// newInstruments holds the information for an i/o operation, no mater
|
||||
// if is for reading or writing.
|
||||
func newInstruments(prefix string,
|
||||
attrT []attribute.KeyValue, attrM []attribute.KeyValue,
|
||||
tracer trace.Tracer, meter metric.Meter,
|
||||
) *instruments {
|
||||
if prefix == "" {
|
||||
prefix = "io."
|
||||
}
|
||||
strSizeCount := fmt.Sprintf("%ssize", prefix)
|
||||
strSizeHistogram := fmt.Sprintf("%ssize-hist", prefix)
|
||||
strTimeCount := fmt.Sprintf("%stime", prefix)
|
||||
strTimeHistogram := fmt.Sprintf("%stime-hist", prefix)
|
||||
strErrorsMeter := fmt.Sprintf("%serrors", prefix)
|
||||
|
||||
nopMProvider := noopmetric.NewMeterProvider()
|
||||
nopM := nopMProvider.Meter(fmt.Sprintf("%snop-tracker", prefix))
|
||||
sizeCount, _ := nopM.Int64Counter(strSizeCount)
|
||||
sizeHistogram, _ := nopM.Int64Histogram(strSizeHistogram, kotelconfig.SizeBucketsOpt)
|
||||
timeCount, _ := nopM.Float64Counter(strTimeCount)
|
||||
timeHistogram, _ := nopM.Float64Histogram(strTimeHistogram, kotelconfig.TimeBucketsOpt)
|
||||
errorsMeter, _ := nopM.Int64Counter(strErrorsMeter)
|
||||
|
||||
if meter != nil {
|
||||
if bsc, err := meter.Int64Counter(strSizeCount, metric.WithUnit("b")); err == nil {
|
||||
sizeCount = bsc
|
||||
}
|
||||
if bsh, err := meter.Int64Histogram(strSizeHistogram, kotelconfig.SizeBucketsOpt, metric.WithUnit("b")); err == nil {
|
||||
sizeHistogram = bsh
|
||||
}
|
||||
if brtc, err := meter.Float64Counter(strTimeCount, metric.WithUnit("s")); err == nil {
|
||||
timeCount = brtc
|
||||
}
|
||||
if brth, err := meter.Float64Histogram(strTimeHistogram, kotelconfig.TimeBucketsOpt, metric.WithUnit("s")); err == nil {
|
||||
timeHistogram = brth
|
||||
}
|
||||
if re, err := meter.Int64Counter(strErrorsMeter); err == nil {
|
||||
errorsMeter = re
|
||||
}
|
||||
}
|
||||
|
||||
var metricFixedAttrs []attribute.KeyValue
|
||||
if len(attrM) > 0 {
|
||||
metricFixedAttrs = make([]attribute.KeyValue, len(attrM))
|
||||
copy(metricFixedAttrs, attrM)
|
||||
}
|
||||
var traceFixedAttrs []attribute.KeyValue
|
||||
if len(attrT) > 0 {
|
||||
traceFixedAttrs = make([]attribute.KeyValue, len(attrT))
|
||||
copy(traceFixedAttrs, attrT)
|
||||
}
|
||||
|
||||
// precompute the metric options with its attributes
|
||||
l := len(metricFixedAttrs)
|
||||
metricAttrsWithErr := make([]attribute.KeyValue, l+1)
|
||||
copy(metricAttrsWithErr, metricFixedAttrs)
|
||||
metricAttrsWithErr[l] = attribute.String("error", "true")
|
||||
|
||||
attrSet := attribute.NewSet(metricFixedAttrs...)
|
||||
attrSetWithErr := attribute.NewSet(metricAttrsWithErr...)
|
||||
|
||||
if tracer == nil {
|
||||
tracer = nooptrace.NewTracerProvider().Tracer("read-tracer")
|
||||
}
|
||||
|
||||
return &instruments{
|
||||
metricFixedAttrs: metricFixedAttrs,
|
||||
traceFixedAttrs: traceFixedAttrs,
|
||||
sizeCount: sizeCount,
|
||||
sizeHistogram: sizeHistogram,
|
||||
timeCount: timeCount,
|
||||
timeHistogram: timeHistogram,
|
||||
errorsMeter: errorsMeter,
|
||||
tracer: tracer,
|
||||
traceSizeAttrName: prefix + "size",
|
||||
traceTimeAttrName: prefix + "time",
|
||||
traceName: prefix + "tracker",
|
||||
metricAttributeSetOpt: metric.WithAttributeSet(attrSet),
|
||||
metricAttributeSetWithErrorOpt: metric.WithAttributeSet(attrSetWithErr),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package otelio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var _ io.ReadCloser = (*instrumentedReader)(nil)
|
||||
|
||||
// we can finish the reading by receiving an `io.EOF` error
|
||||
// (the full buffer was read), or by a `Close` call, when
|
||||
// no more data will be read by the consumer
|
||||
type instrumentedReader struct {
|
||||
reader io.ReadCloser
|
||||
track ioTracking
|
||||
}
|
||||
|
||||
// NewInstrumentedReaderFactory creates a function that can wrap a reader with
|
||||
// an instrumented reader. Is better than the [NewIntrumentedReader] call because
|
||||
// the instruments here are only created once.
|
||||
func NewInstrumentedReaderFactory(prefix string, attrT []attribute.KeyValue, attrM []attribute.KeyValue,
|
||||
tracer trace.Tracer, meter metric.Meter,
|
||||
) func(io.Reader, context.Context) *instrumentedReader {
|
||||
if prefix == "" {
|
||||
prefix = "read."
|
||||
}
|
||||
instr := newInstruments(prefix, attrT, attrM, tracer, meter)
|
||||
|
||||
return func(r io.Reader, ctx context.Context) *instrumentedReader {
|
||||
rc, ok := r.(io.ReadCloser)
|
||||
if !ok {
|
||||
rc = io.NopCloser(r)
|
||||
}
|
||||
return &instrumentedReader{
|
||||
reader: rc,
|
||||
track: ioTracking{
|
||||
instr: instr,
|
||||
ctx: ctx,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewInstrumentedReader wraps a reader with an instrumented reader.
|
||||
// Is better to use [NewInstrumentedReaderFactory].
|
||||
func NewInstrumentedReader(prefix string, r io.Reader, ctx context.Context,
|
||||
attrT []attribute.KeyValue, attrM []attribute.KeyValue,
|
||||
tracer trace.Tracer, meter metric.Meter,
|
||||
) *instrumentedReader {
|
||||
return NewInstrumentedReaderFactory(prefix, attrT, attrM, tracer, meter)(r, ctx)
|
||||
}
|
||||
|
||||
// Read wraps the reader's Read and keeps track of the
|
||||
// read bytes. In case an error happens it will automatically
|
||||
// end the span and report the metrics.
|
||||
func (t *instrumentedReader) Read(b []byte) (int, error) {
|
||||
t.track.start()
|
||||
n, err := t.reader.Read(b)
|
||||
t.track.incSize(int64(n), err)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close wraps the reader Close and ends the trace and reports
|
||||
// the metrics.
|
||||
func (t *instrumentedReader) Close() error {
|
||||
err := t.reader.Close()
|
||||
t.track.end(err)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package otelio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/metric/metricdata"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
sdktracetest "go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
)
|
||||
|
||||
func TestReaderHappyPath(t *testing.T) {
|
||||
var err error
|
||||
|
||||
spanRecorder := sdktracetest.NewSpanRecorder()
|
||||
tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spanRecorder))
|
||||
tracer := tracerProvider.Tracer("test-tracer")
|
||||
ctx, _ := tracer.Start(context.Background(), "test-body-tracker")
|
||||
|
||||
metricReader := sdkmetric.NewManualReader()
|
||||
metricProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(metricReader))
|
||||
meter := metricProvider.Meter("test-meter")
|
||||
|
||||
// this is the buffer where we want to write something:
|
||||
var bw bytes.Buffer
|
||||
bw.Grow(64 * 1024)
|
||||
|
||||
payload := "foo bar"
|
||||
bw.WriteString(payload)
|
||||
|
||||
attrsT := []attribute.KeyValue{
|
||||
attribute.Int("http.status_code", 201),
|
||||
attribute.String("url.path.pattern", "/this/{matched}/path"),
|
||||
attribute.String("url.path", "/this/some_value/path"),
|
||||
}
|
||||
attrsM := []attribute.KeyValue{
|
||||
attribute.Int("http.status_code", 201),
|
||||
attribute.String("url.path", "/this/{matched}/path"),
|
||||
}
|
||||
reader := NewInstrumentedReader(&bw, ctx, attrsT, attrsM, tracer, meter)
|
||||
|
||||
_, err = io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Errorf("cannot read: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
endedSpans := spanRecorder.Ended()
|
||||
if len(endedSpans) != 1 {
|
||||
t.Errorf("num ended spans, want: 1, got: %d", len(endedSpans))
|
||||
for idx, s := range endedSpans {
|
||||
t.Errorf("%d -> %#v", idx, s)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
span := endedSpans[0]
|
||||
gotTraceAttrs := span.Attributes()
|
||||
// trace will have static attributes, plus the size and time it
|
||||
// took to read the data:
|
||||
if len(gotTraceAttrs) != len(attrsT)+2 {
|
||||
t.Errorf("trace attributes, want %d, got %d", len(attrsT), len(gotTraceAttrs))
|
||||
return
|
||||
}
|
||||
|
||||
// check the reported metrics
|
||||
rm := metricdata.ResourceMetrics{}
|
||||
err = metricReader.Collect(context.Background(), &rm)
|
||||
if err != nil {
|
||||
t.Errorf("collecting metrics err: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if len(rm.ScopeMetrics) != 1 {
|
||||
t.Errorf("wrong amount of metrics, want: 1, got: %d", len(rm.ScopeMetrics))
|
||||
for idx, sm := range rm.ScopeMetrics {
|
||||
t.Errorf("%d -> %#v", idx, sm)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// --> check that we have all the metrics we want to report
|
||||
sm := rm.ScopeMetrics[0]
|
||||
wantedMetrics := map[string]bool{
|
||||
"read-size": false,
|
||||
"read-size-hist": false,
|
||||
"read-time": false,
|
||||
"read-time-hist": false,
|
||||
// "reader-errors": false,
|
||||
}
|
||||
numWantedMetrics := len(wantedMetrics)
|
||||
gotMetrics := make(map[string]metricdata.Metrics, numWantedMetrics)
|
||||
for _, m := range sm.Metrics {
|
||||
gotMetrics[m.Name] = m
|
||||
}
|
||||
for k := range wantedMetrics {
|
||||
if _, ok := gotMetrics[k]; !ok {
|
||||
t.Errorf("missing metric %s", k)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// --> check that the metrics have the expected attributes set
|
||||
readSize := gotMetrics["read-size"]
|
||||
readSizeSum, ok := readSize.Data.(metricdata.Sum[int64])
|
||||
if !ok {
|
||||
t.Errorf("cannot access read size aggregation: %#v", readSize.Data)
|
||||
return
|
||||
}
|
||||
if len(readSizeSum.DataPoints) != 1 {
|
||||
t.Errorf("read sum data points, want: 1, got: %d", len(readSizeSum.DataPoints))
|
||||
return
|
||||
}
|
||||
dp := readSizeSum.DataPoints[0]
|
||||
|
||||
if dp.Attributes.Len() != 2 {
|
||||
t.Errorf("missing attributes")
|
||||
return
|
||||
}
|
||||
if dp.Value != int64(len(payload)) {
|
||||
t.Errorf("metric size, want: %d, got: %d", len(payload), dp.Value)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func TestReaderTimeout(t *testing.T) {
|
||||
var err error
|
||||
ctx := context.Background()
|
||||
|
||||
spanRecorder := sdktracetest.NewSpanRecorder()
|
||||
tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spanRecorder))
|
||||
tracer := tracerProvider.Tracer("test-tracer")
|
||||
ctx, _ = tracer.Start(ctx, "test-body-tracker")
|
||||
|
||||
metricReader := sdkmetric.NewManualReader()
|
||||
metricProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(metricReader))
|
||||
meter := metricProvider.Meter("test-meter")
|
||||
|
||||
// this is the buffer where we want to write something:
|
||||
payload := "foo bar"
|
||||
var bw bytes.Buffer
|
||||
bw.WriteString(payload)
|
||||
innerReader := iotest.TimeoutReader(&bw)
|
||||
|
||||
attrsT := []attribute.KeyValue{
|
||||
attribute.Int("http.status_code", 201),
|
||||
attribute.String("url.path.pattern", "/this/{matched}/path"),
|
||||
attribute.String("url.path", "/this/some_value/path"),
|
||||
}
|
||||
attrsM := []attribute.KeyValue{
|
||||
attribute.Int("http.status_code", 201),
|
||||
attribute.String("url.path", "/this/{matched}/path"),
|
||||
}
|
||||
reader := NewInstrumentedReader(innerReader, ctx, attrsT, attrsM, tracer, meter)
|
||||
|
||||
fourBytes := make([]byte, 4)
|
||||
_, err = reader.Read(fourBytes)
|
||||
if err != nil {
|
||||
t.Errorf("cannot read: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
_, err = io.ReadAll(reader)
|
||||
if err == nil {
|
||||
t.Errorf("expected timeout error")
|
||||
return
|
||||
}
|
||||
|
||||
endedSpans := spanRecorder.Ended()
|
||||
if len(endedSpans) != 1 {
|
||||
t.Errorf("num ended spans, want: 1, got: %d", len(endedSpans))
|
||||
for idx, s := range endedSpans {
|
||||
t.Errorf("%d -> %#v", idx, s)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
span := endedSpans[0]
|
||||
gotTraceAttrs := span.Attributes()
|
||||
// trace will have static attributes, plus the size and time it
|
||||
// took to read the data.
|
||||
if len(gotTraceAttrs) != len(attrsT)+2 {
|
||||
t.Errorf("trace attributes, want %d, got %d", len(attrsT), len(gotTraceAttrs))
|
||||
return
|
||||
}
|
||||
spanStatus := span.Status()
|
||||
if spanStatus.Code != codes.Error {
|
||||
t.Errorf("expected error status")
|
||||
return
|
||||
}
|
||||
|
||||
// check the reported metrics
|
||||
rm := metricdata.ResourceMetrics{}
|
||||
err = metricReader.Collect(context.Background(), &rm)
|
||||
if err != nil {
|
||||
t.Errorf("collecting metrics err: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if len(rm.ScopeMetrics) != 1 {
|
||||
t.Errorf("wrong amount of metrics, want: 1, got: %d", len(rm.ScopeMetrics))
|
||||
for idx, sm := range rm.ScopeMetrics {
|
||||
t.Errorf("%d -> %#v", idx, sm)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// --> check that we have all the metrics we want to report
|
||||
sm := rm.ScopeMetrics[0]
|
||||
wantedMetrics := map[string]bool{
|
||||
"read-size": false,
|
||||
"read-size-hist": false,
|
||||
"read-time": false,
|
||||
"read-time-hist": false,
|
||||
// "reader-errors": false,
|
||||
}
|
||||
numWantedMetrics := len(wantedMetrics)
|
||||
gotMetrics := make(map[string]metricdata.Metrics, numWantedMetrics)
|
||||
for _, m := range sm.Metrics {
|
||||
gotMetrics[m.Name] = m
|
||||
}
|
||||
for k := range wantedMetrics {
|
||||
if _, ok := gotMetrics[k]; !ok {
|
||||
t.Errorf("missing metric %s", k)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// --> check that the metrics have the expected attributes set
|
||||
readSize := gotMetrics["read-size"]
|
||||
readSizeSum, ok := readSize.Data.(metricdata.Sum[int64])
|
||||
if !ok {
|
||||
t.Errorf("cannot access read size aggregation: %#v", readSize.Data)
|
||||
return
|
||||
}
|
||||
if len(readSizeSum.DataPoints) != 1 {
|
||||
t.Errorf("read sum data points, want: 1, got: %d", len(readSizeSum.DataPoints))
|
||||
return
|
||||
}
|
||||
dp := readSizeSum.DataPoints[0]
|
||||
|
||||
// attributes are the 2 from the fixed attributes + the error=true
|
||||
if dp.Attributes.Len() != 3 {
|
||||
t.Errorf("missing attributes")
|
||||
return
|
||||
}
|
||||
if dp.Value != 4 {
|
||||
t.Errorf("metric size, want: 4, got: %d", dp.Value)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package otelio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// ioTracking contains the information to be reported
|
||||
// as metrics or traces
|
||||
type ioTracking struct {
|
||||
instr *instruments
|
||||
ctx context.Context
|
||||
span trace.Span
|
||||
started time.Time // when the first call to Read / Write happened
|
||||
finished bool // to avoid several calls to close send repeated metrics
|
||||
gotError error // in case an error happended when reading the data
|
||||
size int64 // bytes
|
||||
}
|
||||
|
||||
func (t *ioTracking) start() {
|
||||
if t.started.IsZero() {
|
||||
t.started = time.Now()
|
||||
t.ctx, t.span = t.instr.tracer.Start(t.ctx, t.instr.traceName)
|
||||
if len(t.instr.traceFixedAttrs) > 0 {
|
||||
t.span.SetAttributes(t.instr.traceFixedAttrs...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ioTracking) incSize(size int64, err error) {
|
||||
t.size += size
|
||||
if err != nil {
|
||||
t.end(err)
|
||||
if err != io.EOF {
|
||||
t.gotError = err
|
||||
}
|
||||
t.end(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ioTracking) end(err error) {
|
||||
if t.finished {
|
||||
return
|
||||
}
|
||||
t.finished = true
|
||||
secs := float64(time.Since(t.started)) / float64(time.Second)
|
||||
|
||||
if err != nil && err != io.EOF {
|
||||
t.gotError = err
|
||||
}
|
||||
|
||||
var metricAttrOpts metric.MeasurementOption // <- we can move this to the instr parte
|
||||
if t.gotError != nil {
|
||||
metricAttrOpts = t.instr.metricAttributeSetWithErrorOpt
|
||||
t.span.RecordError(t.gotError)
|
||||
t.span.SetStatus(codes.Error, t.gotError.Error())
|
||||
} else {
|
||||
t.span.SetStatus(codes.Ok, "")
|
||||
metricAttrOpts = t.instr.metricAttributeSetOpt
|
||||
}
|
||||
|
||||
// There could be a difference between the received Content-Length
|
||||
// that might be tracked at the request level (not inside the io reader)
|
||||
// and the read / written bytes (a client could read less than the content-length
|
||||
// bytes).
|
||||
t.instr.sizeCount.Add(t.ctx, t.size, metricAttrOpts)
|
||||
t.instr.sizeHistogram.Record(t.ctx, t.size, metricAttrOpts)
|
||||
|
||||
t.instr.timeCount.Add(t.ctx, secs, metricAttrOpts)
|
||||
t.instr.timeHistogram.Record(t.ctx, secs, metricAttrOpts)
|
||||
|
||||
t.span.SetAttributes(
|
||||
attribute.Int64(t.instr.traceSizeAttrName, t.size),
|
||||
attribute.Float64(t.instr.traceTimeAttrName, secs))
|
||||
t.span.End()
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package otelio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var _ io.WriteCloser = (*instrumentedWriter)(nil)
|
||||
|
||||
// InstrumentedWriter keeps track of the number of bytes and
|
||||
// and time it takes to write ot an io.Writer.
|
||||
//
|
||||
// We want the start of the span to happen not when the instrumentedWriter
|
||||
// is created, but on the first call to Write.
|
||||
//
|
||||
// we can finish the writing by receiving an `io.EOF` error
|
||||
// (the full buffer was written), or by a `Close` call, when
|
||||
// no more data will be written by the consumer
|
||||
type instrumentedWriter struct {
|
||||
writer io.Writer
|
||||
track ioTracking
|
||||
}
|
||||
|
||||
// NewInstrumentedWriterFactory creates a function that can wrap a writer with
|
||||
// an instrumented writer. Is better than the [NewIntrumentedWriter] call because
|
||||
// the instruments here are only created once.
|
||||
func NewInstrumentedWriterFactory(prefix string, attrT []attribute.KeyValue, attrM []attribute.KeyValue,
|
||||
tracer trace.Tracer, meter metric.Meter,
|
||||
) func(io.Writer, context.Context) *instrumentedWriter {
|
||||
if prefix == "" {
|
||||
prefix = "written."
|
||||
}
|
||||
instr := newInstruments(prefix, attrT, attrM, tracer, meter)
|
||||
|
||||
return func(w io.Writer, ctx context.Context) *instrumentedWriter {
|
||||
return &instrumentedWriter{
|
||||
writer: w,
|
||||
track: ioTracking{
|
||||
instr: instr,
|
||||
ctx: ctx,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewInstrumentedWriter wraps a writer with an instrumented writer.
|
||||
// Is better to use [NewInstrumentedWriterFactory].
|
||||
func NewInstrumentedWriter(prefix string, w io.Writer, ctx context.Context,
|
||||
attrT []attribute.KeyValue, attrM []attribute.KeyValue,
|
||||
tracer trace.Tracer, meter metric.Meter,
|
||||
) *instrumentedWriter {
|
||||
return NewInstrumentedWriterFactory(prefix, attrT, attrM, tracer, meter)(w, ctx)
|
||||
}
|
||||
|
||||
// Write wraps the writer Write and keeps track of the
|
||||
// written bytes. In case an error happens it will automatically
|
||||
// end the span and report the metrics.
|
||||
func (t *instrumentedWriter) Write(b []byte) (int, error) {
|
||||
t.track.start()
|
||||
n, err := t.writer.Write(b)
|
||||
t.track.incSize(int64(n), err)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close wraps the writer Close and ends the trace and reports
|
||||
// the metrics.
|
||||
func (t *instrumentedWriter) Close() error {
|
||||
var err error
|
||||
if cl, ok := t.writer.(io.Closer); ok {
|
||||
err = cl.Close()
|
||||
}
|
||||
t.track.end(err)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package otelio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/metric/metricdata"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
sdktracetest "go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
)
|
||||
|
||||
func TestWriterHappyPath(t *testing.T) {
|
||||
var err error
|
||||
|
||||
spanRecorder := sdktracetest.NewSpanRecorder()
|
||||
tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spanRecorder))
|
||||
tracer := tracerProvider.Tracer("io-writer-test-tracer")
|
||||
ctx, _ := tracer.Start(context.Background(), "io-writer-test-body-tracker")
|
||||
|
||||
metricReader := sdkmetric.NewManualReader()
|
||||
metricProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(metricReader))
|
||||
meter := metricProvider.Meter("io-writer-test-meter")
|
||||
|
||||
// this is the buffer where we want to write something:
|
||||
var bw bytes.Buffer
|
||||
bw.Grow(64 * 1024)
|
||||
payload := "foo bar"
|
||||
|
||||
initialStartedSpansLen := len(spanRecorder.Started())
|
||||
|
||||
attrsT := []attribute.KeyValue{
|
||||
attribute.Int("http.status_code", 201),
|
||||
attribute.String("url.path.pattern", "/this/{matched}/path"),
|
||||
attribute.String("url.path", "/this/some_value/path"),
|
||||
}
|
||||
attrsM := []attribute.KeyValue{
|
||||
attribute.Int("http.status_code", 201),
|
||||
attribute.String("url.path", "/this/{matched}/path"),
|
||||
}
|
||||
writer := NewInstrumentedWriter(&bw, ctx, attrsT, attrsM, tracer, meter)
|
||||
|
||||
startedSpans := spanRecorder.Started()
|
||||
if len(startedSpans) > initialStartedSpansLen {
|
||||
t.Errorf("span cannot start until first write: %#v", startedSpans)
|
||||
return
|
||||
}
|
||||
_, err = io.WriteString(writer, payload)
|
||||
if err != nil {
|
||||
t.Errorf("cannot write: %s", err.Error())
|
||||
return
|
||||
}
|
||||
startedSpans = spanRecorder.Started()
|
||||
if len(startedSpans) < initialStartedSpansLen+1 {
|
||||
t.Errorf("the span should have started after writing: %#v", startedSpans)
|
||||
return
|
||||
}
|
||||
endedSpans := spanRecorder.Ended()
|
||||
if len(endedSpans) > 0 {
|
||||
t.Errorf("the writing does not end a span until its closed")
|
||||
return
|
||||
}
|
||||
|
||||
writer.Close()
|
||||
endedSpans = spanRecorder.Ended()
|
||||
if len(endedSpans) != 1 {
|
||||
t.Errorf("num ended spans, want: 1, got: %d", len(endedSpans))
|
||||
for idx, s := range endedSpans {
|
||||
t.Errorf("%d -> %#v", idx, s)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
span := endedSpans[0]
|
||||
gotTraceAttrs := span.Attributes()
|
||||
// trace will have static attributes, plus the size and time it
|
||||
// took to read the data:
|
||||
if len(gotTraceAttrs) != len(attrsT)+2 {
|
||||
t.Errorf("trace attributes, want %d, got %d (%#v)",
|
||||
len(attrsT)+2, len(gotTraceAttrs), gotTraceAttrs)
|
||||
return
|
||||
}
|
||||
|
||||
// check the reported metrics
|
||||
rm := metricdata.ResourceMetrics{}
|
||||
err = metricReader.Collect(context.Background(), &rm)
|
||||
if err != nil {
|
||||
t.Errorf("collecting metrics err: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if len(rm.ScopeMetrics) != 1 {
|
||||
t.Errorf("wrong amount of metrics, want: 1, got: %d", len(rm.ScopeMetrics))
|
||||
for idx, sm := range rm.ScopeMetrics {
|
||||
t.Errorf("%d -> %#v", idx, sm)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// --> check that we have all the metrics we want to report
|
||||
sm := rm.ScopeMetrics[0]
|
||||
wantedMetrics := map[string]bool{
|
||||
"written-size": false,
|
||||
"written-size-hist": false,
|
||||
"written-time": false,
|
||||
"written-time-hist": false,
|
||||
// "written-errors": false,
|
||||
}
|
||||
numWantedMetrics := len(wantedMetrics)
|
||||
gotMetrics := make(map[string]metricdata.Metrics, numWantedMetrics)
|
||||
for _, m := range sm.Metrics {
|
||||
gotMetrics[m.Name] = m
|
||||
}
|
||||
for k := range wantedMetrics {
|
||||
if _, ok := gotMetrics[k]; !ok {
|
||||
t.Errorf("missing metric %s", k)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// --> check that the metrics have the expected attributes set
|
||||
writtenSize := gotMetrics["written-size"]
|
||||
writtenSizeSum, ok := writtenSize.Data.(metricdata.Sum[int64])
|
||||
if !ok {
|
||||
t.Errorf("cannot access written size aggregation: %#v", writtenSize.Data)
|
||||
return
|
||||
}
|
||||
if len(writtenSizeSum.DataPoints) != 1 {
|
||||
t.Errorf("written sum data points, want: 1, got: %d", len(writtenSizeSum.DataPoints))
|
||||
return
|
||||
}
|
||||
dp := writtenSizeSum.DataPoints[0]
|
||||
|
||||
if dp.Attributes.Len() != 2 {
|
||||
t.Errorf("missing attributes")
|
||||
return
|
||||
}
|
||||
if dp.Value != int64(len(payload)) {
|
||||
t.Errorf("metric size, want: %d, got: %d", len(payload), dp.Value)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package lura
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/semconv/v1.21.0"
|
||||
|
||||
"github.com/luraproject/lura/v2/config"
|
||||
|
||||
kotelconfig "github.com/krakend/krakend-otel/config"
|
||||
)
|
||||
|
||||
// backendConfigAttributes returnsa list of attributes
|
||||
// that will be set for both traces and
|
||||
// metrics, as those are expected to have low cardinality
|
||||
// - the method: one of the `GET`, `POST`, `PUT` .. etc
|
||||
// - the "path" , that is actually the path "template" to not have different values
|
||||
// for different params but the same endpoint.
|
||||
// - server address: the host for the request
|
||||
func backendConfigAttributes(cfg *config.Backend) []attribute.KeyValue {
|
||||
urlPattern := kotelconfig.NormalizeURLPattern(cfg.URLPattern)
|
||||
parentEndpoint := kotelconfig.NormalizeURLPattern(cfg.ParentEndpoint)
|
||||
|
||||
attrs := []attribute.KeyValue{
|
||||
semconv.HTTPRequestMethodKey.String(cfg.Method),
|
||||
semconv.HTTPRoute(urlPattern), // <- for traces we can use URLFull to not have the matched path
|
||||
attribute.String("krakend.endpoint", parentEndpoint),
|
||||
attribute.String("krakend.endpoint_method", cfg.ParentEndpointMethod),
|
||||
}
|
||||
numHosts := len(cfg.Host)
|
||||
if numHosts > 0 {
|
||||
if numHosts == 1 {
|
||||
attrs = append(attrs, semconv.ServerAddress(cfg.Host[0]))
|
||||
} else {
|
||||
hosts := make([]string, 0, numHosts)
|
||||
copy(hosts, cfg.Host)
|
||||
sort.StringSlice(hosts).Sort()
|
||||
strHosts := strings.Join(hosts, "_")
|
||||
attrs = append(attrs, semconv.ServerAddress(strHosts))
|
||||
}
|
||||
}
|
||||
|
||||
return attrs
|
||||
}
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
package lura
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
|
||||
"github.com/luraproject/lura/v2/config"
|
||||
transport "github.com/luraproject/lura/v2/transport/http/client"
|
||||
|
||||
otelconfig "github.com/krakend/krakend-otel/config"
|
||||
clienthttp "github.com/krakend/krakend-otel/http/client"
|
||||
otelstate "github.com/krakend/krakend-otel/state"
|
||||
)
|
||||
|
||||
var defaultOpts = otelconfig.BackendOpts{
|
||||
Metrics: &otelconfig.BackendMetricOpts{
|
||||
DisableStage: false,
|
||||
RoundTrip: true,
|
||||
ReadPayload: true,
|
||||
DetailedConnection: true,
|
||||
StaticAttributes: make(otelconfig.Attributes, 0),
|
||||
},
|
||||
Traces: &otelconfig.BackendTraceOpts{
|
||||
DisableStage: false,
|
||||
RoundTrip: true,
|
||||
ReadPayload: true,
|
||||
DetailedConnection: true,
|
||||
StaticAttributes: make(otelconfig.Attributes, 0),
|
||||
},
|
||||
}
|
||||
|
||||
// HTTPRequestExecutorFromConfig creates an HTTPRequestExecutor to be used
|
||||
// for the backend requests.
|
||||
func HTTPRequestExecutorFromConfig(clientFactory transport.HTTPClientFactory,
|
||||
cfg *config.Backend, opts *otelconfig.BackendOpts, skipPaths []string,
|
||||
getState otelstate.GetterFn,
|
||||
) transport.HTTPRequestExecutor {
|
||||
cf := InstrumentedHTTPClientFactory(clientFactory, cfg, opts, skipPaths, getState)
|
||||
return transport.DefaultHTTPRequestExecutor(cf)
|
||||
}
|
||||
|
||||
func InstrumentedHTTPClientFactory(clientFactory transport.HTTPClientFactory,
|
||||
cfg *config.Backend, opts *otelconfig.BackendOpts, skipPaths []string,
|
||||
getState otelstate.GetterFn,
|
||||
) transport.HTTPClientFactory {
|
||||
for _, sp := range skipPaths {
|
||||
if cfg.ParentEndpoint == sp {
|
||||
return clientFactory
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.Enabled() {
|
||||
// no configuration for the backend, then .. no metrics nor tracing:
|
||||
return clientFactory
|
||||
}
|
||||
|
||||
if opts == nil {
|
||||
opts = &defaultOpts
|
||||
}
|
||||
|
||||
if opts.Metrics == nil {
|
||||
opts.Metrics = defaultOpts.Metrics
|
||||
}
|
||||
if opts.Traces == nil {
|
||||
opts.Traces = defaultOpts.Traces
|
||||
}
|
||||
|
||||
urlPattern := otelconfig.NormalizeURLPattern(cfg.URLPattern)
|
||||
attrs := backendConfigAttributes(cfg)
|
||||
|
||||
metricAttrs := attrs
|
||||
if len(opts.Metrics.StaticAttributes) > 0 {
|
||||
for _, kv := range opts.Metrics.StaticAttributes {
|
||||
if len(kv.Key) > 0 && len(kv.Value) > 0 {
|
||||
metricAttrs = append(metricAttrs, attribute.String(kv.Key, kv.Value))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
traceAttrs := make([]attribute.KeyValue, len(attrs),
|
||||
len(attrs)+1+len(opts.Traces.StaticAttributes))
|
||||
copy(traceAttrs, attrs)
|
||||
traceAttrs = append(traceAttrs, attribute.String("krakend.stage", "backend-request"))
|
||||
if len(opts.Traces.StaticAttributes) > 0 {
|
||||
for _, kv := range opts.Traces.StaticAttributes {
|
||||
if len(kv.Key) > 0 && len(kv.Value) > 0 {
|
||||
traceAttrs = append(traceAttrs, attribute.String(kv.Key, kv.Value))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t := clienthttp.TransportOptions{
|
||||
MetricsOpts: clienthttp.TransportMetricsOptions{
|
||||
RoundTrip: opts.Metrics.RoundTrip,
|
||||
ReadPayload: opts.Metrics.ReadPayload,
|
||||
DetailedConnection: opts.Metrics.DetailedConnection,
|
||||
FixedAttributes: metricAttrs,
|
||||
},
|
||||
TracesOpts: clienthttp.TransportTracesOptions{
|
||||
RoundTrip: opts.Traces.RoundTrip,
|
||||
ReadPayload: opts.Traces.ReadPayload,
|
||||
DetailedConnection: opts.Traces.DetailedConnection,
|
||||
FixedAttributes: traceAttrs,
|
||||
ReportHeaders: opts.Traces.ReportHeaders,
|
||||
},
|
||||
OTELInstance: getState,
|
||||
}
|
||||
|
||||
return func(ctx context.Context) *http.Client {
|
||||
return clienthttp.InstrumentedHTTPClient(clientFactory(ctx), &t, urlPattern)
|
||||
}
|
||||
}
|
||||
+194
@@ -0,0 +1,194 @@
|
||||
package lura
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/semconv/v1.21.0"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"github.com/luraproject/lura/v2/config"
|
||||
"github.com/luraproject/lura/v2/proxy"
|
||||
|
||||
kotelconfig "github.com/krakend/krakend-otel/config"
|
||||
"github.com/krakend/krakend-otel/state"
|
||||
)
|
||||
|
||||
func middlewareProxy(next proxy.Proxy, tracer trace.Tracer, urlPattern string, duration metric.Float64Histogram,
|
||||
tAttrs []attribute.KeyValue, mAttrs metric.MeasurementOption, reportHeaders bool,
|
||||
) func(ctx context.Context, req *proxy.Request) (*proxy.Response, error) {
|
||||
return func(ctx context.Context, req *proxy.Request) (*proxy.Response, error) {
|
||||
var span trace.Span
|
||||
if tracer != nil {
|
||||
ctx, span = tracer.Start(ctx, urlPattern)
|
||||
span.SetAttributes(tAttrs...)
|
||||
span.SetAttributes(semconv.HTTPRequestMethodKey.String(req.Method))
|
||||
if reportHeaders {
|
||||
for hk, hv := range req.Headers {
|
||||
span.SetAttributes(attribute.StringSlice("http.request.header."+strings.ToLower(hk), hv))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
startedAt := time.Now()
|
||||
resp, err := next(ctx, req)
|
||||
|
||||
durationInSecs := float64(time.Since(startedAt)) / float64(time.Second)
|
||||
if duration != nil {
|
||||
isErr := false
|
||||
isCanceled := false
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
isCanceled = true
|
||||
} else {
|
||||
isErr = true
|
||||
}
|
||||
}
|
||||
metricDynAttrs := metric.WithAttributes(
|
||||
semconv.HTTPRequestMethodKey.String(req.Method),
|
||||
attribute.Bool("error", isErr),
|
||||
attribute.Bool("canceled", isCanceled),
|
||||
attribute.Bool("complete", resp != nil && resp.IsComplete))
|
||||
duration.Record(ctx, durationInSecs, mAttrs, metricDynAttrs)
|
||||
}
|
||||
|
||||
if tracer != nil {
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
span.SetAttributes(attribute.Bool("canceled", true))
|
||||
} else {
|
||||
span.SetAttributes(attribute.String("error", err.Error()))
|
||||
}
|
||||
span.SetAttributes(semconv.HTTPResponseStatusCodeKey.Int(500))
|
||||
} else if resp != nil {
|
||||
span.SetAttributes(semconv.HTTPResponseStatusCodeKey.Int(resp.Metadata.StatusCode))
|
||||
if reportHeaders {
|
||||
for hk, hv := range resp.Metadata.Headers {
|
||||
span.SetAttributes(attribute.StringSlice("http.response.header."+strings.ToLower(hk), hv))
|
||||
}
|
||||
}
|
||||
}
|
||||
span.SetAttributes(attribute.Bool("complete", resp != nil && resp.IsComplete))
|
||||
span.End()
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
}
|
||||
|
||||
// Middleware creates a proxy that instruments the proxy it wraps by creating an span if enabled,
|
||||
// and report the duration of this stage in metrics if enabled.
|
||||
func Middleware(gsf state.GetterFn, metricsEnabled bool, tracesEnabled bool,
|
||||
stageName string, urlPattern string, staticAttrs []attribute.KeyValue,
|
||||
reportHeaders bool,
|
||||
) proxy.Middleware {
|
||||
mAttrs := make([]attribute.KeyValue, 0, len(staticAttrs)+1)
|
||||
tAttrs := make([]attribute.KeyValue, 0, len(staticAttrs)+1)
|
||||
|
||||
mAttrs = append(mAttrs, semconv.HTTPRoute(urlPattern))
|
||||
tAttrs = append(tAttrs, attribute.String("krakend.stage", stageName))
|
||||
|
||||
for _, sa := range staticAttrs {
|
||||
mAttrs = append(mAttrs, sa)
|
||||
tAttrs = append(tAttrs, sa)
|
||||
}
|
||||
|
||||
metricAttrs := metric.WithAttributes(mAttrs...)
|
||||
|
||||
return func(next ...proxy.Proxy) proxy.Proxy {
|
||||
if len(next) > 1 {
|
||||
panic(proxy.ErrTooManyProxies)
|
||||
}
|
||||
if len(next) < 1 {
|
||||
panic(proxy.ErrNotEnoughProxies)
|
||||
}
|
||||
gs := gsf()
|
||||
if gs == nil {
|
||||
return next[0] // no instrumentation available
|
||||
}
|
||||
|
||||
var duration metric.Float64Histogram
|
||||
if metricsEnabled {
|
||||
meter := gs.Meter()
|
||||
var err error
|
||||
duration, err = meter.Float64Histogram("krakend."+stageName+".duration", kotelconfig.TimeBucketsOpt)
|
||||
if err != nil {
|
||||
duration = nil
|
||||
}
|
||||
}
|
||||
|
||||
var tracer trace.Tracer
|
||||
if tracesEnabled {
|
||||
tracer = gs.Tracer()
|
||||
}
|
||||
return middlewareProxy(next[0], tracer, urlPattern, duration, tAttrs, metricAttrs, reportHeaders)
|
||||
}
|
||||
}
|
||||
|
||||
// ProxyFactory returns a pipe stage factory that wraps the provided proxy factory with the
|
||||
// instrumentation [Middleware] based on the configuration options.
|
||||
func ProxyFactory(pf proxy.Factory, gsfn state.GetterFn, opts *kotelconfig.PipeOpts,
|
||||
skipPaths []string,
|
||||
) proxy.FactoryFunc {
|
||||
if opts == nil {
|
||||
return pf.New
|
||||
}
|
||||
if gsfn == nil {
|
||||
gsfn = state.GlobalState
|
||||
}
|
||||
|
||||
metricsEnabled := !opts.DisableMetrics
|
||||
tracesEnabled := !opts.DisableTraces
|
||||
|
||||
if !metricsEnabled && !tracesEnabled {
|
||||
return pf.New
|
||||
}
|
||||
|
||||
return func(cfg *config.EndpointConfig) (proxy.Proxy, error) {
|
||||
next, err := pf.New(cfg)
|
||||
if err != nil {
|
||||
return next, err
|
||||
}
|
||||
for _, sp := range skipPaths {
|
||||
if cfg.Endpoint == sp {
|
||||
return next, nil
|
||||
}
|
||||
}
|
||||
|
||||
urlPattern := kotelconfig.NormalizeURLPattern(cfg.Endpoint)
|
||||
return Middleware(gsfn, metricsEnabled, tracesEnabled, "proxy", urlPattern,
|
||||
[]attribute.KeyValue{}, opts.ReportHeaders)(next), nil
|
||||
}
|
||||
}
|
||||
|
||||
// BackendFactory returns a backend factory that wraps the provided backend factory with the
|
||||
// instrumentation [Middleware] based on the configuration options.
|
||||
func BackendFactory(bf proxy.BackendFactory, gsfn state.GetterFn, opts *kotelconfig.BackendOpts,
|
||||
skipPaths []string,
|
||||
) proxy.BackendFactory {
|
||||
if opts == nil || (opts.Metrics.DisableStage && opts.Traces.DisableStage) {
|
||||
return bf
|
||||
}
|
||||
return OTELBackendFactory(bf, gsfn, !opts.Metrics.DisableStage, !opts.Traces.DisableStage,
|
||||
skipPaths, opts.Traces.ReportHeaders)
|
||||
}
|
||||
|
||||
func OTELBackendFactory(bf proxy.BackendFactory, gsfn state.GetterFn, metricsEnabled bool, tracesEnabled bool,
|
||||
skipPaths []string, reportHeaders bool,
|
||||
) proxy.BackendFactory {
|
||||
return func(cfg *config.Backend) proxy.Proxy {
|
||||
next := bf(cfg)
|
||||
for _, sp := range skipPaths {
|
||||
if cfg.ParentEndpoint == sp {
|
||||
return next
|
||||
}
|
||||
}
|
||||
staticAttrs := backendConfigAttributes(cfg)
|
||||
urlPattern := kotelconfig.NormalizeURLPattern(cfg.URLPattern)
|
||||
return Middleware(gsfn, metricsEnabled, tracesEnabled, "backend", urlPattern,
|
||||
staticAttrs, reportHeaders)(next)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package lura
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
kotelconfig "github.com/krakend/krakend-otel/config"
|
||||
kotelhttpserver "github.com/krakend/krakend-otel/http/server"
|
||||
"github.com/krakend/krakend-otel/state"
|
||||
luraconfig "github.com/luraproject/lura/v2/config"
|
||||
"github.com/luraproject/lura/v2/logging"
|
||||
luragin "github.com/luraproject/lura/v2/router/gin"
|
||||
)
|
||||
|
||||
func GlobalRunServer(_ logging.Logger, obsConfig *kotelconfig.Config, stateFn state.GetterFn, next luragin.RunServerFunc) luragin.RunServerFunc {
|
||||
return func(ctx context.Context, cfg luraconfig.ServiceConfig, h http.Handler) error {
|
||||
wrappedH := kotelhttpserver.NewTrackingHandler(h, obsConfig, stateFn)
|
||||
return next(ctx, cfg, wrappedH)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
// Package kotel adds opentelemetry instrumentation to a KrakenD instance
|
||||
// (or some other [Lura](https://github.com/luraproject/lura) based softawre)
|
||||
//
|
||||
// In the KrakenD project, we can differentiate 3 main stages in the process
|
||||
// of handling a request:
|
||||
// - the "router" stage: the part where the router plugins are run, and
|
||||
// is the part from receiving the request, up to the point where the request
|
||||
// enters the Lura's pipeline.
|
||||
// - the "proxy" stage: is for the processing endpoint part, up to the point
|
||||
// - the "backend" stage: is the part for each one of the backends that will
|
||||
// be used for a given endpoint.
|
||||
package kotel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
|
||||
lconfig "github.com/luraproject/lura/v2/config"
|
||||
lcore "github.com/luraproject/lura/v2/core"
|
||||
|
||||
"github.com/krakend/krakend-otel/config"
|
||||
"github.com/krakend/krakend-otel/exporter"
|
||||
"github.com/krakend/krakend-otel/state"
|
||||
)
|
||||
|
||||
// Register uses the ServiceConfig to instantiate the configured exporters.
|
||||
// It also sets the global exporter instances, the global propagation method, and
|
||||
// the global KrakenD otel state, so it can be used from anywhere.
|
||||
func Register(ctx context.Context, srvCfg lconfig.ServiceConfig) (func(), error) {
|
||||
cfg, err := config.FromLura(srvCfg)
|
||||
if err != nil {
|
||||
if errors.Is(err, config.ErrNoConfig) {
|
||||
return func() {}, nil
|
||||
}
|
||||
return func() {}, err
|
||||
}
|
||||
return RegisterWithConfig(ctx, cfg)
|
||||
}
|
||||
|
||||
// RegisterWithConfig instantiates the configured exporters from an already
|
||||
// parsed config: sets the global exporter instances, the global propagation method, and
|
||||
// the global KrakenD otel state, so it can be used from anywhere.
|
||||
func RegisterWithConfig(ctx context.Context, cfg *config.Config) (func(), error) {
|
||||
shutdownFn := func() {}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return shutdownFn, err
|
||||
}
|
||||
cfg.UnsetFieldsToDefaults()
|
||||
|
||||
me, te, err := exporter.Instances(ctx, cfg)
|
||||
if err != nil {
|
||||
return shutdownFn, err
|
||||
}
|
||||
exporter.SetGlobalExporterInstances(me, te)
|
||||
return RegisterGlobalInstance(ctx, me, te, *cfg.MetricReportingPeriod, *cfg.TraceSampleRate, cfg.ServiceName)
|
||||
}
|
||||
|
||||
// RegisterGlobalInstance creates the instance that will be used to report metrics and traces
|
||||
func RegisterGlobalInstance(ctx context.Context, me map[string]exporter.MetricReader, te map[string]exporter.SpanExporter,
|
||||
metricReportingPeriod int, traceSampleRate float64, serviceName string,
|
||||
) (func(), error) {
|
||||
shutdownFn := func() {}
|
||||
prop := propagation.NewCompositeTextMapPropagator(
|
||||
propagation.TraceContext{},
|
||||
propagation.Baggage{},
|
||||
)
|
||||
otel.SetTextMapPropagator(prop)
|
||||
|
||||
globalStateCfg := &state.OTELStateConfig{
|
||||
MetricReportingPeriod: metricReportingPeriod,
|
||||
TraceSampleRate: traceSampleRate,
|
||||
MetricProviders: make([]string, 0, len(me)),
|
||||
TraceProviders: make([]string, 0, len(te)),
|
||||
}
|
||||
for k, v := range me {
|
||||
if v.MetricDefaultReporting() {
|
||||
globalStateCfg.MetricProviders = append(globalStateCfg.MetricProviders, k)
|
||||
}
|
||||
}
|
||||
for k, v := range te {
|
||||
if v.TraceDefaultReporting() {
|
||||
globalStateCfg.TraceProviders = append(globalStateCfg.TraceProviders, k)
|
||||
}
|
||||
}
|
||||
s, err := state.NewWithVersion(serviceName, globalStateCfg, lcore.KrakendVersion, me, te)
|
||||
if err != nil {
|
||||
return shutdownFn, err
|
||||
}
|
||||
shutdownFn = func() { s.Shutdown(ctx) }
|
||||
state.SetGlobalState(s)
|
||||
return shutdownFn, nil
|
||||
}
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
package kotel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
lconfig "github.com/luraproject/lura/v2/config"
|
||||
)
|
||||
|
||||
func TestGlobalConfig(t *testing.T) {
|
||||
cfg := lconfig.ServiceConfig{
|
||||
ExtraConfig: map[string]interface{}{
|
||||
"telemetry/otel": map[string]interface{}{
|
||||
"sample_rate": 2,
|
||||
"reporting_period": 4,
|
||||
"enabled_layers": map[string]interface{}{
|
||||
"router": true,
|
||||
"proxy": true,
|
||||
"backend": true,
|
||||
},
|
||||
"exporters": map[string]interface{}{},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err := Register(context.Background(), cfg)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
func TestGetAggregatedPathForMetrics(t *testing.T) {
|
||||
for i, tc := range []struct {
|
||||
cfg *config.EndpointConfig
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
cfg: &config.EndpointConfig{Endpoint: "/api/:foo/:bar"},
|
||||
expected: "/api/{foo}/{bar}",
|
||||
},
|
||||
{
|
||||
cfg: &config.EndpointConfig{
|
||||
Endpoint: "/api/:foo/:bar",
|
||||
ExtraConfig: config.ExtraConfig{
|
||||
Namespace: map[string]interface{}{"path_aggregation": "pattern"},
|
||||
},
|
||||
},
|
||||
expected: "/api/{foo}/{bar}",
|
||||
},
|
||||
{
|
||||
cfg: &config.EndpointConfig{
|
||||
Endpoint: "/api/:foo/:bar",
|
||||
ExtraConfig: config.ExtraConfig{
|
||||
Namespace: map[string]interface{}{"path_aggregation": "lastparam"},
|
||||
},
|
||||
},
|
||||
expected: "/api/foo/{bar}",
|
||||
},
|
||||
{
|
||||
cfg: &config.EndpointConfig{
|
||||
Endpoint: "/api/:foo/:bar",
|
||||
ExtraConfig: config.ExtraConfig{
|
||||
Namespace: map[string]interface{}{"path_aggregation": "off"},
|
||||
},
|
||||
},
|
||||
expected: "/api/foo/bar",
|
||||
},
|
||||
{
|
||||
expected: "/api/foo/bar",
|
||||
},
|
||||
} {
|
||||
extractor := GetAggregatedPathForMetrics(tc.cfg)
|
||||
r, _ := http.NewRequest("GET", "http://example.tld/api/foo/bar", nil)
|
||||
if tag := extractor(r); tag != tc.expected {
|
||||
t.Errorf("tc-%d: unexpected result: %s", i, tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAggregatedPathForBackendMetrics(t *testing.T) {
|
||||
for i, tc := range []struct {
|
||||
cfg *config.Backend
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
cfg: &config.Backend{URLPattern: "/api/{{.Foo}}/{{.Bar}}"},
|
||||
expected: "/api/{foo}/{bar}",
|
||||
},
|
||||
{
|
||||
cfg: &config.Backend{
|
||||
URLPattern: "/api/{{.Foo}}/{{.Bar}}",
|
||||
ExtraConfig: config.ExtraConfig{
|
||||
Namespace: map[string]interface{}{"path_aggregation": "pattern"},
|
||||
},
|
||||
},
|
||||
expected: "/api/{foo}/{bar}",
|
||||
},
|
||||
{
|
||||
cfg: &config.Backend{
|
||||
URLPattern: "/api/{{.Foo}}/{{.Bar}}",
|
||||
ExtraConfig: config.ExtraConfig{
|
||||
Namespace: map[string]interface{}{"path_aggregation": "lastparam"},
|
||||
},
|
||||
},
|
||||
expected: "/api/foo/{bar}",
|
||||
},
|
||||
{
|
||||
cfg: &config.Backend{
|
||||
URLPattern: "/api/{{.Foo}}/{{.Bar}}",
|
||||
ExtraConfig: config.ExtraConfig{
|
||||
Namespace: map[string]interface{}{"path_aggregation": "off"},
|
||||
},
|
||||
},
|
||||
expected: "/api/foo/bar",
|
||||
},
|
||||
{
|
||||
expected: "/api/foo/bar",
|
||||
},
|
||||
} {
|
||||
extractor := GetAggregatedPathForBackendMetrics(tc.cfg)
|
||||
r, _ := http.NewRequest("GET", "http://example.tld/api/foo/bar", nil)
|
||||
if tag := extractor(r); tag != tc.expected {
|
||||
t.Errorf("tc-%d: unexpected result: %s", i, tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,38 @@
|
||||
package gin
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/luraproject/lura/v2/config"
|
||||
"github.com/luraproject/lura/v2/proxy"
|
||||
krakendgin "github.com/luraproject/lura/v2/router/gin"
|
||||
|
||||
kconfig "github.com/krakend/krakend-otel/config"
|
||||
kotelserver "github.com/krakend/krakend-otel/http/server"
|
||||
)
|
||||
|
||||
// New wraps a handler factory adding some simple instrumentation to the generated handlers
|
||||
func New(hf krakendgin.HandlerFactory, skipPaths []string) krakendgin.HandlerFactory {
|
||||
return func(cfg *config.EndpointConfig, p proxy.Proxy) gin.HandlerFunc {
|
||||
return HandlerFunc(cfg, skipPaths, hf(cfg, p))
|
||||
}
|
||||
}
|
||||
|
||||
// HandlerFunc task is to fill the "matched endpoint pattern" once we know it, so the
|
||||
// global layer tracking can report it for metrics and traces.
|
||||
func HandlerFunc(cfg *config.EndpointConfig, skipPaths []string, next gin.HandlerFunc,
|
||||
) gin.HandlerFunc {
|
||||
// skip paths will not try to read the propagation header, because nothing
|
||||
// in the downstream pipeline will be instruemented. The header can be passed
|
||||
// using the regular `headers` feature.
|
||||
for _, sp := range skipPaths {
|
||||
if cfg.Endpoint == sp {
|
||||
return next
|
||||
}
|
||||
}
|
||||
|
||||
urlPattern := kconfig.NormalizeURLPattern(cfg.Endpoint)
|
||||
return func(c *gin.Context) {
|
||||
kotelserver.SetEndpointPattern(c.Request.Context(), urlPattern)
|
||||
next(c)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
package state
|
||||
@@ -0,0 +1,27 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
otelMu sync.RWMutex
|
||||
otelState *OTELState
|
||||
)
|
||||
|
||||
var _ GetterFn = GlobalState
|
||||
|
||||
// GlobalState retrieves a configured global state
|
||||
func GlobalState() OTEL {
|
||||
otelMu.RLock()
|
||||
s := otelState
|
||||
otelMu.RUnlock()
|
||||
return s
|
||||
}
|
||||
|
||||
// SetGlobalState set the provided state as the global state.
|
||||
func SetGlobalState(s *OTELState) {
|
||||
otelMu.Lock()
|
||||
otelState = s
|
||||
otelMu.Unlock()
|
||||
}
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
// Package state provides the functionality to "pack" into a single
|
||||
// structure a set of configured instances (exporters, meters, tracers...)
|
||||
package state
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
noopmetric "go.opentelemetry.io/otel/metric/noop"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
sdkresource "go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/semconv/v1.21.0"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
nooptrace "go.opentelemetry.io/otel/trace/noop"
|
||||
|
||||
"github.com/krakend/krakend-otel/exporter"
|
||||
)
|
||||
|
||||
const (
|
||||
providerName string = "io.krakend.krakend-otel"
|
||||
)
|
||||
|
||||
// OTEL defines the interface to obtain observability
|
||||
// instruments for a state.
|
||||
type OTEL interface {
|
||||
Tracer() trace.Tracer
|
||||
// MeterProvider() metric.MeterProvider
|
||||
Meter() metric.Meter
|
||||
Propagator() propagation.TextMapPropagator
|
||||
Shutdown(ctx context.Context)
|
||||
}
|
||||
|
||||
// GetterFn defines a function that will return an [OTEL] instance.
|
||||
type GetterFn func() OTEL
|
||||
|
||||
type OTELStateConfig struct {
|
||||
MetricProviders []string `json:"metric_providers"`
|
||||
TraceProviders []string `json:"trace_providers"`
|
||||
MetricReportingPeriod int `json:"metric_reporting_period"`
|
||||
TraceSampleRate float64 `json:"trace_sample_rate"`
|
||||
}
|
||||
|
||||
// OTELState is the basic implementation of an [OTEL] intstance.
|
||||
type OTELState struct {
|
||||
meterProvider metric.MeterProvider
|
||||
tracerProvider trace.TracerProvider
|
||||
|
||||
// we need not the interface, but the actual implementation
|
||||
// to be able to call shutown:
|
||||
sdkMeterProvider *sdkmetric.MeterProvider
|
||||
sdkTracerProvider *sdktrace.TracerProvider
|
||||
tracer trace.Tracer
|
||||
meter metric.Meter
|
||||
}
|
||||
|
||||
// NewWithVersion create a new OTELState with a version for
|
||||
// the KrakenD service, with the provided metrics and traces exporters
|
||||
func NewWithVersion(serviceName string, cfg *OTELStateConfig, version string,
|
||||
me map[string]exporter.MetricReader, te map[string]exporter.SpanExporter,
|
||||
) (*OTELState, error) {
|
||||
res := sdkresource.NewWithAttributes(
|
||||
semconv.SchemaURL,
|
||||
semconv.ServiceName(serviceName),
|
||||
semconv.ServiceVersion(version))
|
||||
|
||||
reportingPeriod := time.Duration(cfg.MetricReportingPeriod) * time.Second
|
||||
metricOpts := make([]sdkmetric.Option, 0, len(cfg.MetricProviders)+2)
|
||||
for idx, prov := range cfg.MetricProviders {
|
||||
pm, ok := me[prov]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("not found exporter %s for metricprovider %d (me %#v)", prov, idx, me)
|
||||
}
|
||||
reader := pm.MetricReader(reportingPeriod)
|
||||
metricOpts = append(metricOpts, sdkmetric.WithReader(reader))
|
||||
}
|
||||
|
||||
var meterProvider metric.MeterProvider = noopmetric.NewMeterProvider()
|
||||
var sdkMeterProvider *sdkmetric.MeterProvider
|
||||
if len(metricOpts) > 0 {
|
||||
sdkMeterProvider = sdkmetric.NewMeterProvider(metricOpts...)
|
||||
meterProvider = sdkMeterProvider
|
||||
}
|
||||
meter := meterProvider.Meter(providerName)
|
||||
|
||||
// Configure the tracing part
|
||||
traceOpts := make([]sdktrace.TracerProviderOption, 0, len(cfg.TraceProviders)+2)
|
||||
for idx, prov := range cfg.TraceProviders {
|
||||
pt, ok := te[prov]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("not found exporter %s for provider %d for tracing", prov, idx)
|
||||
}
|
||||
traceOpts = append(traceOpts, sdktrace.WithBatcher(pt.SpanExporter()))
|
||||
}
|
||||
|
||||
var tracerProvider trace.TracerProvider = nooptrace.NewTracerProvider()
|
||||
var sdkTracerProvider *sdktrace.TracerProvider
|
||||
if len(traceOpts) > 0 {
|
||||
samplerOpt := sdktrace.WithSampler(sdktrace.AlwaysSample())
|
||||
if cfg.TraceSampleRate > 0.0 && cfg.TraceSampleRate < 1.0 {
|
||||
samplerOpt = sdktrace.WithSampler(sdktrace.ParentBased(
|
||||
sdktrace.TraceIDRatioBased(cfg.TraceSampleRate)))
|
||||
}
|
||||
traceOpts = append(traceOpts, samplerOpt)
|
||||
traceOpts = append(traceOpts, sdktrace.WithResource(res))
|
||||
sdkTracerProvider = sdktrace.NewTracerProvider(traceOpts...)
|
||||
tracerProvider = sdkTracerProvider
|
||||
}
|
||||
tracer := tracerProvider.Tracer(providerName)
|
||||
|
||||
return &OTELState{
|
||||
meterProvider: meterProvider,
|
||||
tracerProvider: tracerProvider,
|
||||
sdkMeterProvider: sdkMeterProvider,
|
||||
sdkTracerProvider: sdkTracerProvider,
|
||||
tracer: tracer,
|
||||
meter: meter,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Tracer returns a tracer to start a span.
|
||||
func (s *OTELState) Tracer() trace.Tracer {
|
||||
return s.tracer
|
||||
}
|
||||
|
||||
// Meter returns a meter to create metric instruments.
|
||||
func (s *OTELState) Meter() metric.Meter {
|
||||
return s.meter
|
||||
}
|
||||
|
||||
// Propagator returns the configured propagator to use.
|
||||
func (s *OTELState) Propagator() propagation.TextMapPropagator {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
return otel.GetTextMapPropagator()
|
||||
}
|
||||
|
||||
// Shutdown performs the clean shutdown to be able to
|
||||
// flush pending traces and / or metrics.
|
||||
func (s *OTELState) Shutdown(ctx context.Context) {
|
||||
if s.sdkTracerProvider != nil {
|
||||
s.sdkTracerProvider.Shutdown(ctx)
|
||||
}
|
||||
if s.sdkMeterProvider != nil {
|
||||
s.sdkMeterProvider.Shutdown(ctx)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user