diff --git a/.gitignore b/.gitignore index b844ae1..a584c90 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,3 @@ web/dist web/tmp adapters plugins -!qwdata/.gitkeep -qwdata -!grafana-storage/plugins -grafana-storage diff --git a/Cargo.lock b/Cargo.lock index 6d84615..88d49f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7229,6 +7229,7 @@ dependencies = [ "futures 0.3.28", "plugin-api", "rand 0.8.5", + "rayon", "reqwest 0.11.18", "serde", "serde_json", diff --git a/crates/web-api-plugin-tester/Cargo.toml b/crates/web-api-plugin-tester/Cargo.toml index 7fc5c08..562fefc 100644 --- a/crates/web-api-plugin-tester/Cargo.toml +++ b/crates/web-api-plugin-tester/Cargo.toml @@ -18,3 +18,4 @@ reqwest = { version = "0.11.18", features = ['json', 'serde_json', 'stream', 'ru toml = { version = "0.7.4" } tracing = { version = "0.1.37" } anyhow = { version = "1.0.71" } +rayon = { version = "1.7.0" } diff --git a/crates/web-api-plugin-tester/src/main.rs b/crates/web-api-plugin-tester/src/main.rs index 607bb2f..5f47627 100644 --- a/crates/web-api-plugin-tester/src/main.rs +++ b/crates/web-api-plugin-tester/src/main.rs @@ -1,7 +1,6 @@ -use actix_web::http::StatusCode; use actix_web::{get, App, Error, HttpResponse, HttpServer}; +use futures::prelude::*; use plugin_api::{AppConfig, Plugin}; -use rand::prelude::SliceRandom; use reqwest::Client; use serde::*; use telemetry_plugin::TelemetryPlugin; @@ -16,7 +15,7 @@ async fn main() -> std::io::Result<()> { let mut t = toml::Table::with_capacity(3); t.insert( "endpoint".to_string(), - toml::Value::String("http://localhost:16686".to_string()), + toml::Value::String("http://0.0.0.0:16686".to_string()), ); t.insert( "log_level".to_string(), @@ -49,19 +48,19 @@ const BASE_API_URL: &'static str = "https://jsonplaceholder.typicode.com"; #[get("/posts")] async fn get_posts() -> Result { // Randomly simulate errors in request handling - let choices = [200, 400, 401, 200, 500, 501, 200, 500]; - let mut rng = rand::thread_rng(); - let choice = choices.choose(&mut rng).unwrap().clone(); - match choice { - 400..=401 => Ok(HttpResponse::new(StatusCode::from_u16(choice).unwrap())), - 500..=501 => Ok(HttpResponse::new(StatusCode::from_u16(choice).unwrap())), - _ => { - let posts = fetch_posts(20) - .await - .map_err(actix_web::error::ErrorInternalServerError)?; - Ok(HttpResponse::Ok().json(posts)) - } - } + // let choices = [200, 400, 401, 200, 500, 501, 200, 500]; + // let mut rng = rand::thread_rng(); + // let choice = choices.choose(&mut rng).unwrap().clone(); + // match choice { + // 400..=401 => Ok(HttpResponse::new(StatusCode::from_u16(choice).unwrap())), + // 500..=501 => Ok(HttpResponse::new(StatusCode::from_u16(choice).unwrap())), + // _ => { + let posts = fetch_posts(20) + .await + .map_err(actix_web::error::ErrorInternalServerError)?; + Ok(HttpResponse::Ok().json(posts)) + // } + // } } // Fetching posts with a limit. @@ -77,11 +76,23 @@ async fn fetch_posts(limit: usize) -> anyhow::Result> { .map(|(idx, post)| (idx, post.id)) .collect(); - // fetch post comments one after another. - for (index, post_id) in post_idx_to_ids { - let comments = fetch_comments(&client, post_id).await?; - posts[index].comments = comments + let mut res = futures::stream::iter(post_idx_to_ids.iter()) + .map(|(idx, post_id)| { + let client = client.clone(); + async move { + let comments = fetch_comments(&client, *post_id).await.unwrap(); + (*idx, comments) + } + }) + .buffer_unordered(3); + + for (idx, comments) in res.next().await { + posts[idx].comments = comments; } + // fetch post comments one after another. + // for (index, post_id) in post_idx_to_ids { + // + // } Ok(posts) } @@ -89,8 +100,7 @@ async fn fetch_posts(limit: usize) -> anyhow::Result> { #[instrument(level = "info", name = "fetch_comments")] async fn fetch_comments(client: &Client, id: i64) -> anyhow::Result> { let url = format!("{BASE_API_URL}/posts/{id}/comments"); - let mut comments: Vec = client.get(url).send().await?.json().await?; - Ok(comments) + Ok(client.get(url).send().await?.json().await?) } #[derive(Debug, Serialize, Deserialize)] diff --git a/docker-compose.yml b/docker-compose.yml index 9fde7c8..cac616e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: quickwit: - image: quickwit/quickwit:latest + image: quickwit/quickwit:v0.5.2 command: run restart: always environment: @@ -10,11 +10,11 @@ services: ports: - '7280:7280' - '7281:7281' - volumes: - - ./qwdata:/quickwit/qwdata +# volumes: +# - ./qwdata:/quickwit/qwdata jaeger: - image: jaegertracing/jaeger-query:latest + image: jaegertracing/jaeger-query:1.45 restart: always depends_on: - quickwit @@ -25,9 +25,8 @@ services: - '16686:16686' grafana: - image: grafana/grafana-enterprise:latest + image: grafana/grafana-enterprise:10.0.0 restart: always - user: root depends_on: - quickwit environment: @@ -35,4 +34,4 @@ services: ports: - '3000:3000' volumes: - - ./grafana-storage:/var/lib/grafana + - ./grafana/plugins:/var/lib/grafana/plugins diff --git a/grafana-storage/alerting/1/__default__.tmpl b/grafana-storage/alerting/1/__default__.tmpl deleted file mode 100644 index b8633d1..0000000 --- a/grafana-storage/alerting/1/__default__.tmpl +++ /dev/null @@ -1,53 +0,0 @@ - -{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ if gt (.Alerts.Resolved | len) 0 }}, RESOLVED:{{ .Alerts.Resolved | len }}{{ end }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }} - -{{ define "__text_values_list" }}{{ if len .Values }}{{ $first := true }}{{ range $refID, $value := .Values -}} -{{ if $first }}{{ $first = false }}{{ else }}, {{ end }}{{ $refID }}={{ $value }}{{ end -}} -{{ else }}[no value]{{ end }}{{ end }} - -{{ define "__text_alert_list" }}{{ range . }} -Value: {{ template "__text_values_list" . }} -Labels: -{{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }} -{{ end }}Annotations: -{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }} -{{ end }}{{ if gt (len .GeneratorURL) 0 }}Source: {{ .GeneratorURL }} -{{ end }}{{ if gt (len .SilenceURL) 0 }}Silence: {{ .SilenceURL }} -{{ end }}{{ if gt (len .DashboardURL) 0 }}Dashboard: {{ .DashboardURL }} -{{ end }}{{ if gt (len .PanelURL) 0 }}Panel: {{ .PanelURL }} -{{ end }}{{ end }}{{ end }} - -{{ define "default.title" }}{{ template "__subject" . }}{{ end }} - -{{ define "default.message" }}{{ if gt (len .Alerts.Firing) 0 }}**Firing** -{{ template "__text_alert_list" .Alerts.Firing }}{{ if gt (len .Alerts.Resolved) 0 }} - -{{ end }}{{ end }}{{ if gt (len .Alerts.Resolved) 0 }}**Resolved** -{{ template "__text_alert_list" .Alerts.Resolved }}{{ end }}{{ end }} - - -{{ define "__teams_text_alert_list" }}{{ range . }} -Value: {{ template "__text_values_list" . }} -Labels: -{{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }} -{{ end }} -Annotations: -{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }} -{{ end }} -{{ if gt (len .GeneratorURL) 0 }}Source: [{{ .GeneratorURL }}]({{ .GeneratorURL }}) - -{{ end }}{{ if gt (len .SilenceURL) 0 }}Silence: [{{ .SilenceURL }}]({{ .SilenceURL }}) - -{{ end }}{{ if gt (len .DashboardURL) 0 }}Dashboard: [{{ .DashboardURL }}]({{ .DashboardURL }}) - -{{ end }}{{ if gt (len .PanelURL) 0 }}Panel: [{{ .PanelURL }}]({{ .PanelURL }}) - -{{ end }} -{{ end }}{{ end }} - - -{{ define "teams.default.message" }}{{ if gt (len .Alerts.Firing) 0 }}**Firing** -{{ template "__teams_text_alert_list" .Alerts.Firing }}{{ if gt (len .Alerts.Resolved) 0 }} - -{{ end }}{{ end }}{{ if gt (len .Alerts.Resolved) 0 }}**Resolved** -{{ template "__teams_text_alert_list" .Alerts.Resolved }}{{ end }}{{ end }} diff --git a/qwdata/queues/partition_id b/qwdata/queues/partition_id deleted file mode 100644 index d74523d..0000000 --- a/qwdata/queues/partition_id +++ /dev/null @@ -1 +0,0 @@ -ingest_partition_01H3FQJ4S8FCJRDTW2KNVMPKBF \ No newline at end of file diff --git a/qwdata/queues/wal-00000000000000000000 b/qwdata/queues/wal-00000000000000000000 deleted file mode 100644 index 0f27d2f..0000000 Binary files a/qwdata/queues/wal-00000000000000000000 and /dev/null differ