From 20a6304ba2b5011b80fa15129893066cb9bb61d8 Mon Sep 17 00:00:00 2001 From: Trevor McMaster Date: Mon, 8 Nov 2021 14:00:51 -0700 Subject: [PATCH] Config logging (#71) * Changed the release version of the histogram to compile for webpack * Updated percent-encodings to v2 - The predefined encodings have been removed in v2 (https://github.com/servo/rust-url/blob/master/UPGRADING.md#upgrading-from-percent-encoding-1x-to-2x) - Created our own encodings to match the old ones and added a new 'non-alphanumeric' encoding that has been added - Updated the docs to include the new encoding * Updated percent-encodings to v2 - The predefined encodings have been removed in v2 (https://github.com/servo/rust-url/blob/master/UPGRADING.md#upgrading-from-percent-encoding-1x-to-2x) - Created our own encodings to match the old ones and added a new 'non-alphanumeric' encoding that has been added - Updated the docs to include the new encoding * Added logging to the config parser 'from yaml' functions * Added debug and trace logs to the config parser * cargo fmt changes * Fixed wasm build issues by deriving debug so we can log --- Cargo.lock | 12 +- README.md | 4 + guide/build-guide.sh | 2 +- guide/serve-guide.sh | 2 +- guide/src/config/common-types/expressions.md | 1 + lib/config/Cargo.toml | 2 +- lib/config/src/expression_functions.rs | 92 ++++++++++-- lib/config/src/from_yaml.rs | 3 +- lib/config/src/lib.rs | 147 +++++++++++++------ lib/config/src/select_parser.rs | 92 ++++++++++++ 10 files changed, 288 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2a1c1fc8..651c857b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -268,7 +268,7 @@ dependencies = [ "jsonpath_lib", "log", "maplit", - "percent-encoding 1.0.1", + "percent-encoding", "pest", "pest_derive", "rand", @@ -501,7 +501,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" dependencies = [ "matches", - "percent-encoding 2.1.0", + "percent-encoding", ] [[package]] @@ -1140,12 +1140,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "percent-encoding" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" - [[package]] name = "percent-encoding" version = "2.1.0" @@ -1765,7 +1759,7 @@ dependencies = [ "form_urlencoded", "idna", "matches", - "percent-encoding 2.1.0", + "percent-encoding", ] [[package]] diff --git a/README.md b/README.md index 260d3ae0..2a991a26 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ Changes: - Changed the default try script output to log `headers_all` rather than `headers`. There were complaints about not seeing duplicate headers causing confusion over what was being sent. - Change try script output to go through stdout instead of stderr. - Modified the Config WebAssembly (config-wasm) to also return file body paths from the `getInputFiles()` method. +- Added a new `encode()` option. `encode(value, "non-alphanumeric")` will encode all characters that are not an ASCII letter or digit. + +Bug fixes: +- Upgrade percent-encoding and other dependencies ### v0.5.9 Bug fixes: diff --git a/guide/build-guide.sh b/guide/build-guide.sh index 3f936008..52148743 100755 --- a/guide/build-guide.sh +++ b/guide/build-guide.sh @@ -30,7 +30,7 @@ WASM_OUTPUT_DIR=$(realpath $RESULTS_VIEWER_DIR/lib/hdr-histogram-wasm) cd $WASM_LIB_DIR wasm-pack build --release -t web -d $WASM_OUTPUT_DIR --scope fs cd $WASM_OUTPUT_DIR -sed -i 's/input = .*import\.meta\.url.*/import(".\/hdr_histogram_wasm_bg.wasm");\ninput = require.resolve(".\/hdr_histogram_wasm_bg.wasm")[0][0];/' hdr_histogram_wasm.js +sed -i 's/input = .*import\.meta\.url.*/import(".\/hdr_histogram_wasm_bg.wasm");\n input = require.resolve(".\/hdr_histogram_wasm_bg.wasm")[0][0];/' hdr_histogram_wasm.js # build the results viewer (which includes putting the output into the book's src) cd $RESULTS_VIEWER_DIR diff --git a/guide/serve-guide.sh b/guide/serve-guide.sh index cab46c9f..0a5d44bb 100755 --- a/guide/serve-guide.sh +++ b/guide/serve-guide.sh @@ -30,7 +30,7 @@ WASM_OUTPUT_DIR=$(realpath $RESULTS_VIEWER_DIR/lib/hdr-histogram-wasm) cd $WASM_LIB_DIR wasm-pack build --release -t web -d $WASM_OUTPUT_DIR --scope fs cd $WASM_OUTPUT_DIR -sed -i 's/input = .*import\.meta\.url.*/import(".\/hdr_histogram_wasm_bg.wasm");\ninput = require.resolve(".\/hdr_histogram_wasm_bg.wasm")[0][0];/' hdr_histogram_wasm.js +sed -i 's/input = .*import\.meta\.url.*/import(".\/hdr_histogram_wasm_bg.wasm");\n input = require.resolve(".\/hdr_histogram_wasm_bg.wasm")[0][0];/' hdr_histogram_wasm.js # build the results viewer (which includes putting the output into the book's src) cd $RESULTS_VIEWER_DIR diff --git a/guide/src/config/common-types/expressions.md b/guide/src/config/common-types/expressions.md index 5f10e043..469ff5bc 100644 --- a/guide/src/config/common-types/expressions.md +++ b/guide/src/config/common-types/expressions.md @@ -68,6 +68,7 @@ Encode a string with the given encoding. - `"percent"` - Percent encodes every ASCII character less than hexidecimal 20 and greater than 7E in addition to ` `, `"`, `#`, `>`, `<`, `` ` ``, `?`, `{` and `}` (space, doublequote, hash, greater than, less than, backtick, question mark, open curly brace and close curly brace). - `"percent-path"` - Percent encodes every ASCII character less than hexidecimal 20 and greater than 7E in addition to ` `, `"`, `#`, `>`, `<`, `` ` ``, `?`, `{`, `}`, `%` and `/` (space, doublequote, hash, greater than, less than, backtick, question mark, open curly brace, close curly brace, percent and forward slash). - `"percent-userinfo"` - Percent encodes every ASCII character less than hexidecimal 20 and greater than 7E in addition to ` `, `"`, `#`, `>`, `<`, `` ` ``, `?`, `{`, `}`, `/`, `:`, `;`, `=`, `@`, `\`, `[`, `]`, `^`, and `|` (space, doublequote, hash, greater than, less than, backtick, question mark, open curly brace, close curly brace, forward slash, colon, semi-colon, equal sign, at sign, backslash, open square bracket, close square bracket, caret and pipe).

+- `"non-alphanumeric"` - Non-Alphanumeric encodes every ASCII character that is not an ASCII letter or digit. **Example**: with the value `foo=bar` from a provider named `baz`, then the template `https://localhost/abc?${encode(baz, "percent-userinfo"}` would resolve to `https://localhost/abc?foo%3Dbar`. diff --git a/lib/config/Cargo.toml b/lib/config/Cargo.toml index 6b7abe15..fff7c7d5 100644 --- a/lib/config/Cargo.toml +++ b/lib/config/Cargo.toml @@ -17,7 +17,7 @@ itertools = "0.10" # pinned to 0.2.3 because newer versions made a semver incompatible change by making `Node` no longer # public, which we depend on https://github.com/freestrings/jsonpath/issues/36 jsonpath_lib = "=0.2.3" -percent-encoding = "1" +percent-encoding = "2" pest = "2" pest_derive = "2" rand = "0.8" diff --git a/lib/config/src/expression_functions.rs b/lib/config/src/expression_functions.rs index 1b813367..ef8b798a 100644 --- a/lib/config/src/expression_functions.rs +++ b/lib/config/src/expression_functions.rs @@ -7,6 +7,7 @@ use crate::select_parser::ProviderStream; use ether::{Either, Either3, EitherExt}; use futures::{stream, Stream, StreamExt, TryStreamExt}; use jsonpath_lib as json_path; +use percent_encoding::AsciiSet; use rand::distributions::{Distribution, Uniform}; use regex::Regex; use serde_json as json; @@ -144,31 +145,79 @@ enum Encoding { Percent, PercentPath, PercentUserinfo, + NonAlphanumeric, } +// https://github.com/servo/rust-url/blob/master/UPGRADING.md +// Prepackaged encoding sets, like QUERY_ENCODE_SET and PATH_SEGMENT_ENCODE_SET, are no longer provided. +// You will need to read the specifications relevant to your domain and construct your own encoding sets +// by using the percent_encoding::AsciiSet builder methods on either of the base encoding sets, +// percent_encoding::CONTROLS or percent_encoding::NON_ALPHANUMERIC. + +/// This encode set is used in the URL parser for query strings. +/// +/// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html), +/// space, double quote ("), hash (#), and inequality qualifiers (<), (>) are encoded. +const QUERY_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS + .add(b' ') + .add(b'"') + .add(b'#') + .add(b'<') + .add(b'>'); +/// This encode set is used for path components. +/// +/// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html), +/// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`), +/// question mark (?), and curly brackets ({), (}) are encoded. +const DEFAULT_ENCODE_SET: &AsciiSet = &QUERY_ENCODE_SET.add(b'`').add(b'?').add(b'{').add(b'}'); +/// This encode set is used for on '/'-separated path segment +/// +/// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html), +/// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`), +/// question mark (?), and curly brackets ({), (}), percent sign (%), forward slash (/) are +/// encoded. +const PATH_SEGMENT_ENCODE_SET: &AsciiSet = &DEFAULT_ENCODE_SET.add(b'%').add(b'/'); +/// This encode set is used for username and password. +/// +/// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html), +/// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`), +/// question mark (?), and curly brackets ({), (}), forward slash (/), colon (:), semi-colon (;), +/// equality (=), at (@), backslash (\\), square brackets ([), (]), caret (\^), and pipe (|) are +/// encoded. +const USERINFO_ENCODE_SET: &AsciiSet = &DEFAULT_ENCODE_SET + .add(b'/') + .add(b':') + .add(b';') + .add(b'=') + .add(b'@') + .add(b'[') + .add(b'\\') + .add(b']') + .add(b'^') + .add(b'|'); + impl Encoding { fn encode(self, d: &json::Value) -> String { let s = json_value_to_string(Cow::Borrowed(d)); match self { Encoding::Base64 => base64::encode(s.as_str()), Encoding::PercentSimple => { - percent_encoding::utf8_percent_encode(&s, percent_encoding::SIMPLE_ENCODE_SET) - .to_string() + percent_encoding::utf8_percent_encode(&s, percent_encoding::CONTROLS).to_string() } Encoding::PercentQuery => { - percent_encoding::utf8_percent_encode(&s, percent_encoding::QUERY_ENCODE_SET) - .to_string() + percent_encoding::utf8_percent_encode(&s, QUERY_ENCODE_SET).to_string() } Encoding::Percent => { - percent_encoding::utf8_percent_encode(&s, percent_encoding::DEFAULT_ENCODE_SET) - .to_string() + percent_encoding::utf8_percent_encode(&s, DEFAULT_ENCODE_SET).to_string() } Encoding::PercentPath => { - percent_encoding::utf8_percent_encode(&s, percent_encoding::PATH_SEGMENT_ENCODE_SET) - .to_string() + percent_encoding::utf8_percent_encode(&s, PATH_SEGMENT_ENCODE_SET).to_string() } Encoding::PercentUserinfo => { - percent_encoding::utf8_percent_encode(&s, percent_encoding::USERINFO_ENCODE_SET) + percent_encoding::utf8_percent_encode(&s, USERINFO_ENCODE_SET).to_string() + } + Encoding::NonAlphanumeric => { + percent_encoding::utf8_percent_encode(&s, percent_encoding::NON_ALPHANUMERIC) .to_string() } } @@ -182,6 +231,7 @@ impl Encoding { "percent" => Ok(Encoding::Percent), "percent-path" => Ok(Encoding::PercentPath), "percent-userinfo" => Ok(Encoding::PercentUserinfo), + "non-alphanumeric" => Ok(Encoding::NonAlphanumeric), _ => Err(ExecutingExpressionError::InvalidFunctionArguments("encode", marker).into()), } } @@ -1678,6 +1728,11 @@ mod tests { None, j!("asd%20jkl%7C"), ), + ( + vec![j!("asd 123~").into(), j!("non-alphanumeric").into()], + None, + j!("asd%20123%7E"), + ), ( vec!["a".into(), j!("percent-path").into()], Some(j!({"a": "asd/jkl%"})), @@ -1698,6 +1753,11 @@ mod tests { Some(j!({"a": "asd jkl|"})), j!("asd%20jkl%7C"), ), + ( + vec!["a".into(), j!("non-alphanumeric").into()], + Some(j!({"a": "asd 123~"})), + j!("asd%20123%7E"), + ), ( vec!["a".into(), j!("base64").into()], Some(j!({"a": "foo"})), @@ -1741,6 +1801,11 @@ mod tests { j!({"a": "asd jkl|"}), vec![j!("asd%20jkl%7C")], ), + ( + vec!["a".into(), j!("non-alphanumeric").into()], + j!({"a": "asd 123~"}), + vec![j!("asd%20123%7E")], + ), ( vec!["a".into(), j!("base64").into()], j!({"a": "foo"}), @@ -1783,7 +1848,11 @@ mod tests { vec!["d".into(), j!("percent-userinfo").into()], j!("asd%20jkl%7C"), ), - (vec!["e".into(), j!("base64").into()], j!("Zm9v")), + ( + vec!["e".into(), j!("non-alphanumeric").into()], + j!("asd%20123%7E"), + ), + (vec!["f".into(), j!("base64").into()], j!("Zm9v")), ]; let providers = btreemap!( @@ -1791,7 +1860,8 @@ mod tests { "b".to_string() => literals(vec!(j!("asd\njkl#"))), "c".to_string() => literals(vec!(j!("asd\njkl{"))), "d".to_string() => literals(vec!(j!("asd jkl|"))), - "e".to_string() => literals(vec!(j!("foo"))), + "e".to_string() => literals(vec!(j!("asd 123~"))), + "f".to_string() => literals(vec!(j!("foo"))), ); let providers = Arc::new(providers); diff --git a/lib/config/src/from_yaml.rs b/lib/config/src/from_yaml.rs index d7e1ca82..12a83a07 100644 --- a/lib/config/src/from_yaml.rs +++ b/lib/config/src/from_yaml.rs @@ -340,7 +340,8 @@ impl Insert for Vec { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] pub struct TupleVec(pub Vec<(K, V)>); impl Default for TupleVec { diff --git a/lib/config/src/lib.rs b/lib/config/src/lib.rs index 7ef98bf3..8ef356dd 100644 --- a/lib/config/src/lib.rs +++ b/lib/config/src/lib.rs @@ -116,7 +116,8 @@ impl Limit { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] pub enum HitsPer { Second(f32), Minute(f32), @@ -170,7 +171,8 @@ trait DefaultWithMarker { fn default(marker: Marker) -> Self; } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] enum LoadPatternPreProcessed { Linear(LinearBuilderPreProcessed), } @@ -186,6 +188,7 @@ impl FromYaml for LoadPatternPreProcessed { let ret = match event.into_string() { Ok(s) if s.as_str() == "linear" => { let (linear, marker) = FromYaml::parse(decoder)?; + log::debug!("LoadPatternPreProcessed.parse linear: {:?}", linear); (LoadPatternPreProcessed::Linear(linear), marker) } Ok(s) => return Err(Error::UnrecognizedKey(s, None, marker)), @@ -200,7 +203,8 @@ impl FromYaml for LoadPatternPreProcessed { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] struct LinearBuilderPreProcessed { from: Option, to: PrePercent, @@ -240,14 +244,17 @@ impl FromYaml for LinearBuilderPreProcessed { YamlEvent::Scalar(s, ..) => match s.as_str() { "from" => { let c = FromYaml::parse_into(decoder)?; + log::debug!("LinearBuilderPreProcessed.parse from: {:?}", c); from = Some(c); } "to" => { let a = FromYaml::parse_into(decoder)?; + log::debug!("LinearBuilderPreProcessed.parse to: {:?}", a); to = Some(a); } "over" => { let b = FromYaml::parse_into(decoder)?; + log::debug!("LinearBuilderPreProcessed.parse over: {:?}", b); over = Some(b); } _ => return Err(Error::UnrecognizedKey(s, None, marker)), @@ -281,8 +288,7 @@ impl LoadPattern { } } -#[cfg_attr(debug_assertions, derive(Debug))] -#[derive(Clone, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct ListWithOptions { pub random: bool, pub repeat: bool, @@ -324,21 +330,25 @@ impl FromYaml for ListWithOptions { "random" => { let (r, _): (bool, _) = FromYaml::parse(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("ListWithOptions.parse random: {:?}", r); random = r; } "repeat" => { let (r, _) = FromYaml::parse(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("ListWithOptions.parse repeat: {:?}", r); repeat = r; } "values" => { let (v, _) = FromYaml::parse(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("ListWithOptions.parse values: {:?}", v); values = Some(v); } "unique" => { let (u, _) = FromYaml::parse(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("ListWithOptions.parse unique: {:?}", u); unique = u; } _ => return Err(Error::UnrecognizedKey(s, None, marker)), @@ -357,8 +367,7 @@ impl FromYaml for ListWithOptions { } } -#[cfg_attr(debug_assertions, derive(Debug))] -#[derive(Clone, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum ListProvider { WithOptions(ListWithOptions), DefaultOptions(Vec), @@ -376,11 +385,13 @@ impl FromYaml for ListProvider { match event { YamlEvent::SequenceStart => { let (e, marker) = FromYaml::parse(decoder)?; + log::debug!("ListProvider.parse SequenceStart: {:?}", e); let value = (ListProvider::DefaultOptions(e), marker); Ok(value) } YamlEvent::MappingStart => { let (i, marker) = FromYaml::parse(decoder)?; + log::debug!("ListProvider.parse MappingStart: {:?}", i); let value = (ListProvider::WithOptions(i), marker); Ok(value) } @@ -446,7 +457,8 @@ impl Iterator for ListRepeatRandomIterator { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] enum ProviderPreProcessed { File(FileProviderPreProcessed), Range(RangeProviderPreProcessed), @@ -492,21 +504,25 @@ impl FromYaml for ProviderPreProcessed { "file" => { let (c, marker) = FromYaml::parse(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("ProviderPreProcessed.parse file: {:?}", c); break (ProviderPreProcessed::File(c), marker); } "range" => { let (c, marker) = FromYaml::parse(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("ProviderPreProcessed.parse range: {:?}", c); break (ProviderPreProcessed::Range(c), marker); } "response" => { let (c, marker) = FromYaml::parse(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("ProviderPreProcessed.parse response: {:?}", c); break (ProviderPreProcessed::Response(c), marker); } "list" => { let (c, marker) = FromYaml::parse(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("ProviderPreProcessed.parse list: {:?}", c); break (ProviderPreProcessed::List(c), marker); } _ => return Err(Error::UnrecognizedKey(s, None, marker)), @@ -569,8 +585,7 @@ impl fmt::Display for RangeProvider { } } -#[cfg_attr(debug_assertions, derive(Debug))] -#[derive(Clone, PartialEq, Serialize)] +#[derive(Clone, Debug, PartialEq, Serialize)] pub struct RangeProviderPreProcessed { start: i64, end: i64, @@ -654,8 +669,7 @@ impl FromYaml for RangeProviderPreProcessed { } } -#[cfg_attr(debug_assertions, derive(Debug))] -#[derive(Clone, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum FileFormat { Csv, Json, @@ -681,8 +695,7 @@ impl Default for FileFormat { } } -#[cfg_attr(debug_assertions, derive(Debug))] -#[derive(Clone, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum CsvHeader { Bool(bool), String(String), @@ -719,8 +732,7 @@ fn from_yaml_char_u8>(decoder: &mut YamlDecoder) -> } } -#[cfg_attr(debug_assertions, derive(Debug))] -#[derive(Clone, Default, PartialEq)] +#[derive(Clone, Debug, Default, PartialEq)] pub struct CsvSettings { pub comment: Option, pub delimiter: Option, @@ -815,7 +827,8 @@ impl FromYaml for CsvSettings { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] struct FileProviderPreProcessed { csv: CsvSettings, auto_return: Option, @@ -928,8 +941,7 @@ impl FromYaml for FileProviderPreProcessed { } } -#[cfg_attr(debug_assertions, derive(Debug))] -#[derive(Clone, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct ResponseProvider { pub auto_return: Option, pub buffer: Limit, @@ -997,7 +1009,8 @@ impl FromYaml for ResponseProvider { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] pub struct LoggerPreProcessed { select: Option>, for_each: Vec>, @@ -1064,36 +1077,43 @@ impl FromYaml for LoggerPreProcessed { "select" => { let c = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoggerPreProcessed.parse select: {:?}", c); select = Some(c); } "for_each" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoggerPreProcessed.parse for_each: {:?}", a); for_each = Some(a); } "where" => { let b = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoggerPreProcessed.parse where: {:?}", b); where_clause = Some(b); } "to" => { let b = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoggerPreProcessed.parse to: {:?}", b); to = Some(b); } "pretty" => { let b = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoggerPreProcessed.parse pretty: {:?}", b); pretty = b; } "limit" => { let b = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoggerPreProcessed.parse limit: {:?}", b); limit = Some(b); } "kill" => { let b = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoggerPreProcessed.parse kill: {:?}", b); kill = b; } _ => return Err(Error::UnrecognizedKey(s, None, marker)), @@ -1116,7 +1136,8 @@ impl FromYaml for LoggerPreProcessed { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] struct LogsPreProcessed { select: WithMarker, for_each: Vec>, @@ -1184,7 +1205,7 @@ impl FromYaml for LogsPreProcessed { } } -#[cfg_attr(debug_assertions, derive(Debug))] +#[derive(Debug)] struct EndpointPreProcessed { declare: BTreeMap, headers: TupleVec>, @@ -1268,72 +1289,86 @@ impl FromYaml for EndpointPreProcessed { "declare" => { let c = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse declare: {:?}", c); declare = Some(c); } "headers" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse headers: {:?}", a); headers = Some(a); } "body" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse body: {:?}", a); body = Some(a); } "load_pattern" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse load_pattern: {:?}", a); load_pattern = Some(a); } "method" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse method: {:?}", a); method = Some(a); } "on_demand" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse on_demand: {:?}", a); on_demand = Some(a); } "peak_load" => { let p = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse peak_load: {:?}", p); let p = PreHitsPer(p); peak_load = Some(p); } "tags" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse tags: {:?}", a); tags = Some(a); } "url" => { let v = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse url: {:?}", v); url = Some(v); } "provides" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse provides: {:?}", a); provides = Some(a); } "logs" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse logs: {:?}", a); logs = Some(a); } "max_parallel_requests" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse max_parallel_requests: {:?}", a); max_parallel_requests = Some(a); } "no_auto_returns" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse no_auto_returns: {:?}", a); no_auto_returns = Some(a); } "request_timeout" => { let a = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("EndpointPreProcessed.parse request_timeout: {:?}", a); request_timeout = Some(a); } _ => return Err(Error::UnrecognizedKey(s, None, marker)), @@ -1371,7 +1406,8 @@ impl FromYaml for EndpointPreProcessed { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] enum Body { String(PreTemplate), File(PreTemplate), @@ -1425,7 +1461,8 @@ impl FromYaml for Body { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] struct BodyMultipartPiece { pub headers: TupleVec, pub body: BodyMultipartPieceBody, @@ -1483,7 +1520,8 @@ impl FromYaml for BodyMultipartPiece { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] enum BodyMultipartPieceBody { String(PreTemplate), File(PreTemplate), @@ -1527,8 +1565,7 @@ impl FromYaml for BodyMultipartPieceBody { } } -#[cfg_attr(debug_assertions, derive(Debug))] -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, Debug, PartialEq)] pub enum EndpointProvidesSendOptions { Block, Force, @@ -1564,7 +1601,8 @@ impl FromYaml for EndpointProvidesSendOptions { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] pub(crate) struct EndpointProvidesPreProcessed { for_each: Vec>, select: WithMarker, @@ -1660,7 +1698,8 @@ pub fn default_auto_buffer_start_size() -> usize { 5 } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] struct ClientConfigPreProcessed { headers: TupleVec, keepalive: PreDuration, @@ -1753,7 +1792,8 @@ pub struct GeneralConfig { pub log_level: Option, } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] struct GeneralConfigPreProcessed { auto_buffer_start_size: usize, bucket_size: PreDuration, @@ -1833,11 +1873,11 @@ impl FromYaml for GeneralConfigPreProcessed { // Going forward it is on by default, we only want to allow turning it off via "false". // 'durations' are the equivalent of "true" and the duration is ignored. Anything else should error. duration_from_string(d.clone()).map_err(|err| { - error!("log_provider_stats error {}/{}: {}", s, d, bool_err); - debug!("log_provider_stats duration_from_string error {}/{}: {}", s, d, err); - // We don't want to return a duration error, we want to just say there was a problem with the "name" - Error::YamlDeserialize(Some(s), marker) - })?; + error!("log_provider_stats error {}/{}: {}", s, d, bool_err); + debug!("log_provider_stats duration_from_string error {}/{}: {}", s, d, err); + // We don't want to return a duration error, we want to just say there was a problem with the "name" + Error::YamlDeserialize(Some(s), marker) + })?; true } }; @@ -1876,7 +1916,8 @@ impl FromYaml for GeneralConfigPreProcessed { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] pub struct ConfigPreProcessed { client: ClientConfigPreProcessed, general: GeneralConfigPreProcessed, @@ -1924,11 +1965,13 @@ impl FromYaml for ConfigPreProcessed { "client" => { let (c, _) = FromYaml::parse(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("ConfigPreProcessed.parse client: {:?}", c); client = Some(c); } "general" => { let (a, _) = FromYaml::parse(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("ConfigPreProcessed.parse general: {:?}", a); general = Some(a); } _ => return Err(Error::UnrecognizedKey(s, None, marker)), @@ -1943,7 +1986,8 @@ impl FromYaml for ConfigPreProcessed { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] struct LoadTestPreProcessed { config: ConfigPreProcessed, endpoints: Vec, @@ -1954,6 +1998,7 @@ struct LoadTestPreProcessed { } impl FromYaml for LoadTestPreProcessed { + // Entry point for parsing the yaml file fn parse>(decoder: &mut YamlDecoder) -> ParseResult { let mut config = None; let mut endpoints = None; @@ -1989,31 +2034,37 @@ impl FromYaml for LoadTestPreProcessed { "config" => { let r = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoadTestPreProcessed.parse config: {:?}", r); config = Some(r); } "endpoints" => { let r = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoadTestPreProcessed.parse endpoints: {:?}", r); endpoints = Some(r); } "load_pattern" => { let v = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoadTestPreProcessed.parse load_pattern: {:?}", v); load_pattern = Some(v); } "providers" => { let v = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoadTestPreProcessed.parse providers: {:?}", v); providers = Some(v); } "loggers" => { let v = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoadTestPreProcessed.parse loggers: {:?}", v); loggers = Some(v); } "vars" => { let v = FromYaml::parse_into(decoder).map_err(map_yaml_deserialize_err(s))?; + log::debug!("LoadTestPreProcessed.parse vars: {:?}", v); vars = Some(v); } _ => return Err(Error::UnrecognizedKey(s, None, marker)), @@ -2038,7 +2089,7 @@ impl FromYaml for LoadTestPreProcessed { } } -#[cfg_attr(debug_assertions, derive(Debug))] +#[derive(Debug)] struct WithMarker { inner: T, marker: Marker, @@ -2076,7 +2127,8 @@ impl FromYaml for WithMarker { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] struct PreValueOrExpression(WithMarker); impl PreValueOrExpression { @@ -2103,7 +2155,8 @@ impl FromYaml for PreValueOrExpression { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] struct PreTemplate(WithMarker, bool); impl PreTemplate { @@ -2150,7 +2203,8 @@ impl FromYaml for PreTemplate { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] pub struct PreVar(WithMarker); impl PreVar { @@ -2240,7 +2294,8 @@ fn duration_from_string2(dur: String, marker: Marker) -> Result Ok(Duration::from_secs(total_secs)) } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] pub struct PreDuration(PreTemplate); impl PreDuration { @@ -2259,7 +2314,8 @@ impl FromYaml for PreDuration { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] struct PrePercent(PreTemplate); impl PrePercent { @@ -2289,7 +2345,7 @@ impl FromYaml for PrePercent { } } -#[cfg_attr(debug_assertions, derive(Debug))] +#[derive(Debug)] struct PreLoadPattern(Vec, Marker); #[cfg(debug_assertions)] @@ -2336,7 +2392,8 @@ impl FromYaml for PreLoadPattern { } } -#[cfg_attr(debug_assertions, derive(Debug, PartialEq))] +#[cfg_attr(debug_assertions, derive(PartialEq))] +#[derive(Debug)] struct PreHitsPer(PreTemplate); impl PreHitsPer { diff --git a/lib/config/src/select_parser.rs b/lib/config/src/select_parser.rs index a39c218c..a55d777f 100644 --- a/lib/config/src/select_parser.rs +++ b/lib/config/src/select_parser.rs @@ -307,6 +307,14 @@ fn index_json<'a>( for_each: Option<&[Cow<'a, json::Value>]>, marker: Marker, ) -> Result, ExecutingExpressionError> { + log::trace!( + "index_json json: {}, index: {:?}, no_err: {}, for_each: {:?}, marker: {:?}", + json, + index, + no_err, + for_each, + marker + ); #[allow(unused_assignments)] let mut holder = None; let str_or_number = match index { @@ -361,6 +369,14 @@ fn index_json2<'a>( for_each: Option<&[Cow<'a, json::Value>]>, marker: Marker, ) -> Result, ExecutingExpressionError> { + log::trace!( + "index_json2 json: {}, indexes: {:?}, no_err: {}, for_each: {:?}, marker: {:?}", + json, + indexes, + no_err, + for_each, + marker + ); for index in indexes.iter() { let r = index.evaluate(Cow::Borrowed(&*json), for_each)?; let o = match (json, r) { @@ -417,6 +433,13 @@ impl Path { no_recoverable_error: bool, for_each: Option<&[Cow<'a, json::Value>]>, ) -> Result, ExecutingExpressionError> { + log::trace!( + "Path::evaluate self: {:?}, d: {:?}, no_recoverable_error: {}, for_each: {:?}", + self, + d, + no_recoverable_error, + for_each + ); let (v, rest) = match (&self.start, for_each) { (PathStart::FunctionCall(f), _) => ( f.evaluate(d, no_recoverable_error, for_each)?, @@ -451,6 +474,13 @@ impl Path { impl Iterator, ExecutingExpressionError>> + Clone, ExecutingExpressionError, > { + log::trace!( + "Path::evaluate_as_iter self: {:?}, d: {:?}, no_recoverable_error: {}, for_each: {:?}", + self, + d, + no_recoverable_error, + for_each + ); let r = match &self.start { PathStart::FunctionCall(fnc) => { let rest = self.rest.clone(); @@ -500,6 +530,12 @@ impl Path { providers: &BTreeMap, no_recoverable_error: bool, ) -> impl Stream), ExecutingExpressionError>> { + log::trace!( + "Path::into_stream self: {:?}, providers: {:?}, no_recoverable_error: {}", + self, + providers.keys(), + no_recoverable_error + ); // TODO: don't we need providers when evaluating `rest`? let rest = self.rest; let marker = self.marker; @@ -577,6 +613,7 @@ impl ValueOrExpression { let pairs = Parser::parse(Rule::entry_point, expr) .map_err(|e| CreatingExpressionError::InvalidExpression(e, marker))?; let e = parse_expression(pairs, providers, static_vars, no_recoverable_error, marker)?; + log::debug!("ValueOrExpression parse_expression: {:?}", e); ValueOrExpression::from_expression(e) } @@ -1144,6 +1181,7 @@ impl Template { no_recoverable_error, marker, )?; + log::debug!("Template parse_expression: {:?}", e); match e.simplify_to_string()? { Either::A(s2) => { if let Some(TemplatePiece::NotExpression(s)) = pieces.last_mut() { @@ -1328,6 +1366,7 @@ impl Select { no_recoverable_error, v.marker(), )?; + log::debug!("Select for_each parse_expression: {:?}", e); if providers2.get_special() & FOR_EACH != 0 { Err(error::Error::RecursiveForEachReference(v.marker())) } else { @@ -1356,6 +1395,7 @@ impl Select { no_recoverable_error, v.marker(), )?; + log::debug!("Select where parse_expression: {:?}", e); where_references_for_each = providers2.get_special() & FOR_EACH != 0; if where_references_for_each && join.is_empty() { return Err(error::Error::MissingForEach(v.marker())); @@ -1568,6 +1608,13 @@ fn parse_select( no_recoverable_error: bool, marker: Marker, ) -> Result { + log::trace!( + "parse_select select: {}, providers: {:?}, no_recoverable_error: {}, marker: {:?}", + select, + providers, + no_recoverable_error, + marker + ); let r = match select { json::Value::Null => ParsedSelect::Null, json::Value::Bool(b) => ParsedSelect::Bool(b), @@ -1607,6 +1654,13 @@ fn parse_function_call( no_recoverable_error: bool, marker: Marker, ) -> Result, CreatingExpressionError> { + log::trace!( + "parse_function_call pair: {}, providers: {:?}, no_recoverable_error: {}, marker: {:?}", + pair, + providers, + no_recoverable_error, + marker + ); let mut ident = None; let mut args = Vec::new(); for pair in pair.into_inner() { @@ -1647,6 +1701,13 @@ fn parse_indexed_property( no_recoverable_error: bool, marker: Marker, ) -> Result { + log::trace!( + "parse_function_call pair: {}, providers: {:?}, no_recoverable_error: {}, marker: {:?}", + pair, + providers, + no_recoverable_error, + marker + ); let pair = pair .into_inner() .next() @@ -1673,6 +1734,13 @@ fn parse_path( no_recoverable_error: bool, marker: Marker, ) -> Result, CreatingExpressionError> { + log::trace!( + "parse_path pair: {}, providers: {:?}, no_recoverable_error: {}, marker: {:?}", + pair, + providers, + no_recoverable_error, + marker + ); let mut start = None; let mut rest = Vec::new(); let mut providers2 = RequiredProviders::new(); @@ -1747,6 +1815,7 @@ fn parse_path( rest, marker, }; + log::trace!("parse_path path: {:?}", p); let r = match p.start { PathStart::Value(_) if providers2.is_empty() => { let a = p @@ -1769,6 +1838,13 @@ fn parse_value( no_recoverable_error: bool, marker: Marker, ) -> Result { + log::trace!( + "parse_value pairs: {}, providers: {:?}, no_recoverable_error: {}, marker: {:?}", + pairs, + providers, + no_recoverable_error, + marker + ); let pair = pairs .next() .expect("Expected 1 rule while parsing indexed property"); @@ -1826,6 +1902,7 @@ fn expression_helper( mut items: Vec, level: u8, ) -> Result { + log::trace!("expression_helper items: {:?}, level: {}", items, level); let i = items.iter().rposition(|eoo| { if let ExpressionOrOperator::Operator(o) = eoo { INFIX_OPERATOR_PRECEDENCE[*o as usize] == level @@ -1892,6 +1969,14 @@ fn parse_expression_pieces( no_recoverable_error: bool, marker: Marker, ) -> Result<(), CreatingExpressionError> { + log::trace!( + "parse_expression_pieces pairs: {}, providers: {:?}, pieces: {:?}, no_recoverable_error: {}, marker: {:?}", + pairs, + providers, + pieces, + no_recoverable_error, + marker + ); let mut not_count = 0; let start_len = pieces.len(); for pair in pairs { @@ -1995,6 +2080,13 @@ fn parse_expression( no_recoverable_error: bool, marker: Marker, ) -> Result { + log::trace!( + "parse_expression pairs: {}, providers: {:?}, no_recoverable_error: {}, marker: {:?}", + pairs, + providers, + no_recoverable_error, + marker + ); let mut pieces = Vec::new(); parse_expression_pieces( pairs,