From 50ca6afb47fb08b63d929ced547935c4937fa23f Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Wed, 6 Nov 2024 16:23:08 -0500 Subject: [PATCH] feat(opnsense-config): Xml parsing now works great to parse a full production config. Only some element ordering that is not consistent across multiple elements of the same type sometimes does not match but moving some stuff around gets us easily to a 100% matching file --- harmony-rs/opnsense-config/src/config.rs | 11 +- .../opnsense-config/src/infra/generic_xml.rs | 44 +++- harmony-rs/opnsense-config/src/infra/mod.rs | 2 +- .../opnsense-config/src/infra/yaserde.rs | 20 ++ .../opnsense-config/src/modules/dhcp.rs | 14 +- .../opnsense-config/src/modules/opnsense.rs | 116 ++++---- .../src/tests/data/config-full-1.xml | 249 +++++++++--------- 7 files changed, 261 insertions(+), 195 deletions(-) create mode 100644 harmony-rs/opnsense-config/src/infra/yaserde.rs diff --git a/harmony-rs/opnsense-config/src/config.rs b/harmony-rs/opnsense-config/src/config.rs index 233ae69..947e774 100644 --- a/harmony-rs/opnsense-config/src/config.rs +++ b/harmony-rs/opnsense-config/src/config.rs @@ -170,13 +170,16 @@ impl Config { #[cfg(test)] mod tests { + use crate::infra::yaserde::to_xml_str; + use super::*; use std::path::PathBuf; + use pretty_assertions::assert_eq; #[tokio::test] async fn test_load_config_from_local_file() { let mut test_file_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - test_file_path.push("src/tests/data/config-structure.xml"); + test_file_path.push("src/tests/data/config-full-1.xml"); let config_file_path = test_file_path.to_str().unwrap().to_string(); println!("File path {config_file_path}"); @@ -188,13 +191,9 @@ mod tests { println!("Config {:?}", config); - let yaserde_cfg = yaserde::ser::Config { perform_indent: true, - .. Default::default() - }; - let serialized = yaserde::ser::to_string_with_config(&config.opnsense, &yaserde_cfg).unwrap(); + let serialized = to_xml_str(&config.opnsense).unwrap(); fs::write("/tmp/serialized.xml", &serialized).unwrap(); - std::process::Command::new("xmllint").arg("/tmp/serialized.xml").arg("--output").arg("/tmp/serialized.xmllint.xml").status().expect("xmllint failed"); assert_eq!(config_file_str, serialized); } diff --git a/harmony-rs/opnsense-config/src/infra/generic_xml.rs b/harmony-rs/opnsense-config/src/infra/generic_xml.rs index 1e70a71..dfeae73 100644 --- a/harmony-rs/opnsense-config/src/infra/generic_xml.rs +++ b/harmony-rs/opnsense-config/src/infra/generic_xml.rs @@ -118,7 +118,10 @@ impl YaSerializeTrait for RawXml { #[cfg(test)] mod test { + use crate::infra::yaserde::to_xml_str; + use super::*; + use pretty_assertions::assert_eq; use yaserde_derive::YaDeserialize; use yaserde_derive::YaSerialize; @@ -172,9 +175,46 @@ mod test { #[test] fn rawxml_should_serialize_complex_documents() { - let xml = r#"1060s00101024102401ignore204816384"#; + let xml = r#" + + + + + + + + + + 1 + 0 + 60s + + 0 + 0 + 1 + + 0 + + + 1024 + + + 1024 + + + 0 + + 1 + ignore + 2048 + 16384 + + + + +"#; let rawxml: RawXml = yaserde::de::from_str(xml).unwrap(); - assert_eq!(yaserde::ser::to_string(&rawxml).unwrap(), xml); + assert_eq!(to_xml_str(&rawxml).unwrap(), xml); } #[test] diff --git a/harmony-rs/opnsense-config/src/infra/mod.rs b/harmony-rs/opnsense-config/src/infra/mod.rs index 2563d9c..9c67f9a 100644 --- a/harmony-rs/opnsense-config/src/infra/mod.rs +++ b/harmony-rs/opnsense-config/src/infra/mod.rs @@ -1,4 +1,4 @@ - pub mod generic_xml; pub mod maybe_string; +pub mod yaserde; diff --git a/harmony-rs/opnsense-config/src/infra/yaserde.rs b/harmony-rs/opnsense-config/src/infra/yaserde.rs new file mode 100644 index 0000000..bdd2fcd --- /dev/null +++ b/harmony-rs/opnsense-config/src/infra/yaserde.rs @@ -0,0 +1,20 @@ +use yaserde::YaSerialize; + +pub fn to_xml_str(model: &T) -> Result { + let yaserde_cfg = yaserde::ser::Config { + perform_indent: true, + write_document_declaration: false, + pad_self_closing: false, + ..Default::default() + }; + let serialized = yaserde::ser::to_string_with_config::(model, &yaserde_cfg)?; + + // Opnsense does not specify encoding in the document declaration + // + // yaserde / xml-rs does not allow disabling the encoding attribute in the + // document declaration + // + // So here we just manually prefix the xml document with the exact document declaration + // that opnsense uses + Ok(format!("\n{serialized}\n")) +} diff --git a/harmony-rs/opnsense-config/src/modules/dhcp.rs b/harmony-rs/opnsense-config/src/modules/dhcp.rs index 92972dc..ef18aaf 100644 --- a/harmony-rs/opnsense-config/src/modules/dhcp.rs +++ b/harmony-rs/opnsense-config/src/modules/dhcp.rs @@ -67,6 +67,8 @@ pub struct DhcpRange { #[cfg(test)] mod test { + use crate::infra::yaserde::to_xml_str; + use super::*; use pretty_assertions::assert_eq; @@ -75,19 +77,15 @@ mod test { let dhcpd: Dhcpd = yaserde::de::from_str(SERIALIZED_DHCPD).expect("Deserialize Dhcpd failed"); - let yaserde_cfg = yaserde::ser::Config { - perform_indent: true, - write_document_declaration: false, - ..Default::default() - }; assert_eq!( - yaserde::ser::to_string_with_config(&dhcpd, &yaserde_cfg) + to_xml_str(&dhcpd) .expect("Serialize Dhcpd failed"), SERIALIZED_DHCPD ); } - const SERIALIZED_DHCPD: &str = " + const SERIALIZED_DHCPD: &str = " + 1 192.168.20.1 @@ -140,5 +138,5 @@ mod test { -"; +\n"; } diff --git a/harmony-rs/opnsense-config/src/modules/opnsense.rs b/harmony-rs/opnsense-config/src/modules/opnsense.rs index 9daa437..78011b0 100644 --- a/harmony-rs/opnsense-config/src/modules/opnsense.rs +++ b/harmony-rs/opnsense-config/src/modules/opnsense.rs @@ -73,9 +73,11 @@ pub struct Revision { #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] pub struct Options { - pub path: MaybeString, - pub host: MaybeString, - pub code: MaybeString, + pub path: Option, + pub host: Option, + pub code: Option, + pub send: Option, + pub expect: Option, } #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] @@ -87,36 +89,39 @@ pub struct Filters { #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] pub struct Rule { #[yaserde(attribute)] - pub uuid: MaybeString, + pub uuid: Option, #[yaserde(rename = "associated-rule-id")] - pub associated_rule_id: MaybeString, + pub associated_rule_id: Option, #[yaserde(rename = "type")] - pub r#type: MaybeString, + pub r#type: Option, pub interface: String, pub ipprotocol: String, - pub statetype: String, - pub descr: String, - pub direction: MaybeString, - pub category: MaybeString, - pub quick: MaybeString, - pub protocol: String, + pub statetype: Option, + pub descr: Option, + pub direction: Option, + pub category: Option, + pub quick: Option, + pub protocol: Option, pub source: Source, + pub icmptype: Option, pub destination: Destination, pub updated: Option, - pub created: Created, - pub disabled: Option, + pub created: Option, + pub disabled: Option, } #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] pub struct Source { pub any: Option, + pub network: Option, } #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] pub struct Destination { - pub network: MaybeString, - pub address: MaybeString, - pub port: Option, + pub network: Option, + pub address: Option, + pub port: Option, + pub any: Option, } #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] @@ -343,7 +348,7 @@ pub struct Snmpd { #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] pub struct Syslog { - pub reverse: Option, + pub reverse: Option, pub preservelogs: i32, } @@ -368,10 +373,11 @@ pub struct NatRule { pub ipprotocol: String, pub descr: MaybeString, pub tag: MaybeString, - pub tagged: Option, + pub tagged: Option, pub poolopts: PoolOpts, #[yaserde(rename = "associated-rule-id")] - pub associated_rule_id: String, + pub associated_rule_id: Option, + pub disabled: Option, pub target: String, #[yaserde(rename = "local-port")] pub local_port: i32, @@ -439,7 +445,7 @@ pub struct OPNsenseConfig { #[derive(Debug, YaSerialize, YaDeserialize, PartialEq)] #[yaserde(rename = "IDS")] -struct IDS { +pub struct IDS { #[yaserde(attribute)] version: String, rules: MaybeString, @@ -501,30 +507,30 @@ pub struct ConfigInterfaces { } #[derive(Debug, YaSerialize, YaDeserialize, PartialEq)] -struct Vxlan { +pub struct Vxlan { #[yaserde(attribute)] version: String, } #[derive(Debug, YaSerialize, YaDeserialize, PartialEq)] -struct Loopback { +pub struct Loopback { #[yaserde(attribute)] version: String, } #[derive(Debug, YaSerialize, YaDeserialize, PartialEq)] #[yaserde(rename = "monit")] -struct Monit { +pub struct Monit { #[yaserde(attribute)] version: String, general: GeneralMonit, alert: Option, - service: Option, - test: Option, + service: Vec, + test: Vec, } #[derive(Debug, YaSerialize, YaDeserialize, PartialEq)] -struct GeneralMonit { +pub struct GeneralMonit { enabled: u8, interval: u32, startdelay: u32, @@ -537,20 +543,30 @@ struct GeneralMonit { sslverify: u8, logfile: String, statefile: MaybeString, - eventqueuePath: MaybeString, - eventqueueSlots: MaybeString, - httpdEnabled: u8, - httpdUsername: String, - httpdPassword: String, - httpdPort: u16, - httpdAllow: MaybeString, - mmonitUrl: MaybeString, - mmonitTimeout: u32, - mmonitRegisterCredentials: u8, + #[yaserde(rename = "eventqueuePath")] + event_queue_path: MaybeString, + #[yaserde(rename = "eventqueueSlots")] + event_queue_slots: MaybeString, + #[yaserde(rename = "httpdEnabled")] + httpd_enabled: u8, + #[yaserde(rename = "httpdUsername")] + httpd_username: String, + #[yaserde(rename = "httpdPassword")] + httpd_password: String, + #[yaserde(rename = "httpdPort")] + httpd_port: u16, + #[yaserde(rename = "httpdAllow")] + httpd_allow: MaybeString, + #[yaserde(rename = "mmonitUrl")] + mmonit_url: MaybeString, + #[yaserde(rename = "mmonitTimeout")] + mmonit_timeout: u32, + #[yaserde(rename = "mmonitRegisterCredentials")] + mmonit_register_credentials: u8, } #[derive(Debug, YaSerialize, YaDeserialize, PartialEq)] -struct Alert { +pub struct Alert { #[yaserde(attribute)] uuid: String, enabled: u8, @@ -563,7 +579,7 @@ struct Alert { } #[derive(Debug, YaSerialize, YaDeserialize, PartialEq)] -struct Service { +pub struct Service { #[yaserde(attribute)] uuid: String, enabled: u8, @@ -587,7 +603,7 @@ struct Service { } #[derive(Debug, YaSerialize, YaDeserialize, PartialEq)] -struct Test { +pub struct Test { #[yaserde(attribute)] uuid: String, name: String, @@ -635,7 +651,7 @@ pub struct Capture { #[yaserde(rename = "interfaces")] pub interfaces: MaybeString, #[yaserde(rename = "egress_only")] - pub egress_only: Option, + pub egress_only: MaybeString, #[yaserde(rename = "version")] pub version: MaybeString, #[yaserde(rename = "targets")] @@ -725,7 +741,7 @@ pub struct Firewall { #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] pub struct LvTemplate { #[yaserde(attribute)] - pub version: MaybeString, + pub version: String, #[yaserde(rename = "templates")] pub templates: Option, } @@ -733,7 +749,7 @@ pub struct LvTemplate { #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] pub struct Category { #[yaserde(attribute)] - pub version: MaybeString, + pub version: String, #[yaserde(rename = "categories")] pub categories: Option, } @@ -747,7 +763,7 @@ pub struct Categories { #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] pub struct CategoryItem { #[yaserde(attribute)] - pub uuid: MaybeString, + pub uuid: String, #[yaserde(rename = "name")] pub name: MaybeString, #[yaserde(rename = "auto")] @@ -759,7 +775,7 @@ pub struct CategoryItem { #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] pub struct Alias { #[yaserde(attribute)] - pub version: MaybeString, + pub version: String, #[yaserde(rename = "geoip")] pub geoip: Option, #[yaserde(rename = "aliases")] @@ -786,13 +802,13 @@ pub struct AliasItem { pub name: String, #[yaserde(rename = "type")] pub r#type: String, + pub proto: MaybeString, pub interface: MaybeString, pub counters: String, pub updatefreq: MaybeString, pub content: String, pub categories: MaybeString, pub description: MaybeString, - pub proto: MaybeString, } #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] @@ -1707,19 +1723,21 @@ pub struct HAProxyHealthCheck { pub tcp_send_value: MaybeString, #[yaserde(rename = "tcp_matchType")] pub tcp_match_type: MaybeString, + pub tcp_negate: MaybeString, #[yaserde(rename = "tcp_matchValue")] pub tcp_match_value: MaybeString, - pub tcp_negate: MaybeString, - #[yaserde(rename = "agentPort")] pub agent_port: MaybeString, pub mysql_user: MaybeString, pub mysql_post41: MaybeString, pub pgsql_user: MaybeString, - #[yaserde(alias = "smtpDomain")] pub smtp_domain: MaybeString, pub esmtp_domain: MaybeString, + #[yaserde(rename = "agentPort")] + pub agent_port_uppercase: MaybeString, #[yaserde(rename = "dbUser")] pub db_user: MaybeString, + #[yaserde(rename = "smtpDomain")] + pub smtp_domain_uppercase: MaybeString, } #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)] diff --git a/harmony-rs/opnsense-config/src/tests/data/config-full-1.xml b/harmony-rs/opnsense-config/src/tests/data/config-full-1.xml index 1feaa2e..fd2853d 100644 --- a/harmony-rs/opnsense-config/src/tests/data/config-full-1.xml +++ b/harmony-rs/opnsense-config/src/tests/data/config-full-1.xml @@ -28,28 +28,22 @@ default - - Source routing is another way for an attacker to try to reach non-routable addresses behind your box. + Source routing is another way for an attacker to try to reach non-routable addresses behind your box. It can also be used to probe for information about your internal networks. These functions come enabled - as part of the standard FreeBSD core system. - + as part of the standard FreeBSD core system. net.inet.ip.sourceroute default - - Source routing is another way for an attacker to try to reach non-routable addresses behind your box. + Source routing is another way for an attacker to try to reach non-routable addresses behind your box. It can also be used to probe for information about your internal networks. These functions come enabled - as part of the standard FreeBSD core system. - + as part of the standard FreeBSD core system. net.inet.ip.accept_sourceroute default - - This option turns off the logging of redirect packets because there is no limit and this could fill - up your logs consuming your whole hard drive. - + This option turns off the logging of redirect packets because there is no limit and this could fill + up your logs consuming your whole hard drive. net.inet.icmp.log_redirect default @@ -190,17 +184,14 @@ Enable/disable sending of ICMP redirects in response to IP packets for which a better, - and for the sender directly reachable, route and next hop is known. - + and for the sender directly reachable, route and next hop is known. net.inet.ip.redirect 0 - - Redirect attacks are the purposeful mass-issuing of ICMP type 5 packets. In a normal network, redirects + Redirect attacks are the purposeful mass-issuing of ICMP type 5 packets. In a normal network, redirects to the end stations should not be required. This option enables the NIC to drop all inbound ICMP redirect - packets without returning a response. - + packets without returning a response. net.inet.icmp.drop_redirect 1 @@ -1039,10 +1030,10 @@ in 1 icmp - echoreq 1 + echoreq wanip @@ -1058,20 +1049,20 @@ + nat_618812d37b8193.31302503 + wan + inet + keep state + + + tcp 1 - wan - keep state - tcp - inet
x3690_3
22 - - - nat_618812d37b8193.31302503 root@192.168.1.118 @@ -1079,20 +1070,20 @@ + nat_64fa19f4acba11.80049900 + wan + inet + keep state + + + tcp 1 - wan - keep state - tcp - inet
192.168.20.150
7860 - - - nat_64fa19f4acba11.80049900 root@172.12.0.10 @@ -1100,20 +1091,20 @@ + nat_64fb1fbba71e29.76190279 + wan + inet + keep state + + + tcp 1 - wan - keep state - tcp - inet
192.168.20.150
7861 - - - nat_64fb1fbba71e29.76190279 root@172.12.0.10 @@ -1121,20 +1112,20 @@ + nat_64fb1fcea6d8b7.62653343 + wan + inet + keep state + + + tcp 1 - wan - keep state - tcp - inet
192.168.20.150
7862 - - - nat_64fb1fcea6d8b7.62653343 root@172.12.0.10 @@ -1142,20 +1133,20 @@ + nat_64fb1fdb48ff18.28912920 + wan + inet + keep state + + + tcp 1 - wan - keep state - tcp - inet
192.168.20.150
7863 - - - nat_64fb1fdb48ff18.28912920 root@172.12.0.10 @@ -1163,20 +1154,20 @@ + nat_651ffc35e573d9.09092618 + wan + inet + keep state + + + tcp 1 - wan - keep state - tcp - inet
192.168.20.140
22 - - - nat_651ffc35e573d9.09092618 root@172.12.0.11 @@ -1185,19 +1176,19 @@ nat_65aed5a66c4f65.25454286 + wan + inet + keep state + + + tcp 1 - wan - keep state - tcp - inet
192.168.20.140
8081 - - root@192.168.20.100 @@ -1206,19 +1197,19 @@ nat_65aed67d497580.58958916 + wan + inet + keep state + + + tcp 1 - wan - keep state - tcp - inet
192.168.20.160
8080 - - root@192.168.20.100 @@ -1227,19 +1218,19 @@ nat_65aed6961c4ea7.81903986 + wan + inet + keep state + + + tcp 1 - wan - keep state - tcp - inet
192.168.20.160
4000 - - root@192.168.20.100 @@ -1248,19 +1239,19 @@ nat_65f4a5b928c9a7.52477383 + wan + inet + keep state + + + tcp 1 - wan - keep state - tcp - inet
192.168.20.115
5000 - - root@192.168.20.100 @@ -1269,19 +1260,19 @@ nat_662bb59baf7573.98640354 + wan + inet + keep state + Redirecting to someservice1 on somehost9 + + tcp 1 - wan - keep state - tcp - inet
192.168.20.115
11434 - Redirecting to someservice1 on somehost9 - root@192.168.20.100 @@ -1290,19 +1281,19 @@ nat_663c3458b7b5e4.19986620 + wan + inet + keep state + Redirecting to someservice81 on somehost9 + + tcp 1 - wan - keep state - tcp - inet
192.168.20.115
27017 - Redirecting to someservice81 on somehost9 - root@172.12.0.8 @@ -1311,19 +1302,19 @@ nat_663d85b2e3b364.53108170 + wan + inet + keep state + Redirecting to someservice858 on somehost545 + + tcp 1 - wan - keep state - tcp - inet
192.168.20.163
8888 - Redirecting to someservice858 on somehost545 - root@172.12.0.10 @@ -1332,19 +1323,19 @@ nat_666c8932142ed6.34062700 + wan + inet + keep state + Redirecting to someservice858 on somehost9 + + tcp 1 - wan - keep state - tcp - inet
192.168.20.115
8888 - Redirecting to someservice858 on somehost9 - root@192.168.20.100 @@ -1353,19 +1344,19 @@ nat_667cd97504d870.57128970 + wan + inet + keep state + Redirecting to someservice858 on somehost545 + + tcp 1 - wan - keep state - tcp - inet
192.168.20.163
8889 - Redirecting to someservice858 on somehost545 - root@172.12.0.8 @@ -1401,9 +1392,9 @@ pass + lan inet Default allow LAN to any rule - lan lan @@ -1413,9 +1404,9 @@ pass + lan inet6 Default allow LAN IPv6 to any rule - lan lan @@ -1474,19 +1465,19 @@ nat_6709763b6a6748.85579760 + wan + inet + keep state + port forwarding for reconfig of someservice2 somehost3 + + tcp 1 - wan - keep state - tcp - inet
192.168.20.132
55555 - port forwarding for reconfig of someservice2 somehost3 - root@172.12.0.12 @@ -1496,19 +1487,19 @@ nat_670979b3279551.73601303 + wan + inet + keep state + port forwarding for virtual ip for someservice2 servers + + tcp 1 - wan - keep state - tcp - inet
192.168.20.1
55555 - port forwarding for virtual ip for someservice2 servers - root@172.12.0.12