Page MenuHomePhabricator

Upgrade Fastnetmon to 1.2.1
Closed, ResolvedPublic

Description

Same as T257035.

1.1.8 changelog: https://github.com/pavel-odintsov/fastnetmon/releases/tag/v1.1.8
1.1.9 changelog: https://github.com/pavel-odintsov/fastnetmon/releases/tag/v1.1.9
1.2.0 changelog: https://github.com/pavel-odintsov/fastnetmon/releases/tag/v1.2.0
1.2.1 changelog: https://github.com/pavel-odintsov/fastnetmon/releases/tag/v1.2.1

Most notable is the addition of v6 support and performance improvements.

Nothing of direct interest for us in 1.2.1

Related Objects

Event Timeline

ayounsi created this task.
Restricted Application added a subscriber: Aklapper. · View Herald Transcript
ayounsi renamed this task from Upgrade Fastnetmon to 1.1.9 to Upgrade Fastnetmon to 1.2.0.Feb 8 2021, 10:33 AM
ayounsi updated the task description. (Show Details)
ayounsi renamed this task from Upgrade Fastnetmon to 1.2.0 to Upgrade Fastnetmon to 1.2.1.May 16 2022, 7:12 AM
ayounsi updated the task description. (Show Details)

@ayounsi I've built a backport of fastnetmon 1.2.1 for bullseye-wikimedia. It's not yet uploaded to apt.wikimedia.org, let's sync up for some smoke testing when you're back from vacation and find the time.

Mentioned in SAL (#wikimedia-operations) [2022-05-31T07:55:20Z] <moritzm> upgrade fastnetmon on netflow4002 to 1.2.1 T271228

This was the debconf diff for the puppetised fastnetmon.conf as presented by dpkg. We should check whether some new options should be covered in our puppetised config file.

--- /etc/fastnetmon.conf	2021-12-14 09:48:12.185287948 +0000
+++ /etc/fastnetmon.conf.dpkg-new	2022-05-08 12:49:47.000000000 +0000
@@ -1,5 +1,3 @@
-# This file is managed by Puppet
-# Full configuration options at https://github.com/pavel-odintsov/fastnetmon/blob/v1.1.3/src/fastnetmon.conf
 ###
 ### Main configuration params
 ###
@@ -7,32 +5,31 @@
 ### Logging configuration
 
 # enable this option if you want to send logs to local syslog facility
-logging:local_syslog_logging = on
+logging:local_syslog_logging = off
 
 # enable this option if you want to send logs to a remote syslog server via UDP
 logging:remote_syslog_logging = off
 
+# specify a custom server and port for remote logging
+logging:remote_syslog_server = 10.10.10.10
+logging:remote_syslog_port = 514
+
 # Enable/Disable any actions in case of attack
 enable_ban = on
 
+# Enable ban for IPv6
+enable_ban_ipv6 = on
+
 # disable processing for certain direction of traffic
 process_incoming_traffic = on
-process_outgoing_traffic = off
+process_outgoing_traffic = on
 
 # How many packets will be collected from attack traffic
-ban_details_records_count = 1000
+ban_details_records_count = 50
 
 # How long (in seconds) we should keep an IP in blocked state
 # If you set 0 here it completely disables unban capability
-#
-# Every (ban_time/2) seconds, a background thread scans the list of active bans
-# (basically the list of believed-ongoing attacks).  First it checks if the
-# ban is old enough to consider unbanning it.  If so, if unban_only_if_attack_finished
-# is set, then it checks if there's still elevated traffic to that IP.
-#
-# We set a short ban_time so that our Icinga integration can know quickly when
-# an attack has ended.
-ban_time = 120
+ban_time = 1900
 
 # Check if the attack is still active, before triggering an unban callback with this option
 # If the attack is still active, check each run of the unban watchdog
@@ -40,17 +37,20 @@
 
 # enable per subnet speed meters
 # For each subnet, list track speed in bps and pps for both directions
-enable_subnet_counters = on
+enable_subnet_counters = off
 
 # list of all your networks in CIDR format
 networks_list_path = /etc/networks_list
 
+# list networks in CIDR format which will be not monitored for attacks
+white_list_path = /etc/networks_whitelist
+
 # redraw period for client's screen
 check_period = 1
 
 # Connection tracking is very useful for attack detection because it provides huge amounts of information,
 # but it's very CPU intensive and not recommended in big networks
-enable_connection_tracking = off
+enable_connection_tracking = on
 
 # Different approaches to attack detection
 ban_for_pps = on
@@ -58,13 +58,22 @@
 ban_for_flows = off
 
 # Limits for Dos/DDoS attacks
-threshold_pps = 1250000
-threshold_mbps = 6000
+threshold_pps = 20000
+threshold_mbps = 1000
+threshold_flows = 3500
 
 # Per protocol attack thresholds
 # We don't implement per protocol flow limits, sorry :(
 # These limits should be smaller than global pps/mbps limits
 
+threshold_tcp_mbps = 100000
+threshold_udp_mbps = 100000
+threshold_icmp_mbps = 100000
+
+threshold_tcp_pps = 100000
+threshold_udp_pps = 100000
+threshold_icmp_pps = 100000
+
 ban_for_tcp_bandwidth = off
 ban_for_udp_bandwidth = off
 ban_for_icmp_bandwidth = off
@@ -77,37 +86,48 @@
 ### Traffic capture methods
 ###
 
-# PF_RING traffic capture, fast enough but the wirespeed version needs a paid license
-mirror = off
-
 # Netmap traffic capture (very fast but needs patched drivers)
 mirror_netmap = off
 
-# SnabbSwitch traffic capture
-mirror_snabbswitch = off
-
 # AF_PACKET capture engine
 # Please use it only with modern Linux kernels (3.6 and more)
 # And please install birq for irq ditribution over cores
 mirror_afpacket = off
 
+# You can use this option to multiply all incoming traffc by this value
+# It may be useful for sampled mirror ports
+mirror_af_packet_custom_sampling_rate = 1
+
+# AF_PACKET fanout mode mode, http://man7.org/linux/man-pages/man7/packet.7.html
+# Available modes: cpu, lb, hash, random, rollover, queue_mapping 
+mirror_af_packet_fanout_mode = cpu
+
+# This option should be enabled if you are using Juniper with mirroring of the first X bytes of packet: maximum-packet-length 110;
+af_packet_read_packet_length_from_ip_header = off 
+
+# Port mirroring sampling ratio
+netmap_sampling_ratio = 1
+
+# This option should be enabled if you are using Juniper with mirroring of the first X bytes of packet: maximum-packet-length 110;
+netmap_read_packet_length_from_ip_header = off
+
 # Pcap mode, very slow and thus not suitable for production
 pcap = off
 # Netflow capture method with v5, v9 and IPFIX support
-netflow = on
+netflow = off
 # sFLOW capture suitable for switches
 sflow = off
 
-# PF_RING configuration
-# If you have a license for PF_RING ZC, enable this mode and it might achieve wire speed for 10GE
-enable_pf_ring_zc_mode = off
+# Configuration for netmap, mirror, pcap modes
+# For pcap we could specify "any"
+# For Netmap we could specify multiple interfaces separated by comma
+interfaces = eth3,eth4
 
 # We use average values for traffic speed to certain IP and we calculate average over this time slice
-# See https://fastnetmon.com/docs/junos_integration/
-average_calculation_time = 20
+average_calculation_time = 5
 
 # We use average values for traffic speed for subnet and we calculate average over this time slice
-average_calculation_time_for_subnets = 20
+average_calculation_time_for_subnets = 5
 
 # Delay between traffic recalculation attempts
 speed_calculation_delay = 1
@@ -125,18 +145,25 @@
 # Netflow v9 and IPFIX agents use different and very complex approaches for notifying about sample ratio
 # Here you could specify a sampling ratio for all this agents
 # For NetFLOW v5 we extract sampling ratio from packets directely and this option not used
-netflow_sampling_ratio = 1000
+netflow_sampling_ratio = 1
+
+# sFlow configuration
+
+# It's possible to specify multiple ports here, using commas as delimiter
+sflow_port = 6343
+# sflow_port = 6343,6344
+sflow_host = 0.0.0.0
+
+# Some vendors may lie about full packet length in sFlow packet. To avoid this issue we can switch to using IP packet length from parsed header
+sflow_read_packet_length_from_ip_header = off 
 
-# In some cases with NetFlow we could get huge bursts related to aggregated data nature
-# We could try to get smoother data with this option, i.e. we will divide counters on collection interval time
-netflow_divide_counters_on_interval_length = off
 
 ###
 ### Actions when attack detected
 ###
 
 # This script executed for ban, unban and attack detail collection
-notify_script_path = /usr/local/bin/fastnetmon_notify
+notify_script_path = /usr/local/bin/notify_about_attack.sh
 
 # pass attack details to notify_script via stdin
 # Pass details only in case of "ban" call
@@ -146,63 +173,134 @@
 # collect a full dump of the attack with full payload in pcap compatible format
 collect_attack_pcap_dumps = off
 
-# Execute Deep Packet Inspection on captured PCAP packets
-process_pcap_attack_dumps_with_dpi = off
-
 # Save attack details to Redis
 redis_enabled = off
 
+# Redis configuration
+redis_port = 6379
+redis_host = 127.0.0.1
+
+# specify a custom prefix here
+redis_prefix = mydc1
+
 # We could store attack information to MongoDB
 mongodb_enabled = off
+mongodb_host = localhost
+mongodb_port = 27017
+mongodb_database_name = fastnetmon
 
 # announce blocked IPs with BGP protocol with ExaBGP
 exabgp = off
+exabgp_command_pipe = /var/run/exabgp.cmd
+exabgp_community = 65001:666
+
+# specify multiple communities with this syntax:
+# exabgp_community = [65001:666 65001:777]
+
+# specify different communities for host and subnet announces
+# exabgp_community_subnet = 65001:667
+# exabgp_community_host = 65001:668
+
+exabgp_next_hop = 10.0.3.114
 
 # In complex cases you could have both options enabled and announce host and subnet simultaneously
 
+# Announce /32 host itself with BGP
+exabgp_announce_host = on
+
+# Announce origin subnet of IP address instead IP itself
+exabgp_announce_whole_subnet = off
+
 # Announce Flow Spec rules when we could detect certain attack type
 # Please we aware! Flow Spec announce triggered when we collect some details about attack,
 # i.e. when we call attack_details script
 # Please disable exabgp_announce_host and exabgp_announce_whole_subnet if you want to use this feature
-# Please use ExaBGP v4 only (Git version), for more details: https://github.com/FastVPSEestiOu/fastnetmon/blob/master/docs/BGP_FLOW_SPEC.md
+# Please use ExaBGP v4 only (Git version), for more details: https://github.com/pavel-odintsov/fastnetmon/blob/master/docs/BGP_FLOW_SPEC.md
 exabgp_flow_spec_announces = off
 
 # GoBGP intergation
 gobgp = off
 
+# Configuration for IPv4 announces
+gobgp_next_hop = 0.0.0.0
+gobgp_announce_host = on
+gobgp_announce_whole_subnet = off
+
+gobgp_community_host = 65001:666
+gobgp_community_subnet = 65001:777
+
+# Configuration for IPv6 announces
+gobgp_next_hop_ipv6 = 100::1
+gobgp_announce_host_ipv6 = off
+gobgp_announce_whole_subnet_ipv6 = off
+
+gobgp_community_host_ipv6 = 65001:666
+gobgp_community_subnet_ipv6 = 65001:777
+
 # Graphite monitoring
 # InfluxDB is also supported, please check our reference:
-# https://github.com/FastVPSEestiOu/fastnetmon/blob/master/docs/INFLUXDB_INTEGRATION.md
-
+# https://github.com/pavel-odintsov/fastnetmon/blob/master/docs/INFLUXDB_INTEGRATION.md
 graphite = off
-
+# Please use only IP because domain names are not allowed here
+graphite_host = 127.0.0.1
+graphite_port = 2003
 
 # Default namespace for Graphite data
 graphite_prefix = fastnetmon
 
+# How often we export metrics to Graphite
+graphite_push_period = 1
+
+# Before using InfluxDB you need to create database using influx tool:
+# create database fastnetmon
+
+# InfluxDB
+influxdb = off
+influxdb_host = 127.0.0.1
+influxdb_port = 8086
+influxdb_database = fastnetmon
+
+# InfluxDB auth
+influxdb_auth = off
+influxdb_user = fastnetmon
+influxdb_password = secure
+
+# How often we export metrics to InfluxDB
+influxdb_push_period = 1
+
 # Add local IP addresses and aliases to monitoring list
 # Works only for Linux
-monitor_local_ip_addresses = off
+monitor_local_ip_addresses = on
+
+# Add IP addresses for OpenVZ / Virtuozzo VEs to network monitoring list
+monitor_openvz_vps_ip_addresses = off
 
-hostgroup = wmcs_eqiad_gw:185.15.56.1/32
-wmcs_eqiad_gw_enable_ban = on
+# Create group of hosts with non-standard thresholds
+# You should create this group before (in configuration file) specifying any limits
+# hostgroup = my_hosts:10.10.10.221/32,10.10.10.222/32
 
-wmcs_eqiad_gw_ban_for_pps = on
-wmcs_eqiad_gw_threshold_pps = 200000
+# Configure this group
+my_hosts_enable_ban = off
 
-wmcs_eqiad_gw_ban_for_bandwidth = on
-wmcs_eqiad_gw_threshold_mpbs = 1000
+my_hosts_ban_for_pps = off
+my_hosts_ban_for_bandwidth = off
+my_hosts_ban_for_flows = off
 
-wmcs_eqiad_gw_ban_for_flows = off
+my_hosts_threshold_pps = 20000
+my_hosts_threshold_mbps = 1000
+my_hosts_threshold_flows = 3500
 
 # Path to pid file for checking "if another copy of tool is running", it's useful when you run multiple instances of tool
 pid_path = /var/run/fastnetmon.pid
 
-# Path to file where we store information for fastnetmon_client
+# Path to file where we store IPv4 traffic information for fastnetmon_client
 cli_stats_file_path = /tmp/fastnetmon.dat
 
+# Path to file where we store IPv6 traffic information for fastnetmon_client
+cli_stats_ipv6_file_path = /tmp/fastnetmon_ipv6.dat
+
 # Enable gRPC api (required for fastnetmon_api_client tool)
-enable_api = off
+enable_api = on
 
 ###
 ### Client configuration

Mentioned in SAL (#wikimedia-operations) [2022-05-31T08:07:02Z] <moritzm> imported fastnetmon 1.2.1-1~deb11u1 to apt.wikimedia.org T271228

Great, there is nothing of immediate interest in the diff. IPv6 will probably be the next step here in a different task.

Mentioned in SAL (#wikimedia-operations) [2022-05-31T08:35:33Z] <XioNoX> upgrade fastnetmon to 1.2.1 in drmrs - T271228

Mentioned in SAL (#wikimedia-operations) [2022-05-31T08:39:36Z] <XioNoX> upgrade fastnetmon to 1.2.1 in codfw - T271228

left are eqiad/esams/eqsin. I'll take care of them later today or tomorrow.

Mentioned in SAL (#wikimedia-operations) [2022-06-01T10:49:45Z] <XioNoX> upgrade fastnetmon to 1.2.1 in eqiad - T271228

Mentioned in SAL (#wikimedia-operations) [2022-06-01T10:51:45Z] <XioNoX> upgrade fastnetmon to 1.2.1 in esams - T271228

Mentioned in SAL (#wikimedia-operations) [2022-06-01T10:54:30Z] <XioNoX> upgrade fastnetmon to 1.2.1 in eqsin - T271228

ayounsi claimed this task.

All done!