Notes: 1) This was copied from a lot of different places and some of it may not (probably is not) the best way to do this. It has been working for me though ;) 2) I am running apache as a proxy so that I can provide encryption to the clients, this is why elasticsearch is bound to localhost. 3) If you will be using Kibana you don't want to use "remove_field", you will need those. source s_bro_conn { file("/nsm/bro/logs/current/conn.log" flags(no-parse) program_override("bro_conn")); }; source s_bro_http { file("/nsm/bro/logs/current/http.log" flags(no-parse) program_override("bro_http")); }; source s_bro_dns { file("/nsm/bro/logs/current/dns.log" flags(no-parse) program_override("bro_dns")); }; source s_bro_software { file("/nsm/bro/logs/current/software.log" flags(no-parse) program_override("bro_software")); }; source s_bro_ssl { file("/nsm/bro/logs/current/ssl.log" flags(no-parse) program_override("bro_ssl")); }; source s_bro_ssh { file("/nsm/bro/logs/current/ssh.log" flags(no-parse) program_override("bro_ssh")); }; source s_bro_notice { file("/nsm/bro/logs/current/notice.log" flags(no-parse) program_override("bro_notice")); }; source s_bro_files { file("/nsm/bro/logs/current/files.log" flags(no-parse) program_override("bro_files")); }; source s_bro_smtp { file("/nsm/bro/logs/current/smtp.log" flags(no-parse) program_override("bro_smtp")); }; source s_prads { file("/nsm/prads/prads-asset.log" flags(no-parse) program_override("prads")); }; destination d_logs { tcp("10.0.0.1" port(514)); }; log { source(s_bro_conn); destination(d_logs); }; log { source(s_bro_http); destination(d_logs); }; log { source(s_bro_dns); destination(d_logs); }; log { source(s_bro_software); destination(d_logs); }; log { source(s_bro_ssl); destination(d_logs); }; log { source(s_bro_ssh); destination(d_logs); }; log { source(s_bro_notice); destination(d_logs); }; log { source(s_bro_files); destination(d_logs); }; log { source(s_bro_smtp); destination(d_logs); }; log { source(s_prads); destination(d_logs); }; This is what syslog-ng looks like on the collector (10.0.0.1): ### Sources source s_local { system(); internal(); }; source net_udp { udp(); }; source net_tcp { tcp(); }; ### Destinations destination d_local { file("/var/log/messages"); }; # Windows destination d_event_logs { file("/var/log/log_windows"); }; # Bro destination d_bro_log_conn { file("/var/log/log_bro_conn"); }; destination d_bro_log_dns { file("/var/log/log_bro_dns"); }; destination d_bro_log_http { file("/var/log/log_bro_http"); }; destination d_bro_log_software { file("/var/log/log_bro_software"); }; destination d_bro_log_ssl { file("/var/log/log_bro_ssl"); }; destination d_bro_log_notice { file("/var/log/log_bro_notice"); }; destination d_bro_log_files { file("/var/log/log_bro_files"); }; destination d_bro_log_smtp { file("/var/log/log_bro_smtp"); }; destination d_bro_log_ssh { file("/var/log/log_bro_ssh"); }; # Barracuda destination d_spam_logs { file("/var/log/log_spam"); }; # Prads destination d_prads { file("/var/log/prads"); }; ### Filters # Windows filter f_event_logs { host(dc1) or host(dc2) or host(dc3) or host(dc4) or host(dc5) or host(dc6) or host(dc7) or host(dc8) or host(dc9) or host(dc10); }; # Bro filter f_bro_log_conn { program("bro_conn"); }; filter f_bro_log_dns { program("bro_dns"); }; filter f_bro_log_http { program("bro_http"); }; filter f_bro_log_software { program("bro_software"); }; filter f_bro_log_ssl { program("bro_ssl"); }; filter f_bro_log_notice { program("bro_notice"); }; filter f_bro_log_files { program("bro_files"); }; filter f_bro_log_smtp { program("bro_smtp"); }; filter f_bro_log_ssh { program("bro_ssh"); }; # Barracuda filter f_spam_logs { netmask(192.168.1.1/32) or netmask(192.168.1.2/32); }; # Prads filter f_prads { program("prads"); }; ### Loggers log { source(s_local); destination(d_local); }; # Windows log { source(net_udp); filter(f_event_logs); destination(d_event_logs); }; # Bro log { source(net_tcp); filter(f_bro_log_conn); destination(d_bro_log_conn); }; log { source(net_tcp); filter(f_bro_log_dns); destination(d_bro_log_dns); }; log { source(net_tcp); filter(f_bro_log_http); destination(d_bro_log_http); }; log { source(net_tcp); filter(f_bro_log_software); destination(d_bro_log_software); }; log { source(net_tcp); filter(f_bro_log_ssl); destination(d_bro_log_ssl); }; log { source(net_tcp); filter(f_bro_log_notice); destination(d_bro_log_notice); }; log { source(net_tcp); filter(f_bro_log_files); destination(d_bro_log_files); }; log { source(net_tcp); filter(f_bro_log_smtp); destination(d_bro_log_smtp); }; log { source(net_tcp); filter(f_bro_log_ssh); destination(d_bro_log_ssh); }; # Barracuda log { source(net_udp); filter(f_spam_logs); destination(d_spam_logs); }; # Prads log { source(net_tcp); filter(f_prads); destination(d_prads); }; This is what logstash.conf looks like on the collector: input { # Windows file { path => "/var/log/log_windows" type => windows } # PRADS file { path => "/var/log/prads" type => prads } # Bro file { path => "/var/log/log_bro_conn" type => bro_conn } file { path => "/var/log/log_bro_dns" type => bro_dns } file { path => "/var/log/log_bro_files" type => bro_files } file { path => "/var/log/log_bro_http" type => bro_http } file { path => "/var/log/log_bro_notice" type => bro_notice } file { path => "/var/log/log_bro_software" type => bro_software } file { path => "/var/log/log_bro_ssh" type => bro_ssh } file { path => "/var/log/log_bro_ssl" type => bro_ssl } } filter { ## Bro Conn if [type] == "bro_conn" { grok { match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<src_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dst_port>(.*?))\t(?<proto>(.*?))\t(?<service>(.*?))\t(?<duration>(.*?))\t(?<orig_bytes>(.*?))\t(?<resp_bytes>(.*?))\t(?<conn_state>(.*?))\t(?<local_orig>(.*?))\t(?<missed_bytes>(.*?))\t(?<history>(.*?))\t(?<orig_pkts>(.*?))\t(?<orig_ip_bytes>(.*?))\t(?<resp_pkts>(.*?))\t(?<resp_ip_bytes>(.*?))\t(?<tunnel_parents>(.*))"] remove_field => [ "[message]", "[@version]", "[host]", "[path]" ] } } ## Bro DNS if [type] == "bro_dns" { grok { match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<src_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dst_port>(.*?))\t(?<proto>(.*?))\t(?<trans_id>(.*?))\t(?<query>(.*?))\t(?<qclass>(.*?))\t(?<qclass_name>(.*?))\t(?<qtype>(.*?))\t(?<qtype_name>(.*?))\t(?<rcode>(.*?))\t(?<rcode_name>(.*?))\t(?<AA>(.*?))\t(?<TC>(.*?))\t(?<RD>(.*?))\t(?<RA>(.*?))\t(?<Z>(.*?))\t(?<answers>(.*?))\t(?<TTLs>(.*?))\t(?<rejected>(.*))"] remove_field => [ "[message]", "[@version]", "[host]", "[path]" ] } } ## Bro Files if [type] == "bro_files" { grok { match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<fuid>(.*?))\t(?<src_ip>(.*?))\t(?<dst_ip>(.*?))\t(?<conn_uids>(.*?))\t(?<source>(.*?))\t(?<depth>(.*?))\t(?<analyzers>(.*?))\t(?<mime_type>(.*?))\t(?<filename>(.*?))\t(?<duration>(.*?))\t(?<local_orig>(.*?))\t(?<is_orig>(.*?))\t(?<seen_bytes>(.*?))\t(?<total_bytes>(.*?))\t(?<missing_bytes>(.*?))\t(?<overflow_bytes>(.*?))\t(?<timed_out>(.*?))\t(?<parent_fuid>(.*?))\t(?<md5>(.*?))\t(?<sha1>(.*?))\t(?<sha256>(.*?))\t(?<extracted>(.*))"] remove_field => [ "[message]", "[@version]", "[host]", "[path]" ] } } ## Bro HTTP if [type] == "bro_http" { grok { match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<source_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dest_port>(.*?))\t(?<trans_depth>(.*?))\t(?<method>(.*?))\t(?<domain>(.*?))\t(?<uri>(.*?))\t(?<referrer>(.*?))\t(?<user_agent>(.*?))\t(?<request_body_len>(.*?))\t(?<response_body_len>(.*?))\t(?<status_code>(.*?))\t(?<status_msg>(.*?))\t(?<info_code>(.*?))\t(?<info_msg>(.*?))\t(?<filename>(.*?))\t(?<tags>(.*?))\t(?<username>(.*?))\t(?<password>(.*?))\t(?<proxied>(.*?))\t(?<orig_fuids>(.*?))\t(?<orig_mime_types>(.*?))\t(?<resp_fuids>(.*?))\t(?<resp_mime_types>(.*))"] remove_field => [ "[message]", "[@version]", "[host]", "[path]" ] } mutate { add_field => ["domain_short", "%{domain}"] } mutate { gsub => ["domain_short", ".*\.(.*\.{1}\w*$)", "\1"] } } ## Bro Notice if [type] == "bro_notice" { grok { match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<src_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dst_port>(.*?))\t(?<fuid>(.*?))\t(?<file_mime_type>(.*?))\t(?<file_desc>(.*?))\t(?<proto>(.*?))\t(?<note>(.*?))\t(?<msg>(.*?))\t(?<sub>(.*?))\t(?<src>(.*?))\t(?<dst>(.*?))\t(?<p>(.*?))\t(?<n>(.*?))\t(?<peer_descr>(.*?))\t(?<actions>(.*?))\t(?<suppress_for>(.*?))\t(?<dropped>(.*?))\t(?<remote_location.country_code>(.*?))\t(?<remote_location.region>(.*?))\t(?<remote_location.city>(.*?))\t(?<remote_location.latitude>(.*?))\t(?<remote_location.longitude>(.*))"] remove_field => [ "[message]", "[@version]", "[host]", "[path]", "[remote_location.country_code]", "[remote_location.region]", "[remote_location.city]", "[remote_location.latitude", "[remote_location.longitude]" ] } } ## Bro Software if [type] == "bro_software" { grok { match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<host>(.*?))\t(?<host_p>(.*?))\t(?<software_type>(.*?))\t(?<name>(.*?))\t(?<version.major>(.*?))\t(?<version.minor>(.*?))\t(?<version.minor2>(.*?))\t(?<version.minor3>(.*?))\t(?<version.addl>(.*?))\t(?<unparsed_version>(.*))"] remove_field => [ "[message]", "[@version]", "[host]", "[path]" ] } } ## Bro SSH if [type] == "bro_ssh" { grok { match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<src_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dst_port>(.*?))\t(?<status>(.*?))\t(?<direction>(.*?))\t(?<client>(.*?))\t(?<server>(.*?))\t(?<remote_location.country_code>(.*?))\t(?<remote_location.region>(.*?))\t(?<remote_location.city>(.*?))\t(?<remote_location.latitude>(.*?))\t(?<remote_location.longitude>(.*))"] remove_field => [ "[message]", "[@version]", "[host]", "[path]", "[remote_location.country_code]", "[remote_location.region]", "[remote_location.city]", "[remote_location.latitude]", "[remote_location.longitude]" ] } } ## Bro SSL if [type] == "bro_ssl" { grok { match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<src_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dst_port>(.*?))\t(?<version>(.*?))\t(?<cipher>(.*?))\t(?<server_name>(.*?))\t(?<session_id>(.*?))\t(?<subject>(.*?))\t(?<issuer_subject>(.*?))\t(?<not_valid_before>(.*?))\t(?<not_valid_after>(.*?))\t(?<last_alert>(.*?))\t(?<client_subject>(.*?))\t(?<client_issuer_subject>(.*?))\t(?<cert_hash>(.*?))\t(?<validation_status>(.*))"] remove_field => [ "[message]", "[@version]", "[host]", "[path]" ] } } ## PRADS if [type] == "prads" { grok { match => ["message", "(?<asset>([\w\.:]+)),(?<vlan>([\d]{1,4})),(?<port>([\d]{1,5})),(?<proto>([\d]{1,3})),(?<service>(\S+?)),(?<service-info>\[(.*)\]),(?<distance>([\d]{1,3})),(?<timestamp>(\d{10}))"] remove_field => [ "[message]", "[@version]", "[host]", "[path]" ] } } # Add geo to any src_ip or dst_ip if [src_ip] =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ { geoip { source => "src_ip" target => "src_geoip" fields => [ "city_name", "country_code2", "country_name", "longitude", "latitude" ] add_field => [ "src_cc", "%{[src_geoip][country_code2]}" ] add_field => [ "src_city", "%{[src_geoip][city_name]}" ] add_field => [ "src_country", "%{[src_geoip][country_name]}" ] add_field => [ "src_coordinates", "%{[src_geoip][longitude]},%{[src_geoip][latitude]}" ] remove_field => [ "src_geoip"] } } if [dst_ip] =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ { geoip { source => "dst_ip" target => "dst_geoip" fields => [ "city_name", "country_code2", "country_name", "longitude", "latitude" ] add_field => [ "dst_cc", "%{[dst_geoip][country_code2]}" ] add_field => [ "dst_city", "%{[dst_geoip][city_name]}" ] add_field => [ "dst_country", "%{[dst_geoip][country_name]}" ] add_field => [ "dst_coordinates", "%{[dst_geoip][longitude]},%{[dst_geoip][latitude]}" ] remove_field => [ "dst_geoip"] } } } output { elasticsearch { embedded => true host => "127.0.0.1" bind_host => "127.0.0.1" } } |