syslog-ng_logstash

Notes: 

1) This was copied from a lot of different places and some of it may not (probably is not) the best way to do this. It has been working for me though ;) 

2) I am running apache as a proxy so that I can provide encryption to the clients, this is why elasticsearch is bound to localhost.

3) If you will be using Kibana you don't want to use "remove_field", you will need those.

This is what syslog-ng looks like on the sensor:

source s_bro_conn { file("/nsm/bro/logs/current/conn.log" flags(no-parse) program_override("bro_conn")); };
source s_bro_http { file("/nsm/bro/logs/current/http.log" flags(no-parse) program_override("bro_http")); };
source s_bro_dns { file("/nsm/bro/logs/current/dns.log" flags(no-parse) program_override("bro_dns")); };
source s_bro_software { file("/nsm/bro/logs/current/software.log" flags(no-parse) program_override("bro_software")); };
source s_bro_ssl { file("/nsm/bro/logs/current/ssl.log" flags(no-parse) program_override("bro_ssl")); };
source s_bro_ssh { file("/nsm/bro/logs/current/ssh.log" flags(no-parse) program_override("bro_ssh")); };
source s_bro_notice { file("/nsm/bro/logs/current/notice.log" flags(no-parse) program_override("bro_notice")); };
source s_bro_files { file("/nsm/bro/logs/current/files.log" flags(no-parse) program_override("bro_files")); };
source s_bro_smtp { file("/nsm/bro/logs/current/smtp.log" flags(no-parse) program_override("bro_smtp")); };
source s_prads { file("/nsm/prads/prads-asset.log" flags(no-parse) program_override("prads")); };

destination d_logs { tcp("10.0.0.1" port(514)); };

log { source(s_bro_conn); destination(d_logs); };
log { source(s_bro_http); destination(d_logs); };
log { source(s_bro_dns); destination(d_logs); };
log { source(s_bro_software); destination(d_logs); };
log { source(s_bro_ssl); destination(d_logs); };
log { source(s_bro_ssh); destination(d_logs); };
log { source(s_bro_notice); destination(d_logs); };
log { source(s_bro_files); destination(d_logs); };
log { source(s_bro_smtp); destination(d_logs); };
log { source(s_prads); destination(d_logs); };


This is what syslog-ng looks like on the collector (10.0.0.1):

### Sources

source s_local {
  system();
  internal();
};

source net_udp {
  udp();
};

source net_tcp {
  tcp();
};

### Destinations

destination d_local {
  file("/var/log/messages");
};

# Windows

destination d_event_logs {
  file("/var/log/log_windows");
};

# Bro

destination d_bro_log_conn {
  file("/var/log/log_bro_conn");
};

destination d_bro_log_dns {
  file("/var/log/log_bro_dns");
};

destination d_bro_log_http {
  file("/var/log/log_bro_http");
};

destination d_bro_log_software {
  file("/var/log/log_bro_software");
};

destination d_bro_log_ssl {
  file("/var/log/log_bro_ssl");
};

destination d_bro_log_notice {
  file("/var/log/log_bro_notice");
};

destination d_bro_log_files {
  file("/var/log/log_bro_files");
};

destination d_bro_log_smtp {
  file("/var/log/log_bro_smtp");
};

destination d_bro_log_ssh {
  file("/var/log/log_bro_ssh");
};

# Barracuda
destination d_spam_logs {
  file("/var/log/log_spam");
};

# Prads
destination d_prads {
  file("/var/log/prads");
};

### Filters

# Windows
filter f_event_logs {
  host(dc1) or
  host(dc2) or
  host(dc3) or
  host(dc4) or
  host(dc5) or
  host(dc6) or
  host(dc7) or
  host(dc8) or
  host(dc9) or
  host(dc10);
};

# Bro
filter f_bro_log_conn {
  program("bro_conn");
};

filter f_bro_log_dns {
  program("bro_dns");
};

filter f_bro_log_http {
  program("bro_http");
};

filter f_bro_log_software {
  program("bro_software");
};

filter f_bro_log_ssl {
  program("bro_ssl");
};

filter f_bro_log_notice {
  program("bro_notice");
};

filter f_bro_log_files {
  program("bro_files");
};

filter f_bro_log_smtp {
  program("bro_smtp");
};

filter f_bro_log_ssh {
  program("bro_ssh");
};

# Barracuda
filter f_spam_logs {
  netmask(192.168.1.1/32) or
  netmask(192.168.1.2/32);
};

# Prads
filter f_prads {
  program("prads");
};

### Loggers

log {
  source(s_local);
  destination(d_local);
};

# Windows
log {
  source(net_udp); filter(f_event_logs); destination(d_event_logs);
};

# Bro
log {
  source(net_tcp); filter(f_bro_log_conn); destination(d_bro_log_conn);
};

log {
  source(net_tcp); filter(f_bro_log_dns); destination(d_bro_log_dns);
};

log {
  source(net_tcp); filter(f_bro_log_http); destination(d_bro_log_http);
};

log {
  source(net_tcp); filter(f_bro_log_software); destination(d_bro_log_software);
};

log {
  source(net_tcp); filter(f_bro_log_ssl); destination(d_bro_log_ssl);
};

log {
  source(net_tcp); filter(f_bro_log_notice); destination(d_bro_log_notice);
};

log {
  source(net_tcp); filter(f_bro_log_files); destination(d_bro_log_files);
};

log {
  source(net_tcp); filter(f_bro_log_smtp); destination(d_bro_log_smtp);
};

log {
  source(net_tcp); filter(f_bro_log_ssh); destination(d_bro_log_ssh);
};

# Barracuda
log {
  source(net_udp); filter(f_spam_logs); destination(d_spam_logs);
};

# Prads
log {
  source(net_tcp); filter(f_prads); destination(d_prads);
};

This is what logstash.conf looks like on the collector:

input {

  # Windows

  file {
    path => "/var/log/log_windows"
    type => windows
  }

  # PRADS

  file {
    path => "/var/log/prads"
    type => prads
  }

  # Bro

  file {
    path => "/var/log/log_bro_conn"
    type => bro_conn
  }
  file {
    path => "/var/log/log_bro_dns"
    type => bro_dns
  }
  file {
    path => "/var/log/log_bro_files"
    type => bro_files
  }
  file {
    path => "/var/log/log_bro_http"
    type => bro_http
  }
  file {
    path => "/var/log/log_bro_notice"
    type => bro_notice
  }
  file {
    path => "/var/log/log_bro_software"
    type => bro_software
  }
  file {
    path => "/var/log/log_bro_ssh"
    type => bro_ssh
  }
  file {
    path => "/var/log/log_bro_ssl"
    type => bro_ssl
  }
}

filter {

  ## Bro Conn

  if [type] == "bro_conn" {
    grok {
      match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<src_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dst_port>(.*?))\t(?<proto>(.*?))\t(?<service>(.*?))\t(?<duration>(.*?))\t(?<orig_bytes>(.*?))\t(?<resp_bytes>(.*?))\t(?<conn_state>(.*?))\t(?<local_orig>(.*?))\t(?<missed_bytes>(.*?))\t(?<history>(.*?))\t(?<orig_pkts>(.*?))\t(?<orig_ip_bytes>(.*?))\t(?<resp_pkts>(.*?))\t(?<resp_ip_bytes>(.*?))\t(?<tunnel_parents>(.*))"]
      remove_field => [ "[message]", "[@version]", "[host]", "[path]" ]
    }
  }

  ## Bro DNS

  if [type] == "bro_dns" {
    grok {
      match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<src_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dst_port>(.*?))\t(?<proto>(.*?))\t(?<trans_id>(.*?))\t(?<query>(.*?))\t(?<qclass>(.*?))\t(?<qclass_name>(.*?))\t(?<qtype>(.*?))\t(?<qtype_name>(.*?))\t(?<rcode>(.*?))\t(?<rcode_name>(.*?))\t(?<AA>(.*?))\t(?<TC>(.*?))\t(?<RD>(.*?))\t(?<RA>(.*?))\t(?<Z>(.*?))\t(?<answers>(.*?))\t(?<TTLs>(.*?))\t(?<rejected>(.*))"]
      remove_field => [ "[message]", "[@version]", "[host]", "[path]" ]
    }
  }

  ## Bro Files

  if [type] == "bro_files" {
    grok {
      match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<fuid>(.*?))\t(?<src_ip>(.*?))\t(?<dst_ip>(.*?))\t(?<conn_uids>(.*?))\t(?<source>(.*?))\t(?<depth>(.*?))\t(?<analyzers>(.*?))\t(?<mime_type>(.*?))\t(?<filename>(.*?))\t(?<duration>(.*?))\t(?<local_orig>(.*?))\t(?<is_orig>(.*?))\t(?<seen_bytes>(.*?))\t(?<total_bytes>(.*?))\t(?<missing_bytes>(.*?))\t(?<overflow_bytes>(.*?))\t(?<timed_out>(.*?))\t(?<parent_fuid>(.*?))\t(?<md5>(.*?))\t(?<sha1>(.*?))\t(?<sha256>(.*?))\t(?<extracted>(.*))"]
      remove_field => [ "[message]", "[@version]", "[host]", "[path]" ]
    }
  }

  ## Bro HTTP

  if [type] == "bro_http" {
    grok {
      match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<source_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dest_port>(.*?))\t(?<trans_depth>(.*?))\t(?<method>(.*?))\t(?<domain>(.*?))\t(?<uri>(.*?))\t(?<referrer>(.*?))\t(?<user_agent>(.*?))\t(?<request_body_len>(.*?))\t(?<response_body_len>(.*?))\t(?<status_code>(.*?))\t(?<status_msg>(.*?))\t(?<info_code>(.*?))\t(?<info_msg>(.*?))\t(?<filename>(.*?))\t(?<tags>(.*?))\t(?<username>(.*?))\t(?<password>(.*?))\t(?<proxied>(.*?))\t(?<orig_fuids>(.*?))\t(?<orig_mime_types>(.*?))\t(?<resp_fuids>(.*?))\t(?<resp_mime_types>(.*))"]
      remove_field => [ "[message]", "[@version]", "[host]", "[path]" ]
    }

    mutate {
      add_field    => ["domain_short", "%{domain}"]
    }

    mutate {
      gsub => ["domain_short", ".*\.(.*\.{1}\w*$)", "\1"]
    }
  }

  ## Bro Notice

  if [type] == "bro_notice" {
    grok {
      match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<src_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dst_port>(.*?))\t(?<fuid>(.*?))\t(?<file_mime_type>(.*?))\t(?<file_desc>(.*?))\t(?<proto>(.*?))\t(?<note>(.*?))\t(?<msg>(.*?))\t(?<sub>(.*?))\t(?<src>(.*?))\t(?<dst>(.*?))\t(?<p>(.*?))\t(?<n>(.*?))\t(?<peer_descr>(.*?))\t(?<actions>(.*?))\t(?<suppress_for>(.*?))\t(?<dropped>(.*?))\t(?<remote_location.country_code>(.*?))\t(?<remote_location.region>(.*?))\t(?<remote_location.city>(.*?))\t(?<remote_location.latitude>(.*?))\t(?<remote_location.longitude>(.*))"]
      remove_field => [ "[message]", "[@version]", "[host]", "[path]", "[remote_location.country_code]", "[remote_location.region]", "[remote_location.city]", "[remote_location.latitude", "[remote_location.longitude]" ]
    }
  }

  ## Bro Software

  if [type] == "bro_software" {
    grok {
      match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<host>(.*?))\t(?<host_p>(.*?))\t(?<software_type>(.*?))\t(?<name>(.*?))\t(?<version.major>(.*?))\t(?<version.minor>(.*?))\t(?<version.minor2>(.*?))\t(?<version.minor3>(.*?))\t(?<version.addl>(.*?))\t(?<unparsed_version>(.*))"]
      remove_field => [ "[message]", "[@version]", "[host]", "[path]" ]
    }
  }

  ## Bro SSH

  if [type] == "bro_ssh" {
    grok {
      match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<src_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dst_port>(.*?))\t(?<status>(.*?))\t(?<direction>(.*?))\t(?<client>(.*?))\t(?<server>(.*?))\t(?<remote_location.country_code>(.*?))\t(?<remote_location.region>(.*?))\t(?<remote_location.city>(.*?))\t(?<remote_location.latitude>(.*?))\t(?<remote_location.longitude>(.*))"]
      remove_field => [ "[message]", "[@version]", "[host]", "[path]", "[remote_location.country_code]", "[remote_location.region]", "[remote_location.city]", "[remote_location.latitude]", "[remote_location.longitude]" ]
    }
  }

  ## Bro SSL

  if [type] == "bro_ssl" {
    grok {
      match => ["message", "(?<timestamp>(\d{10}\.\d{6}))\t(?<uid>(.*?))\t(?<src_ip>(.*?))\t(?<src_port>(.*?))\t(?<dst_ip>(.*?))\t(?<dst_port>(.*?))\t(?<version>(.*?))\t(?<cipher>(.*?))\t(?<server_name>(.*?))\t(?<session_id>(.*?))\t(?<subject>(.*?))\t(?<issuer_subject>(.*?))\t(?<not_valid_before>(.*?))\t(?<not_valid_after>(.*?))\t(?<last_alert>(.*?))\t(?<client_subject>(.*?))\t(?<client_issuer_subject>(.*?))\t(?<cert_hash>(.*?))\t(?<validation_status>(.*))"]
      remove_field => [ "[message]", "[@version]", "[host]", "[path]" ]
    }
  }

  ## PRADS

    if [type] == "prads" {
    grok {
      match => ["message", "(?<asset>([\w\.:]+)),(?<vlan>([\d]{1,4})),(?<port>([\d]{1,5})),(?<proto>([\d]{1,3})),(?<service>(\S+?)),(?<service-info>\[(.*)\]),(?<distance>([\d]{1,3})),(?<timestamp>(\d{10}))"]
      remove_field => [ "[message]", "[@version]", "[host]", "[path]" ]
    }
  }

  # Add geo to any src_ip or dst_ip

  if [src_ip] =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ {
    geoip {
      source       => "src_ip"
      target       => "src_geoip"
      fields       => [ "city_name", "country_code2", "country_name", "longitude", "latitude" ]
      add_field    => [ "src_cc", "%{[src_geoip][country_code2]}" ]
      add_field    => [ "src_city", "%{[src_geoip][city_name]}" ]
      add_field    => [ "src_country", "%{[src_geoip][country_name]}" ]
      add_field    => [ "src_coordinates", "%{[src_geoip][longitude]},%{[src_geoip][latitude]}" ]
      remove_field => [ "src_geoip"]
    }
  }

  if [dst_ip] =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ {
    geoip {
      source       => "dst_ip"
      target       => "dst_geoip"
      fields       => [ "city_name", "country_code2", "country_name", "longitude", "latitude" ]
      add_field    => [ "dst_cc", "%{[dst_geoip][country_code2]}" ]
      add_field    => [ "dst_city", "%{[dst_geoip][city_name]}" ]
      add_field    => [ "dst_country", "%{[dst_geoip][country_name]}" ]
      add_field    => [ "dst_coordinates", "%{[dst_geoip][longitude]},%{[dst_geoip][latitude]}" ]
      remove_field => [ "dst_geoip"]
    }
  }
}

output {
  elasticsearch {
    embedded => true
    host => "127.0.0.1"
    bind_host => "127.0.0.1"
  }
}



Comments