nginx-naxsi-ui 1.4.6-1ubuntu3 / usr / share / nginx-naxsi / naxsi-ui / nx_extract.py

#!/usr/bin/env python

from ConfigParser import ConfigParser
from twisted.web import http
from twisted.internet import protocol
from twisted.internet import reactor, threads
from twisted.web.server import Site
from twisted.web.static import File
from twisted.web.resource import Resource
from twisted.web.error import NoResource
from zope.interface import implements
from twisted.cred.portal import IRealm, Portal
from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse, ICredentialsChecker
from twisted.web.guard import HTTPAuthSessionWrapper, DigestCredentialFactory
from twisted.web.resource import IResource
from twisted.cred import credentials
from ordereddict import OrderedDict # don't lose compatibility with python < 2.7

import SQLWrapper
import pprint
import re
import getopt
import sys
import datetime
import time
import cgi
import os


glob_allow=True
glob_rules_file="/etc/nginx/naxsi_core.rules"
glob_conf_file = ''
glob_username = ''
glob_pass = ''
glob_fileList = []

class rules_extractor(object):
   def __init__(self, page_hit, rules_hit, rules_file, conf_file='naxsi-ui.conf'):
       
       self.wrapper = SQLWrapper.SQLWrapper(glob_conf_file)
       self.wrapper.connect()
       self.wrapper.setRowToDict()
           
       self.rules_list = []
       self.final_rules = []
       self.base_rules = []
       self.page_hit = page_hit
       self.rules_hit = rules_hit
       self.core_msg = {}
       self.extract_core(glob_rules_file)
       
   def extract_core(self, rules_file):
      try:
         fd = open(glob_rules_file, 'r')
         for i in fd:
            if i.startswith('MainRule'):
               pos = i.find('id:')
               pos_msg = i.find('msg:')
               self.core_msg[i[pos + 3:i[pos + 3].find(';') - 1]] = i[pos_msg + 4:][:i[pos_msg + 4:].find('"')]
         fd.close()
      except:
         print ("Unable to open rules file.")
         pass

   def gen_basic_rules(self,url=None, srcip=None, dsthost=None,
                rule_id=None, exception_md5=None,
                exception_id=None):

     tmp_rules = []
     #self.rules_list = self.wrapper.getWhitelist()     
     self.base_rules = self.rules_list[:]
#     pprint.pprint(self.base_rules)

   def transform_to_dict(self, l):
      d = {}
      for i in l:
         if not d.has_key(i[0]):
            d[i[0]] = []
         d[i[0]].append(i[1])
      #elimininate duplicate ids in each value
      for i in d:
         d[i] = list(set(d[i]))
      return d


   def get_partial_match_dict(self, d, to_find):
      for i, current_dict in enumerate(d):
        if all(key in current_dict and current_dict[key] == val 
                for key, val in to_find.iteritems()):
            return i


   def opti_rules_back(self):
      # rules of requests extracting optimized whitelists, from 
      # more restrictive to less restrictive.
      opti_select_DESC = [
         # select on url+var_name+zone+rule_id
         ("select  count(*) as ct, e.rule_id, e.zone, e.var_name, u.url, count(distinct c.peer_ip) as peer_count, "
          "(select count(distinct peer_ip) from connections) as ptot, "
          "(select count(*) from connections) as tot "
          "from exceptions as e, urls as u, connections as c where c.url_id "
          "= u.url_id and c.id_exception = e.exception_id GROUP BY u.url, e.var_name,"
          "e.zone, e.rule_id HAVING (ct) > ((select count(*) from connections)/1000)"),
         # select on var_name+zone+rule_id (unpredictable URL)
         ("select  count(*) as ct, e.rule_id, e.zone, e.var_name, '' as url, count(distinct c.peer_ip) as peer_count, "
          "(select count(distinct peer_ip) from connections) as ptot, "
          "(select count(*) from connections) as tot "
          "from exceptions as e, urls as u, connections as c where c.url_id = u.url_id and c.id_exception = "
          "e.exception_id GROUP BY e.var_name,  e.zone, e.rule_id HAVING (ct) > "
          "((select count(*) from connections)/1000)"),
         # select on zone+url+rule_id (unpredictable arg_name)
         ("select  count(*) as ct, e.rule_id, e.zone, '' as var_name, u.url, count(distinct c.peer_ip) as peer_count, "
          "(select count(distinct peer_ip) from connections) as ptot, "
          "(select count(*) from connections) as tot "
          "from exceptions as e, urls as u, connections as c where c.url_id "
          "= u.url_id and c.id_exception = e.exception_id GROUP BY u.url, "
          "e.zone, e.rule_id HAVING (ct) > ((select count(*) from connections)/1000)"),
        # select on zone+url+var_name (unpredictable id)
         ("select  count(*) as ct, 0 as rule_id, e.zone, e.var_name, u.url, count(distinct c.peer_ip) as peer_count, "
          "(select count(distinct peer_ip) from connections) as ptot, "
          "(select count(*) from connections) as tot "
          "from exceptions as e, urls as u, connections as c where c.url_id "
          "= u.url_id and c.id_exception = e.exception_id GROUP BY u.url, "
          "e.zone, e.var_name HAVING (ct) > ((select count(*) from connections)/1000)")
         ]
      
      for req in opti_select_DESC:
         self.wrapper.execute(req)
         res = self.wrapper.getResults()
         for r in res:
            #r += "# total_count:"+str(i['count'])+" ("+str(round((i['count'] / float(i['total'])) * 100,2))+"% of total) peer_count:"+str(i['peer_count'])+"\n"
            if len(r['var_name']) > 0:
               self.try_append({'url': r['url'], 'rule_id': r['rule_id'], 'zone': r['zone'],  'var_name': r['var_name'], 
                                'hcount':  r['ct'], 'htotal': r['tot'], 'pcount':r['peer_count'], 'ptotal':r['ptot'],
                                'pratio': round((r['peer_count'] / float(r['ptot'])) * 100,2),
                                'hratio': round((r['ct'] / float(r['tot'])) * 100,2)
                                })
            else:
               self.try_append({'url': r['url'], 'rule_id': r['rule_id'], 'zone': r['zone'], 'var_name': '', 
                                'hcount': r['ct'],  'htotal': r['tot'], 'ptotal':r['ptot'],
                                'pratio': round((r['peer_count'] / float(r['ptot'])) * 100,2),
                                'hratio': round((r['ct'] / float(r['tot'])) * 100,2),
                                'pcount':r['peer_count']})
      return self.base_rules, self.final_rules

#returns true if whitelist 'target' is already handled by final_rules
#does a dummy comparison and compares the counters
   def try_append(self, target, delmatch=False):
      count=0
      nb_rule=0
      for z in self.final_rules[:]:
         if len(target['url']) > 0 and len(z['url']) > 0 and target['url'] != z['url']:
            continue
         if target['rule_id'] != 0 and z['rule_id'] != 0 and target['rule_id'] != z['rule_id']:
            continue
         if len(target['zone']) > 0 and len(z['zone']) > 0 and target['zone'] != z['zone']:
            continue
         if len(target['var_name']) > 0 and len(z['var_name']) > 0 and target['var_name'] != z['var_name']:
            continue
         if delmatch is True:
            self.final_rules.remove(z)
         else:
            nb_rule += 1
            count += int(z['hcount'])
      if delmatch is True:
         return
      if (target['hcount'] > count) or (target['hcount'] >= count and nb_rule > self.rules_hit):
         pprint.pprint(target)
         self.try_append(target, True)
         self.final_rules.append(target)
         return

   def generate_stats(self):
      stats = ""
      self.wrapper.execute("select count(distinct exception_id) as uniq_exception from exceptions")
      uniq_ex = self.wrapper.getResults()[0]['uniq_exception']
      self.wrapper.execute("select count(distinct peer_ip) as uniq_peer from connections")
      uniq_peer = self.wrapper.getResults()[0]['uniq_peer']
      return "<ul><li>There is currently %s unique exceptions.</li></ul><ul><li>There is currently %s different peers that triggered rules.</li></ul>" % (uniq_ex, uniq_peer)


class NaxsiUI(Resource):
   def __init__(self):
      Resource.__init__(self)
      #twisted will handle static content for me
      self.putChild('bootstrap', File('./bootstrap'))
      self.putChild('js', File('./js'))
      #make the correspondance between the path and the object to call
      self.page_handler = {'/' : Index, '/graphs': GraphView, '/get_rules': GenWhitelist, '/map': WootMap}

   def getChild(self, name, request):
      handler = self.page_handler.get(request.path)
      if handler is not None:
         return handler()
      else:
         return NoResource()
         

class Index(Resource):
   def __init__(self):
      Resource.__init__(self)
      self.ex = rules_extractor(0,0, None)

   def render_GET(self, request):
      fd = open('index.tpl', 'r')
      helpmsg = ''
      for i in fd:
         helpmsg += i
      fd.close()
      helpmsg = helpmsg.replace('__STATS__', self.ex.generate_stats())
      helpmsg = helpmsg.replace('__HOSTNAME__', request.getHeader('Host'))
      return helpmsg


class WootMap(Resource):
   isLeaf = True
   def __init__(self):
      self.has_geoip = False
      try:
         import GeoIP
         self.has_geoip = True
      except:
         print "No GeoIP module, no map"
         return
      Resource.__init__(self)
      self.ex = rules_extractor(0,0, None)
      self.gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
   def render_GET(self, request):
      if self.has_geoip is False:
         return "No GeoIP module/database installed."
      render = open('map.tpl').read()
      self.ex.wrapper.execute('select peer_ip as p, count(*) as c from connections group by peer_ip')
      ips = self.ex.wrapper.getResults()
      fd = open("country2coords.txt", "r")
      bycn = {}
      for ip in ips:
         country = self.gi.country_code_by_addr(ip['p'])
         if country is None or len(country) < 2:
            country = "CN"
         if country not in bycn:
            bycn[country] = {'count': int(ip['c']), 'coords': ''}
            fd.seek(0)
            for cn in fd:
               if country in cn:
                  bycn[country]['coords'] = cn[len(country)+1:-1]
                  break
            if len(bycn[country]['coords']) < 1:
               bycn[country]['coords'] = "37.090240,-95.7128910"
         else:
            bycn[country]['count'] += ip['c']
            pprint.pprint(bycn[country])
      base_array = 'citymap["__CN__"] = {center: new google.maps.LatLng(__COORDS__), population: __COUNT__};\n'
      citymap = ''
      for cn in bycn.keys():
         citymap += base_array.replace('__CN__', cn).replace('__COORDS__', bycn[cn]['coords']).replace('__COUNT__', 
                                                                                                       str(bycn[cn]['count']))
      render = render.replace('__CITYMAP__', citymap)
      return render
   
class GraphView(Resource):
   isLeaf = True
   
   def __init__(self):
      Resource.__init__(self)
      self.ex = rules_extractor(0,0, None)


   def render_GET(self, request):

      fd = open('graphs.tpl')
      html = ''
      for i in fd:
         html += i
      fd.close()
      
      array_excep, _ = self.build_js_array()
      sqli_array, sql_count = self.build_js_array(1000, 1099)
      xss_array, xss_count = self.build_js_array(1300, 1399)
      rfi_array, rfi_count = self.build_js_array(1100, 1199)
      upload_array, upload_count = self.build_js_array(1500, 1599)
      dt_array, dt_count = self.build_js_array(1200, 1299)
      evade_array, evade_count = self.build_js_array(1400, 1499)
      intern_array, intern_count = self.build_js_array(0, 10)

      self.ex.wrapper.execute('select peer_ip as ip, count(id_exception) as c from connections group by peer_ip order by count(id_exception) DESC limit 10')
      top_ten = self.ex.wrapper.getResults()
      top_ten_html = '<table class="table table-bordered" border="1" ><thead><tr><th>IP</th><th>Rule Hits</th></tr></thead><tbody>'
      for i in top_ten:
         top_ten_html += '<tr><td>' + cgi.escape(i['ip']) + ' </td><td> ' + str(i['c']) + '</td></tr>'
      top_ten_html += '</tbody></table>'

      top_ten_page_html = ''

      self.ex.wrapper.execute('select distinct u.url as url, count(id_exception) as c from connections  join urls as u on (u.url_id = connections.url_id) group by u.url order by count(id_exception) DESC limit 10;')
      top_ten_page = self.ex.wrapper.getResults()
      top_ten_page_html = '<table class="table table-bordered" border="1" ><thead><tr><th>URI</th><th>Exceptions Count</th></tr></thead><tbody>'
      
      for i in top_ten_page:
          top_ten_page_html += '<tr><td>' + cgi.escape(i['url']).replace('\'', '\\\'') + ' </td><td> ' + str(i['c']) + '</td></tr>'
      top_ten_page_html += '</tbody></table>'

      dict_replace = {'__TOPTEN__': top_ten_html, '__TOPTENPAGE__': top_ten_page_html, '__TOTALEXCEP__': array_excep, '__SQLCOUNT__': str(sql_count),  '__XSSCOUNT__': str(xss_count), '__DTCOUNT__': str(dt_count), '__RFICOUNT__': str(rfi_count), '__EVCOUNT__': str(evade_count), '__UPCOUNT__': str(upload_count), '__INTCOUNT__': str(intern_count), '__SQLIEXCEP__': sqli_array, '__XSSEXCEP__': xss_array, '__RFIEXCEP__': rfi_array, '__DTEXCEP__': dt_array, '__UPLOADEXCEP__': upload_array, '__EVADEEXCEP__': evade_array, '__INTERNEXCEP__': intern_array}

      html = reduce(lambda html,(b, c): html.replace(b, c), dict_replace.items(), html)
      return html

   def create_js_array(self, res):
      array = '['
      for i in res:
          
          d = i.replace('/', '-')
          date_begin = str(d).split('-')
          date_begin[1] = str(int(date_begin[1]) - 1)
          date_begin = ','.join(date_begin)
          array += '[Date.UTC(' + date_begin  + '),' + str(res[i]).replace('/', '-') + '],'
      if array != '[':
         array = array[:-1] + ']'
      else:
         array += ']'
      return array

   def build_dict(self, res):
      d = OrderedDict()
      for i in res:
         if i['d'] not in d.keys():
            d[i['d']] = i['ex']
      return d


   def build_js_array(self, id_beg = None, id_end = None):
      if id_beg is None or id_end is None:
         self.ex.wrapper.execute('select substr(date,1,10) as d, count(id_exception) as ex from connections group by substr(date,1,10)')
      else:
          self.ex.wrapper.execute('select substr(date,1, 10) as d, count(id_exception) as ex from connections join exceptions as e on (e.exception_id = id_exception) where e.rule_id >= %s and e.rule_id <= %s group by substr(date, 1, 10)', (str(id_beg), str(id_end)))
      count = self.ex.wrapper.getResults()
      mydict = self.build_dict(count)
      total_hit = 0
      for i in count:
         total_hit += i['ex']
      myarray = self.create_js_array(mydict)
      return myarray, total_hit

class GenWhitelist(Resource):

    def render_GET(self, request):
      request.setHeader('content-type', 'text/plain')
      ex = rules_extractor(int(request.args.get('page_hit', ['10'])[0]), 
                           int(request.args.get('rules_hit', ['10'])[0]), 
                           glob_rules_file)
      ex.gen_basic_rules()
      base_rules, opti_rules = ex.opti_rules_back()
      opti_rules.sort(lambda a,b: (b['hratio']+(b['pratio']*3)) < (a['hratio']+(a['pratio']*3)))
      pprint.pprint(opti_rules)
      r = '########### Optimized Rules Suggestion ##################\n'
      if not len(opti_rules):
         r+= "#No rules to be generated\n"
         return
      opti_rules.sort(key=lambda k: (k['hratio'], k['pratio']))
      _i = len(opti_rules)-1
      while _i >= 0:
         i = opti_rules[_i]
         _i = _i - 1
         r += ("# total_count:"+str(i['hcount'])+" ("+str(i['hratio'])+
               "%), peer_count:"+str(i['pcount'])+" ("+str(i['pratio'])+"%)")
         r += " | "+ex.core_msg.get(str(i['rule_id']), "?")+"\n"
         if (i['hratio'] < 5 and i['pratio'] < 5) or (i['pratio'] < 5):
            r += '#'
         r += 'BasicRule wl:' + str(i['rule_id']) + ' "mz:'
         if i['url'] is not None and len(i['url']) > 0:
            r += '$URL:' + i['url']
         if i['zone'] is not None and len(i['zone']) > 0:
            if i['url']:
               r += '|'
            r += i['zone']
         if i['var_name'] is not None and len(i['var_name']) > 0:
            # oooh, that must be bad.
            r = r[:-len(i['zone'])]+"$"+r[-len(i['zone']):]
            r += "_VAR:"+i['var_name']
         r += '";\n'      
      return r

class HTTPRealm(object):
   implements(IRealm)

   def requestAvatar(self, avatarID, mind, *interfaces):
      return (IResource, NaxsiUI(), lambda: None)

def daemonize (stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
    try: 
        pid = os.fork() 
        if pid > 0:
            sys.exit(0)
    except OSError, e: 
        sys.stderr.write ("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror) )
        sys.exit(1)

#    os.chdir("/") 
    os.umask(0) 
    os.setsid() 

    try: 
        pid = os.fork() 
        if pid > 0:
            sys.exit(0)
    except OSError, e: 
        sys.stderr.write ("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror) )
        sys.exit(1)

    si = open(stdin, 'r')
    so = open(stdout, 'a+')
    se = open(stderr, 'a+', 0)
    os.dup2(si.fileno(), sys.stdin.fileno())
    os.dup2(so.fileno(), sys.stdout.fileno())
    os.dup2(se.fileno(), sys.stderr.fileno())
         

      
def usage():
   print 'Usage : python nx_extract /path/to/conf/file'


if __name__  == '__main__':
   if len(sys.argv) != 2:
      usage()
      exit(42)
   glob_conf_file = sys.argv[1]
   fd = open(sys.argv[1], 'r')
   conf = ConfigParser()
   conf.readfp(fd)
   try:
      port = int(conf.get('nx_extract', 'port'))
   except:
      print "No port in conf file ! Using default port (8081)"
      port = 8081
   try:
      glob_rules_file = conf.get('nx_extract', 'rules_path')
   except:
      print "No rules path in conf file ! Using default (/etc/nginx/sec-rules/core.rules)"

   try:
      glob_user = conf.get('nx_extract', 'username')
   except:
      print 'No username for web access ! Nx_extract will exit.'
      exit(-1)

   try:
      glob_pass = conf.get('nx_extract', 'password')
   except:
      print 'No password for web access ! Nx_extract will exit.'
      exit(-1)
   fd.close()

   credshandler = InMemoryUsernamePasswordDatabaseDontUse()  # i know there is DontUse in the name
   credshandler.addUser(glob_user, glob_pass)
   portal = Portal(HTTPRealm(), [credshandler])
   credentialFactory = DigestCredentialFactory("md5", "Naxsi-UI")

   webroot = HTTPAuthSessionWrapper(portal, [credentialFactory])
   
   factory = Site(webroot)
   reactor.listenTCP(port, factory)

#   daemonize(stdout = '/tmp/nx_extract_output', stderr = '/tmp/nx_extract_error')
   reactor.run()