Iptables speedup

systemvm/patches/debian/config/opt/cloud/bin/configure.py

@@ -17,27 +17,16 @@
 # specific language governing permissions and limitations
 # under the License.
 import sys
-import os
 import base64
 
-from merge import DataBag
-from pprint import pprint
-import subprocess
-import logging
 import re
-import time
-import shutil
-import os.path
-import os
 from fcntl import flock, LOCK_EX, LOCK_UN
 
-from cs.CsDatabag import CsDataBag, CsCmdLine
-import cs.CsHelper
+from cs.CsDatabag import CsDataBag
 from cs.CsNetfilter import CsNetfilters
 from cs.CsDhcp import CsDhcp
 from cs.CsRedundant import *
 from cs.CsFile import CsFile
-from cs.CsApp import CsApache, CsDnsmasq
 from cs.CsMonitor import CsMonitor
 from cs.CsLoadBalancer import CsLoadBalancer
 from cs.CsConfig import CsConfig
@@ -207,7 +196,23 @@ def create(self):
 
         def process(self, direction, rule_list, base):
             count = base
-            for i in rule_list:
+            rule_list_splitted = []
+            for rule in rule_list:
+                if ',' in rule['cidr']:
+                    cidrs = rule['cidr'].split(',')
+                    for cidr in cidrs:
+                        new_rule = {
+                            'cidr': cidr,
+                            'last_port': rule['last_port'],
+                            'type': rule['type'],
+                            'first_port': rule['first_port'],
+                            'allowed': rule['allowed']
+                        }
+                        rule_list_splitted.append(new_rule)
+                else:
+                    rule_list_splitted.append(rule)
+
+            for i in rule_list_splitted:
                 r = self.AclRule(direction, self, i, self.config, count)
                 r.create()
                 count += 1
@@ -260,7 +265,7 @@ def create(self):
                     rstr = "%s -m icmp --icmp-type %s" % (rstr, self.icmp_type)
                 rstr = "%s %s -j %s" % (rstr, self.dport, self.action)
                 rstr = rstr.replace("  ", " ").lstrip()
-                self.fw.append([self.table, self.count, rstr])
+                self.fw.append([self.table, "", rstr])
 
     def flushAllowAllEgressRules(self):
         logging.debug("Flush allow 'all' egress firewall rule")
@@ -483,7 +488,7 @@ def configure_iptables(self, dev, obj):
         self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])])
         self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 4500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])])
         self.fw.append(["", "front", "-A INPUT -i %s -p esp -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])])
-        self.fw.append(["nat", "front", "-A POSTROUTING -t nat -o %s -m mark --mark 0x525 -j ACCEPT" % dev])
+        self.fw.append(["nat", "front", "-A POSTROUTING -o %s -m mark --mark 0x525 -j ACCEPT" % dev])
         for net in obj['peer_guest_cidr_list'].lstrip().rstrip().split(','):
             self.fw.append(["mangle", "front",
                             "-A FORWARD -s %s -d %s -j MARK --set-xmark 0x525/0xffffffff" % (obj['local_guest_cidr'], net)])
@@ -809,7 +814,7 @@ def forward_vr(self, rule):
                 rule['internal_ip'],
                 internal_fwports
               )
-        fw4 = "-j SNAT --to-source %s -A POSTROUTING -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \
+        fw4 = "-A POSTROUTING -j SNAT --to-source %s -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \
               (
                 self.getGuestIp(),
                 self.getNetworkByIp(rule['internal_ip']),
@@ -1011,7 +1016,7 @@ def main(argv):
             lb.process()
 
             logging.debug("Configuring iptables rules")
-            nf = CsNetfilters()
+            nf = CsNetfilters(False)
             nf.compare(config.get_fw())
 
             logging.debug("Configuring iptables rules done ...saving rules")

systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py

@@ -401,8 +401,7 @@ def fw_router(self):
 
         self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"])
-        self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" %
-                        self.dev])
+        self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev])
         self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"])
 
@@ -436,6 +435,13 @@ def fw_router(self):
     def fw_vpcrouter(self):
         if not self.config.is_vpc():
             return
+        self.fw.append(["mangle", "front", "-A PREROUTING " +
+                        "-m state --state RELATED,ESTABLISHED " +
+                        "-j CONNMARK --restore-mark --nfmask 0xffffffff --ctmask 0xffffffff"])
+
+        self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"])
+        self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"])
+        self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"])
 
         self.fw.append(["filter", "", "-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT"])
 
@@ -484,22 +490,14 @@ def fw_vpcrouter(self):
                             (guestNetworkCidr, self.dev, self.address['public_ip'])])
 
         if self.get_type() in ["public"]:
-            self.fw.append(["", "front",
-                            "-A FORWARD -o %s -d %s -j ACL_INBOUND_%s" % (
-                                self.dev, self.address['network'], self.dev)
-                            ])
             self.fw.append(
                 ["mangle", "", "-A FORWARD -j VPN_STATS_%s" % self.dev])
             self.fw.append(
                 ["mangle", "", "-A VPN_STATS_%s -o %s -m mark --mark 0x525/0xffffffff" % (self.dev, self.dev)])
             self.fw.append(
                 ["mangle", "", "-A VPN_STATS_%s -i %s -m mark --mark 0x524/0xffffffff" % (self.dev, self.dev)])
             self.fw.append(
-                ["", "front", "-A FORWARD -j NETWORK_STATS_%s" % self.dev])
-
-        self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"])
-        self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"])
-        self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"])
+                ["", "front", "-A FORWARD -j NETWORK_STATS_eth1"])
 
         self.fw.append(["", "", "-A NETWORK_STATS -i eth0 -o eth2 -p tcp"])
         self.fw.append(["", "", "-A NETWORK_STATS -i eth2 -o eth0 -p tcp"])
@@ -508,7 +506,8 @@ def fw_vpcrouter(self):
 
         self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"])
-
+        self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev])
+        self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"])
 

systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py

@@ -60,7 +60,7 @@ def process(self):
 
         # We restart DNSMASQ every time the configure.py is called in order to avoid lease problems.
         if not self.cl.is_redundant() or self.cl.is_master():
-            CsHelper.service("dnsmasq", "restart")
+            CsHelper.execute2("service dnsmasq restart")
 
     def configure_server(self):
         # self.conf.addeq("dhcp-hostsfile=%s" % DHCP_HOSTS)

systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py

@@ -71,14 +71,15 @@ def _configure_firewall(self, add_rules, remove_rules, stat_rules):
             port = path[1]
             firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
 
-        for rules in remove_rules:
+        for rules in stat_rules:
             path = rules.split(':')
             ip = path[0]
             port = path[1]
-            firewall.append(["filter", "", "-D INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
+            firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
 
-        for rules in stat_rules:
+        for rules in remove_rules:
             path = rules.split(':')
             ip = path[0]
             port = path[1]
-            firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
+            if ["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)] in firewall:
+                firewall.remove(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])

systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py

@@ -15,10 +15,12 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+from __future__ import print_function
+
 import CsHelper
-from pprint import pprint
-from CsDatabag import CsDataBag, CsCmdLine
+from CsDatabag import CsCmdLine
 import logging
+from cs_iptables_save import Tables
 
 
 class CsChain(object):
@@ -81,6 +83,7 @@ class CsNetfilters(object):
 
     def __init__(self, load=True):
         self.rules = []
+        self.iptablerules = []
         self.table = CsTable()
         self.chain = CsChain()
         if load:
@@ -91,7 +94,10 @@ def get_all_rules(self):
             if i.startswith('*'):  # Table
                 self.table.add(i[1:])
             if i.startswith(':'):  # Chain
-                self.chain.add(self.table.last(), i[1:].split(' ')[0])
+                string = i[1:].split(' ')[0]
+                cmd = "iptables -t %s -N %s" % (self.table.last(), string)
+                self.iptablerules.append(cmd)
+                self.chain.add(self.table.last(), string)
             if i.startswith('-A'):  # Rule
                 self.chain.add_rule(i.split()[1])
                 rule = CsNetfilter()
@@ -125,22 +131,25 @@ def has_rule(self, new_rule):
     def get_unseen(self):
         del_list = [x for x in self.rules if x.unseen()]
         for r in del_list:
-            cmd = "iptables -t %s %s" % (r.get_table(), r.to_str(True))
-            logging.debug("unseen cmd:  %s ", cmd)
-            CsHelper.execute(cmd)
-            # print "Delete rule %s from table %s" % (r.to_str(True), r.get_table())
+            self.delete(r)
             logging.info("Delete rule %s from table %s", r.to_str(True), r.get_table())
 
     def compare(self, list):
         """ Compare reality with what is needed """
-        # PASS 1:  Ensure all chains are present
+        for c in self.chain.get("filter"):
+            # Ensure all inbound/outbound chains have a default drop rule
+            if c.startswith("ACL_INBOUND") or c.startswith("ACL_OUTBOUND"):
+                list.append(["filter", "", "-A %s -j DROP" % c])
+        # PASS 1:  Ensure all chains are present and cleanup unused rules.
         for fw in list:
             new_rule = CsNetfilter()
             new_rule.parse(fw[2])
             new_rule.set_table(fw[0])
-            self.add_chain(new_rule)
+            self.has_rule(new_rule)
+
+        self.del_standard()
+        self.get_unseen()
 
-        ruleSet = set()
         # PASS 2: Create rules
         for fw in list:
             tupledFw = tuple(fw)
@@ -154,36 +163,33 @@ def compare(self, list):
             if isinstance(fw[1], int):
                 new_rule.set_count(fw[1])
 
-            rule_chain = new_rule.get_chain()
-
-            logging.debug("Checking if the rule already exists: rule=%s table=%s chain=%s", new_rule.get_rule(), new_rule.get_table(), new_rule.get_chain())
-            if self.has_rule(new_rule):
-                logging.debug("Exists: rule=%s table=%s", fw[2], new_rule.get_table())
-            else:
-                # print "Add rule %s in table %s" % ( fw[2], new_rule.get_table())
-                logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table())
-                # front means insert instead of append
-                cpy = fw[2]
-                if fw[1] == "front":
-                    cpy = cpy.replace('-A', '-I')
-                if isinstance(fw[1], int):
-                    # if the rule is for ACLs, we want to insert them in order, right before the DROP all
-                    if rule_chain.startswith("ACL_INBOUND") or rule_chain.startswith("ACL_OUTBOUND"):
-                        rule_count = self.chain.get_count(rule_chain)
-                        cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), rule_count))
-                    else:
-                        cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1]))
-                CsHelper.execute("iptables -t %s %s" % (new_rule.get_table(), cpy))
-                ruleSet.add(tupledFw)
-                self.chain.add_rule(rule_chain)
-        self.del_standard()
-        self.get_unseen()
+            logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table())
+            # front means insert instead of append
+            cpy = fw[2]
+            if fw[1] == "front":
+                cpy = cpy.replace('-A', '-I')
+            if isinstance(fw[1], int):
+                cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1]))
+
+            self.iptablerules.append("iptables -t %s %s" % (new_rule.get_table(), cpy))
+        self.apply_rules()
+
+    def apply_rules(self):
+        s = []
+        for r in self.iptablerules:
+            r.replace('  ', ' ')  # Remove duplicate spaces
+            if r not in s:
+                s.append(r)
 
-    def add_chain(self, rule):
-        """ Add the given chain if it is not already present """
-        if not self.has_chain(rule.get_table(), rule.get_chain()):
-            CsHelper.execute("iptables -t %s -N %s" % (rule.get_table(), rule.get_chain()))
-            self.chain.add(rule.get_table(), rule.get_chain())
+        chains = Tables(s)
+        chains.table_printout()
+
+        # COMMIT all rules.
+        result = CsHelper.execute("iptables-restore < /tmp/rules.save")
+        if result:
+            logging.info("iptables-restore result: %s", result)
+        else:
+            logging.info("iptables-restore result: success!")
 
     def del_standard(self):
         """ Del rules that are there but should not be deleted