Remove dynamic imports (#335)

* Remove plugins
Current usage of plugins is not pluggable and includes logging
stuff.
Move this to conf/logging.
* Removed dynamic imports
* Add tests for hunters registration

kube_hunter/README.md

@@ -5,8 +5,6 @@ First, let's go through kube-hunter's basic architecture.
 ### Directory Structure  
 ~~~  
 kube-hunter/  
-        plugins/  
-           # your plugin
         kube_hunter/  
             core/  
             modules/  

kube_hunter/conf/__init__.py

@@ -6,5 +6,3 @@ config = parse_args()
 setup_logger(config.log)
 
 __all__ = [config]
-
-import plugins  # noqa

kube_hunter/conf/logging.py

@@ -5,6 +5,10 @@ DEFAULT_LEVEL = logging.INFO
 DEFAULT_LEVEL_NAME = logging.getLevelName(DEFAULT_LEVEL)
 LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s %(message)s"
 
+# Suppress logging from scapy
+logging.getLogger("scapy.runtime").setLevel(logging.CRITICAL)
+logging.getLogger("scapy.loading").setLevel(logging.CRITICAL)
+
 
 def setup_logger(level_name):
     # Remove any existing handlers

kube_hunter/core/events/types.py

@@ -62,10 +62,6 @@ class Event(object):
         return history
 
 
-""" Event Types """
-# TODO: make proof an abstract method.
-
-
 class Service(object):
     def __init__(self, name, path="", secure=True):
         self.name = name
@@ -126,8 +122,6 @@ global event_id_count_lock
 event_id_count_lock = threading.Lock()
 event_id_count = 0
 
-""" Discovery/Hunting Events """
-
 
 class NewHostEvent(Event):
     def __init__(self, host, cloud=None):
@@ -195,9 +189,6 @@ class ReportDispatched(Event):
     pass
 
 
-""" Core Vulnerabilities """
-
-
 class K8sVersionDisclosure(Vulnerability, Event):
     """The kubernetes version could be obtained from the {} endpoint """
 

kube_hunter/core/events/types/__init__.py

@@ -1,10 +0,0 @@
-from os.path import dirname, basename, isfile
-import glob
-
-from .common import *  # noqa
-
-# dynamically importing all modules in folder
-files = glob.glob(dirname(__file__) + "/*.py")
-for module_name in (basename(f)[:-3] for f in files if isfile(f) and not f.endswith("__init__.py")):
-    if module_name != "handler":
-        exec("from .{} import *".format(module_name))

kube_hunter/modules/discovery/__init__.py

@@ -1,8 +1,21 @@
-from os.path import dirname, basename, isfile
-import glob
+from . import (
+    apiserver,
+    dashboard,
+    etcd,
+    hosts,
+    kubectl,
+    kubelet,
+    ports,
+    proxy,
+)
 
-# dynamically importing all modules in folder
-files = glob.glob(dirname(__file__) + "/*.py")
-for module_name in (basename(f)[:-3] for f in files if isfile(f) and not f.endswith("__init__.py")):
-    if not module_name.startswith("test_"):
-        exec("from .{} import *".format(module_name))
+__all__ = [
+    apiserver,
+    dashboard,
+    etcd,
+    hosts,
+    kubectl,
+    kubelet,
+    ports,
+    proxy,
+]

kube_hunter/modules/hunting/__init__.py

@@ -1,7 +1,31 @@
-from os.path import dirname, basename, isfile
-import glob
+from . import (
+    aks,
+    apiserver,
+    arp,
+    capabilities,
+    certificates,
+    cves,
+    dashboard,
+    dns,
+    etcd,
+    kubelet,
+    mounts,
+    proxy,
+    secrets,
+)
 
-# dynamically importing all modules in folder
-files = glob.glob(dirname(__file__) + "/*.py")
-for module_name in (basename(f)[:-3] for f in files if isfile(f) and not f.endswith("__init__.py")):
-    exec(f"from .{module_name} import *")
+__all__ = [
+    aks,
+    apiserver,
+    arp,
+    capabilities,
+    certificates,
+    cves,
+    dashboard,
+    dns,
+    etcd,
+    kubelet,
+    mounts,
+    proxy,
+    secrets,
+]

plugins/README.md

@@ -1,14 +0,0 @@
-# Plugins
-
-This folder contains modules that will load before any parsing of arguments by kube-hunter main module.
-
-An example for using a plugin to add an argument:
-```python
-# example.py
-from kube_hunter.conf import config
-
-config.parser.add_argument('--exampleflag', action="store_true", help="enables active hunting")
-```
-What we did here was just add a file to the `plugins/` folder, import the parser, and adding an argument.
-
-All plugins in this folder gets imported right after the main arguments are added, and right before they are getting parsed, so you can add an argument that will later be used in your [hunting/discovery module](../kube_hunter/README.md).

plugins/__init__.py

@@ -1,7 +0,0 @@
-from os.path import dirname, basename, isfile
-import glob
-
-# dynamically importing all modules in folder
-files = glob.glob(dirname(__file__) + "/*.py")
-for module_name in (basename(f)[:-3] for f in files if isfile(f) and not f.endswith("__init__.py")):
-    exec(f"from .{module_name} import *")

plugins/logging_mod.py

@@ -1,5 +0,0 @@
-import logging
-
-# Suppress logging from scapy
-logging.getLogger("scapy.runtime").setLevel(logging.CRITICAL)
-logging.getLogger("scapy.loading").setLevel(logging.CRITICAL)

tests/core/test_cloud.py

@@ -1,7 +1,7 @@
 import requests_mock
 import json
 
-from kube_hunter.core.events.types.common import NewHostEvent
+from kube_hunter.core.events.types import NewHostEvent
 
 
 # Testing if it doesn't try to run get_cloud if the cloud type is already set.

tests/core/test_handler.py

@@ -0,0 +1,112 @@
+from kube_hunter.core.events.handler import handler
+from kube_hunter.modules.discovery.apiserver import ApiServiceDiscovery
+from kube_hunter.modules.discovery.dashboard import KubeDashboard as KubeDashboardDiscovery
+from kube_hunter.modules.discovery.etcd import EtcdRemoteAccess as EtcdRemoteAccessDiscovery
+from kube_hunter.modules.discovery.hosts import FromPodHostDiscovery, HostDiscovery
+from kube_hunter.modules.discovery.kubectl import KubectlClientDiscovery
+from kube_hunter.modules.discovery.kubelet import KubeletDiscovery
+from kube_hunter.modules.discovery.ports import PortDiscovery
+from kube_hunter.modules.discovery.proxy import KubeProxy as KubeProxyDiscovery
+from kube_hunter.modules.hunting.aks import AzureSpnHunter, ProveAzureSpnExposure
+from kube_hunter.modules.hunting.apiserver import (
+    AccessApiServer,
+    ApiVersionHunter,
+    AccessApiServerActive,
+    AccessApiServerWithToken,
+)
+from kube_hunter.modules.hunting.arp import ArpSpoofHunter
+from kube_hunter.modules.hunting.capabilities import PodCapabilitiesHunter
+from kube_hunter.modules.hunting.certificates import CertificateDiscovery
+from kube_hunter.modules.hunting.cves import K8sClusterCveHunter, KubectlCVEHunter
+from kube_hunter.modules.hunting.dashboard import KubeDashboard
+from kube_hunter.modules.hunting.dns import DnsSpoofHunter
+from kube_hunter.modules.hunting.etcd import EtcdRemoteAccess, EtcdRemoteAccessActive
+from kube_hunter.modules.hunting.kubelet import (
+    ReadOnlyKubeletPortHunter,
+    SecureKubeletPortHunter,
+    ProveRunHandler,
+    ProveContainerLogsHandler,
+    ProveSystemLogs,
+)
+from kube_hunter.modules.hunting.mounts import VarLogMountHunter, ProveVarLogMount
+from kube_hunter.modules.hunting.proxy import KubeProxy, ProveProxyExposed, K8sVersionDisclosureProve
+from kube_hunter.modules.hunting.secrets import AccessSecrets
+
+PASSIVE_HUNTERS = {
+    ApiServiceDiscovery,
+    KubeDashboardDiscovery,
+    EtcdRemoteAccessDiscovery,
+    FromPodHostDiscovery,
+    HostDiscovery,
+    KubectlClientDiscovery,
+    KubeletDiscovery,
+    PortDiscovery,
+    KubeProxyDiscovery,
+    AzureSpnHunter,
+    AccessApiServer,
+    AccessApiServerWithToken,
+    ApiVersionHunter,
+    PodCapabilitiesHunter,
+    CertificateDiscovery,
+    K8sClusterCveHunter,
+    KubectlCVEHunter,
+    KubeDashboard,
+    EtcdRemoteAccess,
+    ReadOnlyKubeletPortHunter,
+    SecureKubeletPortHunter,
+    VarLogMountHunter,
+    KubeProxy,
+    AccessSecrets,
+}
+
+ACTIVE_HUNTERS = {
+    ProveAzureSpnExposure,
+    AccessApiServerActive,
+    ArpSpoofHunter,
+    DnsSpoofHunter,
+    EtcdRemoteAccessActive,
+    ProveRunHandler,
+    ProveContainerLogsHandler,
+    ProveSystemLogs,
+    ProveVarLogMount,
+    ProveProxyExposed,
+    K8sVersionDisclosureProve,
+}
+
+
+def remove_test_hunters(hunters):
+    return {hunter for hunter in hunters if not hunter.__module__.startswith("test")}
+
+
+def test_passive_hunters_registered():
+    expected_missing = set()
+    expected_odd = set()
+
+    registered_passive = remove_test_hunters(handler.passive_hunters.keys())
+    actual_missing = PASSIVE_HUNTERS - registered_passive
+    actual_odd = registered_passive - PASSIVE_HUNTERS
+
+    assert expected_missing == actual_missing, "Passive hunters are missing"
+    assert expected_odd == actual_odd, "Unexpected passive hunters are registered"
+
+
+# TODO (#334): Active hunters registration cannot be tested since it requires `config.active` to be set
+# def test_active_hunters_registered():
+#     expected_missing = set()
+#     expected_odd = set()
+#
+#     registered_active = remove_test_hunters(handler.active_hunters.keys())
+#     actual_missing = ACTIVE_HUNTERS - registered_active
+#     actual_odd = registered_active - ACTIVE_HUNTERS
+#
+#     assert expected_missing == actual_missing, "Active hunters are missing"
+#     assert expected_odd == actual_odd, "Unexpected active hunters are registered"
+
+
+def test_all_hunters_registered():
+    # TODO: Enable active hunting mode in testing
+    # expected = PASSIVE_HUNTERS | ACTIVE_HUNTERS
+    expected = PASSIVE_HUNTERS
+    actual = remove_test_hunters(handler.all_hunters.keys())
+
+    assert expected == actual