Merge pull request #382 from 0xdabbad00/redshift_and_es_support

Redshift and es support

Author: 0xdabbad00

cloudmapper.py

@@ -31,7 +31,7 @@
 import importlib
 import commands
 
-__version__ = "2.5.4"
+__version__ = "2.5.5"
 
 
 def show_help(commands):

commands/prepare.py

@@ -32,7 +32,7 @@
 from netaddr import IPNetwork, IPAddress
 from shared.common import get_account, get_regions, is_external_cidr
 from shared.query import query_aws, get_parameter_file
-from shared.nodes import Account, Region, Vpc, Az, Subnet, Ec2, Elb, Elbv2, Rds, VpcEndpoint, Ecs, Lambda, Cidr, Connection
+from shared.nodes import Account, Region, Vpc, Az, Subnet, Ec2, Elb, Elbv2, Rds, VpcEndpoint, Ecs, Lambda, Redshift, ElasticSearch, Cidr, Connection
 
 __description__ = "Generate network connection information file"
 
@@ -117,6 +117,22 @@ def get_lambda_functions(region):
     return pyjq.all('.Functions[]|select(.VpcConfig!=null)', functions)
 
 
+def get_redshift(region):
+    clusters = query_aws(region.account, "redshift-describe-clusters", region.region)
+    return pyjq.all('.Clusters[]', clusters)
+
+
+def get_elasticsearch(region):
+    es_domains = []
+    domain_json = query_aws(region.account, "es-list-domain-names", region.region)
+    domains = pyjq.all('.DomainNames[]', domain_json)
+    for domain in domains:
+        es = get_parameter_file(region, 'es', 'describe-elasticsearch-domain', domain['DomainName'])['DomainStatus']
+        if 'VPCOptions' in es:
+            es_domains.append(es)
+    return es_domains
+
+
 def get_sgs(vpc):
     sgs = query_aws(vpc.account, "ec2-describe-security-groups", vpc.region)
     return pyjq.all('.SecurityGroups[] | select(.VpcId == "{}")'.format(vpc.local_id), sgs)
@@ -351,6 +367,16 @@ def build_data_structure(account_data, config, outputfilter):
             node = Lambda(region, lambda_json)
             nodes[node.arn] = node
 
+        # Redshift clusters
+        for node_json in get_redshift(region):
+            node = Redshift(region, node_json)
+            nodes[node.arn] = node
+
+        # ElasticSearch clusters
+        for node_json in get_elasticsearch(region):
+            node = ElasticSearch(region, node_json)
+            nodes[node.arn] = node
+
         # Filter out nodes based on tags
         if len(outputfilter.get("tags", [])) > 0:
             for node_id in list(nodes):
@@ -374,7 +400,7 @@ def build_data_structure(account_data, config, outputfilter):
                 # If there were no matches, remove the node
                 if not has_match:
                     del nodes[node_id]
-        
+
         # Add the nodes to their respective subnets
         for node_arn in list(nodes):
             node = nodes[node_arn]

shared/nodes.py

@@ -653,6 +653,90 @@ def __init__(self, parent, json_blob):
         super(Lambda, self).__init__(parent, json_blob)
 
 
+class Redshift(Leaf):
+    @property
+    def ips(self):
+        ips = []
+        for cluster_node in self._json_blob['ClusterNodes']:
+            ips.append(cluster_node['PrivateIPAddress'])
+            ips.append(cluster_node['PublicIPAddress'])
+        return ips
+
+    @property
+    def can_egress(self):
+        return False
+
+    @property
+    def subnets(self):
+        return []
+
+    @property
+    def tags(self):
+        return pyjq.all('.Tags[]', self._json_blob)
+
+    @property
+    def is_public(self):
+        for ip in self.ips:
+            if is_public_ip(ip):
+                return True
+        return False
+
+    @property
+    def security_groups(self):
+        return pyjq.all('.VpcSecurityGroups[].VpcSecurityGroupId', self._json_blob)
+
+    def __init__(self, parent, json_blob):
+        self._type = "redshift"
+
+        # Set the parent to a VPC
+        # Redshift has no subnet
+        assert(parent._type == "region")
+        for vpc in parent.children:
+            if vpc.local_id == json_blob['VpcId']:
+                self._parent = vpc
+
+        self._local_id = json_blob['ClusterIdentifier']
+        self._arn = json_blob['Endpoint']['Address']
+        self._name = truncate(json_blob['ClusterIdentifier'])
+        super(Redshift, self).__init__(self._parent, json_blob)
+
+
+class ElasticSearch(Leaf):
+    @property
+    def ips(self):
+        return []
+
+    @property
+    def can_egress(self):
+        return False
+
+    @property
+    def subnets(self):
+        return pyjq.all('.VPCOptions.SubnetIds[]', self._json_blob)
+
+    @property
+    def tags(self):
+        # TODO Custom collection is required for the tags because list-domains returns a domain name,
+        # but getting the tags requires calling `es list-tags --arn ARN` and you get the ARN from
+        # the  `es-describe-elasticsearch-domain` files
+        return []
+
+    @property
+    def is_public(self):
+        return False
+
+    @property
+    def security_groups(self):
+        return pyjq.all('.VPCOptions.SecurityGroupIds[]', self._json_blob)
+
+    def __init__(self, parent, json_blob):
+        self._type = "elasticsearch"
+
+        self._local_id = json_blob['ARN']
+        self._arn = json_blob['ARN']
+        self._name = truncate(json_blob['DomainName'])
+        super(ElasticSearch, self).__init__(parent, json_blob)
+
 class Cidr(Leaf):
     def ips(self):
         return [self._local_id]

shared/public.py

@@ -122,6 +122,9 @@ def get_public_nodes(account, config, use_cache=False):
             for ip in target_node['node_data']['ips']:
                 if is_public_ip(ip):
                     target['hostname'] = ip
+        elif target_node['type'] == 'redshift':
+            target['type'] = 'redshift'
+            target['hostname'] = target_node['node_data'].get('Endpoint', {}).get('Address', '')
         else:
             # Unknown node
             raise Exception('Unknown type: {}'.format(target_node['type']))

web/icons/aws/elasticsearch.svg

@@ -265,6 +265,15 @@
         "background-clip": "none"
         }
     },
+    {
+        "selector": "[type = \"elasticsearch\"]",
+        "css": {
+        "background-opacity": 0,
+        "background-image": "./icons/aws/elasticsearch.svg",
+        "background-fit": "contain",
+        "background-clip": "none"
+        }
+    },
     {
         "selector": "[type = \"kinesis\"]",
         "css": {