Release v1.1 (#93)

Author: austinbyers

docs/source/adding-yara-rules.rst

@@ -8,6 +8,8 @@ Included Rules
 BinaryAlert includes a number of `custom YARA rules <https://github.com/airbnb/binaryalert/tree/master/rules/public>`_  written by Airbnb's analysts which detect a variety of hacktools, malware, and ransomware. All included rules have been tested against a corpus of more than 2 million binaries to ensure the highest fidelity.
 
 
+.. _clone-yara-rules:
+
 Clone Rules From Other Projects
 -------------------------------
 BinaryAlert makes it easy to clone YARA rules from other open-source projects:
@@ -16,11 +18,7 @@ BinaryAlert makes it easy to clone YARA rules from other open-source projects:
 
   $ ./manage.py clone_rules
 
-This will copy a subset of YARA rules from each of the following repositories:
-
-* `Neo23x0/signature-base <https://github.com/Neo23x0/signature-base>`_
-* `YARA-Rules/rules <https://github.com/YARA-Rules/rules>`_
-
+This will copy a subset of YARA rules from several :ref:`open-source collections <yara-credits>`.
 You can add more rule sources in `rules/clone_rules.py <https://github.com/airbnb/binaryalert/blob/master/rules/clone_rules.py>`_
 
 

docs/source/conf.py

@@ -63,9 +63,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.0'
+version = '1.1'
 # The full version, including alpha/beta/rc tags.
-release = '1.0.0'
+release = '1.1.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/credits.rst

@@ -0,0 +1,46 @@
+Credits
+=======
+
+People
+------
+BinaryAlert is brought to you by `Airbnb <http://airbnb.io>`_:
+
+- `Austin Byers <https://github.com/austinbyers>`_ (Architect, Primary Engineer)
+- `mime-frame <https://github.com/mime-frame>`_ (Concept, Design Review, YARA Rules)
+- `Daimon <https://github.com/fusionrace>`_ (YARA Rules)
+- And many others in the `full list of contributors <https://github.com/airbnb/binaryalert/graphs/contributors>`_
+
+
+.. _yara-credits:
+
+YARA Rules
+----------
+When :ref:`cloning YARA rules from other projects <clone-yara-rules>`, subsets of the following
+collections are included by default:
+
+- `Neo23x0/signature-base <https://github.com/Neo23x0/signature-base>`_
+- `YARA-Rules/rules <https://github.com/YARA-Rules/rules>`_
+
+
+Open-Source Tools
+-----------------
+We are proud to contribute to the open-source community, without which BinaryAlert would not be
+possible. BinaryAlert relies on several open-source tools and libraries:
+
+- `backoff <https://github.com/litl/backoff>`_: Function decoration for backoff and retry
+- `boto3 <https://boto3.readthedocs.io>`_: AWS SDK for Python
+- `cbapi <https://cbapi.readthedocs.io>`_: Carbon Black API for Python
+- `pyhcl <https://github.com/virtuald/pyhcl>`_: Python parser for HCL (e.g. Terraform configuration)
+- `terraform <https://www.terraform.io/>`_: Infrastructure-as-Code
+- `yara <http://virustotal.github.io/yara/>`_: Pattern matching for malware analysis
+- `yara-python <https://github.com/VirusTotal/yara-python>`_: The Python interface for YARA
+- `yextend <https://github.com/BayshoreNetworks/yextend>`_: YARA analysis of archive data
+
+
+Bundled Software
+................
+The following tools are pre-compiled for use in Lambda and included in the BinaryAlert repo:
+
+- `cbapi-python <https://github.com/carbonblack/cbapi-python>`_ | `LICENSE <https://github.com/carbonblack/cbapi-python/blob/master/LICENSE>`__
+- `yara-python <https://github.com/VirusTotal/yara-python>`_  | `LICENSE <https://github.com/VirusTotal/yara-python/blob/master/LICENSE>`__
+- `yextend <https://github.com/BayshoreNetworks/yextend>`_  | `LICENSE <https://github.com/BayshoreNetworks/yextend/blob/master/LICENSE>`__

docs/source/index.rst

@@ -46,3 +46,4 @@ Table of Contents
    yara-matches
    metrics-and-monitoring
    troubleshooting-faq
+   credits

docs/source/troubleshooting-faq.rst

@@ -33,6 +33,8 @@ Why did my live test fail?
 --------------------------
 Check the :ref:`Lambda execution logs <cloudwatch_logs>` and the :ref:`BinaryAlert dashboard <cloudwatch_dashboard>` for abnormalities. A common problem is that the BinaryAlert analyzers don't understand the compiled YARA rules file. Make sure your `virtual environment <getting-started.html>`_ is set up correctly and that your YARA rules only use the :ref:`supported modules <supported_yara_modules>`. It is also possible that one or more AWS components might be down.
 
+It may take 1-3 minutes after a deploy before the Lambda functions are ready to go. If a live test fails immediately after a deploy, wait a few minutes and try again.
+
 
 How do I setup YARA match / metric alarm alerts?
 ------------------------------------------------

lambda_functions/analyzer/README.rst

@@ -72,4 +72,4 @@ and install ``yara-python`` and ``yextend`` as follows:
     zip -r yara3.7.0_yextend1.6.zip *
 
 
-Then ``scp`` the newzipfile to replace the one in the repo.
+Then ``scp`` the new zipfile to replace the one in the repo.

lambda_functions/analyzer/yara_analyzer.py

@@ -7,6 +7,11 @@
 
 import yara
 
+if __package__:
+    from lambda_functions.analyzer.common import LOGGER
+else:
+    from common import LOGGER  # type: ignore
+
 
 # YARA matches from both yara-python and yextend are stored in this generic YaraMatch tuple.
 YaraMatch = collections.namedtuple(
@@ -109,9 +114,13 @@ def analyze(self, target_file: str, original_target_path: str = '') -> List[Yara
 
         # Yextend matches
         os.environ['LD_LIBRARY_PATH'] = os.environ['LAMBDA_TASK_ROOT']
-        yextend_output = subprocess.check_output(
-            ['./yextend', '-r', self._compiled_rules_file, '-t', target_file, '-j'])
-        yextend_list = json.loads(yextend_output.decode('utf-8'))
-        yextend_matches = _convert_yextend_to_yara_match(yextend_list[0])
+        try:
+            yextend_output = subprocess.check_output(
+                ['./yextend', '-r', self._compiled_rules_file, '-t', target_file, '-j'])
+            yextend_list = json.loads(yextend_output.decode('utf-8'))
+        except (json.JSONDecodeError, subprocess.CalledProcessError):
+            LOGGER.exception('Fatal error when running yextend')
+            return yara_python_matches
 
+        yextend_matches = _convert_yextend_to_yara_match(yextend_list[0])
         return yara_python_matches + yextend_matches

manage.py

@@ -21,7 +21,7 @@
 from tests import live_test
 
 # BinaryAlert version.
-VERSION = '1.1.0.beta'
+VERSION = '1.1.0'
 
 # File locations.
 PROJECT_DIR = os.path.dirname(os.path.realpath(__file__))  # Directory containing this file.

terraform/lambda_iam.tf

@@ -117,9 +117,14 @@ data "aws_iam_policy_document" "binaryalert_analyzer_policy" {
   }
 
   statement {
-    sid       = "GetFromBinaryAlertBucket"
-    effect    = "Allow"
-    actions   = ["s3:GetObject"]
+    sid    = "GetFromBinaryAlertBucket"
+    effect = "Allow"
+
+    actions = [
+      "s3:GetObject",
+      "s3:HeadObject",
+    ]
+
     resources = ["${aws_s3_bucket.binaryalert_binaries.arn}/*"]
   }
 

terraform/terraform.tfvars

@@ -58,7 +58,7 @@ sqs_retention_minutes = 60
 // Number of S3 object keys to pack into a single SQS message.
 // Each downstream analyzer will process at most 10 SQS messages, each with this many objects.
 // Higher values allow for higher throughput, but are constrained by analyzer execution time limit.
-lambda_batch_objects_per_message = 15
+lambda_batch_objects_per_message = 5
 
 // Memory limit (MB) for the batching Lambda function. 128 is the minimum allowed by Lambda.
 lambda_batch_memory_mb = 128
@@ -68,7 +68,7 @@ lambda_batch_memory_mb = 128
 lambda_dispatch_frequency_minutes = 2
 
 // Maximum number of analyzers that can be asynchronously invoked during one dispatcher run.
-// Higher values allow for more throughtput, but if too many analyzers are invoked too quickly,
+// Higher values allow for more throughput, but if too many analyzers are invoked too quickly,
 // Lambda invocations may be throttled.
 lambda_dispatch_limit = 500