From 237259ecf7abf005cddb6ea3185374ca27b75207 Mon Sep 17 00:00:00 2001 From: Lisa Hilmes <55513548+lhilmes-cb@users.noreply.github.com> Date: Wed, 6 Nov 2019 10:44:24 -0700 Subject: [PATCH 001/197] Updated readme to edit PSC Did not change any code that included PSC. Changed two instances of PSC in body text to Carbon Black Cloud. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c7297dd3..2404c8a3 100755 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ For distinction between credentials of different Carbon Black products, use the * ``credentials.response`` for CB Response * ``credentials.protection`` for CB Protection -For example, if you use a PSC product, you should have created a credentials file in one of these locations: +For example, if you use a Carbon Black Cloud product, you should have created a credentials file in one of these locations: * ``/etc/carbonblack/credentials.psc`` * ``~/.carbonblack/credentials.psc`` @@ -147,7 +147,7 @@ The possible options for each credential profile are: different tokens for each. * **ssl_verify**: True or False; controls whether the SSL/TLS certificate presented by the server is validated against the local trusted CA store. -* **org_key**: The organization key. This is required to access the PSC, and can be found in the console. The format is ``123ABC45``. +* **org_key**: The organization key. This is required to access the Carbon Black Cloud, and can be found in the console. The format is ``123ABC45``. * **proxy**: A proxy specification that will be used when connecting to the CB server. The format is: ``http://myusername:mypassword@proxy.company.com:8001/`` where the hostname of the proxy is ``proxy.company.com``, port 8001, and using username/password ``myusername`` and ``mypassword`` respectively. From 1f1170d9f26484f048ef3ac7713992ba6ad81850 Mon Sep 17 00:00:00 2001 From: Ruchir Arya Date: Mon, 11 Nov 2019 11:16:40 -0500 Subject: [PATCH 002/197] FIX: List object is not callable. --- src/cbapi/psc/threathunter/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/psc/threathunter/models.py b/src/cbapi/psc/threathunter/models.py index 1c2e7e5c..25807ad9 100644 --- a/src/cbapi/psc/threathunter/models.py +++ b/src/cbapi/psc/threathunter/models.py @@ -1070,7 +1070,7 @@ def download_url(self, expiration_seconds=3600): raise InvalidObjectError("{} should be retried".format(self.sha256)) else: return next((item.url - for item in downloads.found() + for item in downloads.found if self.sha256 == item.sha256), None) From b654400d37e77f1f0c99356dedf1b1a4f0d6d172 Mon Sep 17 00:00:00 2001 From: Becca Vasil Date: Tue, 12 Nov 2019 11:26:08 -0700 Subject: [PATCH 003/197] version bump for hotfix --- README.md | 2 +- docs/changelog.rst | 7 +++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c7297dd3..c6a33cd4 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.5.4** +**Latest Version: 1.5.5** These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. diff --git a/docs/changelog.rst b/docs/changelog.rst index bd3b10dd..fc07bce9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,5 +1,12 @@ CbAPI Changelog =============== +CbAPI 1.5.5 - Released November 12, 2019 +---------------------------------------- + +Updates + +* CB ThreatHunter + * Fix List object that was not callable. CbAPI 1.5.4 - Released October 24, 2019 ---------------------------------------- diff --git a/docs/conf.py b/docs/conf.py index e7d7ba46..a4c3aff1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.5' # The full version, including alpha/beta/rc tags. -release = u'1.5.4' +release = u'1.5.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 7d20c2f3..3b2aaf3c 100755 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ setup( name='cbapi', - version='1.5.4', + version='1.5.5', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index db4780f0..f7169448 100755 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2019 Carbon Black' -__version__ = '1.5.4' +__version__ = '1.5.5' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From aa71ec197ab9277748a5398baf18b30a62891a69 Mon Sep 17 00:00:00 2001 From: Becca Vasil Date: Mon, 18 Nov 2019 11:41:41 -0700 Subject: [PATCH 004/197] changed PSC to Carbon Black Cloud for rebranding to VMware --- bin/cbapi-psc | 2 +- docs/getting-started.rst | 2 +- docs/index.rst | 4 ++-- docs/threathunter-api.rst | 2 +- src/cbapi/psc/livequery/rest_api.py | 2 +- src/cbapi/psc/threathunter/rest_api.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/cbapi-psc b/bin/cbapi-psc index a52b461e..18bbc6e6 100644 --- a/bin/cbapi-psc +++ b/bin/cbapi-psc @@ -40,7 +40,7 @@ def configure(opts): if not os.path.exists(credential_path): os.makedirs(credential_path, 0o700) - url = input("URL to the Cb PSC API server (do not include '/integrationServices') [https://hostname]: ") + url = input("URL to the Carbon Black Cloud API server (do not include '/integrationServices') [https://hostname]: ") ssl_verify = True diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 1742a151..5ce22b15 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -36,7 +36,7 @@ Alternatively, if you're using Windows (change ``c:\python27`` if Python is inst This configuration script will walk you through entering your API credentials and will save them to your current user's credential file location, which is located in the ``.carbonblack`` directory in your user's home directory. -If using cbapi-psc, you will also be asked to provide an org key. An org key is required to access the PSC, and can be found in the console under Settings -> API Keys. +If using cbapi-psc, you will also be asked to provide an org key. An org key is required to access the Carbon Black Cloud, and can be found in the console under Settings -> API Keys. Your First Query ---------------- diff --git a/docs/index.rst b/docs/index.rst index 3196c1d0..bbb3ee87 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -137,7 +137,7 @@ For distinction between credentials of different Carbon Black products, use the * ``credentials.response`` for CB Response * ``credentials.protection`` for CB Protection -For example, if you use a PSC product, you should have created a credentials file in one of these locations: +For example, if you use a Carbon Black Cloud product, you should have created a credentials file in one of these locations: * ``/etc/carbonblack/credentials.psc`` * ``~/.carbonblack/credentials.psc`` @@ -170,7 +170,7 @@ The possible options for each credential profile are: different tokens for each. * **ssl_verify**: True or False; controls whether the SSL/TLS certificate presented by the server is validated against the local trusted CA store. -* **org_key**: The organization key. This is required to access the PSC, and can be found in the console. The format is ``123ABC45``. +* **org_key**: The organization key. This is required to access the Carbon Black Cloud, and can be found in the console. The format is ``123ABC45``. * **proxy**: A proxy specification that will be used when connecting to the CB server. The format is: ``http://myusername:mypassword@proxy.company.com:8001/`` where the hostname of the proxy is ``proxy.company.com``, port 8001, and using username/password ``myusername`` and ``mypassword`` respectively. diff --git a/docs/threathunter-api.rst b/docs/threathunter-api.rst index 01ff3eeb..25a399d5 100644 --- a/docs/threathunter-api.rst +++ b/docs/threathunter-api.rst @@ -4,7 +4,7 @@ CB ThreatHunter API =================== This page documents the public interfaces exposed by cbapi when communicating with a -Carbon Black PSC ThreatHunter server. +Carbon Black Cloud ThreatHunter server. Main Interface -------------- diff --git a/src/cbapi/psc/livequery/rest_api.py b/src/cbapi/psc/livequery/rest_api.py index 9e50fc14..d8e94db0 100644 --- a/src/cbapi/psc/livequery/rest_api.py +++ b/src/cbapi/psc/livequery/rest_api.py @@ -7,7 +7,7 @@ class CbLiveQueryAPI(BaseAPI): - """The main entry point into the Cb PSC LiveQuery API. + """The main entry point into the Carbon Black Cloud LiveQuery API. :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. Uses the profile named 'default' when not specified. diff --git a/src/cbapi/psc/threathunter/rest_api.py b/src/cbapi/psc/threathunter/rest_api.py index 463d6966..60a865ac 100644 --- a/src/cbapi/psc/threathunter/rest_api.py +++ b/src/cbapi/psc/threathunter/rest_api.py @@ -8,7 +8,7 @@ class CbThreatHunterAPI(BaseAPI): - """The main entry point into the Cb ThreatHunter PSC API. + """The main entry point into the Carbon Black Cloud ThreatHunter API. :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. Uses the profile named 'default' when not specified. From 4690d9d2aa750c1a382cff485a8c13eb9ca23f4a Mon Sep 17 00:00:00 2001 From: Becca Vasil Date: Tue, 19 Nov 2019 10:05:40 -0700 Subject: [PATCH 005/197] One more change to Carbon Black Cloud --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index bbb3ee87..0795513b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -184,9 +184,9 @@ Environment Variable Support The latest cbapi for python supports specifying API credentials in the following three environment variables: -`CBAPI_TOKEN` the envar for holding the CbR/CbP api token or the ConnectorId/APIKEY combination for CB Defense/PSC. +`CBAPI_TOKEN` the envar for holding the CbR/CbP api token or the ConnectorId/APIKEY combination for CB Defense/Carbon Black Cloud. -The `CBAPI_URL` envar holds the FQDN of the target, a CbR , CBD, or CbD/PSC server specified just as they are in the +The `CBAPI_URL` envar holds the FQDN of the target, a CbR , CBD, or CbD/Carbon Black Cloud server specified just as they are in the configuration file format specified above. The optional `CBAPI_SSL_VERIFY` envar can be used to control SSL validation(True/False or 0/1), which will default to ON when From 95209b7193619032038130a3a9e831144c11535c Mon Sep 17 00:00:00 2001 From: Becca Vasil Date: Tue, 19 Nov 2019 10:52:00 -0700 Subject: [PATCH 006/197] version bump to 1.5.6 --- README.md | 2 +- docs/changelog.rst | 8 ++++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6d40df68..7fb36671 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.5.5** +**Latest Version: 1.5.6** These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. diff --git a/docs/changelog.rst b/docs/changelog.rst index fc07bce9..1a57eace 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,5 +1,13 @@ CbAPI Changelog =============== +CbAPI 1.5.6 - Released November 19, 2019 +---------------------------------------- + +Updates + +* General + * Name change to Carbon Black Cloud from PSC. + CbAPI 1.5.5 - Released November 12, 2019 ---------------------------------------- diff --git a/docs/conf.py b/docs/conf.py index a4c3aff1..7acc801d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.5' # The full version, including alpha/beta/rc tags. -release = u'1.5.5' +release = u'1.5.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 3b2aaf3c..e491e338 100755 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ setup( name='cbapi', - version='1.5.5', + version='1.5.6', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index f7169448..d6c75863 100755 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2019 Carbon Black' -__version__ = '1.5.5' +__version__ = '1.5.6' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From 40b9e60503f70614e79efe8c6053c199e0f8d5be Mon Sep 17 00:00:00 2001 From: Ruchir Arya Date: Fri, 22 Nov 2019 00:17:56 -0500 Subject: [PATCH 007/197] FIX: Py3 - NameError: name 'long' is not defined PEP 237: Essentially, long renamed to int. That is, there is only one built-in integral type, named int; but it behaves mostly like the old long type. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ File "C:\Program Files\Python38\lib\site-packages\cbapi\models.py", line 147, in __get__ if type(d) is float or type(d) is int or type(d) is long: NameError: name 'long' is not defined ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ --- src/cbapi/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/models.py b/src/cbapi/models.py index 4143d899..72faaca3 100644 --- a/src/cbapi/models.py +++ b/src/cbapi/models.py @@ -144,7 +144,7 @@ def __init__(self, field_name, multiplier=1.0): def __get__(self, instance, instance_type=None): d = super(EpochDateTimeFieldDescriptor, self).__get__(instance, instance_type) - if type(d) is float or type(d) is int or type(d) is long: + if type(d) is float or type(d) is int: epoch_seconds = d / self.multiplier return datetime.utcfromtimestamp(epoch_seconds) else: From 55eb8dc099f4855353ace26c8559e08da7f37b49 Mon Sep 17 00:00:00 2001 From: Ruchir Arya Date: Fri, 22 Nov 2019 23:46:22 -0500 Subject: [PATCH 008/197] Added check to compare Python version --- src/cbapi/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cbapi/models.py b/src/cbapi/models.py index 72faaca3..3aaf1b44 100644 --- a/src/cbapi/models.py +++ b/src/cbapi/models.py @@ -6,6 +6,7 @@ from cbapi.six import python_2_unicode_compatible +import sys import base64 import os.path from cbapi.six import iteritems, add_metaclass @@ -144,7 +145,8 @@ def __init__(self, field_name, multiplier=1.0): def __get__(self, instance, instance_type=None): d = super(EpochDateTimeFieldDescriptor, self).__get__(instance, instance_type) - if type(d) is float or type(d) is int: + long = long if sys.version_info < (3, 0) else int + if type(d) is float or type(d) is int or type(d) is long: epoch_seconds = d / self.multiplier return datetime.utcfromtimestamp(epoch_seconds) else: From a602e23b9d16afe2840d40de61c8d7b2ef77c20d Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 23 Sep 2019 13:54:52 -0600 Subject: [PATCH 009/197] CBAPI-1085: move live response to PSC level Also move deviceInfo and create a new REST API object at the PSC level. --- src/cbapi/psc/__init__.py | 6 + src/cbapi/psc/{defense => }/cblr.py | 0 src/cbapi/psc/defense/rest_api.py | 18 +- src/cbapi/psc/livequery/rest_api.py | 7 +- src/cbapi/psc/models.py | 127 ++++++++++++++ src/cbapi/psc/models/deviceInfo.yaml | 221 +++++++++++++++++++++++++ src/cbapi/psc/rest_api.py | 29 ++++ src/cbapi/psc/threathunter/rest_api.py | 7 +- 8 files changed, 392 insertions(+), 23 deletions(-) rename src/cbapi/psc/{defense => }/cblr.py (100%) create mode 100755 src/cbapi/psc/models.py create mode 100755 src/cbapi/psc/models/deviceInfo.yaml create mode 100755 src/cbapi/psc/rest_api.py diff --git a/src/cbapi/psc/__init__.py b/src/cbapi/psc/__init__.py index e69de29b..93149b5c 100644 --- a/src/cbapi/psc/__init__.py +++ b/src/cbapi/psc/__init__.py @@ -0,0 +1,6 @@ +# Exported public API for the Cb PSC API + +from __future__ import absolute_import + +from .rest_api import CbPSCBaseAPI +from cbapi.psc.models import Device as DeviceV6 diff --git a/src/cbapi/psc/defense/cblr.py b/src/cbapi/psc/cblr.py similarity index 100% rename from src/cbapi/psc/defense/cblr.py rename to src/cbapi/psc/cblr.py diff --git a/src/cbapi/psc/defense/rest_api.py b/src/cbapi/psc/defense/rest_api.py index cf28e67b..db208c6f 100644 --- a/src/cbapi/psc/defense/rest_api.py +++ b/src/cbapi/psc/defense/rest_api.py @@ -1,8 +1,7 @@ from cbapi.utils import convert_query_params from cbapi.query import PaginatedQuery -from .cblr import LiveResponseSessionManager -from cbapi.connection import BaseAPI +from cbapi.psc import CbPSCBaseAPI import logging import time @@ -14,7 +13,7 @@ def convert_to_kv_pairs(q): return k, v -class CbDefenseAPI(BaseAPI): +class CbDefenseAPI(CbPSCBaseAPI): """The main entry point into the Cb Defense API. :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. @@ -26,8 +25,7 @@ class CbDefenseAPI(BaseAPI): >>> cb = CbDefenseAPI(profile="production") """ def __init__(self, *args, **kwargs): - super(CbDefenseAPI, self).__init__(product_name="psc", *args, **kwargs) - self._lr_scheduler = None + super(CbDefenseAPI, self).__init__(*args, **kwargs) def _perform_query(self, cls, query_string=None): return Query(cls, self, query_string) @@ -50,16 +48,6 @@ def get_notifications(self): res = self.get_object("/integrationServices/v3/notification") return res.get("notifications", []) - @property - def live_response(self): - if self._lr_scheduler is None: - self._lr_scheduler = LiveResponseSessionManager(self) - - return self._lr_scheduler - - def _request_lr_session(self, sensor_id): - return self.live_response.request_session(sensor_id) - class Query(PaginatedQuery): """Represents a prepared query to the Cb Defense server. diff --git a/src/cbapi/psc/livequery/rest_api.py b/src/cbapi/psc/livequery/rest_api.py index d8e94db0..e67d8621 100644 --- a/src/cbapi/psc/livequery/rest_api.py +++ b/src/cbapi/psc/livequery/rest_api.py @@ -1,12 +1,12 @@ from cbapi.psc.livequery.models import Run, RunHistory -from cbapi.connection import BaseAPI +from cbapi.psc import CbPSCBaseAPI from cbapi.errors import CredentialError, ApiError import logging log = logging.getLogger(__name__) -class CbLiveQueryAPI(BaseAPI): +class CbLiveQueryAPI(CbPSCBaseAPI): """The main entry point into the Carbon Black Cloud LiveQuery API. :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. @@ -18,8 +18,7 @@ class CbLiveQueryAPI(BaseAPI): >>> cb = CbLiveQueryAPI(profile="production") """ def __init__(self, *args, **kwargs): - super(CbLiveQueryAPI, self).__init__(product_name="psc", *args, **kwargs) - self._lr_scheduler = None + super(CbLiveQueryAPI, self).__init__(*args, **kwargs) if not self.credentials.get("org_key", None): raise CredentialError("No organization key specified") diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py new file mode 100755 index 00000000..0ffe8349 --- /dev/null +++ b/src/cbapi/psc/models.py @@ -0,0 +1,127 @@ +from cbapi.models import MutableBaseModel +from cbapi.errors import ServerError + +from copy import deepcopy +import logging +import json + +log = logging.getLogger(__name__) + + +class PSCMutableModel(MutableBaseModel): + _change_object_http_method = "PATCH" + _change_object_key_name = None + + def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=False): + super(PSCMutableModel, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, + force_init=force_init, full_doc=full_doc) + if not self._change_object_key_name: + self._change_object_key_name = self.primary_key + + def _parse(self, obj): + if type(obj) == dict and self.info_key in obj: + return obj[self.info_key] + + def _update_object(self): + if self._change_object_http_method != "PATCH": + return self._update_entire_object() + else: + return self._patch_object() + + def _update_entire_object(self): + if self.__class__.primary_key in self._dirty_attributes.keys() or self._model_unique_id is None: + new_object_info = deepcopy(self._info) + try: + if not self._new_object_needs_primary_key: + del(new_object_info[self.__class__.primary_key]) + except Exception: + pass + log.debug("Creating a new {0:s} object".format(self.__class__.__name__)) + ret = self._cb.api_json_request(self.__class__._new_object_http_method, self.urlobject, + data={self.info_key: new_object_info}) + else: + log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) + ret = self._cb.api_json_request(self.__class__._change_object_http_method, + self._build_api_request_uri(), data={self.info_key: self._info}) + + return self._refresh_if_needed(ret) + + def _patch_object(self): + if self.__class__.primary_key in self._dirty_attributes.keys() or self._model_unique_id is None: + log.debug("Creating a new {0:s} object".format(self.__class__.__name__)) + ret = self._cb.api_json_request(self.__class__._new_object_http_method, self.urlobject, + data=self._info) + else: + updates = {} + for k in self._dirty_attributes.keys(): + updates[k] = self._info[k] + log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) + ret = self._cb.api_json_request(self.__class__._change_object_http_method, + self._build_api_request_uri(), data=updates) + + return self._refresh_if_needed(ret) + + def _refresh_if_needed(self, request_ret): + refresh_required = True + + if request_ret.status_code not in range(200, 300): + try: + message = json.loads(request_ret.text)[0] + except: + message = request_ret.text + + raise ServerError(request_ret.status_code, message, + result="Did not update {} record.".format(self.__class__.__name__)) + else: + try: + message = request_ret.json() + log.debug("Received response: %s" % message) + if not isinstance(message, dict): + raise ServerError(request_ret.status_code, message, + result="Unknown error updating {0:s} record.".format(self.__class__.__name__)) + else: + if message.get("success", False): + if isinstance(message.get(self.info_key, None), dict): + self._info = message.get(self.info_key) + self._full_init = True + refresh_required = False + else: + if self._change_object_key_name in message.keys(): + # if all we got back was an ID, try refreshing to get the entire record. + log.debug("Only received an ID back from the server, forcing a refresh") + self._info[self.primary_key] = message[self._change_object_key_name] + refresh_required = True + else: + # "success" is False + raise ServerError(request_ret.status_code, message.get("message", ""), + result="Did not update {0:s} record.".format(self.__class__.__name__)) + except: + pass + + self._dirty_attributes = {} + if refresh_required: + self.refresh() + return self._model_unique_id + + +class Device(PSCMutableModel): + urlobject = "/integrationServices/v3/device" + primary_key = "deviceId" + info_key = "deviceInfo" + swagger_meta_file = "psc/models/deviceInfo.yaml" + + def __init__(self, cb, model_unique_id, initial_data=None): + super(Device, self).__init__(cb, model_unique_id, initial_data) + + def lr_session(self): + """ + Retrieve a Live Response session object for this Device. + + :return: Live Response session object + :rtype: :py:class:`cbapi.defense.cblr.LiveResponseSession` + :raises ApiError: if there is an error establishing a Live Response session for this Device + + """ + return self._cb._request_lr_session(self._model_unique_id) + + diff --git a/src/cbapi/psc/models/deviceInfo.yaml b/src/cbapi/psc/models/deviceInfo.yaml new file mode 100755 index 00000000..7d7106a7 --- /dev/null +++ b/src/cbapi/psc/models/deviceInfo.yaml @@ -0,0 +1,221 @@ +type: object +properties: + osVersion: + type: string + activationCode: + type: string + organizationId: + type: integer + format: int64 + deviceId: + type: integer + format: int64 + deviceSessionId: + type: integer + format: int64 + deviceOwnerId: + type: integer + format: int64 + deviceGuid: + type: string + format: uuid + email: + type: string + format: email + assignedToId: + type: integer + format: int64 + assignedToName: + type: string + deviceType: + type: string + x-nullable: true + enum: + - "MAC" + - "WINDOWS" + firstName: + type: string + lastName: + type: string + middleName: + type: string + createTime: + type: integer + format: epoch-ms-date-time + policyId: + type: integer + format: int64 + policyName: + type: string + quarantined: + type: boolean + targetPriorityType: + type: string + x-nullable: true + enum: + - "HIGH" + - "LOW" + - "MEDIUM" + - "MISSION_CRITICAL" + lastVirusActivityTime: + type: integer + format: epoch-ms-date-time + firstVirusActivityTime: + type: integer + format: epoch-ms-date-time + activationCodeExpiryTime: + type: integer + format: epoch-ms-date-time + organizationName: + type: string + sensorVersion: + type: string + registeredTime: + type: integer + format: epoch-ms-date-time + lastContact: + type: integer + format: epoch-ms-date-time + lastReportedTime: + type: integer + format: epoch-ms-date-time + windowsPlatform: + type: string + x-nullable: true + enum: + - "CLIENT_X64" + - "CLIENT_X86" + - "SERVER_X6" + - "SERVER_X86" + vdiBaseDevice: + type: integer + format: int64 + avStatus: + type: array + items: + type: string + x-nullable: true + enum: + - "AV_ACTIVE" + - "AV_BYPASS" + - "AV_DEREGISTERED" + - "AV_NOT_REGISTERED" + - "AV_REGISTERED" + - "FULLY_DISABLED" + - "FULLY_ENABLED" + - "INSTALLED" + - "INSTALLED_SERVER" + - "NOT_INSTALLED" + - "ONACCESS_SCAN_DISABLED" + - "ONDEMAND_SCAN_DISABLED" + - "ONDEMOND_SCAN_DISABLED" + - "PRODUCT_UPDATE_DISABLED" + - "SIGNATURE_UPDATE_DISABLED" + - "UNINSTALLED" + - "UNINSTALLED_SERVER" + deregisteredTime: + type: integer + format: epoch-ms-date-time + sensorStates: + type: array + items: + type: string + x-nullable: true + enum: + - "ACTIVE" + - "CSR_ACTION" + - "DB_CORRUPTION_DETECTED" + - "DRIVER_INIT_ERROR" + - "LOOP_DETECTED" + - "PANICS_DETECTED" + - "REMGR_INIT_ERROR" + - "REPUX_ACTION" + - "SENSOR_MAINTENANCE" + - "SENSOR_RESET_IN_PROGRESS" + - "SENSOR_SHUTDOWN" + - "SENSOR_UNREGISTERED" + - "SENSOR_UPGRADE_IN_PROGRESS" + - "UNSUPPORTED_OS" + - "WATCHDOG" + messages: + type: array + items: + type: object + properties: + message: + type: string + time: + type: integer + format: epoch-ms-date-time + rootedBySensor: + type: boolean + rootedBySensorTime: + type: integer + format: epoch-ms-date-time + lastInternalIpAddress: + type: string + lastExternalIpAddress: + type: string + lastLocation: + type: string + x-nullable: true + enum: + - "OFFSITE" + - "ONSITE" + - "UNKNOWN" + avUpdateServers: + type: array + items: + type: string + passiveMode: + type: boolean + lastResetTime: + type: integer + format: epoch-ms-date-time + lastShutdownTime: + type: integer + format: epoch-ms-date-time + scanStatus: + type: string + scanLastActionTime: + type: integer + format: epoch-ms-date-time + scanLastCompleteTime: + type: integer + format: epoch-ms-date-time + linuxKernelVersion: + type: string + avEngine: + type: string + avLastScanTime: + type: integer + format: epoch-ms-date-time + rootedByAnalytics: + type: boolean + rootedByAnalyticsTime: + type: integer + format: epoch-ms-date-time + testId: + type: integer + avMaster: + type: boolean + uninstalledTime: + type: integer + format: epoch-ms-date-time + name: + type: string + status: + type: string + x-nullable: true + enum: + - "ACTIVE" + - "ALL" + - "BYPASS" + - "BYPASS_ON" + - "DEREGISTERED" + - "ERROR" + - "INACTIVE" + - "PENDING" + - "QUARANTINE" + - "REGISTERED" + - "UNINSTALLED" diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py new file mode 100755 index 00000000..747b88ea --- /dev/null +++ b/src/cbapi/psc/rest_api.py @@ -0,0 +1,29 @@ +from cbapi.connection import BaseAPI +from .cblr import LiveResponseSessionManager +import logging + +log = logging.getLogger(__name__) + +class CbPSCBaseAPI(BaseAPI): + """The main entry point into the Cb PSC API. + + :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. + Uses the profile named 'default' when not specified. + + Usage:: + + >>> from cbapi import CbPSCBaseAPI + >>> cb = CbPSCBaseAPI(profile="production") + """ + def __init__(self, *args, **kwargs): + super(CbPSCBaseAPI, self).__init__(product_name="psc", *args, **kwargs) + self._lr_scheduler = None + + @property + def live_response(self): + if self._lr_scheduler is None: + self._lr_scheduler = LiveResponseSessionManager(self) + return self._lr_scheduler + + def _request_lr_session(self, sensor_id): + return self.live_response.request_session(sensor_id) diff --git a/src/cbapi/psc/threathunter/rest_api.py b/src/cbapi/psc/threathunter/rest_api.py index 60a865ac..a52e7d85 100644 --- a/src/cbapi/psc/threathunter/rest_api.py +++ b/src/cbapi/psc/threathunter/rest_api.py @@ -1,5 +1,5 @@ from cbapi.psc.threathunter.query import Query -from cbapi.connection import BaseAPI +from cbapi.psc import CbPSCBaseAPI from cbapi.psc.threathunter.models import ReportSeverity from cbapi.errors import CredentialError import logging @@ -7,7 +7,7 @@ log = logging.getLogger(__name__) -class CbThreatHunterAPI(BaseAPI): +class CbThreatHunterAPI(CbPSCBaseAPI): """The main entry point into the Carbon Black Cloud ThreatHunter API. :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. @@ -19,8 +19,7 @@ class CbThreatHunterAPI(BaseAPI): >>> cb = CbThreatHunterAPI(profile="production") """ def __init__(self, *args, **kwargs): - super(CbThreatHunterAPI, self).__init__(product_name="psc", *args, **kwargs) - self._lr_scheduler = None + super(CbThreatHunterAPI, self).__init__(*args, **kwargs) if not self.credentials.get("org_key", None): raise CredentialError("No organization key specified") From 925e168ece26f65be5bd433b4af453101f3198af Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 23 Sep 2019 14:25:31 -0600 Subject: [PATCH 010/197] screwed up the imports in the subpackages --- src/cbapi/psc/defense/rest_api.py | 2 +- src/cbapi/psc/livequery/rest_api.py | 2 +- src/cbapi/psc/threathunter/rest_api.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cbapi/psc/defense/rest_api.py b/src/cbapi/psc/defense/rest_api.py index db208c6f..12db10f9 100644 --- a/src/cbapi/psc/defense/rest_api.py +++ b/src/cbapi/psc/defense/rest_api.py @@ -1,7 +1,7 @@ from cbapi.utils import convert_query_params from cbapi.query import PaginatedQuery -from cbapi.psc import CbPSCBaseAPI +from cbapi.psc.rest_api import CbPSCBaseAPI import logging import time diff --git a/src/cbapi/psc/livequery/rest_api.py b/src/cbapi/psc/livequery/rest_api.py index e67d8621..824677f0 100644 --- a/src/cbapi/psc/livequery/rest_api.py +++ b/src/cbapi/psc/livequery/rest_api.py @@ -1,5 +1,5 @@ from cbapi.psc.livequery.models import Run, RunHistory -from cbapi.psc import CbPSCBaseAPI +from cbapi.psc.rest_api import CbPSCBaseAPI from cbapi.errors import CredentialError, ApiError import logging diff --git a/src/cbapi/psc/threathunter/rest_api.py b/src/cbapi/psc/threathunter/rest_api.py index a52e7d85..8de417bc 100644 --- a/src/cbapi/psc/threathunter/rest_api.py +++ b/src/cbapi/psc/threathunter/rest_api.py @@ -1,5 +1,5 @@ from cbapi.psc.threathunter.query import Query -from cbapi.psc import CbPSCBaseAPI +from cbapi.psc.rest_api import CbPSCBaseAPI from cbapi.psc.threathunter.models import ReportSeverity from cbapi.errors import CredentialError import logging From b6d4f06c15b0843e0195d54f8051620210f6d5e0 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 23 Sep 2019 14:36:30 -0600 Subject: [PATCH 011/197] minor commit for purpose of rerunning checks request-checks: true --- src/cbapi/psc/threathunter/rest_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cbapi/psc/threathunter/rest_api.py b/src/cbapi/psc/threathunter/rest_api.py index 8de417bc..b53ec429 100644 --- a/src/cbapi/psc/threathunter/rest_api.py +++ b/src/cbapi/psc/threathunter/rest_api.py @@ -6,7 +6,6 @@ log = logging.getLogger(__name__) - class CbThreatHunterAPI(CbPSCBaseAPI): """The main entry point into the Carbon Black Cloud ThreatHunter API. From c0b975f0a418125175582ed19a404d1ce8644d23 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 23 Sep 2019 14:41:03 -0600 Subject: [PATCH 012/197] changed the export definitions in PSC package request-checks: true --- src/cbapi/psc/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cbapi/psc/__init__.py b/src/cbapi/psc/__init__.py index 93149b5c..2b0c1fe0 100644 --- a/src/cbapi/psc/__init__.py +++ b/src/cbapi/psc/__init__.py @@ -2,5 +2,5 @@ from __future__ import absolute_import -from .rest_api import CbPSCBaseAPI -from cbapi.psc.models import Device as DeviceV6 +from cbapi.psc.rest_api import CbPSCBaseAPI +from cbapi.psc.models import Device From fbcf6929ad836f169e17586bb048c462755ff364 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 23 Sep 2019 14:54:19 -0600 Subject: [PATCH 013/197] adding the API object to the top level __init__.py as well request-checks: true --- src/cbapi/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index d6c75863..8139d481 100755 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -11,6 +11,7 @@ # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI from cbapi.protection.rest_api import CbEnterpriseProtectionAPI, CbProtectionAPI +from cbapi.psc import CbPSCBaseAPI from cbapi.psc.defense import CbDefenseAPI from cbapi.psc.threathunter import CbThreatHunterAPI from cbapi.psc.livequery import CbLiveQueryAPI From 7b648c1b3c94dbd6e91331f525decc3359913696 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 23 Sep 2019 15:16:12 -0600 Subject: [PATCH 014/197] something tells me we've got an import loop...trying to correct it --- src/cbapi/psc/cblr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/psc/cblr.py b/src/cbapi/psc/cblr.py index 6341a9f0..dff534c7 100644 --- a/src/cbapi/psc/cblr.py +++ b/src/cbapi/psc/cblr.py @@ -6,7 +6,7 @@ from cbapi.errors import TimeoutError from cbapi.live_response_api import CbLRManagerBase, CbLRSessionBase, poll_status -from cbapi.psc.defense.models import Device +from cbapi.psc.models import Device OS_LIVE_RESPONSE_ENUM = { "WINDOWS": 1, From f91988e80c70b3b171bea733c2aeac58ea337cb7 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 23 Sep 2019 15:17:02 -0600 Subject: [PATCH 015/197] retrying that commit request-checks: true --- src/cbapi/psc/cblr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cbapi/psc/cblr.py b/src/cbapi/psc/cblr.py index dff534c7..ab1b710c 100644 --- a/src/cbapi/psc/cblr.py +++ b/src/cbapi/psc/cblr.py @@ -8,6 +8,7 @@ from cbapi.live_response_api import CbLRManagerBase, CbLRSessionBase, poll_status from cbapi.psc.models import Device + OS_LIVE_RESPONSE_ENUM = { "WINDOWS": 1, "LINUX": 2, From cea8d08cdf94e764dbff8e04f4d2953586ea2757 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Fri, 27 Sep 2019 15:21:12 -0600 Subject: [PATCH 016/197] CBAPI-1061: initial implementation of Devices v6 APIs --- src/cbapi/connection.py | 15 + src/cbapi/psc/livequery/query.py | 384 +---------------------- src/cbapi/psc/models.py | 73 ++++- src/cbapi/psc/models/deviceInfo.yaml | 434 +++++++++++++++----------- src/cbapi/psc/query.py | 440 +++++++++++++++++++++++++++ src/cbapi/psc/rest_api.py | 100 ++++++ 6 files changed, 889 insertions(+), 557 deletions(-) create mode 100755 src/cbapi/psc/query.py diff --git a/src/cbapi/connection.py b/src/cbapi/connection.py index 5f3727f9..98e5243d 100644 --- a/src/cbapi/connection.py +++ b/src/cbapi/connection.py @@ -268,6 +268,21 @@ def get_object(self, uri, query_parameters=None, default=None): return default else: raise ServerError(error_code=result.status_code, message="Unknown error: {0}".format(result.content)) + + def get_raw_data(self, uri, query_parameters=None, default=None): + if query_parameters: + if isinstance(query_parameters, dict): + query_parameters = convert_query_params(query_parameters) + uri += '?%s' % (urllib.parse.urlencode(sorted(query_parameters))) + + result = self.api_json_request("GET", uri) + if result.status_code == 200: + return result + elif result.status_code == 204: + # empty response + return default + else: + raise ServerError(error_code=result.status_code, message="Unknown error: {0}".format(result.content)) def api_json_request(self, method, uri, **kwargs): headers = kwargs.pop("headers", {}) diff --git a/src/cbapi/psc/livequery/query.py b/src/cbapi/psc/livequery/query.py index 4ee5df99..6471e552 100644 --- a/src/cbapi/psc/livequery/query.py +++ b/src/cbapi/psc/livequery/query.py @@ -1,199 +1,13 @@ -from cbapi.errors import ApiError, MoreThanOneResultError +from cbapi.errors import ApiError +from cbapi.psc.query import QueryBuilder, PSCQueryBase +from cbapi.psc.query import QueryBuilderSupportMixin, IterableQueryMixin import logging -import functools from six import string_types -from solrq import Q log = logging.getLogger(__name__) -class QueryBuilder(object): - """ - Provides a flexible interface for building prepared queries for the CB - LiveQuqery backend. - - This object can be instantiated directly, or can be managed implicitly - through the :py:meth:`CbLiveQuqeryAPI.select` API. - """ - - def __init__(self, **kwargs): - if kwargs: - self._query = Q(**kwargs) - else: - self._query = None - self._raw_query = None - - def _guard_query_params(func): - """Decorates the query construction methods of *QueryBuilder*, preventing - them from being called with parameters that would result in an intetnally - inconsistent query. - """ - - @functools.wraps(func) - def wrap_guard_query_change(self, q, **kwargs): - if self._raw_query is not None and (kwargs or isinstance(q, Q)): - raise ApiError("Cannot modify a raw query with structured parameters") - if self._query is not None and isinstance(q, string_types): - raise ApiError("Cannot modify a structured query with a raw parameter") - return func(self, q, **kwargs) - - return wrap_guard_query_change - - @_guard_query_params - def where(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - if self._raw_query is None: - self._raw_query = [] - self._raw_query.append(q) - elif isinstance(q, Q) or kwargs: - if self._query is not None: - raise ApiError("Use .and_() or .or_() for an extant solrq.Q object") - if kwargs: - q = Q(**kwargs) - self._query = q - else: - raise ApiError(".where() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def and_(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - self.where(q) - elif isinstance(q, Q) or kwargs: - if kwargs: - q = Q(**kwargs) - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".and_() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def or_(self, q, **kwargs): - """Adds a disjunctive filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - q = Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query | q - else: - raise ApiError(".or_() only accepts solrq.Q objects") - - return self - - @_guard_query_params - def not_(self, q, **kwargs): - """Adds a negative filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - q = ~Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".not_() only accepts solrq.Q objects") - - def _collapse(self): - """The query can be represented by either an array of strings - (_raw_query) which is concatenated and passed directly to Solr, or - a solrq.Q object (_query) which is then converted into a string to - pass to Solr. This function will perform the appropriate conversions to - end up with the 'q' string sent into the POST request to the - PSC-R query endpoint.""" - if self._raw_query is not None: - return " ".join(self._raw_query) - elif self._query is not None: - return str(self._query) - else: - return None # return everything - - -class LiveQueryBase: - """ - Represents the base of all LiveQuery query classes. - """ - - def __init__(self, doc_class, cb): - self._doc_class = doc_class - self._cb = cb - - -class IterableQueryMixin: - """ - A mix-in to provide iterability to a query. - """ - - def all(self): - return self._perform_query() - - def first(self): - allres = list(self) - res = allres[:1] - if not len(res): - return None - return res[0] - - def one(self): - allres = list(self) - res = allres[:2] - if len(res) == 0: - raise MoreThanOneResultError( - message="0 results for query {0:s}".format(self._query) - ) - if len(res) > 1: - raise MoreThanOneResultError( - message="{0:d} results found for query {1:s}".format( - len(self), self._query - ) - ) - return res[0] - - def __len__(self): - return 0 - - def __getitem__(self, item): - return None - - def __iter__(self): - return self._perform_query() - - -class RunQuery(LiveQueryBase): +class RunQuery(PSCQueryBase): """ Represents a query that either creates or retrieves the status of a LiveQuery run. @@ -293,7 +107,7 @@ def submit(self): return self._doc_class(self._cb, initial_data=resp.json()) -class RunHistoryQuery(LiveQueryBase, IterableQueryMixin): +class RunHistoryQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): """ Represents a query that retrieves historic LiveQuery runs. """ @@ -302,66 +116,6 @@ def __init__(self, doc_class, cb): self._query_builder = QueryBuilder() self._sort = {} - def where(self, q=None, **kwargs): - """Add a filter to this query. - - :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q: - return self - if isinstance(q, QueryBuilder): - self._query_builder = q - else: - self._query_builder.where(q, **kwargs) - return self - - def and_(self, q=None, **kwargs): - """Add a conjunctive filter to this query. - - :param q: Query string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") - - self._query_builder.and_(q, **kwargs) - return self - - def or_(self, q=None, **kwargs): - """Add a disjunctive filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".or_() expects a solrq.Q or kwargs") - - self._query_builder.or_(q, **kwargs) - return self - - def not_(self, q=None, **kwargs): - """Adds a negated filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q and not kwargs: - raise ApiError(".not_() expects a solrq.Q, or kwargs") - - self._query_builder.not_(q, **kwargs) - return self - def sort_by(self, key, direction="ASC"): """Sets the sorting behavior on a query's results. @@ -435,7 +189,7 @@ def _perform_query(self, start=0, rows=0): break -class ResultQuery(LiveQueryBase, IterableQueryMixin): +class ResultQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): """ Represents a query that retrieves results from a LiveQuery run. """ @@ -447,68 +201,6 @@ def __init__(self, doc_class, cb): self._batch_size = 100 self._run_id = None - def where(self, q=None, **kwargs): - """Add a filter to this query. - - :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError( - ".where() expects a string, a QueryBuilder, a solrq.Q, or kwargs" - ) - - if isinstance(q, QueryBuilder): - self._query_builder = q - else: - self._query_builder.where(q, **kwargs) - return self - - def and_(self, q=None, **kwargs): - """Add a conjunctive filter to this query. - - :param q: Query string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") - - self._query_builder.and_(q, **kwargs) - return self - - def or_(self, q=None, **kwargs): - """Add a disjunctive filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".or_() expects a solrq.Q or kwargs") - - self._query_builder.or_(q, **kwargs) - return self - - def not_(self, q=None, **kwargs): - """Adds a negated filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q and not kwargs: - raise ApiError(".not_() expects a solrq.Q, or kwargs") - - self._query_builder.not_(q, **kwargs) - return self - def criteria(self, **kwargs): """Sets the filter criteria on a query's results. @@ -609,7 +301,7 @@ def _perform_query(self, start=0, rows=0): break -class FacetQuery(LiveQueryBase, IterableQueryMixin): +class FacetQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): """ Represents a query that receives facet information from a LiveQuery run. """ @@ -620,68 +312,6 @@ def __init__(self, doc_class, cb): self._criteria = {} self._run_id = None - def where(self, q=None, **kwargs): - """Add a filter to this query. - - :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError( - ".where() expects a string, a QueryBuilder, a solrq.Q, or kwargs" - ) - - if isinstance(q, QueryBuilder): - self._query_builder = q - else: - self._query_builder.where(q, **kwargs) - return self - - def and_(self, q=None, **kwargs): - """Add a conjunctive filter to this query. - - :param q: Query string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") - - self._query_builder.and_(q, **kwargs) - return self - - def or_(self, q=None, **kwargs): - """Add a disjunctive filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".or_() expects a solrq.Q or kwargs") - - self._query_builder.or_(q, **kwargs) - return self - - def not_(self, q=None, **kwargs): - """Adds a negated filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q and not kwargs: - raise ApiError(".not_() expects a solrq.Q, or kwargs") - - self._query_builder.not_(q, **kwargs) - return self - def facet_field(self, field): """Sets the facet fields to be received by this query. diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 0ffe8349..785c0ccd 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -1,9 +1,11 @@ from cbapi.models import MutableBaseModel from cbapi.errors import ServerError +from cbapi.psc.query import DeviceSearchQuery from copy import deepcopy import logging import json +import time log = logging.getLogger(__name__) @@ -105,14 +107,26 @@ def _refresh_if_needed(self, request_ret): class Device(PSCMutableModel): - urlobject = "/integrationServices/v3/device" - primary_key = "deviceId" - info_key = "deviceInfo" + urlobject = "/appservices/v6/orgs/{0}/devices" + urlobject_single = "/appservices/v6/orgs/{0}/devices/{1}" + primary_key = "device_id" + #info_key = "deviceInfo" swagger_meta_file = "psc/models/deviceInfo.yaml" def __init__(self, cb, model_unique_id, initial_data=None): super(Device, self).__init__(cb, model_unique_id, initial_data) + @classmethod + def _query_implementation(cls, cb): + return DeviceSearchQuery(cls, cb) + + def _refresh(self): + url = self.urlobject_single.format(self._cb.credentials.org_key, self._model_unique_id) + resp = self._cb.get_object(url) + self._info = resp + self._last_refresh_time = time.time() + return True + def lr_session(self): """ Retrieve a Live Response session object for this Device. @@ -124,4 +138,57 @@ def lr_session(self): """ return self._cb._request_lr_session(self._model_unique_id) + def background_scan(self, flag): + """ + Set the background scan option for this device. + + :param boolean flag: True to turn background scan on, False to turn it off. + """ + return self._cb.device_background_scan([ self._model_unique_id ], flag) + + def bypass(self, flag): + """ + Set the bypass option for this device. + + :param boolean flag: True to enable bypass, False to disable it. + """ + return self._cb.device_bypass([ self._model_unique_id ], flag) + + def delete_sensor(self): + """ + Delete this sensor device. + """ + return self._cb.device_delete_sensor([ self._model_unique_id ]) + + def deregister_sensor(self): + """ + Deregister this sensor device. + """ + return self._cb.device_deregister_sensor([ self._model_unique_id ]) + + def quarantine(self, flag): + """ + Set the quarantine option for this device. + + :param boolean flag: True to enable quarantine, False to disable it. + """ + return self._cb.device_quarantine([ self._model_unique_id ], flag) + + def update_policy(self, policy_id): + """ + Set the current policy for this device. + + :param int policy_id: ID of the policy to set for the devices. + """ + return self._cb.device_update_policy([ self._model_unique_id ], policy_id) + + def update_sensor_version(self, sensor_version): + """ + Update the sensor version for this device. + + :param dict sensor_version: New version of the sensor; + specified as { "OS": "versionnumber" } + """ + return self._cb.device_update_sensor_version([ self._model_unique_id ], sensor_version) + \ No newline at end of file diff --git a/src/cbapi/psc/models/deviceInfo.yaml b/src/cbapi/psc/models/deviceInfo.yaml index 7d7106a7..4aa2f197 100755 --- a/src/cbapi/psc/models/deviceInfo.yaml +++ b/src/cbapi/psc/models/deviceInfo.yaml @@ -1,221 +1,301 @@ type: object properties: - osVersion: + activation_code: type: string - activationCode: + description: Device activation code + activation_code_expiry_time: type: string - organizationId: + description: When the expiration code expires and cannot be used to register a device + ad_group_id: type: integer format: int64 - deviceId: - type: integer - format: int64 - deviceSessionId: + description: Device's AD group + av_ave_version: + type: string + description: AVE version (part of AV Version) + av_engine: + type: string + example: '4.3.0.203-ave.8.3.42.106:avpack.8.4.2.36:vdf.8.12.142.100' + description: Current AV version + av_last_scan_time: + type: string + description: Last AV scan time + av_master: + type: boolean + description: Whether the device is an AV Master (?) + av_pack_version: + type: string + description: Pack version (part of AV Version) + av_product_version: + type: string + description: AV Product version (part of AV Version) + av_status: + type: array + description: AV Statuses + items: + type: string + enum: + - AV_NOT_REGISTERED + - AV_REGISTERED + - AV_DEREGISTERED + - AV_ACTIVE + - AV_BYPASS + - NOT_INSTALLED + - INSTALLED + - UNINSTALLED + - INSTALLED_SERVER + - UNINSTALLED_SERVER + - FULLY_ENABLED + - FULLY_DISABLED + - SIGNATURE_UPDATE_DISABLED + - ONACCESS_SCAN_DISABLED + - ONDEMOND_SCAN_DISABLED + - ONDEMAND_SCAN_DISABLED + - PRODUCT_UPDATE_DISABLED + av_update_servers: + type: array + description: Device's AV servers + items: + type: string + av_vdf_version: + type: string + description: VDF version (part of AV Version) + current_sensor_policy_name: + type: string + description: Current MSM policy name + deregistered_time: + type: string + description: When the device was deregistered with the PSC backend + device_id: type: integer format: int64 - deviceOwnerId: + description: ID of the device + device_meta_data_item_list: + type: array + description: MSM Device metadata + items: + type: object + properties: + key_name: + type: string + key_value: + type: string + position: + type: integer + format: int32 + device_owner_id: type: integer format: int64 - deviceGuid: + description: ID of the user who owns the device + device_type: type: string - format: uuid + example: WINDOWS + description: Device type + enum: + - WINDOWS + - ANDROID + - MAC + - IOS + - LINUX + - OTHER email: type: string - format: email - assignedToId: - type: integer - format: int64 - assignedToName: + description: Email of the user who owns the device + encoded_activation_code: + type: string + description: Encoded device activation code + first_name: type: string - deviceType: + description: First name of the user who owns the device + last_contact_time: type: string - x-nullable: true + description: Time the device last checked into the PSC backend + last_device_policy_changed_time: + type: string + description: Last time the device's policy was changed + last_device_policy_requested_time: + type: string + description: Last time the device requested policy updates + last_external_ip_address: + type: string + description: Device's external IP + last_internal_ip_address: + type: string + description: Device's internal IP + last_location: + type: string + description: Location of the device (on-/off-premises) enum: - - "MAC" - - "WINDOWS" - firstName: + - UNKNOWN + - ONSITE + - OFFSITE + last_name: + type: string + description: Last name of the user who owns the device + last_policy_updated_time: type: string - lastName: + description: Last time the device was MSM processed + last_reported_time: type: string - middleName: + description: Time when device last reported an event to PSC backend + last_reset_time: type: string - createTime: + description: When the sensor was last reset + last_shutdown_time: + type: string + description: When the device last shut down + linux_kernel_version: + type: string + description: Linux kernel version + login_user_name: + type: string + description: Last acive logged in username + mac_address: + type: string + description: Device's hardware MAC address + middle_name: + type: string + description: Middle name of the user who owns the device + name: + type: string + description: Device Hostname + organization_id: type: integer - format: epoch-ms-date-time - policyId: + format: int64 + example: 1000 + description: Org ID to which the device belongs + organization_name: + type: string + description: Name of the org that owns this device + os_version: + type: string + example: 'Windows 7 x86 SP: 1' + description: Version of the OS + passive_mode: + type: boolean + description: Whether the device is in passive mode (bypass?) + policy_id: type: integer format: int64 - policyName: + description: ID of the policy this device is using + policy_name: type: string + description: Name of the policy this device is using + policy_override: + type: boolean + description: Manually assigned policy (overrides mass sensor management) quarantined: type: boolean - targetPriorityType: + registered_time: type: string - x-nullable: true - enum: - - "HIGH" - - "LOW" - - "MEDIUM" - - "MISSION_CRITICAL" - lastVirusActivityTime: - type: integer - format: epoch-ms-date-time - firstVirusActivityTime: - type: integer - format: epoch-ms-date-time - activationCodeExpiryTime: - type: integer - format: epoch-ms-date-time - organizationName: + description: When the device was registered with the PSC backend + rooted_by_analytics: + type: boolean + rooted_by_analytics_time: type: string - sensorVersion: + format: date-time + rooted_by_sensor: + type: boolean + scan_last_action_time: type: string - registeredTime: - type: integer - format: epoch-ms-date-time - lastContact: - type: integer - format: epoch-ms-date-time - lastReportedTime: - type: integer - format: epoch-ms-date-time - windowsPlatform: + description: When the background scan was last active + scan_last_complete_time: + type: string + description: When the background scan was last completed + scan_status: type: string - x-nullable: true + description: Background scan status enum: - - "CLIENT_X64" - - "CLIENT_X86" - - "SERVER_X6" - - "SERVER_X86" - vdiBaseDevice: - type: integer - format: int64 - avStatus: - type: array - items: - type: string - x-nullable: true - enum: - - "AV_ACTIVE" - - "AV_BYPASS" - - "AV_DEREGISTERED" - - "AV_NOT_REGISTERED" - - "AV_REGISTERED" - - "FULLY_DISABLED" - - "FULLY_ENABLED" - - "INSTALLED" - - "INSTALLED_SERVER" - - "NOT_INSTALLED" - - "ONACCESS_SCAN_DISABLED" - - "ONDEMAND_SCAN_DISABLED" - - "ONDEMOND_SCAN_DISABLED" - - "PRODUCT_UPDATE_DISABLED" - - "SIGNATURE_UPDATE_DISABLED" - - "UNINSTALLED" - - "UNINSTALLED_SERVER" - deregisteredTime: - type: integer - format: epoch-ms-date-time - sensorStates: + - NEVER_RUN + - STOPPED + - IN_PROGRESS + - COMPLETED + sensor_out_of_date: + type: boolean + description: Whether the device is out of date + sensor_states: type: array + description: Active sensor states items: type: string - x-nullable: true enum: - - "ACTIVE" - - "CSR_ACTION" - - "DB_CORRUPTION_DETECTED" - - "DRIVER_INIT_ERROR" - - "LOOP_DETECTED" - - "PANICS_DETECTED" - - "REMGR_INIT_ERROR" - - "REPUX_ACTION" - - "SENSOR_MAINTENANCE" - - "SENSOR_RESET_IN_PROGRESS" - - "SENSOR_SHUTDOWN" - - "SENSOR_UNREGISTERED" - - "SENSOR_UPGRADE_IN_PROGRESS" - - "UNSUPPORTED_OS" - - "WATCHDOG" - messages: - type: array - items: - type: object - properties: - message: - type: string - time: - type: integer - format: epoch-ms-date-time - rootedBySensor: - type: boolean - rootedBySensorTime: - type: integer - format: epoch-ms-date-time - lastInternalIpAddress: - type: string - lastExternalIpAddress: + - ACTIVE + - PANICS_DETECTED + - LOOP_DETECTED + - DB_CORRUPTION_DETECTED + - CSR_ACTION + - REPUX_ACTION + - DRIVER_INIT_ERROR + - REMGR_INIT_ERROR + - UNSUPPORTED_OS + - SENSOR_UPGRADE_IN_PROGRESS + - SENSOR_UNREGISTERED + - WATCHDOG + - SENSOR_RESET_IN_PROGRESS + - DRIVER_INIT_REBOOT_REQUIRED + - SENSOR_SHUTDOWN + - SENSOR_MAINTENANCE + - DEBUG_MODE_ENABLED + - AUTO_UPDATE_DISABLED + - SELF_PROTECT_DISABLED + - VDI_MODE_ENABLED + - POC_MODE_ENABLED + - SECURITY_CENTER_OPTLN_DISABLED + - LIVE_RESPONSE_RUNNING + - LIVE_RESPONSE_NOT_RUNNING + - LIVE_RESPONSE_KILLED + - LIVE_RESPONSE_NOT_KILLED + - LIVE_RESPONSE_ENABLED + - LIVE_RESPONSE_DISABLED + sensor_version: type: string - lastLocation: + example: 3.4.0.0 + description: Version of the PSC sensor + status: type: string - x-nullable: true + description: Device status enum: - - "OFFSITE" - - "ONSITE" - - "UNKNOWN" - avUpdateServers: - type: array - items: - type: string - passiveMode: - type: boolean - lastResetTime: - type: integer - format: epoch-ms-date-time - lastShutdownTime: - type: integer - format: epoch-ms-date-time - scanStatus: + - PENDING + - REGISTERED + - UNINSTALLED + - DEREGISTERED + - ACTIVE + - INACTIVE + - ERROR + - ALL + - BYPASS_ON + - BYPASS + - QUARANTINE + - SENSOR_OUTOFDATE + - DELETED + - LIVE + target_priority_type: type: string - scanLastActionTime: - type: integer - format: epoch-ms-date-time - scanLastCompleteTime: - type: integer - format: epoch-ms-date-time - linuxKernelVersion: - type: string - avEngine: + example: MISSION_CRITICAL + description: Priority of the device + enum: + - LOW + - MEDIUM + - HIGH + - MISSION_CRITICAL + uninstall_code: type: string - avLastScanTime: - type: integer - format: epoch-ms-date-time - rootedByAnalytics: - type: boolean - rootedByAnalyticsTime: + description: Code to enter to uninstall this device + vdi_base_device: type: integer - format: epoch-ms-date-time - testId: - type: integer - avMaster: + format: int64 + description: VDI Base device + virtual_machine: type: boolean - uninstalledTime: - type: integer - format: epoch-ms-date-time - name: + description: Whether this device is a Virtual Machine (VMware AppDefense integration + virtualization_provider: type: string - status: + description: VM Virtualization Provider + windows_platform: type: string - x-nullable: true + description: 'Type of windows platform (client/server, x86/x64)' enum: - - "ACTIVE" - - "ALL" - - "BYPASS" - - "BYPASS_ON" - - "DEREGISTERED" - - "ERROR" - - "INACTIVE" - - "PENDING" - - "QUARANTINE" - - "REGISTERED" - - "UNINSTALLED" + - CLIENT_X86 + - CLIENT_X64 + - SERVER_X86 + - SERVER_X64 diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py new file mode 100755 index 00000000..2d278f9c --- /dev/null +++ b/src/cbapi/psc/query.py @@ -0,0 +1,440 @@ +from cbapi.errors import ApiError, MoreThanOneResultError +import logging +import functools +from six import string_types +from solrq import Q +from builtins import isinstance + +log = logging.getLogger(__name__) + +class QueryBuilder(object): + """ + Provides a flexible interface for building prepared queries for the CB + PSC backend. + + This object can be instantiated directly, or can be managed implicitly + through the :py:meth:`select` API. + """ + + def __init__(self, **kwargs): + if kwargs: + self._query = Q(**kwargs) + else: + self._query = None + self._raw_query = None + + def _guard_query_params(func): + """Decorates the query construction methods of *QueryBuilder*, preventing + them from being called with parameters that would result in an internally + inconsistent query. + """ + + @functools.wraps(func) + def wrap_guard_query_change(self, q, **kwargs): + if self._raw_query is not None and (kwargs or isinstance(q, Q)): + raise ApiError("Cannot modify a raw query with structured parameters") + if self._query is not None and isinstance(q, string_types): + raise ApiError("Cannot modify a structured query with a raw parameter") + return func(self, q, **kwargs) + + return wrap_guard_query_change + + @_guard_query_params + def where(self, q, **kwargs): + """Adds a conjunctive filter to a query. + + :param q: string or `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: QueryBuilder object + :rtype: :py:class:`QueryBuilder` + """ + if isinstance(q, string_types): + if self._raw_query is None: + self._raw_query = [] + self._raw_query.append(q) + elif isinstance(q, Q) or kwargs: + if self._query is not None: + raise ApiError("Use .and_() or .or_() for an extant solrq.Q object") + if kwargs: + q = Q(**kwargs) + self._query = q + else: + raise ApiError(".where() only accepts strings or solrq.Q objects") + + return self + + @_guard_query_params + def and_(self, q, **kwargs): + """Adds a conjunctive filter to a query. + + :param q: string or `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: QueryBuilder object + :rtype: :py:class:`QueryBuilder` + """ + if isinstance(q, string_types): + self.where(q) + elif isinstance(q, Q) or kwargs: + if kwargs: + q = Q(**kwargs) + if self._query is None: + self._query = q + else: + self._query = self._query & q + else: + raise ApiError(".and_() only accepts strings or solrq.Q objects") + + return self + + @_guard_query_params + def or_(self, q, **kwargs): + """Adds a disjunctive filter to a query. + + :param q: `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: QueryBuilder object + :rtype: :py:class:`QueryBuilder` + """ + if kwargs: + q = Q(**kwargs) + + if isinstance(q, Q): + if self._query is None: + self._query = q + else: + self._query = self._query | q + else: + raise ApiError(".or_() only accepts solrq.Q objects") + + return self + + @_guard_query_params + def not_(self, q, **kwargs): + """Adds a negative filter to a query. + + :param q: `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: QueryBuilder object + :rtype: :py:class:`QueryBuilder` + """ + if kwargs: + q = ~Q(**kwargs) + + if isinstance(q, Q): + if self._query is None: + self._query = q + else: + self._query = self._query & q + else: + raise ApiError(".not_() only accepts solrq.Q objects") + + def _collapse(self): + """The query can be represented by either an array of strings + (_raw_query) which is concatenated and passed directly to Solr, or + a solrq.Q object (_query) which is then converted into a string to + pass to Solr. This function will perform the appropriate conversions to + end up with the 'q' string sent into the POST request to the + PSC-R query endpoint.""" + if self._raw_query is not None: + return " ".join(self._raw_query) + elif self._query is not None: + return str(self._query) + else: + return None # return everything + +class PSCQueryBase: + """ + Represents the base of all LiveQuery query classes. + """ + + def __init__(self, doc_class, cb): + self._doc_class = doc_class + self._cb = cb + + +class QueryBuilderSupportMixin: + """ + A mixin that supplies wrapper methods to access the _query_builder. + """ + def where(self, q=None, **kwargs): + """Add a filter to this query. + + :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: Query object + :rtype: :py:class:`Query` + """ + + if not q: + return self + if isinstance(q, QueryBuilder): + self._query_builder = q + else: + self._query_builder.where(q, **kwargs) + return self + + def and_(self, q=None, **kwargs): + """Add a conjunctive filter to this query. + + :param q: Query string or `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: Query object + :rtype: :py:class:`Query` + """ + if not q and not kwargs: + raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") + + self._query_builder.and_(q, **kwargs) + return self + + def or_(self, q=None, **kwargs): + """Add a disjunctive filter to this query. + + :param q: `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: Query object + :rtype: :py:class:`Query` + """ + if not q and not kwargs: + raise ApiError(".or_() expects a solrq.Q or kwargs") + + self._query_builder.or_(q, **kwargs) + return self + + def not_(self, q=None, **kwargs): + """Adds a negated filter to this query. + + :param q: `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: Query object + :rtype: :py:class:`Query` + """ + + if not q and not kwargs: + raise ApiError(".not_() expects a solrq.Q, or kwargs") + + self._query_builder.not_(q, **kwargs) + return self + + +class IterableQueryMixin: + """ + A mix-in to provide iterability to a query. + """ + def all(self): + return self._perform_query() + + def first(self): + allres = list(self) + res = allres[:1] + if not len(res): + return None + return res[0] + + def one(self): + allres = list(self) + res = allres[:2] + if len(res) == 0: + raise MoreThanOneResultError( + message="0 results for query {0:s}".format(self._query) + ) + if len(res) > 1: + raise MoreThanOneResultError( + message="{0:d} results found for query {1:s}".format( + len(self), self._query + ) + ) + return res[0] + + def __len__(self): + return 0 + + def __getitem__(self, item): + return None + + def __iter__(self): + return self._perform_query() + + +class DeviceSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): + """ + Represents a query that is used to locate Device objects. + """ + valid_statuses = ["PENDING", "REGISTERED", "UNINSTALLED", "DEREGISTERED", + "ACTIVE", "INACTIVE", "ERROR", "ALL", "BYPASS_ON", + "BYPASS", "QUARANTINE", "SENSOR_OUTOFDATE", + "DELETED", "LIVE"] + valid_priorities = ["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"] + valid_sort_keys = ["target_priority", "policy_name", "name", + "last_contact_time", "av_pack_version"] + valid_directions = ["ASC", "DESC"] + + def __init__(self, doc_class, cb): + super().__init__(doc_class, cb) + self._query_builder = QueryBuilder() + self._query_body = {} + self._sortcriteria = {} + + def ad_group_ids(self, ad_group_ids): + """ + Restricts the devices that this query is performed on to the specified + AD group IDs. + + :param ad_group_ids: list of ints + :return: This instance + """ + if not all(isinstance(ad_group_id, int) for ad_group_id in ad_group_ids): + raise ApiError("One or more invalid AD group IDs") + self._query_body["ad_group_ids"] = ad_group_ids + return self + + def policy_ids(self, policy_ids): + """ + Restricts the devices that this query is performed on to the specified + policy IDs. + + :param policy_ids: list of ints + :return: This instance + """ + if not all(isinstance(policy_id, int) for policy_id in policy_ids): + raise ApiError("One or more invalid AD group IDs") + self._query_body["policy_ids"] = policy_ids + return self + + def status(self, statuses): + """ + Restricts the devices that this query is performed on to the specified + status values. + + :param statuses: list of strings + :return: This instance + """ + if not all((stat in DeviceSearchQuery.valid_statuses) for stat in statuses): + raise ApiError("One or more invalid status values") + self._query_body["status"] = statuses + return self + + def target_priorities(self, target_priorities): + """ + Restricts the devices that this query is performed on to the specified + target priority values. + + :param target_priorities: list of strings + :return: This instance + """ + if not all((prio in DeviceSearchQuery.valid_priorities) for prio in target_priorities): + raise ApiError("One or more invalid target priority values") + self._query_body["target_priorities"] = target_priorities + return self + + def sort_by(self, key, direction="ASC"): + """Sets the sorting behavior on a query's results. + + Example:: + + >>> cb.select(Device).sort_by("name") + + :param key: the key in the schema to sort by + :param direction: the sort order, either "ASC" or "DESC" + :rtype: :py:class:`DeviceSearchQuery` + """ + if key not in DeviceSearchQuery.valid_sort_keys: + raise ApiError("Invalid sort key specified") + if direction not in DeviceSearchQuery.valid_directions: + raise ApiError("invalid sort direction specified") + self._sortcriteria = {"field_name": key, "sort_order": direction} + return self + + def _build_request(self): + request = self._query_body + request["query_string"] = self._query_builder._collapse() + if not self._sortcriteria.is_empty(): + request["sort"] = self._sortcriteria + return request + + def _build_url(self, from_row, max_rows, tail_end): + url = self._doc_class.urlobject.format( + self._cb.credentials.org_key) + tail_end + query_params = [] + if from_row > 0: + query_params.append("from_row={0:i}".format(from_row)) + if max_rows >= 0: + query_params.append("max_rows={0:i}".format(max_rows)) + if not query_params.is_empty(): + url = url + "?" + "&".join(query_params) + return url + + def _count(self): + if self._count_valid: + return self._total_results + + url = self._build_url(0, -1, "/_search") + request = self._build_request() + resp = self._cb.post_object(url, body=request) + result = resp.json() + + self._total_results = result["num_found"] + self._count_valid = True + + return self._total_results + + def _perform_query(self, from_row=0, max_rows=-1): + request = self._build_request() + current = from_row + numrows = 0 + still_querying = True + while still_querying: + url = self._build_url(from_row, max_rows, "/_search") + resp = self._cb.post_object(url, body=request) + result = resp.json() + + self._total_results = result["num_found"] + self._count_valid = True + + results = result.get("results", []) + for item in results: + yield self._doc_class(self._cb, item["device_id"], item) + current += 1 + numrows += 1 + + if max_rows > 0 and numrows == max_rows: + still_querying = False + break + + from_row = current + if current >= self._total_results: + still_querying = False + break + + def download(self): + """ + Uses the query parameters that have been set to download all + device listings in CSV format. + + Example:: + + >>> cb.select(Device).status(["ALL"]).download() + + :return: The CSV raw data as returned from the server. + """ + tmp = self._query_body.get("status",[]) + if tmp.is_empty(): + raise ApiError("at least one status must be specified to download") + query_params = { "device_status": ",".join(tmp) } + tmp = self._query_body.get("ad_group_ids", []) + if not tmp.is_empty(): + query_params["ad_group_id"] = ",".join(tmp) + tmp = self._query_body.get("policy_ids", []) + if not tmp.is_empty(): + query_params["policy_id"] = ",".join(tmp) + tmp = self._query_body.get("target_priorities", []) + if not tmp.is_empty(): + query_params["target_priority"] = ",".join(tmp) + tmp = self._query_builder._collapse() + if not tmp.is_empty(): + query_params["query_string"] = tmp + if not self._sortcriteria.is_empty(): + query_params["sort_field"] = self._sortcriteria["field_name"] + query_params["sort_order"] = self._sortcriteria["sort_order"] + url = self._build_url(0, -1, "/_search/download") + return self._cb.get_raw_data(url, query_params) diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 747b88ea..e756cf79 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -1,5 +1,7 @@ from cbapi.connection import BaseAPI +from cbapi.errors import ServerError from .cblr import LiveResponseSessionManager +from .models import Device import logging log = logging.getLogger(__name__) @@ -18,6 +20,8 @@ class CbPSCBaseAPI(BaseAPI): def __init__(self, *args, **kwargs): super(CbPSCBaseAPI, self).__init__(product_name="psc", *args, **kwargs) self._lr_scheduler = None + + #---- LiveOps @property def live_response(self): @@ -27,3 +31,99 @@ def live_response(self): def _request_lr_session(self, sensor_id): return self.live_response.request_session(sensor_id) + + #---- Device API + + def get_device(self, device_id): + """ + Locate a device with the specified device ID. + + :param int device_id: The ID of the device to look up. + :return: The new device object. + """ + rc = Device(self, device_id) + rc.refresh() + return rc + + def _device_action(self, device_ids, action_type, options=None): + request = { "action_type": action_type, "device_id": device_ids } + if options: + request["options"] = options + url = "/appservices/v6/orgs/{0}/device_actions".format(self.credentials.org_key) + resp = self._cb.post_object(url, body=request) + if resp.status_code == 200: + return resp.json() + elif resp.status_code == 204: + return None + else: + raise ServerError(error_code=resp.status_code, message="Device action error: {0}".format(resp.content)) + + def _action_toggle(self, flag): + if flag: + return { "toggle": "ON" } + else: + return { "toggle": "OFF" } + + def device_background_scan(self, device_ids, flag): + """ + Set the background scan option for the specified devices. + + :param list device_ids: List of IDs of devices to be set. + :param boolean flag: True to turn background scan on, False to turn it off. + """ + return self._device_action(device_ids, "BACKGROUND_SCAN", self._action_toggle(flag)) + + def device_bypass(self, device_ids, flag): + """ + Set the bypass option for the specified devices. + + :param list device_ids: List of IDs of devices to be set. + :param boolean flag: True to enable bypass, False to disable it. + """ + return self._device_action(device_ids, "BYPASS", self._action_toggle(flag)) + + def device_delete_sensor(self, device_ids): + """ + Delete the specified sensor devices. + + :param list device_ids: List of IDs of devices to be deleted. + """ + return self._device_action(device_ids, "DELETE_SENSOR") + + def device_deregister_sensor(self, device_ids): + """ + Deregister the specified sensor devices. + + :param list device_ids: List of IDs of devices to be deregistered. + """ + return self._device_action(device_ids, "DEREGISTER_SENSOR") + + def device_quarantine(self, device_ids, flag): + """ + Set the quarantine option for the specified devices. + + :param list device_ids: List of IDs of devices to be set. + :param boolean flag: True to enable quarantine, False to disable it. + """ + return self._device_action(device_ids, "QUARANTINE", self._action_toggle(flag)) + + def device_update_policy(self, device_ids, policy_id): + """ + Set the current policy for the specified devices. + + :param list device_ids: List of IDs of devices to be changed. + :param int policy_id: ID of the policy to set for the devices. + """ + return self._device_action(device_ids, "UPDATE_POLICY", { "policy_id": policy_id }) + + def device_update_sensor_version(self, device_ids, sensor_version): + """ + Update the sensor version for the specified devices. + + :param list device_ids: List of IDs of devices to be changed. + :param dict sensor_version: New version of the sensor; + specified as { "OS": "versionnumber" } + """ + return self._device_action(device_ids, "UPDATE_SENSOR_VERSION", + { "sensor_version": sensor_version }) + \ No newline at end of file From 2774ed6b95d188f60b291fd1b88c8c291ba673b0 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Fri, 27 Sep 2019 16:31:56 -0600 Subject: [PATCH 017/197] CBAPI-1061: added unit tests for the new model object --- src/cbapi/psc/__init__.py | 4 +- src/cbapi/psc/rest_api.py | 2 +- test/cbapi/psc/test_models.py | 183 ++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 3 deletions(-) create mode 100755 test/cbapi/psc/test_models.py diff --git a/src/cbapi/psc/__init__.py b/src/cbapi/psc/__init__.py index 2b0c1fe0..239b1eb6 100644 --- a/src/cbapi/psc/__init__.py +++ b/src/cbapi/psc/__init__.py @@ -2,5 +2,5 @@ from __future__ import absolute_import -from cbapi.psc.rest_api import CbPSCBaseAPI -from cbapi.psc.models import Device +from .rest_api import CbPSCBaseAPI +from .models import Device diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index e756cf79..83618872 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -50,7 +50,7 @@ def _device_action(self, device_ids, action_type, options=None): if options: request["options"] = options url = "/appservices/v6/orgs/{0}/device_actions".format(self.credentials.org_key) - resp = self._cb.post_object(url, body=request) + resp = self.post_object(url, body=request) if resp.status_code == 200: return resp.json() elif resp.status_code == 204: diff --git a/test/cbapi/psc/test_models.py b/test/cbapi/psc/test_models.py new file mode 100755 index 00000000..85a903d7 --- /dev/null +++ b/test/cbapi/psc/test_models.py @@ -0,0 +1,183 @@ +import pytest +from cbapi.psc.models import Device +from cbapi.psc.rest_api import CbPSCBaseAPI +from test.mocks import MockResponse, ConnectionMocks + +class MockScheduler: + def __init__(self, expected_id): + self.expected_id = expected_id + self.was_called = False + + def request_session(self, sensor_id): + assert sensor_id == self.expected_id + self.was_called = True + return { "itworks": True } + +def test_Device_lr_session(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + sked = MockScheduler(6023) + api._lr_scheduler = sked + dev = Device(api, 6023) + sess = dev.lr_session() + assert sess["itworks"] + assert sked.was_called + +def test_Device_background_scan(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "BACKGROUND_SCAN" + assert body["device_id"] == [ 6023 ] + t = body["options"] + assert t["toggle"] == "ON" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + dev = Device(api, 6023) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev.background_scan(True) + assert _was_called + +def test_Device_bypass(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "BYPASS" + assert body["device_id"] == [ 6023 ] + t = body["options"] + assert t["toggle"] == "OFF" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + dev = Device(api, 6023) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev.bypass(False) + assert _was_called + +def test_Device_delete_sensor(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "DELETE_SENSOR" + assert body["device_id"] == [ 6023 ] + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + dev = Device(api, 6023) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev.delete_sensor() + assert _was_called + +def test_Device_deregister_sensor(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "DEREGISTER_SENSOR" + assert body["device_id"] == [ 6023 ] + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + dev = Device(api, 6023) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev.deregister_sensor() + assert _was_called + +def test_Device_quarantine(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "QUARANTINE" + assert body["device_id"] == [ 6023 ] + t = body["options"] + assert t["toggle"] == "ON" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + dev = Device(api, 6023) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev.quarantine(True) + assert _was_called + +def test_Device_update_policy(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "UPDATE_POLICY" + assert body["device_id"] == [ 6023 ] + t = body["options"] + assert t["policy_id"] == 8675309 + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + dev = Device(api, 6023) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev.update_policy(8675309) + assert _was_called + +def test_Device_update_sensor_version(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "UPDATE_SENSOR_VERSION" + assert body["device_id"] == [ 6023 ] + t = body["options"] + t2 = t["sensor_version"] + assert t2["RHEL"] == "2.3.4.5" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + dev = Device(api, 6023) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev.update_sensor_version({"RHEL": "2.3.4.5"}) + assert _was_called + \ No newline at end of file From c8417b9f0f8f1fd8d6ec7290228cbfa4f7f30657 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 30 Sep 2019 12:18:59 -0600 Subject: [PATCH 018/197] CBAPI-1061: add unit tests for PSC rest_api object Includes testing the device query. --- src/cbapi/psc/query.py | 20 +-- src/cbapi/psc/rest_api.py | 8 +- test/cbapi/psc/test_rest_api.py | 289 ++++++++++++++++++++++++++++++++ 3 files changed, 306 insertions(+), 11 deletions(-) create mode 100755 test/cbapi/psc/test_rest_api.py diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 2d278f9c..85cb4760 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -348,7 +348,7 @@ def sort_by(self, key, direction="ASC"): def _build_request(self): request = self._query_body request["query_string"] = self._query_builder._collapse() - if not self._sortcriteria.is_empty(): + if self._sortcriteria != {}: request["sort"] = self._sortcriteria return request @@ -360,7 +360,7 @@ def _build_url(self, from_row, max_rows, tail_end): query_params.append("from_row={0:i}".format(from_row)) if max_rows >= 0: query_params.append("max_rows={0:i}".format(max_rows)) - if not query_params.is_empty(): + if query_params != []: url = url + "?" + "&".join(query_params) return url @@ -418,22 +418,22 @@ def download(self): :return: The CSV raw data as returned from the server. """ tmp = self._query_body.get("status",[]) - if tmp.is_empty(): + if tmp == []: raise ApiError("at least one status must be specified to download") query_params = { "device_status": ",".join(tmp) } tmp = self._query_body.get("ad_group_ids", []) - if not tmp.is_empty(): - query_params["ad_group_id"] = ",".join(tmp) + if tmp != []: + query_params["ad_group_id"] = ",".join([str(t) for t in tmp]) tmp = self._query_body.get("policy_ids", []) - if not tmp.is_empty(): - query_params["policy_id"] = ",".join(tmp) + if tmp != []: + query_params["policy_id"] = ",".join([str(t) for t in tmp]) tmp = self._query_body.get("target_priorities", []) - if not tmp.is_empty(): + if tmp != []: query_params["target_priority"] = ",".join(tmp) tmp = self._query_builder._collapse() - if not tmp.is_empty(): + if tmp != []: query_params["query_string"] = tmp - if not self._sortcriteria.is_empty(): + if self._sortcriteria != {}: query_params["sort_field"] = self._sortcriteria["field_name"] query_params["sort_order"] = self._sortcriteria["sort_order"] url = self._build_url(0, -1, "/_search/download") diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 83618872..0ae416f3 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -1,5 +1,5 @@ from cbapi.connection import BaseAPI -from cbapi.errors import ServerError +from cbapi.errors import ApiError, ServerError from .cblr import LiveResponseSessionManager from .models import Device import logging @@ -20,6 +20,12 @@ class CbPSCBaseAPI(BaseAPI): def __init__(self, *args, **kwargs): super(CbPSCBaseAPI, self).__init__(product_name="psc", *args, **kwargs) self._lr_scheduler = None + + def _perform_query(self, cls, **kwargs): + if hasattr(cls, "_query_implementation"): + return cls._query_implementation(self) + else: + raise ApiError("All PSC models should provide _query_implementation") #---- LiveOps diff --git a/test/cbapi/psc/test_rest_api.py b/test/cbapi/psc/test_rest_api.py new file mode 100755 index 00000000..5ae94752 --- /dev/null +++ b/test/cbapi/psc/test_rest_api.py @@ -0,0 +1,289 @@ +import pytest +from cbapi.errors import ApiError +from cbapi.psc.models import Device +from cbapi.psc.rest_api import CbPSCBaseAPI +from test.mocks import ConnectionMocks, MockResponse + + +def test_get_device(monkeypatch): + _was_called = False + + def mock_get_object(url): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/devices/6023" + _was_called = True + return { "device_id": 6023, "organization_name": "thistestworks" } + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + rc = api.get_device(6023) + assert _was_called + assert isinstance(rc, Device) + assert rc.device_id == 6023 + assert rc.organization_name == "thistestworks" + + +def test_device_background_scan(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "BACKGROUND_SCAN" + assert body["device_id"] == [ 6023 ] + t = body["options"] + assert t["toggle"] == "ON" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.device_background_scan([ 6023 ], True) + assert _was_called + + +def test_device_bypass(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "BYPASS" + assert body["device_id"] == [ 6023 ] + t = body["options"] + assert t["toggle"] == "OFF" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.device_bypass([ 6023 ], False) + assert _was_called + + +def test_device_delete_sensor(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "DELETE_SENSOR" + assert body["device_id"] == [ 6023 ] + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.device_delete_sensor([ 6023 ]) + assert _was_called + + +def test_device_deregister_sensor(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "DEREGISTER_SENSOR" + assert body["device_id"] == [ 6023 ] + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.device_deregister_sensor([ 6023 ]) + assert _was_called + + +def test_device_quarantine(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "QUARANTINE" + assert body["device_id"] == [ 6023 ] + t = body["options"] + assert t["toggle"] == "ON" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.device_quarantine([ 6023 ], True) + assert _was_called + + +def test_device_update_policy(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "UPDATE_POLICY" + assert body["device_id"] == [ 6023 ] + t = body["options"] + assert t["policy_id"] == 8675309 + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.device_update_policy([ 6023 ], 8675309) + assert _was_called + + +def test_device_update_sensor_version(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "UPDATE_SENSOR_VERSION" + assert body["device_id"] == [ 6023 ] + t = body["options"] + t2 = t["sensor_version"] + assert t2["RHEL"] == "2.3.4.5" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.device_update_sensor_version([ 6023 ], { "RHEL": "2.3.4.5"}) + assert _was_called + + +def test_query_device_with_all_bells_and_whistles(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/devices/_search" + assert body["ad_group_ids"] == [ 14, 25 ] + assert body["policy_ids"] == [ 8675309 ] + assert body["query_string"] == "foobar" + assert body["sort"] == { "field_name": "name", "sort_order": "DESC" } + assert body["status"] == [ "ALL" ] + assert body["target_priorities"] == [ "HIGH" ] + _was_called = True + body = { "device_id": 6023, "organization_name": "thistestworks" } + envelope = { "results": [ body ], "num_found": 1 } + return MockResponse(envelope) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(Device).where("foobar").ad_group_ids([ 14, 25 ]) \ + .policy_ids([ 8675309 ]).status([ "ALL" ]).target_priorities(["HIGH"]).sort_by("name", "DESC") + d = query.one() + assert _was_called + assert d.device_id == 6023 + assert d.organization_name == "thistestworks" + + +def test_query_device_invalid_ad_group_ids(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).ad_group_ids([ "Bogus" ]) + + +def test_query_device_invalid_policy_ids(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).policy_ids([ "Bogus" ]) + + +def test_query_device_invalid_status(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).status([ "Bogus" ]) + + +def test_query_device_invalid_priority(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).target_priorities([ "Bogus" ]) + + +def test_query_device_invalid_sort_column(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).sort_by("BOGUS") + + +def test_query_device_invalid_sort_direction(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).sort_by("policy_name", "BOGUS") + + +def test_query_device_download(monkeypatch): + _was_called = False + + def mock_get_raw_data(url, query_params): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/devices/_search/download" + assert query_params["device_status"] == "ALL" + assert query_params["ad_group_id"] == "14,25" + assert query_params["policy_id"] == "8675309" + assert query_params["target_priority"] == "HIGH" + assert query_params["query_string"] == "foobar" + assert query_params["sort_field"] == "name" + assert query_params["sort_order"] == "DESC" + _was_called = True + return "123456789,123456789,123456789" + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_raw_data", mock_get_raw_data) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + rc = api.select(Device).where("foobar").ad_group_ids([ 14, 25 ]) \ + .policy_ids([ 8675309 ]).status([ "ALL" ]).target_priorities(["HIGH"]) \ + .sort_by("name", "DESC").download() + assert _was_called + assert rc == "123456789,123456789,123456789" + \ No newline at end of file From a0349d23984c250a887c29d1bbfb0b997d08bb45 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Wed, 2 Oct 2019 15:58:06 -0600 Subject: [PATCH 019/197] CBAPI-1061: add examples & do rework on API code & tests Turns out I was using the wrong Swagger file as a guide to Devices v6 :O. This necessitated rebuilding the model, banging the query into shape, and updating the test code (including adding more tests). At this point, list_devices works, but download_device_list gives a 415 error code. --- examples/psc/device_control.py | 69 ++++++ examples/psc/download_device_list.py | 47 +++++ examples/psc/list_devices.py | 45 ++++ src/cbapi/example_helpers.py | 15 +- src/cbapi/psc/models.py | 11 +- src/cbapi/psc/models/deviceInfo.yaml | 45 ++-- src/cbapi/psc/query.py | 217 +++++++++++++++---- src/cbapi/psc/rest_api.py | 22 +- test/cbapi/psc/test_models.py | 6 +- test/cbapi/psc/test_rest_api.py | 302 +++++++++++++++++++++++++-- 10 files changed, 681 insertions(+), 98 deletions(-) create mode 100755 examples/psc/device_control.py create mode 100755 examples/psc/download_device_list.py create mode 100755 examples/psc/list_devices.py diff --git a/examples/psc/device_control.py b/examples/psc/device_control.py new file mode 100755 index 00000000..91cdc152 --- /dev/null +++ b/examples/psc/device_control.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc import Device + +def toggle_value(args): + if args.on: + return True + if args.off: + return False + raise Exception("Unknown toggle value") + +def main(): + parser = build_cli_parser("Send control messages to device") + parser.add_argument("-d", "--device_id", type=int, required=True, help="The ID of the device to be controlled") + subparsers = parser.add_subparsers(required=True, dest="command", help="Device command help") + + bgscan_p = subparsers.add_parser("background_scan", help="Set background scanning status") + toggle = bgscan_p.add_mutually_exclusive_group(required=True) + group.add_argument("--on", action="store_true", help="Turn background scanning on") + group.add_argument("--off", action="store_true", help="Turn background scanning off") + + bypass_p = subparsers.add_parser("bypass", help="Set bypass mode") + toggle = bypass_p.add_mutually_exclusive_group(required=True) + group.add_argument("--on", action="store_true", help="Enable bypass mode") + group.add_argument("--off", action="store_true", help="Disable bypass mode") + + subparsers.add_parser("delete", help="Delete sensor") + subparsers.add_parser("uninstall", help="Uninstall sensor") + + quarantine_p = subparsers.add_parser("quarantine", help="Set quarantine mode") + toggle = quarantine_p.add_mutually_exclusive_group(required=True) + group.add_argument("--on", action="store_true", help="Enable quarantine mode") + group.add_argument("--off", action="store_true", help="Disable quarantine mode") + + policy_p = subparsers.add_parser("policy", help="Update policy for node") + policy_p.add_argument("-p", "--policy_id", type=int, required=True, help="New policy ID to set for node") + + sensorv_p = subparses_parser("sensor_version", help="Update sensor version for node") + sensorv_p.add_argument("-o", "--os", required=True, help="Operating system for sensor") + sensorv_p.add_argument("-V", "--version", required=True, help="Version number of sensor") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + dev = cb.get_device(args.device_id) + + if args.command == "background_scan": + dev.background_scan(toggle_value(args)) + elif args.command == "bypass": + dev.bypass(toggle_value(args)) + elif args.command == "delete": + dev.delete_sensor() + elif args.command == "uninstall": + dev.uninstall_sensor() + elif args.command == "quarantine": + dev.quarantine(toggle_value(args)) + elif args.command == "policy": + dev.update_policy(args.policy_id) + elif args.command == "sensor_version": + dev.update_sensor_version({args.os: args.version}) + else: + raise NotImplementedError("Unknown command") + print("OK") + + +if __name__ == "__main__": + sys.exit(main()) + \ No newline at end of file diff --git a/examples/psc/download_device_list.py b/examples/psc/download_device_list.py new file mode 100755 index 00000000..38a540cf --- /dev/null +++ b/examples/psc/download_device_list.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc import Device + +def main(): + parser = build_cli_parser("Download device list in CSV format") + parser.add_argument("-q", "--query", help="Query string for looking for devices") + parser.add_argument("-A", "--ad_group_id", action="append", type=int, help="Active Directory Group ID") + parser.add_argument("-p", "--policy_id", action="append", type=int, help="Policy ID") + parser.add_argument("-s", "--status", action="append", help="Status of device") + parser.add_argument("-P", "--priority", action="append", help="Target priority of device") + parser.add_argument("-S", "--sort_by", help="Field to sort the output by") + parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") + parser.add_argument("-O", "--output", help="File to save output to (default stdout)") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + query = cb.select(Device) + if args.query: + query = query.where(args.query) + if args.ad_group_id: + query = query.ad_group_ids(args.ad_group_id) + if args.policy_id: + query = query.policy_ids(args.policy_id) + if args.status: + query = query.status(args.status) + if args.priority: + query = query.target_priorities(args.priority) + if args.sort_by: + direction = "DESC" if args.reverse else "ASC" + query = query.sort_by(args.sort_by, direction) + + data = query.download() + if args.output: + file = open(args.output, "w") + file.write(data) + file.close() + else: + print(data) + + +if __name__ == "__main__": + sys.exit(main()) + \ No newline at end of file diff --git a/examples/psc/list_devices.py b/examples/psc/list_devices.py new file mode 100755 index 00000000..918b0bc3 --- /dev/null +++ b/examples/psc/list_devices.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc import Device + +def main(): + parser = build_cli_parser("List devices") + parser.add_argument("-q", "--query", help="Query string for looking for devices") + parser.add_argument("-A", "--ad_group_id", action="append", type=int, help="Active Directory Group ID") + parser.add_argument("-p", "--policy_id", action="append", type=int, help="Policy ID") + parser.add_argument("-s", "--status", action="append", help="Status of device") + parser.add_argument("-P", "--priority", action="append", help="Target priority of device") + parser.add_argument("-S", "--sort_by", help="Field to sort the output by") + parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + query = cb.select(Device) + if args.query: + query = query.where(args.query) + if args.ad_group_id: + query = query.ad_group_ids(args.ad_group_id) + if args.policy_id: + query = query.policy_ids(args.policy_id) + if args.status: + query = query.status(args.status) + if args.priority: + query = query.target_priorities(args.priority) + if args.sort_by: + direction = "DESC" if args.reverse else "ASC" + query = query.sort_by(args.sort_by, direction) + + devices = list(query) + print("{0:9} {1:40}{2:18}{3}".format("ID", "Hostname", "IP Address", "Last Checkin Time")) + for device in devices: + print("{0:9} {1:40s}{2:18s}{3}".format(device.id, device.name or "None", \ + device.last_internal_ip_address or "Unknown", \ + device.last_contact_time)) + + +if __name__ == "__main__": + sys.exit(main()) + \ No newline at end of file diff --git a/src/cbapi/example_helpers.py b/src/cbapi/example_helpers.py index fb8b2123..97bdb8e6 100644 --- a/src/cbapi/example_helpers.py +++ b/src/cbapi/example_helpers.py @@ -14,6 +14,7 @@ import hashlib from cbapi.protection import CbEnterpriseProtectionAPI +from cbapi.psc import CbPSCBaseAPI from cbapi.psc.defense import CbDefenseAPI from cbapi.psc.threathunter import CbThreatHunterAPI from cbapi.psc.livequery import CbLiveQueryAPI @@ -65,7 +66,6 @@ def get_cb_response_object(args): def get_cb_protection_object(args): if args.verbose: - import logging logging.basicConfig() logging.getLogger("cbapi").setLevel(logging.DEBUG) logging.getLogger("__main__").setLevel(logging.DEBUG) @@ -77,10 +77,21 @@ def get_cb_protection_object(args): return cb +def get_cb_psc_object(args): + if args.verbose: + logging.basicConfig() + logging.getLogger("cbapi").setLevel(logging.DEBUG) + logging.getLogger("__main__").setLevel(logging.DEBUG) + + if args.cburl and args.apitoken: + cb = CbPSCBaseAPI(url=args.cburl, token=args.apitoken, ssl_verify=(not args.no_ssl_verify)) + else: + cb = CbPSCBaseAPI(profile=args.profile) + + return cb def get_cb_defense_object(args): if args.verbose: - import logging logging.basicConfig() logging.getLogger("cbapi").setLevel(logging.DEBUG) logging.getLogger("__main__").setLevel(logging.DEBUG) diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 785c0ccd..4f0bb791 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -109,7 +109,7 @@ def _refresh_if_needed(self, request_ret): class Device(PSCMutableModel): urlobject = "/appservices/v6/orgs/{0}/devices" urlobject_single = "/appservices/v6/orgs/{0}/devices/{1}" - primary_key = "device_id" + primary_key = "id" #info_key = "deviceInfo" swagger_meta_file = "psc/models/deviceInfo.yaml" @@ -160,11 +160,11 @@ def delete_sensor(self): """ return self._cb.device_delete_sensor([ self._model_unique_id ]) - def deregister_sensor(self): + def uninstall_sensor(self): """ - Deregister this sensor device. + Uninstall this sensor device. """ - return self._cb.device_deregister_sensor([ self._model_unique_id ]) + return self._cb.device_uninstall_sensor([ self._model_unique_id ]) def quarantine(self, flag): """ @@ -186,8 +186,7 @@ def update_sensor_version(self, sensor_version): """ Update the sensor version for this device. - :param dict sensor_version: New version of the sensor; - specified as { "OS": "versionnumber" } + :param dict sensor_version: New version properties for the sensor. """ return self._cb.device_update_sensor_version([ self._model_unique_id ], sensor_version) diff --git a/src/cbapi/psc/models/deviceInfo.yaml b/src/cbapi/psc/models/deviceInfo.yaml index 4aa2f197..f03c3a3d 100755 --- a/src/cbapi/psc/models/deviceInfo.yaml +++ b/src/cbapi/psc/models/deviceInfo.yaml @@ -65,6 +65,7 @@ properties: description: Current MSM policy name deregistered_time: type: string + format: date-time description: When the device was deregistered with the PSC backend device_id: type: integer @@ -87,17 +88,6 @@ properties: type: integer format: int64 description: ID of the user who owns the device - device_type: - type: string - example: WINDOWS - description: Device type - enum: - - WINDOWS - - ANDROID - - MAC - - IOS - - LINUX - - OTHER email: type: string description: Email of the user who owns the device @@ -107,14 +97,21 @@ properties: first_name: type: string description: First name of the user who owns the device + id: + type: integer + format: int64 + description: ID of the device last_contact_time: type: string + format: date-time description: Time the device last checked into the PSC backend last_device_policy_changed_time: type: string + format: date-time description: Last time the device's policy was changed last_device_policy_requested_time: type: string + format: date-time description: Last time the device requested policy updates last_external_ip_address: type: string @@ -134,15 +131,19 @@ properties: description: Last name of the user who owns the device last_policy_updated_time: type: string + format: date-time description: Last time the device was MSM processed last_reported_time: type: string + format: date-time description: Time when device last reported an event to PSC backend last_reset_time: type: string + format: date-time description: When the sensor was last reset last_shutdown_time: type: string + format: date-time description: When the device last shut down linux_kernel_version: type: string @@ -167,6 +168,17 @@ properties: organization_name: type: string description: Name of the org that owns this device + os: + type: string + example: WINDOWS + description: Device type + enum: + - WINDOWS + - ANDROID + - MAC + - IOS + - LINUX + - OTHER os_version: type: string example: 'Windows 7 x86 SP: 1' @@ -186,21 +198,18 @@ properties: description: Manually assigned policy (overrides mass sensor management) quarantined: type: boolean + description: Whether the device is quarantined registered_time: - type: string - description: When the device was registered with the PSC backend - rooted_by_analytics: - type: boolean - rooted_by_analytics_time: type: string format: date-time - rooted_by_sensor: - type: boolean + description: When the device was registered with the PSC backend scan_last_action_time: type: string + format: date-time description: When the background scan was last active scan_last_complete_time: type: string + format: date-time description: When the background scan was last completed scan_status: type: string diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 85cb4760..c5676565 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -1,4 +1,4 @@ -from cbapi.errors import ApiError, MoreThanOneResultError +from cbapi.errors import ApiError, MoreThanOneResultError, ServerError import logging import functools from six import string_types @@ -260,21 +260,30 @@ class DeviceSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMix """ Represents a query that is used to locate Device objects. """ + valid_os = [ "WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER" ] valid_statuses = ["PENDING", "REGISTERED", "UNINSTALLED", "DEREGISTERED", "ACTIVE", "INACTIVE", "ERROR", "ALL", "BYPASS_ON", "BYPASS", "QUARANTINE", "SENSOR_OUTOFDATE", "DELETED", "LIVE"] valid_priorities = ["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"] - valid_sort_keys = ["target_priority", "policy_name", "name", - "last_contact_time", "av_pack_version"] valid_directions = ["ASC", "DESC"] def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._query_builder = QueryBuilder() - self._query_body = {} + self._criteria = {} + self._time_filter = {} + self._exclusions = {} self._sortcriteria = {} + def _update_criteria(self, key, newlist): + oldlist = self._criteria.get(key, []) + self._criteria[key] = oldlist + newlist + + def _update_exclusions(self, key, newlist): + oldlist = self._exclusions.get(key, []) + self._exclusions[key] = oldlist + newlist + def ad_group_ids(self, ad_group_ids): """ Restricts the devices that this query is performed on to the specified @@ -285,7 +294,59 @@ def ad_group_ids(self, ad_group_ids): """ if not all(isinstance(ad_group_id, int) for ad_group_id in ad_group_ids): raise ApiError("One or more invalid AD group IDs") - self._query_body["ad_group_ids"] = ad_group_ids + self._update_criteria("ad_group_id", ad_group_ids) + return self + + def device_ids(self, device_ids): + """ + Restricts the devices that this query is performed on to the specified + device IDs. + + :param ad_group_ids: list of ints + :return: This instance + """ + if not all(isinstance(device_id, int) for device_id in device_ids): + raise ApiError("One or more invalid device IDs") + self._update_criteria("id", device_ids) + return self + + def last_contact_time(self, *args, **kwargs): + """ + Restricts the devices that this query is performed on to the specified + last contact time (either specified as a start and end point or as a + range). + + :return: This instance + """ + if kwargs.get("start", None) and kwargs.get("end", None): + if kwargs.get("range", None): + raise ApiError("cannot specify range= in addition to start= and end=") + stime = kwargs["start"] + if not isinstance(stime, str): + stime = stime.isoformat() + etime = kwargs["end"] + if not isinstance(etime, str): + etime = etime.isoformat() + self._time_filter = { "start": stime, "end": etime } + elif kwargs.get("range", None): + if kwargs.get("start", None) or kwargs.get("end", None): + raise ApiError("cannot specify start= or end= in addition to range=") + self._time_filter = { "range": kwargs["range"] } + else: + raise ApiError("must specify either start= and end= or range=") + return self + + def os(self, operating_systems): + """ + Restricts the devices that this query is performed on to the specified + operating systems. + + :param operating_systems: list of operating systems + :return: This instance + """ + if not all((osval in DeviceSearchQuery.valid_os) for osval in operating_systems): + raise ApiError("One or more invalid operating systems") + self._update_criteria("os", operating_systems) return self def policy_ids(self, policy_ids): @@ -297,8 +358,8 @@ def policy_ids(self, policy_ids): :return: This instance """ if not all(isinstance(policy_id, int) for policy_id in policy_ids): - raise ApiError("One or more invalid AD group IDs") - self._query_body["policy_ids"] = policy_ids + raise ApiError("One or more invalid policy IDs") + self._update_criteria("policy_id", policy_ids) return self def status(self, statuses): @@ -311,7 +372,7 @@ def status(self, statuses): """ if not all((stat in DeviceSearchQuery.valid_statuses) for stat in statuses): raise ApiError("One or more invalid status values") - self._query_body["status"] = statuses + self._update_criteria("status", statuses) return self def target_priorities(self, target_priorities): @@ -324,7 +385,20 @@ def target_priorities(self, target_priorities): """ if not all((prio in DeviceSearchQuery.valid_priorities) for prio in target_priorities): raise ApiError("One or more invalid target priority values") - self._query_body["target_priorities"] = target_priorities + self._update_criteria("target_priority", target_priorities) + return self + + def exclude_sensor_versions(self, sensor_versions): + """ + Restricts the devices that this query is performed on to exclude specified + sensor versions. + + :param sensor_versions: List of sensor versions to exclude + :return: This instance + """ + if not all(isinstance(v, str) for v in sensor_versions): + raise ApiError("One or more invalid sensor versions") + self._update_exclusions("sensor_version", sensor_versions) return self def sort_by(self, key, direction="ASC"): @@ -338,38 +412,36 @@ def sort_by(self, key, direction="ASC"): :param direction: the sort order, either "ASC" or "DESC" :rtype: :py:class:`DeviceSearchQuery` """ - if key not in DeviceSearchQuery.valid_sort_keys: - raise ApiError("Invalid sort key specified") if direction not in DeviceSearchQuery.valid_directions: raise ApiError("invalid sort direction specified") - self._sortcriteria = {"field_name": key, "sort_order": direction} + self._sortcriteria = { "field": key, "order": direction } return self - def _build_request(self): - request = self._query_body - request["query_string"] = self._query_builder._collapse() + def _build_request(self, from_row, max_rows): + mycrit = self._criteria + if self._time_filter: + mycrit["last_contact_time"] = self._time_filter + request = { "criteria": mycrit, "exclusions": self._exclusions } + request["query"] = self._query_builder._collapse() + if from_row > 0: + request["start"] = from_row + if max_rows >= 0: + request["rows"] = max_rows if self._sortcriteria != {}: - request["sort"] = self._sortcriteria + request["sort"] = [ self._sortcriteria ] return request - def _build_url(self, from_row, max_rows, tail_end): + def _build_url(self, tail_end): url = self._doc_class.urlobject.format( self._cb.credentials.org_key) + tail_end - query_params = [] - if from_row > 0: - query_params.append("from_row={0:i}".format(from_row)) - if max_rows >= 0: - query_params.append("max_rows={0:i}".format(max_rows)) - if query_params != []: - url = url + "?" + "&".join(query_params) return url def _count(self): if self._count_valid: return self._total_results - url = self._build_url(0, -1, "/_search") - request = self._build_request() + url = self._build_url("/_search") + request = self._build_request(0, -1) resp = self._cb.post_object(url, body=request) result = resp.json() @@ -379,12 +451,12 @@ def _count(self): return self._total_results def _perform_query(self, from_row=0, max_rows=-1): - request = self._build_request() + url = self._build_url("/_search") current = from_row numrows = 0 still_querying = True while still_querying: - url = self._build_url(from_row, max_rows, "/_search") + request = self._build_request(current, max_rows) resp = self._cb.post_object(url, body=request) result = resp.json() @@ -393,7 +465,7 @@ def _perform_query(self, from_row=0, max_rows=-1): results = result.get("results", []) for item in results: - yield self._doc_class(self._cb, item["device_id"], item) + yield self._doc_class(self._cb, item["id"], item) current += 1 numrows += 1 @@ -417,24 +489,83 @@ def download(self): :return: The CSV raw data as returned from the server. """ - tmp = self._query_body.get("status",[]) - if tmp == []: + tmp = self._criteria.get("status", []) + if not tmp: raise ApiError("at least one status must be specified to download") - query_params = { "device_status": ",".join(tmp) } - tmp = self._query_body.get("ad_group_ids", []) - if tmp != []: + query_params = { "status": ",".join(tmp) } + tmp = self._criteria.get("ad_group_id", []) + if tmp: query_params["ad_group_id"] = ",".join([str(t) for t in tmp]) - tmp = self._query_body.get("policy_ids", []) - if tmp != []: + tmp = self._criteria.get("policy_id", []) + if tmp: query_params["policy_id"] = ",".join([str(t) for t in tmp]) - tmp = self._query_body.get("target_priorities", []) - if tmp != []: + tmp = self._criteria.get("target_priority", []) + if tmp: query_params["target_priority"] = ",".join(tmp) tmp = self._query_builder._collapse() - if tmp != []: + if tmp: query_params["query_string"] = tmp - if self._sortcriteria != {}: - query_params["sort_field"] = self._sortcriteria["field_name"] - query_params["sort_order"] = self._sortcriteria["sort_order"] - url = self._build_url(0, -1, "/_search/download") + if self._sortcriteria: + query_params["sort_field"] = self._sortcriteria["field"] + query_params["sort_order"] = self._sortcriteria["order"] + url = self._build_url("/_search/download") return self._cb.get_raw_data(url, query_params) + + def _bulk_device_action(self, action_type, options=None): + request = { "action_type": action_type, "search": self._build_request(0, -1) } + if options: + request["options"] = options + return self._cb._raw_device_action(request) + + def background_scan(self, flag): + """ + Set the background scan option for the specified devices. + + :param boolean flag: True to turn background scan on, False to turn it off. + """ + return self._bulk_device_action("BACKGROUND_SCAN", self._cb._action_toggle(flag)) + + def bypass(self, flag): + """ + Set the bypass option for the specified devices. + + :param boolean flag: True to enable bypass, False to disable it. + """ + return self._bulk_device_action("BYPASS", self._cb._action_toggle(flag)) + + def delete_sensor(self): + """ + Delete the specified sensor devices. + """ + return self._bulk_device_action("DELETE_SENSOR") + + def uninstall_sensor(self): + """ + Uninstall the specified sensor devices. + """ + return self._bulk_device_action("UNINSTALL_SENSOR") + + def quarantine(self, flag): + """ + Set the quarantine option for the specified devices. + + :param boolean flag: True to enable quarantine, False to disable it. + """ + return self._bulk_device_action("QUARANTINE", self._cb._action_toggle(flag)) + + def update_policy(self, policy_id): + """ + Set the current policy for the specified devices. + + :param int policy_id: ID of the policy to set for the devices. + """ + return self._bulk_device_action("UPDATE_POLICY", { "policy_id": policy_id }) + + def update_sensor_version(self, sensor_version): + """ + Update the sensor version for the specified devices. + + :param dict sensor_version: New version properties for the sensor. + """ + return self._bulk_device_action("UPDATE_SENSOR_VERSION", \ + { "sensor_version": sensor_version }) \ No newline at end of file diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 0ae416f3..e53c98f9 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -51,10 +51,7 @@ def get_device(self, device_id): rc.refresh() return rc - def _device_action(self, device_ids, action_type, options=None): - request = { "action_type": action_type, "device_id": device_ids } - if options: - request["options"] = options + def _raw_device_action(self, request): url = "/appservices/v6/orgs/{0}/device_actions".format(self.credentials.org_key) resp = self.post_object(url, body=request) if resp.status_code == 200: @@ -64,6 +61,12 @@ def _device_action(self, device_ids, action_type, options=None): else: raise ServerError(error_code=resp.status_code, message="Device action error: {0}".format(resp.content)) + def _device_action(self, device_ids, action_type, options=None): + request = { "action_type": action_type, "device_id": device_ids } + if options: + request["options"] = options + return self._raw_device_action(request) + def _action_toggle(self, flag): if flag: return { "toggle": "ON" } @@ -96,13 +99,13 @@ def device_delete_sensor(self, device_ids): """ return self._device_action(device_ids, "DELETE_SENSOR") - def device_deregister_sensor(self, device_ids): + def device_uninstall_sensor(self, device_ids): """ - Deregister the specified sensor devices. + Uninstall the specified sensor devices. - :param list device_ids: List of IDs of devices to be deregistered. + :param list device_ids: List of IDs of devices to be uninstalled. """ - return self._device_action(device_ids, "DEREGISTER_SENSOR") + return self._device_action(device_ids, "UNINSTALL_SENSOR") def device_quarantine(self, device_ids, flag): """ @@ -127,8 +130,7 @@ def device_update_sensor_version(self, device_ids, sensor_version): Update the sensor version for the specified devices. :param list device_ids: List of IDs of devices to be changed. - :param dict sensor_version: New version of the sensor; - specified as { "OS": "versionnumber" } + :param dict sensor_version: New version properties for the sensor. """ return self._device_action(device_ids, "UPDATE_SENSOR_VERSION", { "sensor_version": sensor_version }) diff --git a/test/cbapi/psc/test_models.py b/test/cbapi/psc/test_models.py index 85a903d7..af8b930f 100755 --- a/test/cbapi/psc/test_models.py +++ b/test/cbapi/psc/test_models.py @@ -90,13 +90,13 @@ def mock_post_object(url, body, **kwargs): dev.delete_sensor() assert _was_called -def test_Device_deregister_sensor(monkeypatch): +def test_Device_uninstall_sensor(monkeypatch): _was_called = False def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "DEREGISTER_SENSOR" + assert body["action_type"] == "UNINSTALL_SENSOR" assert body["device_id"] == [ 6023 ] _was_called = True return MockResponse(None, 204) @@ -108,7 +108,7 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - dev.deregister_sensor() + dev.uninstall_sensor() assert _was_called def test_Device_quarantine(monkeypatch): diff --git a/test/cbapi/psc/test_rest_api.py b/test/cbapi/psc/test_rest_api.py index 5ae94752..d8193814 100755 --- a/test/cbapi/psc/test_rest_api.py +++ b/test/cbapi/psc/test_rest_api.py @@ -94,13 +94,13 @@ def mock_post_object(url, body, **kwargs): assert _was_called -def test_device_deregister_sensor(monkeypatch): +def test_device_uninstall_sensor(monkeypatch): _was_called = False def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "DEREGISTER_SENSOR" + assert body["action_type"] == "UNINSTALL_SENSOR" assert body["device_id"] == [ 6023 ] _was_called = True return MockResponse(None, 204) @@ -111,7 +111,7 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_deregister_sensor([ 6023 ]) + api.device_uninstall_sensor([ 6023 ]) assert _was_called @@ -191,14 +191,21 @@ def test_query_device_with_all_bells_and_whistles(monkeypatch): def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/devices/_search" - assert body["ad_group_ids"] == [ 14, 25 ] - assert body["policy_ids"] == [ 8675309 ] - assert body["query_string"] == "foobar" - assert body["sort"] == { "field_name": "name", "sort_order": "DESC" } - assert body["status"] == [ "ALL" ] - assert body["target_priorities"] == [ "HIGH" ] + assert body["query"] == "foobar" + t = body.get("criteria", {}) + assert t["ad_group_id"] == [ 14, 25 ] + assert t["os"] == [ "LINUX" ] + assert t["policy_id"] == [ 8675309 ] + assert t["status"] == [ "ALL" ] + assert t["target_priority"] == [ "HIGH" ] + t = body.get("exclusions", {}) + assert t["sensor_version"] == [ "0.1" ] + t = body.get("sort", []) + t2 = t[0] + assert t2["field"] == "name" + assert t2["order"] == "DESC" _was_called = True - body = { "device_id": 6023, "organization_name": "thistestworks" } + body = { "id": 6023, "organization_name": "thistestworks" } envelope = { "results": [ body ], "num_found": 1 } return MockResponse(envelope) @@ -209,13 +216,73 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) query = api.select(Device).where("foobar").ad_group_ids([ 14, 25 ]) \ - .policy_ids([ 8675309 ]).status([ "ALL" ]).target_priorities(["HIGH"]).sort_by("name", "DESC") + .os([ "LINUX" ]).policy_ids([ 8675309 ]).status([ "ALL" ]) \ + .target_priorities(["HIGH"]).exclude_sensor_versions(["0.1"]) \ + .sort_by("name", "DESC") d = query.one() assert _was_called - assert d.device_id == 6023 + assert d.id == 6023 assert d.organization_name == "thistestworks" +def test_query_device_wth_last_contact_time_as_start_end(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/devices/_search" + assert body["query"] == "foobar" + t = body.get("criteria", {}) + t2 = t.get("last_contact_time", {}) + assert t2["start"] == "2019-09-30T12:34:56" + assert t2["end"] == "2019-10-01T12:00:12" + _was_called = True + body = { "id": 6023, "organization_name": "thistestworks" } + envelope = { "results": [ body ], "num_found": 1 } + return MockResponse(envelope) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(Device).where("foobar") \ + .last_contact_time(start="2019-09-30T12:34:56", end="2019-10-01T12:00:12") + d = query.one() + assert _was_called + assert d.id == 6023 + assert d.organization_name == "thistestworks" + + +def test_query_device_with_last_contact_time_as_range(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/devices/_search" + assert body["query"] == "foobar" + t = body.get("criteria", {}) + t2 = t.get("last_contact_time", {}) + assert t2["range"] == "-3w" + _was_called = True + body = { "id": 6023, "organization_name": "thistestworks" } + envelope = { "results": [ body ], "num_found": 1 } + return MockResponse(envelope) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(Device).where("foobar").last_contact_time(range="-3w") + d = query.one() + assert _was_called + assert d.id == 6023 + assert d.organization_name == "thistestworks" + + def test_query_device_invalid_ad_group_ids(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) @@ -228,8 +295,45 @@ def test_query_device_invalid_policy_ids(): org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(Device).policy_ids([ "Bogus" ]) + + +def test_query_device_last_contact_time_no_params_ok(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).last_contact_time() + + +def test_query_device_last_contact_time_range_specified_bad(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).last_contact_time(start="2019-09-30T12:34:56", \ + end="2019-10-01T12:00:12", range="-3w") + + +def test_query_device_last_contact_time_start_specified_bad(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).last_contact_time(start="2019-09-30T12:34:56", \ + range="-3w") +def test_query_device_last_contact_time_end_specified_bad(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).last_contact_time(end="2019-10-01T12:00:12", range="-3w") + + +def test_query_device_invalid_os(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).os([ "COMMODORE_64" ]) + + def test_query_device_invalid_status(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) @@ -244,11 +348,11 @@ def test_query_device_invalid_priority(): api.select(Device).target_priorities([ "Bogus" ]) -def test_query_device_invalid_sort_column(): +def test_query_device_invalid_exclude_sensor_version(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): - api.select(Device).sort_by("BOGUS") + api.select(Device).exclude_sensor_versions([ 12703 ]) def test_query_device_invalid_sort_direction(): @@ -264,7 +368,7 @@ def test_query_device_download(monkeypatch): def mock_get_raw_data(url, query_params): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/devices/_search/download" - assert query_params["device_status"] == "ALL" + assert query_params["status"] == "ALL" assert query_params["ad_group_id"] == "14,25" assert query_params["policy_id"] == "8675309" assert query_params["target_priority"] == "HIGH" @@ -286,4 +390,170 @@ def mock_get_raw_data(url, query_params): .sort_by("name", "DESC").download() assert _was_called assert rc == "123456789,123456789,123456789" - \ No newline at end of file + + +def test_query_device_do_background_scan(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "BACKGROUND_SCAN" + t = body["search"] + assert t["query"] == "foobar" + t = body["options"] + assert t["toggle"] == "ON" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.select(Device).where("foobar").background_scan(True) + assert _was_called + + +def test_query_device_do_bypass(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "BYPASS" + t = body["search"] + assert t["query"] == "foobar" + t = body["options"] + assert t["toggle"] == "OFF" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.select(Device).where("foobar").bypass(False) + assert _was_called + + +def test_query_device_do_delete_sensor(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "DELETE_SENSOR" + t = body["search"] + assert t["query"] == "foobar" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.select(Device).where("foobar").delete_sensor() + assert _was_called + + +def test_query_device_do_uninstall_sensor(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "UNINSTALL_SENSOR" + t = body["search"] + assert t["query"] == "foobar" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.select(Device).where("foobar").uninstall_sensor() + assert _was_called + + +def test_query_device_do_quarantine(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "QUARANTINE" + t = body["search"] + assert t["query"] == "foobar" + t = body["options"] + assert t["toggle"] == "ON" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.select(Device).where("foobar").quarantine(True) + assert _was_called + + +def test_query_device_do_update_policy(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "UPDATE_POLICY" + t = body["search"] + assert t["query"] == "foobar" + t = body["options"] + assert t["policy_id"] == 8675309 + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.select(Device).where("foobar").update_policy(8675309) + assert _was_called + + +def test_query_device_do_update_sensor_version(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body["action_type"] == "UPDATE_SENSOR_VERSION" + t = body["search"] + assert t["query"] == "foobar" + t = body["options"] + t2 = t["sensor_version"] + assert t2["RHEL"] == "2.3.4.5" + _was_called = True + return MockResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api.select(Device).where("foobar").update_sensor_version({ "RHEL": "2.3.4.5"}) + assert _was_called + + \ No newline at end of file From f3750817dcb085b6d9faea338697bdcb40d792a5 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Thu, 3 Oct 2019 12:47:35 -0600 Subject: [PATCH 020/197] some minor glitch cleanup in device_control, and adding logging to download_device_list (this is increasingly looking like a server bug) --- examples/psc/device_control.py | 52 +++++++++++++++------------- examples/psc/download_device_list.py | 3 ++ 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/examples/psc/device_control.py b/examples/psc/device_control.py index 91cdc152..497de7b5 100755 --- a/examples/psc/device_control.py +++ b/examples/psc/device_control.py @@ -14,30 +14,30 @@ def toggle_value(args): def main(): parser = build_cli_parser("Send control messages to device") parser.add_argument("-d", "--device_id", type=int, required=True, help="The ID of the device to be controlled") - subparsers = parser.add_subparsers(required=True, dest="command", help="Device command help") + subparsers = parser.add_subparsers(dest="command", help="Device command help") bgscan_p = subparsers.add_parser("background_scan", help="Set background scanning status") toggle = bgscan_p.add_mutually_exclusive_group(required=True) - group.add_argument("--on", action="store_true", help="Turn background scanning on") - group.add_argument("--off", action="store_true", help="Turn background scanning off") + toggle.add_argument("--on", action="store_true", help="Turn background scanning on") + toggle.add_argument("--off", action="store_true", help="Turn background scanning off") bypass_p = subparsers.add_parser("bypass", help="Set bypass mode") toggle = bypass_p.add_mutually_exclusive_group(required=True) - group.add_argument("--on", action="store_true", help="Enable bypass mode") - group.add_argument("--off", action="store_true", help="Disable bypass mode") + toggle.add_argument("--on", action="store_true", help="Enable bypass mode") + toggle.add_argument("--off", action="store_true", help="Disable bypass mode") subparsers.add_parser("delete", help="Delete sensor") subparsers.add_parser("uninstall", help="Uninstall sensor") quarantine_p = subparsers.add_parser("quarantine", help="Set quarantine mode") toggle = quarantine_p.add_mutually_exclusive_group(required=True) - group.add_argument("--on", action="store_true", help="Enable quarantine mode") - group.add_argument("--off", action="store_true", help="Disable quarantine mode") + toggle.add_argument("--on", action="store_true", help="Enable quarantine mode") + toggle.add_argument("--off", action="store_true", help="Disable quarantine mode") policy_p = subparsers.add_parser("policy", help="Update policy for node") policy_p.add_argument("-p", "--policy_id", type=int, required=True, help="New policy ID to set for node") - sensorv_p = subparses_parser("sensor_version", help="Update sensor version for node") + sensorv_p = subparsers.add_parser("sensor_version", help="Update sensor version for node") sensorv_p.add_argument("-o", "--os", required=True, help="Operating system for sensor") sensorv_p.add_argument("-V", "--version", required=True, help="Version number of sensor") @@ -45,24 +45,26 @@ def main(): cb = get_cb_psc_object(args) dev = cb.get_device(args.device_id) - if args.command == "background_scan": - dev.background_scan(toggle_value(args)) - elif args.command == "bypass": - dev.bypass(toggle_value(args)) - elif args.command == "delete": - dev.delete_sensor() - elif args.command == "uninstall": - dev.uninstall_sensor() - elif args.command == "quarantine": - dev.quarantine(toggle_value(args)) - elif args.command == "policy": - dev.update_policy(args.policy_id) - elif args.command == "sensor_version": - dev.update_sensor_version({args.os: args.version}) + if args.command: + if args.command == "background_scan": + dev.background_scan(toggle_value(args)) + elif args.command == "bypass": + dev.bypass(toggle_value(args)) + elif args.command == "delete": + dev.delete_sensor() + elif args.command == "uninstall": + dev.uninstall_sensor() + elif args.command == "quarantine": + dev.quarantine(toggle_value(args)) + elif args.command == "policy": + dev.update_policy(args.policy_id) + elif args.command == "sensor_version": + dev.update_sensor_version({args.os: args.version}) + else: + raise NotImplementedError("Unknown command") + print("OK") else: - raise NotImplementedError("Unknown command") - print("OK") - + print(dev) if __name__ == "__main__": sys.exit(main()) diff --git a/examples/psc/download_device_list.py b/examples/psc/download_device_list.py index 38a540cf..65fec973 100755 --- a/examples/psc/download_device_list.py +++ b/examples/psc/download_device_list.py @@ -4,6 +4,9 @@ from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc import Device +import logging +logging.basicConfig(level=logging.DEBUG) + def main(): parser = build_cli_parser("Download device list in CSV format") parser.add_argument("-q", "--query", help="Query string for looking for devices") From 2c4ff9c4233dc119e38a69d84ed4f0a3d8aac67a Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Thu, 3 Oct 2019 16:41:19 -0600 Subject: [PATCH 021/197] temporary fix for Content-Type issue with the download API Take that header out once PSC backend is fixed. --- src/cbapi/connection.py | 7 ++++--- src/cbapi/psc/query.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cbapi/connection.py b/src/cbapi/connection.py index 98e5243d..db89431e 100644 --- a/src/cbapi/connection.py +++ b/src/cbapi/connection.py @@ -269,15 +269,16 @@ def get_object(self, uri, query_parameters=None, default=None): else: raise ServerError(error_code=result.status_code, message="Unknown error: {0}".format(result.content)) - def get_raw_data(self, uri, query_parameters=None, default=None): + def get_raw_data(self, uri, query_parameters=None, default=None, **kwargs): if query_parameters: if isinstance(query_parameters, dict): query_parameters = convert_query_params(query_parameters) uri += '?%s' % (urllib.parse.urlencode(sorted(query_parameters))) - result = self.api_json_request("GET", uri) + hdrs = kwargs.pop("headers", {}) + result = self.api_json_request("GET", uri, headers=hdrs) if result.status_code == 200: - return result + return result.text elif result.status_code == 204: # empty response return default diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index c5676565..8467d70d 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -509,7 +509,8 @@ def download(self): query_params["sort_field"] = self._sortcriteria["field"] query_params["sort_order"] = self._sortcriteria["order"] url = self._build_url("/_search/download") - return self._cb.get_raw_data(url, query_params) + # AGRB 10/3/2019 - Header is TEMPORARY until bug is fixed in API. Remove when fix deployed. + return self._cb.get_raw_data(url, query_params, headers={ "Content-Type": "application/json"}) def _bulk_device_action(self, action_type, options=None): request = { "action_type": action_type, "search": self._build_request(0, -1) } From 9dc4f3362fd38907a9dc6d136cfb3e82c2593084 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 7 Oct 2019 14:38:24 -0600 Subject: [PATCH 022/197] documentation for Devices v6 API Mostly a page that ties the class docstrings into the overall structure. --- docs/index.rst | 1 + docs/livequery-examples.rst | 2 +- docs/psc-api.rst | 44 +++++++++++++++++++++++++++++++++++++ src/cbapi/__init__.py | 4 ++-- 4 files changed, 48 insertions(+), 3 deletions(-) create mode 100755 docs/psc-api.rst diff --git a/docs/index.rst b/docs/index.rst index 0795513b..b7ef79ee 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -247,6 +247,7 @@ documenting all of the methods available to you. protection-api defense-api threathunter-api + psc-api livequery-api exceptions diff --git a/docs/livequery-examples.rst b/docs/livequery-examples.rst index 9dff3f5b..a786889f 100644 --- a/docs/livequery-examples.rst +++ b/docs/livequery-examples.rst @@ -1,5 +1,5 @@ CB LiveQuery API Examples -======================== +========================= Let's cover a few example functions that our LiveQuery Python bindings enable. To begin, we need to import the relevant libraries:: diff --git a/docs/psc-api.rst b/docs/psc-api.rst new file mode 100755 index 00000000..9805ceb0 --- /dev/null +++ b/docs/psc-api.rst @@ -0,0 +1,44 @@ +.. _psc_api: + +CB PSC API +========== + +This page documents the public interfaces exposed by cbapi when communicating with +the Carbon Black Predictive Security Cloud (PSC). + +Main Interface +-------------- + +To use cbapi with the Carbon Black PSC, you use CbPSCBaseAPI objects. + +.. autoclass:: cbapi.psc.rest_api.CbPSCBaseAPI + :members: + :inherited-members: + +Device API +---------- + +The PSC can be used to enumerate devices within your organization, and change their +status via a control request. + +You can use the select() method on the CbPSCBaseAPI to create a query object for +Device objects, which can be used to locate a list of Devices. + +*Example:* + + >>> cbapi = CbPSCBaseAPI(...) + >>> devices = cbapi.select(Device).os("LINUX").status("ALL") + +Selects all devices running Linux from the current organization. + +**Query Object:** + +.. autoclass:: cbapi.psc.query.DeviceSearchQuery + :members: + +**Model Object:** + +.. autoclass:: cbapi.psc.models.Device + :members: + :undoc-members: + diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index 8139d481..2c792e43 100755 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -5,8 +5,8 @@ __title__ = 'cbapi' __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' -__copyright__ = 'Copyright 2019 Carbon Black' -__version__ = '1.5.6' +__copyright__ = 'Copyright 2018-2019 Carbon Black' +__version__ = '1.6.0' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From da474115e9c301cecf215f10bb1751e8606bcb73 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 8 Oct 2019 09:58:16 -0600 Subject: [PATCH 023/197] apparently Python 2.7 doesn't like "builtins" --- src/cbapi/psc/query.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 8467d70d..e5fe4c7d 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -3,7 +3,6 @@ import functools from six import string_types from solrq import Q -from builtins import isinstance log = logging.getLogger(__name__) From 6d9ff8d5df226ab51986b6194935cc5e9634e754 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 8 Oct 2019 11:03:31 -0600 Subject: [PATCH 024/197] rename deviceInfo.yaml to device.yaml --- src/cbapi/psc/models.py | 2 +- src/cbapi/psc/models/{deviceInfo.yaml => device.yaml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/cbapi/psc/models/{deviceInfo.yaml => device.yaml} (100%) diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 4f0bb791..bd1e3009 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -111,7 +111,7 @@ class Device(PSCMutableModel): urlobject_single = "/appservices/v6/orgs/{0}/devices/{1}" primary_key = "id" #info_key = "deviceInfo" - swagger_meta_file = "psc/models/deviceInfo.yaml" + swagger_meta_file = "psc/models/device.yaml" def __init__(self, cb, model_unique_id, initial_data=None): super(Device, self).__init__(cb, model_unique_id, initial_data) diff --git a/src/cbapi/psc/models/deviceInfo.yaml b/src/cbapi/psc/models/device.yaml similarity index 100% rename from src/cbapi/psc/models/deviceInfo.yaml rename to src/cbapi/psc/models/device.yaml From 22c8bcb3043cb26efd437fa8cc67750bb87c58ac Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 8 Oct 2019 13:41:18 -0600 Subject: [PATCH 025/197] fixed a mock method to take keyword args for the temporary workaround --- test/cbapi/psc/test_rest_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cbapi/psc/test_rest_api.py b/test/cbapi/psc/test_rest_api.py index d8193814..016371fb 100755 --- a/test/cbapi/psc/test_rest_api.py +++ b/test/cbapi/psc/test_rest_api.py @@ -365,7 +365,7 @@ def test_query_device_invalid_sort_direction(): def test_query_device_download(monkeypatch): _was_called = False - def mock_get_raw_data(url, query_params): + def mock_get_raw_data(url, query_params, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/devices/_search/download" assert query_params["status"] == "ALL" From ec5e933ef6add723c9c119c7c7a9b0377c862f4c Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 14 Oct 2019 12:56:54 -0600 Subject: [PATCH 026/197] de-flake8'ed all source files in root src/cbapi directory; also set up the standard flake8 configuration stanza in setup.cfg --- setup.cfg | 5 +++++ src/cbapi/auth.py | 25 ++++++++++++----------- src/cbapi/connection.py | 23 ++++++++++------------ src/cbapi/defense.py | 2 +- src/cbapi/errors.py | 1 - src/cbapi/event.py | 2 +- src/cbapi/example_helpers.py | 36 +++++++++++++++++++++------------- src/cbapi/live_response_api.py | 29 ++++++++++++++------------- src/cbapi/models.py | 16 +++++++-------- src/cbapi/oldmodels.py | 6 +++--- src/cbapi/query.py | 3 ++- src/cbapi/six.py | 26 ++++++++++++------------ src/cbapi/winerror.py | 17 +++++++++++++++- 13 files changed, 110 insertions(+), 81 deletions(-) diff --git a/setup.cfg b/setup.cfg index fe62be21..c5c82eef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,8 @@ description-file = README.md [bdist_wheel] universal = 1 + +[flake8] +max-doc-length = 120 +max-line-length = 120 + diff --git a/src/cbapi/auth.py b/src/cbapi/auth.py index 5470050d..baba7f8f 100644 --- a/src/cbapi/auth.py +++ b/src/cbapi/auth.py @@ -47,32 +47,35 @@ def __init__(self, *args, **kwargs): if isinstance(x, six.string_types) and x.lower() in _boolean_states: self[k] = _boolean_states[x.lower()] + class CredentialStoreFactory(object): - #CredentialStore(product_name, credential_file=credential_file) + # CredentialStore(product_name, credential_file=credential_file) @staticmethod - def getCredentialStore(product_name,credential_file): - if credential_file is None and os.environ.get('CBAPI_TOKEN',False) and os.environ.get('CBAPI_URL',False): + def getCredentialStore(product_name, credential_file): + if credential_file is None and os.environ.get('CBAPI_TOKEN', False) and os.environ.get('CBAPI_URL', False): log.debug("Using Envar credential store") return EnvarCredentialStore() else: log.debug("Using file credential store") - return FileCredentialStore(**{"product_name":product_name,"credential_file":credential_file}) + return FileCredentialStore(**{"product_name": product_name, "credential_file": credential_file}) + -#A CredentialStore backed by os.environ rather than by a file on disk +# A CredentialStore backed by os.environ rather than by a file on disk class EnvarCredentialStore(object): def __init__(self): - #`CBAPI_URL`, `CBAPI_TOKEN`, `CBAPI_SSL_VERIFY` + # `CBAPI_URL`, `CBAPI_TOKEN`, `CBAPI_SSL_VERIFY` environ = os.environ - self.cbapi_url = environ.get('CBAPI_URL',None) - self.cbapi_token = environ.get('CBAPI_TOKEN',None) - self.cbapi_ssl_verify = environ.get('CBAPI_SSL_VERIFY',True) + self.cbapi_url = environ.get('CBAPI_URL', None) + self.cbapi_token = environ.get('CBAPI_TOKEN', None) + self.cbapi_ssl_verify = environ.get('CBAPI_SSL_VERIFY', True) self.org_key = environ.get('CBAPI_ORG_KEY', None) - self.credentials = Credentials(url=self.cbapi_url,token=self.cbapi_token,ssl_verify=self.cbapi_ssl_verify) + self.credentials = Credentials(url=self.cbapi_url, token=self.cbapi_token, ssl_verify=self.cbapi_ssl_verify) - def get_credentials(self,profile=None): + def get_credentials(self, profile=None): return self.credentials + class FileCredentialStore(object): def __init__(self, product_name, **kwargs): if product_name not in ("response", "protection", "psc"): diff --git a/src/cbapi/connection.py b/src/cbapi/connection.py index db89431e..356f1d30 100644 --- a/src/cbapi/connection.py +++ b/src/cbapi/connection.py @@ -23,8 +23,6 @@ except ImportError: MAX_RETRIES = 5 -from requests.packages.urllib3.poolmanager import PoolManager - import logging import json @@ -32,8 +30,7 @@ from cbapi.six.moves import urllib from .auth import CredentialStoreFactory, Credentials -from .errors import ServerError, TimeoutError, ApiError, ObjectNotFoundError, UnauthorizedError, CredentialError, \ - ConnectionError +from .errors import ServerError, TimeoutError, ApiError, ObjectNotFoundError, UnauthorizedError, ConnectionError from . import __version__ from .cache.lru import lru_cache_function @@ -45,8 +42,8 @@ def check_python_tls_compatibility(): try: - tls_adapter = CbAPISessionAdapter(force_tls_1_2=True) - except Exception as e: + CbAPISessionAdapter(force_tls_1_2=True) + except Exception: ret = "TLSv1.1" if "OP_NO_TLSv1_1" not in ssl.__dict__: @@ -113,8 +110,8 @@ def __init__(self, credentials, integration_name=None, timeout=None, max_retries if credentials.ssl_cert_file: self.ssl_verify = credentials.ssl_cert_file - user_agent = "cbapi/{0:s} Python/{1:d}.{2:d}.{3:d}".format(__version__, - sys.version_info[0], sys.version_info[1], sys.version_info[2]) + user_agent = "cbapi/{0:s} Python/{1:d}.{2:d}.{3:d}" \ + .format(__version__, sys.version_info[0], sys.version_info[1], sys.version_info[2]) if integration_name: user_agent += " {}".format(integration_name) @@ -214,7 +211,7 @@ def __init__(self, *args, **kwargs): product_name = kwargs.pop("product_name", None) credential_file = kwargs.pop("credential_file", None) integration_name = kwargs.pop("integration_name", None) - self.credential_store = CredentialStoreFactory.getCredentialStore(product_name,credential_file) + self.credential_store = CredentialStoreFactory.getCredentialStore(product_name, credential_file) url, token, org_key = kwargs.pop("url", None), kwargs.pop("token", None), kwargs.pop("org_key", None) if url and token: @@ -261,20 +258,20 @@ def get_object(self, uri, query_parameters=None, default=None): if result.status_code == 200: try: return result.json() - except: + except Exception: raise ServerError(result.status_code, "Cannot parse response as JSON: {0:s}".format(result.content)) elif result.status_code == 204: # empty response return default else: raise ServerError(error_code=result.status_code, message="Unknown error: {0}".format(result.content)) - + def get_raw_data(self, uri, query_parameters=None, default=None, **kwargs): if query_parameters: if isinstance(query_parameters, dict): query_parameters = convert_query_params(query_parameters) uri += '?%s' % (urllib.parse.urlencode(sorted(query_parameters))) - + hdrs = kwargs.pop("headers", {}) result = self.api_json_request("GET", uri, headers=hdrs) if result.status_code == 200: @@ -299,7 +296,7 @@ def api_json_request(self, method, uri, **kwargs): try: resp = result.json() - except: + except Exception: return result if "errorMessage" in resp: diff --git a/src/cbapi/defense.py b/src/cbapi/defense.py index 676dcd85..e9a15cb6 100644 --- a/src/cbapi/defense.py +++ b/src/cbapi/defense.py @@ -1,2 +1,2 @@ # Compatibility with old Defense API code -from cbapi.psc.defense import * +from cbapi.psc.defense import * # noqa: F401, F403 diff --git a/src/cbapi/errors.py b/src/cbapi/errors.py index cadcf171..8273b30a 100644 --- a/src/cbapi/errors.py +++ b/src/cbapi/errors.py @@ -102,4 +102,3 @@ class InvalidHashError(Exception): class MoreThanOneResultError(ApiError): """Only one object was requested, but multiple matches were found in the Carbon Black datastore.""" pass - diff --git a/src/cbapi/event.py b/src/cbapi/event.py index fb1982da..48734d56 100644 --- a/src/cbapi/event.py +++ b/src/cbapi/event.py @@ -58,7 +58,7 @@ def run(self): try: callback["func"](*callback["args"], **kwargs) - except Exception as e: + except Exception: with self._error_lock: self._errors.append({"exception": traceback.format_exc(), "timestamp": time.time(), "callback_func": callback["func"].__name__, diff --git a/src/cbapi/example_helpers.py b/src/cbapi/example_helpers.py index 97bdb8e6..a237e35e 100644 --- a/src/cbapi/example_helpers.py +++ b/src/cbapi/example_helpers.py @@ -1,4 +1,5 @@ from __future__ import print_function +from cbapi.six import PY3 import sys import time import argparse @@ -24,12 +25,13 @@ # Example scripts: we want to make sure that sys.stdout is using utf-8 encoding. See issue #36. -from cbapi.six import PY3 if not PY3: sys.stdout = codecs.getwriter('utf8')(sys.stdout) + def eprint(*args, **kwargs): - _print(*args, file=sys.stderr, **kwargs) + print(*args, file=sys.stderr, **kwargs) + def build_cli_parser(description="Cb Example Script"): parser = argparse.ArgumentParser(description=description) @@ -77,6 +79,7 @@ def get_cb_protection_object(args): return cb + def get_cb_psc_object(args): if args.verbose: logging.basicConfig() @@ -87,9 +90,10 @@ def get_cb_psc_object(args): cb = CbPSCBaseAPI(url=args.cburl, token=args.apitoken, ssl_verify=(not args.no_ssl_verify)) else: cb = CbPSCBaseAPI(profile=args.profile) - + return cb + def get_cb_defense_object(args): if args.verbose: logging.basicConfig() @@ -126,7 +130,7 @@ def get_cb_livequery_object(args): if args.cburl and args.apitoken and args.orgkey: cb = CbLiveQueryAPI(url=args.cburl, token=args.apitoken, org_key=args.orgkey, - ssl_verify=(not args.no_ssl_verify)) + ssl_verify=(not args.no_ssl_verify)) else: cb = CbLiveQueryAPI(profile=args.profile) @@ -177,22 +181,28 @@ def read_iocs(cb, file=sys.stdin): eprint("line {}: invalid query".format(idx + 1)) return (report_id.hexdigest(), dict(iocs)) + # # Live Response # + class QuitException(Exception): pass + class CliArgsException(Exception): pass + class CliHelpException(Exception): pass + class CliAttachError(Exception): pass + def split_cli(line): ''' we'd like to use shlex.split() but that doesn't work well for @@ -204,7 +214,6 @@ def split_cli(line): parts = line.split(' ') final = [] - inQuotes = False while len(parts) > 0: tok = parts.pop(0) @@ -288,7 +297,7 @@ def cmdloop(self, intro=None): break except KeyboardInterrupt: break - except CliAttachError as e: + except CliAttachError: print("You must attach to a session") continue except CliArgsException as e: @@ -336,7 +345,6 @@ def _file_path_fixup(self, path): and performs the fixups. ''' - if (self._is_path_absolute(path)): return path elif (self._is_path_drive_relative(path)): @@ -501,11 +509,13 @@ def do_ps(self, line): self._needs_attached() p = CliArgs(usage='ps [OPTIONS]') - p.add_option('-v', '--verbose', default=False, action='store_true', help='Display verbose info about each process') + p.add_option('-v', '--verbose', default=False, action='store_true', + help='Display verbose info about each process') p.add_option('-p', '--pid', default=None, help='Display only the given pid') (opts, args) = p.parse_line(line) - if (opts.pid): opts.pid = int(opts.pid) + if opts.pid: + opts.pid = int(opts.pid) processes = self.lr_session.list_processes() @@ -574,7 +584,7 @@ def do_exec(self, line): exe = tok - #ok - now the command (exe) is in tok + # ok - now the command (exe) is in tok # we need to do some crappy path manipulation # to see what we are supposed to execute if (self._is_path_absolute(exe)): @@ -590,7 +600,7 @@ def do_exec(self, line): # then a file exist in the current working # directory that matches the exe name - execute it exe = ntpath.join(self.cwd, exe) - else : + else: # the cwd + exe does not exist - let windows # resolve the path pass @@ -600,7 +610,7 @@ def do_exec(self, line): cmdline = exe + ' ' + ' '.join(parts) else: cmdline = exe - #print "CMD: %s" % cmdline + # print "CMD: %s" % cmdline if not optWorkDir: optWorkDir = self.cwd @@ -648,7 +658,6 @@ def do_del(self, line): del ''' - self._needs_attached() if line is None or line == '': @@ -668,7 +677,6 @@ def do_mkdir(self, line): mdkir ''' - self._needs_attached() if line is None or line == '': diff --git a/src/cbapi/live_response_api.py b/src/cbapi/live_response_api.py index 6126fd32..8d06f947 100644 --- a/src/cbapi/live_response_api.py +++ b/src/cbapi/live_response_api.py @@ -10,9 +10,9 @@ import shutil -from cbapi.errors import TimeoutError, ObjectNotFoundError, ApiError, ServerError +from cbapi.errors import TimeoutError, ObjectNotFoundError, ApiError from cbapi.six import itervalues -from concurrent.futures import ThreadPoolExecutor, as_completed, _base, wait +from concurrent.futures import _base, wait from cbapi import winerror from cbapi.six.moves.queue import Queue @@ -32,9 +32,9 @@ def __init__(self, details): self.decoded_win32_error = "" # Details object: - # {u'status': u'error', u'username': u'admin', u'sensor_id': 9, u'name': u'kill', u'completion': 1464319733.190924, - # u'object': 1660, u'session_id': 7, u'result_type': u'WinHresult', u'create_time': 1464319733.171967, - # u'result_desc': u'', u'id': 22, u'result_code': 2147942487} + # {u'status': u'error', u'username': u'admin', u'sensor_id': 9, u'name': u'kill', + # u'completion': 1464319733.190924, u'object': 1660, u'session_id': 7, u'result_type': u'WinHresult', + # u'create_time': 1464319733.171967, u'result_desc': u'', u'id': 22, u'result_code': 2147942487} if self.details.get("status") == "error" and self.details.get("result_type") == "WinHresult": # attempt to decode the win32 error @@ -45,11 +45,11 @@ def __init__(self, details): self.decoded_win32_error = winerror.decode_hresult(self.win32_error) if self.decoded_win32_error: win32_error_text += " ({0})".format(self.decoded_win32_error) - except: + except Exception: pass finally: message_list.append(win32_error_text) - + self.message = ": ".join(message_list) def __str__(self): @@ -260,10 +260,10 @@ def walk(self, top, topdown=True, onerror=None, followlinks=False): if not topdown: yield top, [fn["filename"] for fn in dirnames], [fn["filename"] for fn in filenames] - # # Process operations # + def kill_process(self, pid): """ Terminate a process on the remote endpoint @@ -384,7 +384,7 @@ def list_registry_keys_and_values(self, regkey): :Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: - >>> pprint.pprint(lr_session.list_registry_keys_and_values('HKLM\\SYSTEM\\CurrentControlSet\\services\\ACPI')) + >>> pprint.pprint(lr_session.list_registry_keys_and_values('HKLM\\SYSTEM\\CurrentControlSet\\services\\ACPI')) {'sub_keys': [u'Parameters', u'Enum'], 'values': [{u'value_data': 0, u'value_name': u'Start', @@ -588,10 +588,11 @@ def _lr_post_command(self, data): try: error_message = json.loads(e.message) if error_message["status"] == "NOT_FOUND": - self.session_id, self.session_data = self._cblr_manager._get_or_create_session(self.sensor_id) + self.session_id, self.session_data = \ + self._cblr_manager._get_or_create_session(self.sensor_id) retries -= 1 continue - except: + except Exception: pass raise ApiError("Received 404 error from server: {0}".format(e.message)) else: @@ -850,9 +851,9 @@ def _session_keepalive_thread(self): log.debug("Session {0} for sensor {1} not valid any longer, removing from cache" .format(session.session_id, session.sensor_id)) delete_list.append(session.sensor_id) - except: - log.debug("Keepalive on session {0} (sensor {1}) failed with unknown error, removing from cache" - .format(session.session_id, session.sensor_id)) + except Exception: + log.debug(("Keepalive on session {0} (sensor {1}) failed with unknown error, " + + "removing from cache").format(session.session_id, session.sensor_id)) delete_list.append(session.sensor_id) for sensor_id in delete_list: diff --git a/src/cbapi/models.py b/src/cbapi/models.py index 3aaf1b44..e0161339 100644 --- a/src/cbapi/models.py +++ b/src/cbapi/models.py @@ -9,7 +9,7 @@ import sys import base64 import os.path -from cbapi.six import iteritems, add_metaclass +from cbapi.six import iteritems, add_metaclass, integer_types from cbapi.six.moves import range from .response.utils import convert_from_cb, convert_to_cb import yaml @@ -145,8 +145,7 @@ def __init__(self, field_name, multiplier=1.0): def __get__(self, instance, instance_type=None): d = super(EpochDateTimeFieldDescriptor, self).__get__(instance, instance_type) - long = long if sys.version_info < (3, 0) else int - if type(d) is float or type(d) is int or type(d) is long: + if type(d) is float or isinstance(type(d), integer_types): epoch_seconds = d / self.multiplier return datetime.utcfromtimestamp(epoch_seconds) else: @@ -231,7 +230,7 @@ def new_object(cls, cb, item, **kwargs): def __getattr__(self, item): try: - val = super(NewBaseModel, self).__getattribute__(item) + super(NewBaseModel, self).__getattribute__(item) except AttributeError: pass # fall through to the rest of the logic... @@ -414,7 +413,7 @@ def _update_object(self): else: log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) http_method = self.__class__._change_object_http_method - ret = self._cb.api_json_request(http_method,self._build_api_request_uri(http_method=http_method), + ret = self._cb.api_json_request(http_method, self._build_api_request_uri(http_method=http_method), data=self._info) return self._refresh_if_needed(ret) @@ -425,7 +424,7 @@ def _refresh_if_needed(self, request_ret): if request_ret.status_code not in range(200, 300): try: message = json.loads(request_ret.text)[0] - except: + except Exception: message = request_ret.text raise ServerError(request_ret.status_code, message, @@ -450,7 +449,7 @@ def _refresh_if_needed(self, request_ret): refresh_required = True else: self._full_init = True - except: + except Exception: refresh_required = True self._dirty_attributes = {} @@ -488,7 +487,7 @@ def _delete_object(self): if ret.status_code not in (200, 204): try: message = json.loads(ret.text)[0] - except: + except Exception: message = ret.text raise ServerError(ret.status_code, message, result="Did not delete {0:s}.".format(str(self))) @@ -507,4 +506,3 @@ def __repr__(self): if self.is_dirty(): r += " (*)" return r - diff --git a/src/cbapi/oldmodels.py b/src/cbapi/oldmodels.py index 64f6ac88..82cab04a 100644 --- a/src/cbapi/oldmodels.py +++ b/src/cbapi/oldmodels.py @@ -253,7 +253,7 @@ def _update_object(self): if ret.status_code not in (200, 204): try: message = json.loads(ret.text)[0] - except: + except Exception: message = ret.text raise ServerError(ret.status_code, message, @@ -273,7 +273,7 @@ def _update_object(self): else: self._info = json.loads(ret.text) self._full_init = True - except: + except Exception: self.refresh() if not self._model_unique_id: @@ -295,7 +295,7 @@ def _delete_object(self): if ret.status_code not in (200, 204): try: message = json.loads(ret.text)[0] - except: + except Exception: message = ret.text raise ServerError(ret.status_code, message, result="Did not delete {0:s}.".format(str(self))) diff --git a/src/cbapi/query.py b/src/cbapi/query.py index a08b29da..fcb6f937 100644 --- a/src/cbapi/query.py +++ b/src/cbapi/query.py @@ -32,7 +32,8 @@ def one(self): res = self[:2] if len(res) == 0 or len(res) > 1: - raise MoreThanOneResultError(message="{0:d} results found for query {1:s}".format(len(self), str(self._query))) + raise MoreThanOneResultError(message="{0:d} results found for query {1:s}" + .format(len(self), str(self._query))) return res[0] diff --git a/src/cbapi/six.py b/src/cbapi/six.py index 190c0239..77f4957e 100644 --- a/src/cbapi/six.py +++ b/src/cbapi/six.py @@ -46,10 +46,10 @@ MAXSIZE = sys.maxsize else: - string_types = basestring, - integer_types = (int, long) + string_types = basestring, # noqa: F821 + integer_types = (int, long) # noqa: F821 class_types = (type, types.ClassType) - text_type = unicode + text_type = unicode # noqa: F821 binary_type = str if sys.platform.startswith("java"): @@ -223,6 +223,7 @@ def get_code(self, fullname): return None get_source = get_code # same as get_code + _importer = _SixMetaPathImporter(__name__) @@ -479,6 +480,7 @@ class Module_six_moves_urllib(types.ModuleType): def __dir__(self): return ['parse', 'error', 'request', 'response', 'robotparser'] + _importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib") @@ -644,7 +646,7 @@ def b(s): # Workaround for standalone backslash def u(s): - return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") # noqa: F821 unichr = unichr int2byte = chr @@ -727,11 +729,11 @@ def print_(*args, **kwargs): return def write(data): - if not isinstance(data, basestring): + if not isinstance(data, basestring): # noqa: F821 data = str(data) # If the file has an encoding, encode unicode with it. - if (isinstance(fp, file) and - isinstance(data, unicode) and + if (isinstance(fp, file) and # noqa: F821 + isinstance(data, unicode) and # noqa: F821 fp.encoding is not None): errors = getattr(fp, "errors", None) if errors is None: @@ -741,13 +743,13 @@ def write(data): want_unicode = False sep = kwargs.pop("sep", None) if sep is not None: - if isinstance(sep, unicode): + if isinstance(sep, unicode): # noqa: F821 want_unicode = True elif not isinstance(sep, str): raise TypeError("sep must be None or a string") end = kwargs.pop("end", None) if end is not None: - if isinstance(end, unicode): + if isinstance(end, unicode): # noqa: F821 want_unicode = True elif not isinstance(end, str): raise TypeError("end must be None or a string") @@ -755,12 +757,12 @@ def write(data): raise TypeError("invalid keyword arguments to print()") if not want_unicode: for arg in args: - if isinstance(arg, unicode): + if isinstance(arg, unicode): # noqa: F821 want_unicode = True break if want_unicode: - newline = unicode("\n") - space = unicode(" ") + newline = unicode("\n") # noqa: F821 + space = unicode(" ") # noqa: F821 else: newline = "\n" space = " " diff --git a/src/cbapi/winerror.py b/src/cbapi/winerror.py index e63310cb..88811885 100644 --- a/src/cbapi/winerror.py +++ b/src/cbapi/winerror.py @@ -2568,6 +2568,7 @@ class Win32Error(ErrorBaseClass): ERROR_DS_INVALID_SEARCH_FLAG_TUPLE = 8627 ERROR_DS_HIERARCHY_TABLE_TOO_DEEP = 8628 + SEVERITY_SUCCESS = 0 SEVERITY_ERROR = 1 @@ -2996,29 +2997,43 @@ class CommDlgError(ErrorBaseClass): def HRESULT_FROM_WIN32(scode): return -2147024896 | (scode & 65535) + def SUCCEEDED(Status): return ((Status) >= 0) -def FAILED(Status): return (Status<0) + +def FAILED(Status): return (Status < 0) + def HRESULT_CODE(hr): return ((hr) & 65535) + def SCODE_CODE(sc): return ((sc) & 65535) + def HRESULT_FACILITY(hr): return (((hr) >> 16) & 8191) + def SCODE_FACILITY(sc): return (((sc) >> 16) & 8191) + def HRESULT_SEVERITY(hr): return (((hr) >> 31) & 1) + def SCODE_SEVERITY(sc): return (((sc) >> 31) & 1) + FACILITY_NT_BIT = 268435456 + + def HRESULT_FROM_NT(x): return x | FACILITY_NT_BIT + def GetScode(hr): return hr + def ResultFromScode(sc): return sc + def decode_hresult(hresult): if HRESULT_FACILITY(hresult) == Facility.FACILITY_WIN32: return Win32Error.lookup_error(HRESULT_CODE(hresult)) From 8272d43dec1106233b002be161f64e21d19f9d3a Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 14 Oct 2019 13:57:02 -0600 Subject: [PATCH 027/197] de-flake8'ed more of the CBAPI code --- setup.cfg | 1 + src/cbapi/cache/lru.py | 14 +++- src/cbapi/protection/models.py | 6 +- src/cbapi/protection/rest_api.py | 9 ++- src/cbapi/psc/cblr.py | 18 +++-- src/cbapi/psc/defense/__init__.py | 2 +- src/cbapi/psc/defense/models.py | 4 +- src/cbapi/psc/livequery/models.py | 45 ++++++----- src/cbapi/psc/livequery/query.py | 23 +++--- src/cbapi/psc/models.py | 49 ++++++------ src/cbapi/psc/query.py | 104 +++++++++++++------------ src/cbapi/psc/rest_api.py | 53 +++++++------ src/cbapi/psc/threathunter/models.py | 3 +- src/cbapi/psc/threathunter/query.py | 9 ++- src/cbapi/psc/threathunter/rest_api.py | 1 + 15 files changed, 181 insertions(+), 160 deletions(-) diff --git a/setup.cfg b/setup.cfg index c5c82eef..4a92c2b2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,6 +5,7 @@ description-file = README.md universal = 1 [flake8] +per-file-ignores = src/*/__init__.py:F401,F403 max-doc-length = 120 max-line-length = 120 diff --git a/src/cbapi/cache/lru.py b/src/cbapi/cache/lru.py index 47c6d617..3b2476b3 100644 --- a/src/cbapi/cache/lru.py +++ b/src/cbapi/cache/lru.py @@ -31,6 +31,7 @@ def wrapper(func): return LRUCachedFunction(func, LRUCacheDict(max_size, expiration)) return wrapper + def _lock_decorator(func): """ If the LRUCacheDict is concurrent, then we should lock in order to avoid @@ -45,6 +46,7 @@ def withlock(self, *args, **kwargs): withlock.__name__ == func.__name__ return withlock + class LRUCacheDict(object): """ A dictionary-like object, supporting LRU caching semantics. @@ -95,8 +97,10 @@ class EmptyCacheThread(threading.Thread): def __init__(self, cache, peek_duration=60): me = self + def kill_self(o): me + self.ref = weakref.ref(cache) self.peek_duration = peek_duration super(LRUCacheDict.EmptyCacheThread, self).__init__() @@ -136,7 +140,7 @@ def clear(self): self.__access_times.clear() def __contains__(self, key): - return self.has_key(key) + return key in self @_lock_decorator def has_key(self, key): @@ -218,6 +222,7 @@ def cleanup(self): else: return None + class LRUCachedFunction(object): """ A memoized function, backed by an LRU cache. @@ -247,7 +252,8 @@ class LRUCachedFunction(object): >>> f(6) Calling f(6) 6 - >>> f(4) #No longer in cache - 4 is the least recently used, and there are at least 3 others items in cache [3,4,5,6]. + >>> f(4) #No longer in cache - 4 is the least recently used, and there are at least 3 others + items in cache [3,4,5,6]. Calling f(4) 4 @@ -261,7 +267,8 @@ def __init__(self, function, cache=None): self.__name__ = self.function.__name__ def __call__(self, *args, **kwargs): - key = repr( (args, kwargs) ) + "#" + self.__name__ #In principle a python repr(...) should not return any # characters. + key = repr((args, kwargs)) + "#" + self.__name__ + # In principle a python repr(...) should not return any # characters. try: return self.cache[key] except KeyError: @@ -269,6 +276,7 @@ def __call__(self, *args, **kwargs): self.cache[key] = value return value + if __name__ == "__main__": import doctest doctest.testmod() diff --git a/src/cbapi/protection/models.py b/src/cbapi/protection/models.py index 9eeee882..1e282d33 100644 --- a/src/cbapi/protection/models.py +++ b/src/cbapi/protection/models.py @@ -95,10 +95,11 @@ def _build_api_request_uri(self, http_method="GET"): for n in ["debugLevel", "kernelDebugLevel", "debugFlags", "debugDuration", "ccLevel", "ccFlags"]): args.append("changeDiagnostics=true") if any(n in self._dirty_attributes - for n in ["template", "templateCloneCleanupMode", "templateCloneCleanupTime", "templateCloneCleanupTimeScale", "templateTrackModsOnly"]): + for n in ["template", "templateCloneCleanupMode", "templateCloneCleanupTime", + "templateCloneCleanupTimeScale", "templateTrackModsOnly"]): args.append("changeTemplate=true") if "tamperProtectionActive" in self._dirty_attributes: - if self.get("tamperProtectionActive", True) == True: + if self.get("tamperProtectionActive", True): args.append("newTamperProtectionActive=true") else: args.append("newTamperProtectionActive=false") @@ -440,4 +441,3 @@ class UserGroup(MutableBaseModel, CreatableModelMixin): @classmethod def _minimum_server_version(cls): return LooseVersion("8.0") - diff --git a/src/cbapi/protection/rest_api.py b/src/cbapi/protection/rest_api.py index ee722277..f3fa6c0a 100644 --- a/src/cbapi/protection/rest_api.py +++ b/src/cbapi/protection/rest_api.py @@ -30,7 +30,8 @@ def __init__(self, *args, **kwargs): except UnauthorizedError: raise UnauthorizedError(uri=self.url, message="Invalid API token for server {0:s}.".format(self.url)) - log.debug('Connected to Cb server version %s at %s' % (self._server_info['ParityServerVersion'], self.session.server)) + log.debug('Connected to Cb server version %s at %s' + % (self._server_info['ParityServerVersion'], self.session.server)) self.cb_server_version = LooseVersion(self._server_info['ParityServerVersion']) def _perform_query(self, cls, **kwargs): @@ -40,7 +41,8 @@ def _perform_query(self, cls, **kwargs): return Query(cls, self, **kwargs) def _populate_server_info(self): - self._server_info = dict((sc['name'], sc['value']) for sc in self.get_object("/api/bit9platform/v1/serverConfig")) + self._server_info = dict((sc['name'], sc['value']) + for sc in self.get_object("/api/bit9platform/v1/serverConfig")) @property def info(self): @@ -179,7 +181,8 @@ def _count(self): args['q'] = self._query query_args = convert_query_params(args) - self._total_results = int(self._cb.get_object(self._doc_class.urlobject, query_parameters=query_args).get("count", 0)) + self._total_results = int(self._cb.get_object(self._doc_class.urlobject, + query_parameters=query_args).get("count", 0)) self._count_valid = True return self._total_results diff --git a/src/cbapi/psc/cblr.py b/src/cbapi/psc/cblr.py index ab1b710c..e945ba68 100644 --- a/src/cbapi/psc/cblr.py +++ b/src/cbapi/psc/cblr.py @@ -17,12 +17,14 @@ log = logging.getLogger(__name__) + class LiveResponseSession(CbLRSessionBase): def __init__(self, cblr_manager, session_id, sensor_id, session_data=None): super(LiveResponseSession, self).__init__(cblr_manager, session_id, sensor_id, session_data=session_data) device_info = self._cb.select(Device, self.sensor_id) self.os_type = OS_LIVE_RESPONSE_ENUM.get(device_info.deviceType, None) + class WorkItem(object): def __init__(self, fn, sensor_id): self.fn = fn @@ -30,8 +32,10 @@ def __init__(self, fn, sensor_id): self.sensor_id = sensor_id.deviceId else: self.sensor_id = int(sensor_id) - + self.future = _base.Future() + + class CompletionNotification(object): def __init__(self, sensor_id): self.sensor_id = sensor_id @@ -80,6 +84,7 @@ def run_job(self, work_item): except Exception as e: work_item.future.set_exception(e) + class LiveResponseSessionManager(CbLRManagerBase): cblr_base = "/integrationServices/v3/cblr" cblr_session_cls = LiveResponseSession @@ -89,7 +94,7 @@ def submit_job(self, job, sensor): # spawn the scheduler thread self._job_scheduler = LiveResponseJobScheduler(self._cb) self._job_scheduler.start() - + work_item = WorkItem(job, sensor) self._job_scheduler.submit_job(work_item) return work_item.future @@ -115,7 +120,7 @@ def _close_session(self, session_id): try: self._cb.put_object("{cblr_base}/session".format(session_id, cblr_base=self.cblr_base), {"session_id": session_id, "status": "CLOSE"}) - except: + except Exception: pass def _create_session(self, sensor_id): @@ -124,6 +129,7 @@ def _create_session(self, sensor_id): session_id = response["id"] return session_id + class LiveResponseJobScheduler(threading.Thread): daemon = True @@ -224,7 +230,7 @@ def submit_job(self, work_item): def _spawn_new_workers(self): if len(self._job_workers) >= self._max_workers: return - + schedule_max = self._max_workers - len(self._job_workers) ''' sensors = [s for s in self._cb.select(Device) if s.deviceId in self._unscheduled_jobs @@ -233,10 +239,10 @@ def _spawn_new_workers(self): ''' log.debug("spawning new workers to handle unscheduled jobs: {0}".format(self._unscheduled_jobs)) sensors = [s for s in self._cb.select(Device) if s.deviceId in self._unscheduled_jobs - and s.deviceId not in self._job_workers] + and s.deviceId not in self._job_workers] sensors_to_schedule = sensors[:schedule_max] log.debug("Spawning new workers to handle these sensors: {0}".format(sensors_to_schedule)) for sensor in sensors_to_schedule: log.debug("Spawning new JobWorker for sensor id {0}".format(sensor.deviceId)) self._job_workers[sensor.deviceId] = JobWorker(self._cb, sensor.deviceId, self.schedule_queue) - self._job_workers[sensor.deviceId].start() \ No newline at end of file + self._job_workers[sensor.deviceId].start() diff --git a/src/cbapi/psc/defense/__init__.py b/src/cbapi/psc/defense/__init__.py index 97378861..bc37a1d5 100644 --- a/src/cbapi/psc/defense/__init__.py +++ b/src/cbapi/psc/defense/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import from .rest_api import CbDefenseAPI -from cbapi.psc.defense.models import Device, Event, Policy \ No newline at end of file +from cbapi.psc.defense.models import Device, Event, Policy diff --git a/src/cbapi/psc/defense/models.py b/src/cbapi/psc/defense/models.py index b68f34e0..0b02d1b9 100644 --- a/src/cbapi/psc/defense/models.py +++ b/src/cbapi/psc/defense/models.py @@ -68,7 +68,7 @@ def _refresh_if_needed(self, request_ret): if request_ret.status_code not in range(200, 300): try: message = json.loads(request_ret.text)[0] - except: + except Exception: message = request_ret.text raise ServerError(request_ret.status_code, message, @@ -96,7 +96,7 @@ def _refresh_if_needed(self, request_ret): # "success" is False raise ServerError(request_ret.status_code, message.get("message", ""), result="Did not update {0:s} record.".format(self.__class__.__name__)) - except: + except Exception: pass self._dirty_attributes = {} diff --git a/src/cbapi/psc/livequery/models.py b/src/cbapi/psc/livequery/models.py index 82b501da..38514034 100644 --- a/src/cbapi/psc/livequery/models.py +++ b/src/cbapi/psc/livequery/models.py @@ -54,7 +54,7 @@ def _refresh(self): self._info = resp self._last_refresh_time = time.time() return True - + def stop(self): if self._is_deleted: raise ApiError("cannot stop a deleted query") @@ -65,20 +65,21 @@ def stop(self): self._info = result.json() self._last_refresh_time = time.time() return True - except: - raise ServerError(result.status_code, "Cannot parse response as JSON: {0:s}".format(result.content)) + except Exception: + raise ServerError(result.status_code, "Cannot parse response as JSON: {0:s}".format(result.content)) return False - + def delete(self): if self._is_deleted: - return True # already deleted + return True # already deleted url = self.urlobject_single.format(self._cb.credentials.org_key, self.id) result = self._cb.delete_object(url) if result.status_code == 200: self._is_deleted = True return True return False - + + class RunHistory(Run): """ Represents a historical LiveQuery ``Run``. @@ -89,13 +90,14 @@ def __init__(self, cb, initial_data=None): item = initial_data model_unique_id = item.get("id") super(Run, self).__init__(cb, - model_unique_id, initial_data=item, - force_init=False, full_doc=True) + model_unique_id, initial_data=item, + force_init=False, full_doc=True) @classmethod def _query_implementation(cls, cb): return RunHistoryQuery(cls, cb) + class Result(UnrefreshableModel): """ Represents a single result from a LiveQuery ``Run``. @@ -185,13 +187,13 @@ def metrics_(self): Returns the reified ``Result.Metrics`` for this result. """ return self._metrics - + def query_device_summaries(self): return self._cb.select(DeviceSummary).run_id(self._run_id) - + def query_result_facets(self): return self._cb.select(ResultFacet).run_id(self._run_id) - + def query_device_summary_facets(self): return self._cb.select(DeviceSummaryFacet).run_id(self._run_id) @@ -203,7 +205,7 @@ class DeviceSummary(UnrefreshableModel): primary_key = "id" swagger_meta_file = "psc/livequery/models/device_summary.yaml" urlobject = "/livequery/v1/orgs/{}/runs/{}/results/device_summaries/_search" - + class Metrics(UnrefreshableModel): """ Represents the metrics for a result. @@ -215,12 +217,12 @@ def __init__(self, cb, initial_data): initial_data=initial_data, force_init=False, full_doc=True, - ) - + ) + @classmethod def _query_implementation(cls, cb): return ResultQuery(cls, cb) - + def __init__(self, cb, initial_data): super(DeviceSummary, self).__init__( cb, @@ -230,7 +232,7 @@ def __init__(self, cb, initial_data): full_doc=True, ) self._metrics = DeviceSummary.Metrics(cb, initial_data=initial_data["metrics"]) - + @property def metrics_(self): """ @@ -238,6 +240,7 @@ def metrics_(self): """ return self._metrics + class ResultFacet(UnrefreshableModel): """ Represents the summary of results for a single field in a LiveQuery ``Run``. @@ -245,7 +248,7 @@ class ResultFacet(UnrefreshableModel): primary_key = "field" swagger_meta_file = "psc/livequery/models/facet.yaml" urlobject = "/livequery/v1/orgs/{}/runs/{}/results/_facet" - + class Values(UnrefreshableModel): """ Represents the values associated with a field. @@ -258,11 +261,11 @@ def __init__(self, cb, initial_data): force_init=False, full_doc=True, ) - + @classmethod def _query_implementation(cls, cb): return FacetQuery(cls, cb) - + def __init__(self, cb, initial_data): super(ResultFacet, self).__init__( cb, @@ -280,12 +283,12 @@ def values_(self): """ return self._values + class DeviceSummaryFacet(ResultFacet): """ Represents the summary of results for a single device summary in a LiveQuery ``Run``. """ urlobject = "/livequery/v1/orgs/{}/runs/{}/results/device_summaries/_facet" - + def __init__(self, cb, initial_data): super(DeviceSummaryFacet, self).__init__(cb, initial_data) - \ No newline at end of file diff --git a/src/cbapi/psc/livequery/query.py b/src/cbapi/psc/livequery/query.py index 6471e552..6fbf9f17 100644 --- a/src/cbapi/psc/livequery/query.py +++ b/src/cbapi/psc/livequery/query.py @@ -115,7 +115,7 @@ def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._query_builder = QueryBuilder() self._sort = {} - + def sort_by(self, key, direction="ASC"): """Sets the sorting behavior on a query's results. @@ -131,10 +131,10 @@ def sort_by(self, key, direction="ASC"): return self def _build_request(self, start, rows): - request = {"start": start } + request = {"start": start} if self._query_builder: - request["query"] = self._query_builder._collapse(); + request["query"] = self._query_builder._collapse() if rows != 0: request["rows"] = rows if self._sort: @@ -311,14 +311,14 @@ def __init__(self, doc_class, cb): self._facet_fields = [] self._criteria = {} self._run_id = None - + def facet_field(self, field): """Sets the facet fields to be received by this query. - + Example:: - + >>> cb.select(ResultFacet).run_id(my_run).facet_field(["device.policy_name", "device.os"]) - + :param field: Field(s) to be received, either single string or list of strings :return: Query object :rtype: :py:class:`Query` @@ -350,17 +350,17 @@ def run_id(self, run_id): """ self._run_id = run_id return self - + def _build_request(self, rows): - terms = { "fields": self._facet_fields } + terms = {"fields": self._facet_fields} if rows != 0: terms["rows"] = rows request = {"query": self._query_builder._collapse(), "terms": terms} if self._criteria: request["criteria"] = self._criteria return request - - def _perform_query(self, rows=0): + + def _perform_query(self, rows=0): if self._run_id is None: raise ApiError("Can't retrieve results without a run ID") @@ -373,4 +373,3 @@ def _perform_query(self, rows=0): results = result.get("terms", []) for item in results: yield self._doc_class(self._cb, item) - \ No newline at end of file diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index bd1e3009..9bb941ab 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -16,7 +16,7 @@ class PSCMutableModel(MutableBaseModel): def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=False): super(PSCMutableModel, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=force_init, full_doc=full_doc) + force_init=force_init, full_doc=full_doc) if not self._change_object_key_name: self._change_object_key_name = self.primary_key @@ -69,7 +69,7 @@ def _refresh_if_needed(self, request_ret): if request_ret.status_code not in range(200, 300): try: message = json.loads(request_ret.text)[0] - except: + except Exception: message = request_ret.text raise ServerError(request_ret.status_code, message, @@ -97,7 +97,7 @@ def _refresh_if_needed(self, request_ret): # "success" is False raise ServerError(request_ret.status_code, message.get("message", ""), result="Did not update {0:s} record.".format(self.__class__.__name__)) - except: + except Exception: pass self._dirty_attributes = {} @@ -110,7 +110,6 @@ class Device(PSCMutableModel): urlobject = "/appservices/v6/orgs/{0}/devices" urlobject_single = "/appservices/v6/orgs/{0}/devices/{1}" primary_key = "id" - #info_key = "deviceInfo" swagger_meta_file = "psc/models/device.yaml" def __init__(self, cb, model_unique_id, initial_data=None): @@ -119,14 +118,14 @@ def __init__(self, cb, model_unique_id, initial_data=None): @classmethod def _query_implementation(cls, cb): return DeviceSearchQuery(cls, cb) - + def _refresh(self): url = self.urlobject_single.format(self._cb.credentials.org_key, self._model_unique_id) resp = self._cb.get_object(url) self._info = resp self._last_refresh_time = time.time() return True - + def lr_session(self): """ Retrieve a Live Response session object for this Device. @@ -141,53 +140,51 @@ def lr_session(self): def background_scan(self, flag): """ Set the background scan option for this device. - + :param boolean flag: True to turn background scan on, False to turn it off. """ - return self._cb.device_background_scan([ self._model_unique_id ], flag) - + return self._cb.device_background_scan([self._model_unique_id], flag) + def bypass(self, flag): """ Set the bypass option for this device. - + :param boolean flag: True to enable bypass, False to disable it. """ - return self._cb.device_bypass([ self._model_unique_id ], flag) - + return self._cb.device_bypass([self._model_unique_id], flag) + def delete_sensor(self): """ Delete this sensor device. """ - return self._cb.device_delete_sensor([ self._model_unique_id ]) - + return self._cb.device_delete_sensor([self._model_unique_id]) + def uninstall_sensor(self): """ Uninstall this sensor device. """ - return self._cb.device_uninstall_sensor([ self._model_unique_id ]) - + return self._cb.device_uninstall_sensor([self._model_unique_id]) + def quarantine(self, flag): """ Set the quarantine option for this device. - + :param boolean flag: True to enable quarantine, False to disable it. """ - return self._cb.device_quarantine([ self._model_unique_id ], flag) - + return self._cb.device_quarantine([self._model_unique_id], flag) + def update_policy(self, policy_id): """ Set the current policy for this device. - + :param int policy_id: ID of the policy to set for the devices. """ - return self._cb.device_update_policy([ self._model_unique_id ], policy_id) - + return self._cb.device_update_policy([self._model_unique_id], policy_id) + def update_sensor_version(self, sensor_version): """ Update the sensor version for this device. - + :param dict sensor_version: New version properties for the sensor. """ - return self._cb.device_update_sensor_version([ self._model_unique_id ], sensor_version) - - \ No newline at end of file + return self._cb.device_update_sensor_version([self._model_unique_id], sensor_version) diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index e5fe4c7d..d99af06e 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -1,4 +1,4 @@ -from cbapi.errors import ApiError, MoreThanOneResultError, ServerError +from cbapi.errors import ApiError, MoreThanOneResultError import logging import functools from six import string_types @@ -6,6 +6,7 @@ log = logging.getLogger(__name__) + class QueryBuilder(object): """ Provides a flexible interface for building prepared queries for the CB @@ -141,6 +142,7 @@ def _collapse(self): else: return None # return everything + class PSCQueryBase: """ Represents the base of all LiveQuery query classes. @@ -214,7 +216,7 @@ def not_(self, q=None, **kwargs): self._query_builder.not_(q, **kwargs) return self - + class IterableQueryMixin: """ @@ -259,14 +261,14 @@ class DeviceSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMix """ Represents a query that is used to locate Device objects. """ - valid_os = [ "WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER" ] + valid_os = ["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"] valid_statuses = ["PENDING", "REGISTERED", "UNINSTALLED", "DEREGISTERED", "ACTIVE", "INACTIVE", "ERROR", "ALL", "BYPASS_ON", "BYPASS", "QUARANTINE", "SENSOR_OUTOFDATE", "DELETED", "LIVE"] valid_priorities = ["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"] valid_directions = ["ASC", "DESC"] - + def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._query_builder = QueryBuilder() @@ -274,15 +276,15 @@ def __init__(self, doc_class, cb): self._time_filter = {} self._exclusions = {} self._sortcriteria = {} - + def _update_criteria(self, key, newlist): oldlist = self._criteria.get(key, []) self._criteria[key] = oldlist + newlist - + def _update_exclusions(self, key, newlist): oldlist = self._exclusions.get(key, []) self._exclusions[key] = oldlist + newlist - + def ad_group_ids(self, ad_group_ids): """ Restricts the devices that this query is performed on to the specified @@ -295,7 +297,7 @@ def ad_group_ids(self, ad_group_ids): raise ApiError("One or more invalid AD group IDs") self._update_criteria("ad_group_id", ad_group_ids) return self - + def device_ids(self, device_ids): """ Restricts the devices that this query is performed on to the specified @@ -308,13 +310,13 @@ def device_ids(self, device_ids): raise ApiError("One or more invalid device IDs") self._update_criteria("id", device_ids) return self - + def last_contact_time(self, *args, **kwargs): """ Restricts the devices that this query is performed on to the specified last contact time (either specified as a start and end point or as a range). - + :return: This instance """ if kwargs.get("start", None) and kwargs.get("end", None): @@ -326,15 +328,15 @@ def last_contact_time(self, *args, **kwargs): etime = kwargs["end"] if not isinstance(etime, str): etime = etime.isoformat() - self._time_filter = { "start": stime, "end": etime } + self._time_filter = {"start": stime, "end": etime} elif kwargs.get("range", None): if kwargs.get("start", None) or kwargs.get("end", None): raise ApiError("cannot specify start= or end= in addition to range=") - self._time_filter = { "range": kwargs["range"] } + self._time_filter = {"range": kwargs["range"]} else: raise ApiError("must specify either start= and end= or range=") return self - + def os(self, operating_systems): """ Restricts the devices that this query is performed on to the specified @@ -347,7 +349,7 @@ def os(self, operating_systems): raise ApiError("One or more invalid operating systems") self._update_criteria("os", operating_systems) return self - + def policy_ids(self, policy_ids): """ Restricts the devices that this query is performed on to the specified @@ -360,7 +362,7 @@ def policy_ids(self, policy_ids): raise ApiError("One or more invalid policy IDs") self._update_criteria("policy_id", policy_ids) return self - + def status(self, statuses): """ Restricts the devices that this query is performed on to the specified @@ -373,7 +375,7 @@ def status(self, statuses): raise ApiError("One or more invalid status values") self._update_criteria("status", statuses) return self - + def target_priorities(self, target_priorities): """ Restricts the devices that this query is performed on to the specified @@ -386,7 +388,7 @@ def target_priorities(self, target_priorities): raise ApiError("One or more invalid target priority values") self._update_criteria("target_priority", target_priorities) return self - + def exclude_sensor_versions(self, sensor_versions): """ Restricts the devices that this query is performed on to exclude specified @@ -399,7 +401,7 @@ def exclude_sensor_versions(self, sensor_versions): raise ApiError("One or more invalid sensor versions") self._update_exclusions("sensor_version", sensor_versions) return self - + def sort_by(self, key, direction="ASC"): """Sets the sorting behavior on a query's results. @@ -413,42 +415,42 @@ def sort_by(self, key, direction="ASC"): """ if direction not in DeviceSearchQuery.valid_directions: raise ApiError("invalid sort direction specified") - self._sortcriteria = { "field": key, "order": direction } + self._sortcriteria = {"field": key, "order": direction} return self - + def _build_request(self, from_row, max_rows): mycrit = self._criteria if self._time_filter: mycrit["last_contact_time"] = self._time_filter - request = { "criteria": mycrit, "exclusions": self._exclusions } + request = {"criteria": mycrit, "exclusions": self._exclusions} request["query"] = self._query_builder._collapse() if from_row > 0: request["start"] = from_row if max_rows >= 0: request["rows"] = max_rows if self._sortcriteria != {}: - request["sort"] = [ self._sortcriteria ] + request["sort"] = [self._sortcriteria] return request - + def _build_url(self, tail_end): url = self._doc_class.urlobject.format( self._cb.credentials.org_key) + tail_end return url - + def _count(self): if self._count_valid: return self._total_results - + url = self._build_url("/_search") request = self._build_request(0, -1) resp = self._cb.post_object(url, body=request) result = resp.json() - + self._total_results = result["num_found"] self._count_valid = True return self._total_results - + def _perform_query(self, from_row=0, max_rows=-1): url = self._build_url("/_search") current = from_row @@ -458,10 +460,10 @@ def _perform_query(self, from_row=0, max_rows=-1): request = self._build_request(current, max_rows) resp = self._cb.post_object(url, body=request) result = resp.json() - + self._total_results = result["num_found"] self._count_valid = True - + results = result.get("results", []) for item in results: yield self._doc_class(self._cb, item["id"], item) @@ -476,22 +478,22 @@ def _perform_query(self, from_row=0, max_rows=-1): if current >= self._total_results: still_querying = False break - + def download(self): """ Uses the query parameters that have been set to download all device listings in CSV format. - + Example:: - + >>> cb.select(Device).status(["ALL"]).download() - + :return: The CSV raw data as returned from the server. """ - tmp = self._criteria.get("status", []) + tmp = self._criteria.get("status", []) if not tmp: raise ApiError("at least one status must be specified to download") - query_params = { "status": ",".join(tmp) } + query_params = {"status": ",".join(tmp)} tmp = self._criteria.get("ad_group_id", []) if tmp: query_params["ad_group_id"] = ",".join([str(t) for t in tmp]) @@ -509,36 +511,36 @@ def download(self): query_params["sort_order"] = self._sortcriteria["order"] url = self._build_url("/_search/download") # AGRB 10/3/2019 - Header is TEMPORARY until bug is fixed in API. Remove when fix deployed. - return self._cb.get_raw_data(url, query_params, headers={ "Content-Type": "application/json"}) + return self._cb.get_raw_data(url, query_params, headers={"Content-Type": "application/json"}) def _bulk_device_action(self, action_type, options=None): - request = { "action_type": action_type, "search": self._build_request(0, -1) } + request = {"action_type": action_type, "search": self._build_request(0, -1)} if options: request["options"] = options return self._cb._raw_device_action(request) - + def background_scan(self, flag): """ Set the background scan option for the specified devices. - + :param boolean flag: True to turn background scan on, False to turn it off. """ return self._bulk_device_action("BACKGROUND_SCAN", self._cb._action_toggle(flag)) - + def bypass(self, flag): """ Set the bypass option for the specified devices. - + :param boolean flag: True to enable bypass, False to disable it. """ return self._bulk_device_action("BYPASS", self._cb._action_toggle(flag)) - + def delete_sensor(self): """ Delete the specified sensor devices. """ return self._bulk_device_action("DELETE_SENSOR") - + def uninstall_sensor(self): """ Uninstall the specified sensor devices. @@ -548,24 +550,24 @@ def uninstall_sensor(self): def quarantine(self, flag): """ Set the quarantine option for the specified devices. - + :param boolean flag: True to enable quarantine, False to disable it. """ return self._bulk_device_action("QUARANTINE", self._cb._action_toggle(flag)) - + def update_policy(self, policy_id): """ Set the current policy for the specified devices. - + :param int policy_id: ID of the policy to set for the devices. """ - return self._bulk_device_action("UPDATE_POLICY", { "policy_id": policy_id }) - + return self._bulk_device_action("UPDATE_POLICY", {"policy_id": policy_id}) + def update_sensor_version(self, sensor_version): """ Update the sensor version for the specified devices. - + :param dict sensor_version: New version properties for the sensor. """ - return self._bulk_device_action("UPDATE_SENSOR_VERSION", \ - { "sensor_version": sensor_version }) \ No newline at end of file + return self._bulk_device_action("UPDATE_SENSOR_VERSION", + {"sensor_version": sensor_version}) diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index e53c98f9..cf7b7aed 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -6,6 +6,7 @@ log = logging.getLogger(__name__) + class CbPSCBaseAPI(BaseAPI): """The main entry point into the Cb PSC API. @@ -26,8 +27,8 @@ def _perform_query(self, cls, **kwargs): return cls._query_implementation(self) else: raise ApiError("All PSC models should provide _query_implementation") - - #---- LiveOps + + # ---- LiveOps @property def live_response(self): @@ -38,19 +39,19 @@ def live_response(self): def _request_lr_session(self, sensor_id): return self.live_response.request_session(sensor_id) - #---- Device API - + # ---- Device API + def get_device(self, device_id): """ Locate a device with the specified device ID. - + :param int device_id: The ID of the device to look up. :return: The new device object. """ rc = Device(self, device_id) rc.refresh() return rc - + def _raw_device_action(self, request): url = "/appservices/v6/orgs/{0}/device_actions".format(self.credentials.org_key) resp = self.post_object(url, body=request) @@ -60,23 +61,23 @@ def _raw_device_action(self, request): return None else: raise ServerError(error_code=resp.status_code, message="Device action error: {0}".format(resp.content)) - + def _device_action(self, device_ids, action_type, options=None): - request = { "action_type": action_type, "device_id": device_ids } + request = {"action_type": action_type, "device_id": device_ids} if options: request["options"] = options return self._raw_device_action(request) - + def _action_toggle(self, flag): if flag: - return { "toggle": "ON" } + return {"toggle": "ON"} else: - return { "toggle": "OFF" } - + return {"toggle": "OFF"} + def device_background_scan(self, device_ids, flag): """ Set the background scan option for the specified devices. - + :param list device_ids: List of IDs of devices to be set. :param boolean flag: True to turn background scan on, False to turn it off. """ @@ -85,53 +86,51 @@ def device_background_scan(self, device_ids, flag): def device_bypass(self, device_ids, flag): """ Set the bypass option for the specified devices. - + :param list device_ids: List of IDs of devices to be set. :param boolean flag: True to enable bypass, False to disable it. """ return self._device_action(device_ids, "BYPASS", self._action_toggle(flag)) - + def device_delete_sensor(self, device_ids): """ Delete the specified sensor devices. - + :param list device_ids: List of IDs of devices to be deleted. """ return self._device_action(device_ids, "DELETE_SENSOR") - + def device_uninstall_sensor(self, device_ids): """ Uninstall the specified sensor devices. - + :param list device_ids: List of IDs of devices to be uninstalled. """ return self._device_action(device_ids, "UNINSTALL_SENSOR") - + def device_quarantine(self, device_ids, flag): """ Set the quarantine option for the specified devices. - + :param list device_ids: List of IDs of devices to be set. :param boolean flag: True to enable quarantine, False to disable it. """ return self._device_action(device_ids, "QUARANTINE", self._action_toggle(flag)) - + def device_update_policy(self, device_ids, policy_id): """ Set the current policy for the specified devices. - + :param list device_ids: List of IDs of devices to be changed. :param int policy_id: ID of the policy to set for the devices. """ - return self._device_action(device_ids, "UPDATE_POLICY", { "policy_id": policy_id }) + return self._device_action(device_ids, "UPDATE_POLICY", {"policy_id": policy_id}) def device_update_sensor_version(self, device_ids, sensor_version): """ Update the sensor version for the specified devices. - + :param list device_ids: List of IDs of devices to be changed. :param dict sensor_version: New version properties for the sensor. """ - return self._device_action(device_ids, "UPDATE_SENSOR_VERSION", - { "sensor_version": sensor_version }) - \ No newline at end of file + return self._device_action(device_ids, "UPDATE_SENSOR_VERSION", {"sensor_version": sensor_version}) diff --git a/src/cbapi/psc/threathunter/models.py b/src/cbapi/psc/threathunter/models.py index 25807ad9..461380b9 100644 --- a/src/cbapi/psc/threathunter/models.py +++ b/src/cbapi/psc/threathunter/models.py @@ -96,7 +96,8 @@ def tree(self): def parents(self): """Returns a query for parent processes associated with this process. - :return: Returns a Query object with the appropriate search parameters for parent processes, or None if the process has no recorded parent + :return: Returns a Query object with the appropriate search parameters for parent processes, + or None if the process has no recorded parent :rtype: :py:class:`cbapi.psc.threathunter.query.AsyncProcessQuery` or None """ if "parent_guid" in self._info: diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index ae7997af..a2fd62e1 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -1,5 +1,5 @@ from cbapi.query import PaginatedQuery, BaseQuery, SimpleQuery -from cbapi.errors import ServerError, ApiError, TimeoutError +from cbapi.errors import ApiError, TimeoutError import time from solrq import Q from six import string_types @@ -257,7 +257,8 @@ def _get_query_parameters(self): args['q'] = self._query_builder._collapse() if self._query_builder._process_guid is not None: args["cb.process_guid"] = self._query_builder._process_guid - args["fl"] = "*,parent_hash,parent_name,process_cmdline,backend_timestamp,device_external_ip,device_group,device_internal_ip,device_os,process_effective_reputation,process_reputation,ttp" + args["fl"] = "*,parent_hash,parent_name,process_cmdline,backend_timestamp,device_external_ip,device_group," \ + + "device_internal_ip,device_os,process_effective_reputation,process_reputation,ttp" return args @@ -338,7 +339,7 @@ def __init__(self, doc_class, cb): self._query_token = None self._timeout = 0 self._timed_out = False - self._sort_by = "backend_timestamp" # Requires default to prevent unstable fetching of results + self._sort_by = "backend_timestamp" # Requires default to prevent unstable fetching of results self._sort_direction = "ASC" def sort_by(self, key, direction="ASC"): @@ -455,7 +456,7 @@ def _search(self, start=0, rows=0): result_url = '{}?start={}&rows={}'.format( result_url_template, current, - 10 # Batch gets to reduce API calls + 10 # Batch gets to reduce API calls ) result = self._cb.get_object(result_url, query_parameters=query_parameters) diff --git a/src/cbapi/psc/threathunter/rest_api.py b/src/cbapi/psc/threathunter/rest_api.py index b53ec429..8de417bc 100644 --- a/src/cbapi/psc/threathunter/rest_api.py +++ b/src/cbapi/psc/threathunter/rest_api.py @@ -6,6 +6,7 @@ log = logging.getLogger(__name__) + class CbThreatHunterAPI(CbPSCBaseAPI): """The main entry point into the Carbon Black Cloud ThreatHunter API. From 46b67fc94b0edc1febe04bf9338aafc561a9bb75 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 15 Oct 2019 14:18:32 -0600 Subject: [PATCH 028/197] de-flake8'ed the rest of the source files in CBAPI sensor_events.py in response is machine-generated, can't be linted properly - exclude it in setup.cfg --- setup.cfg | 3 + src/cbapi/response/cblr.py | 5 +- src/cbapi/response/event.py | 24 ++--- src/cbapi/response/models.py | 156 +++++++++++++++++---------------- src/cbapi/response/query.py | 13 +-- src/cbapi/response/rest_api.py | 15 ++-- src/cbapi/response/utils.py | 7 +- 7 files changed, 116 insertions(+), 107 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4a92c2b2..e78a0367 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,6 +5,9 @@ description-file = README.md universal = 1 [flake8] +exclude = + src/cbapi/response/sensor_events.py, + .svn,CVS,.bzr,.hg,.git,__pycache__,.tox per-file-ignores = src/*/__init__.py:F401,F403 max-doc-length = 120 max-line-length = 120 diff --git a/src/cbapi/response/cblr.py b/src/cbapi/response/cblr.py index c17a571c..bded6f7d 100644 --- a/src/cbapi/response/cblr.py +++ b/src/cbapi/response/cblr.py @@ -14,7 +14,8 @@ class LiveResponseSessionManager(CbLRManagerBase): cblr_session_cls = LiveResponseSession def _get_or_create_session(self, sensor_id): - sensor_sessions = [s for s in self._cb.get_object("{cblr_base}/session?active_only=true".format(cblr_base=self.cblr_base)) + sensor_sessions = [s for s in self._cb.get_object("{cblr_base}/session?active_only=true" + .format(cblr_base=self.cblr_base)) if s["sensor_id"] == sensor_id and s["status"] in ("pending", "active")] if len(sensor_sessions) > 0: @@ -41,7 +42,7 @@ def _close_session(self, session_id): session_data = self._cb.get_object("{cblr_base}/session/{0}".format(session_id, cblr_base=self.cblr_base)) session_data["status"] = "close" self._cb.put_object("{cblr_base}/session/{0}".format(session_id, cblr_base=self.cblr_base), session_data) - except: + except Exception: pass def _create_session(self, sensor_id): diff --git a/src/cbapi/response/event.py b/src/cbapi/response/event.py index 9dbe59ac..c26c9ffd 100644 --- a/src/cbapi/response/event.py +++ b/src/cbapi/response/event.py @@ -33,18 +33,18 @@ def __init__(self, cb, filename): def run(self): while not self._fp.closed: - l = self._fp.readline() - if l == "": + line = self._fp.readline() + if line == "": # wait for new events sleep(1) continue try: - msg = json.loads(l) + msg = json.loads(line) routing_key = msg.get("type") # log.debug("Received message with routing key %s" % routing_key) - registry.eval_callback(routing_key, l, self._cb) - except: + registry.eval_callback(routing_key, line, self._cb) + except Exception: pass def stop(self): @@ -76,8 +76,8 @@ def __init__(self, cb, listening_address, **kwargs): certfile = kwargs.get("certfile", None) if not keyfile or not certfile: raise CredentialError("Need to specify 'keyfile' and 'certfile' for HTTPS") - self.httpd.socket = ssl.wrap_socket (self.httpd.socket, certfile=certfile, keyfile=keyfile, - server_side=True) + self.httpd.socket = ssl.wrap_socket(self.httpd.socket, certfile=certfile, keyfile=keyfile, + server_side=True) def run(self): self.httpd.serve_forever() @@ -168,7 +168,7 @@ def on_connection_closed(self, connection, reply_code, reply_text): self._connection.ioloop.stop() else: log.warning('Connection closed, reopening in 5 seconds: (%s) %s', - reply_code, reply_text) + reply_code, reply_text) self._connection.add_timeout(5, self.reconnect) def reconnect(self): @@ -231,7 +231,7 @@ def on_channel_closed(self, channel, reply_code, reply_text): """ log.warning('Channel %i was closed: (%s) %s', - channel, reply_code, reply_text) + channel, reply_code, reply_text) self._connection.close() def setup_exchange(self, exchange_name): @@ -282,7 +282,7 @@ def on_queue_declareok(self, method_frame): """ for routing_key in self.ROUTING_KEYS: log.debug('Binding %s to %s with %s', - self.EXCHANGE, self.QUEUE, routing_key) + self.EXCHANGE, self.QUEUE, routing_key) self._channel.queue_bind(self.on_bindok, self.QUEUE, self.EXCHANGE, routing_key) @@ -331,7 +331,7 @@ def on_consumer_cancelled(self, method_frame): """ log.debug('Consumer was cancelled remotely, shutting down: %r', - method_frame) + method_frame) if self._channel: self._channel.close() @@ -419,7 +419,7 @@ def on_message(self, unused_channel, basic_deliver, properties, body): """ log.debug('Received message # %s with properties %s', - basic_deliver.delivery_tag, properties) + basic_deliver.delivery_tag, properties) registry.eval_callback(basic_deliver.routing_key, body, self._cb) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 44351708..5e68920a 100644 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -1,8 +1,6 @@ #!/usr/bin/env python - from __future__ import absolute_import -import contextlib import copy import json from distutils.version import LooseVersion @@ -20,32 +18,31 @@ from cbapi.utils import convert_query_params from ..errors import InvalidObjectError, ApiError, TimeoutError -from ..oldmodels import BaseModel, MutableModel, immutable - -if six.PY3: - long = int - from io import BytesIO as StringIO -else: - from cStringIO import StringIO +from ..oldmodels import BaseModel, immutable from ..models import NewBaseModel, MutableBaseModel, CreatableModelMixin -from ..oldmodels import BaseModel, immutable from .utils import convert_from_cb, convert_from_solr, parse_42_guid, convert_event_time, parse_process_guid, \ convert_to_solr from ..errors import ServerError, InvalidHashError, ObjectNotFoundError from ..query import SimpleQuery, PaginatedQuery from .query import Query -try: - from functools import total_ordering -except ImportError: - from total_ordering import total_ordering - from cbapi.six import python_2_unicode_compatible, iteritems # Get constants for decoding the Netconn events import socket +if six.PY3: + long = int + from io import BytesIO as StringIO +else: + from cStringIO import StringIO + +try: + from functools import total_ordering +except ImportError: + from total_ordering import total_ordering + log = logging.getLogger(__name__) @@ -102,7 +99,7 @@ class StoragePartitionQuery(SimpleQuery): def results(self): if not self._full_init: self._results = [] - for k,v in iteritems(self._cb.get_object(self._urlobject, default={})): + for k, v in iteritems(self._cb.get_object(self._urlobject, default={})): t = self._doc_class.new_object(self._cb, v, full_doc=True) if self._match_query(t): self._results.append(t) @@ -345,7 +342,8 @@ def search_processes(self, min_score=None, max_score=None): :param min_score: minimum feed score :param max_score: maximum feed score - :return: Returns a :py:class:`response.rest_api.Query` object with the appropriate search parameters for processes + :return: Returns a :py:class:`response.rest_api.Query` object with the appropriate + search parameters for processes :rtype: :py:class:`response.rest_api.Query` """ return self._search(Process, min_score, max_score) @@ -355,7 +353,8 @@ def search_binaries(self, min_score=None, max_score=None): Perform a *Binary* search within this feed that satisfies min_score and max_score :param min_score: minimum feed score :param max_score: maximum feed score - :return: Returns a :py:class:`response.rest_api.Query` object within the appropriate search parameters for binaries + :return: Returns a :py:class:`response.rest_api.Query` object within the appropriate + search parameters for binaries :rtype: :py:class:`response.rest_api.Query` """ return self._search(Binary, min_score, max_score) @@ -655,7 +654,7 @@ def network_interfaces(self): for adapter in getattr(self, 'network_adapters', '').split('|'): parts = adapter.split(',') if len(parts) == 2: - out.append(Sensor.NetworkAdapter._make([':'.join(a+b for a,b in zip(parts[1][::2], parts[1][1::2])), + out.append(Sensor.NetworkAdapter._make([':'.join(a+b for a, b in zip(parts[1][::2], parts[1][1::2])), parts[0]])) return out @@ -742,7 +741,6 @@ def flush_events(self): :warning: This may cause a significant amount of network traffic from this sensor to the Cb Response Server """ - # Note that Cb Response 6 requires the date/time stamp to be sent in RFC822 format (not ISO 8601). # since the date/time stamp just needs to be far in the future, we just fake a GMT timezone. self.event_log_flush_time = datetime.now() + timedelta(days=365) @@ -816,7 +814,8 @@ def _update_object(self): # # Note that even though we delete the event_log_flush_time here, it'll get re-initialized when we GET # the sensor after sending the PUT request. - if "event_log_flush_time" in self._dirty_attributes and self._info.get("event_log_flush_time", None) is not None: + if "event_log_flush_time" in self._dirty_attributes and self._info.get("event_log_flush_time", + None) is not None: if self._cb.cb_server_version > LooseVersion("6.0.0"): # since the date/time stamp just needs to be far in the future, we just fake a GMT timezone. try: @@ -882,12 +881,12 @@ def where(self, new_query): @property def results(self): if not self._full_init: - #ZE CB-15681 - REMOVE BLOCK WHEN BUG IS FIXED + # ZE CB-15681 - REMOVE BLOCK WHEN BUG IS FIXED try: full_results = self._cb.get_object(self._urlobject, query_parameters=convert_query_params(self._query)) - except ServerError as se: + except ServerError: full_results = False - #ZE CB-15681 - REMOVE BLOCK WHEN BUG IS FIXED + # ZE CB-15681 - REMOVE BLOCK WHEN BUG IS FIXED if not full_results: self._results = [] else: @@ -945,7 +944,7 @@ def _retrieve_cb_info(self): return info def _update_object(self): - if self._cb.cb_server_version < LooseVersion("6.1.0") or self._info.get("id", None) == None: + if self._cb.cb_server_version < LooseVersion("6.1.0") or self._info.get("id", None) is None: # only include IDs of the teams and not the entire dictionary # - applies to Cb Response server < 6.0 as well as Cb Response servers >= 6.0 where the user hasn't # been created yet. @@ -1074,7 +1073,7 @@ def query(self, new_query): self._reset_query() qt = list(self._query) qt.append(("q", new_query)) - self.search_query = "&".join(("{0}={1}".format(k, urllib.parse.quote(v)) for k,v in qt)) + self.search_query = "&".join(("{0}={1}".format(k, urllib.parse.quote(v)) for k, v in qt)) @property def facets(self): @@ -1257,7 +1256,7 @@ def tag_info(self, tag_name): class ThreatReportQuery(Query): def set_ignored(self, ignored_flag=True): qt = (("cb.urlver", "1"), ("q", self._query)) - search_query = "&".join(("{0}={1}".format(k, urllib.parse.quote(v)) for k,v in qt)) + search_query = "&".join(("{0}={1}".format(k, urllib.parse.quote(v)) for k, v in qt)) payload = {"updates": {"is_ignored": ignored_flag}, "query": search_query} self._cb.post_object("/api/v1/threat_report", payload) @@ -1304,8 +1303,8 @@ def __init__(self, cb, full_id, initial_data=None): try: # fill in feed_id and id (feed_id, report_id) = full_id.split(":") - initial_data = { "feed_id": feed_id, "id": report_id } - except: + initial_data = {"feed_id": feed_id, "id": report_id} + except Exception: raise ApiError("ThreatReport ID must be in form ':'") super(ThreatReport, self).__init__(cb, full_id, initial_data) @@ -1315,7 +1314,7 @@ def feed(self): return self._join(Feed, "feed_id") def _update_object(self): - update_content = { "ids": { str(self.feed_id): [str(self.id)] }, "updates": {}} + update_content = {"ids": {str(self.feed_id): [str(self.id)]}, "updates": {}} for k in self._dirty_attributes.keys(): update_content["updates"][k] = getattr(self, k) @@ -1324,7 +1323,7 @@ def _update_object(self): if ret.status_code not in (200, 204): try: message = json.loads(ret.text)[0] - except: + except Exception: message = ret.text raise ServerError(ret.status_code, message, @@ -1343,7 +1342,7 @@ def _update_object(self): else: self._info = json.loads(ret.text) self._full_init = True - except: + except Exception: self.refresh() self._dirty_attributes = {} @@ -1715,8 +1714,8 @@ def endpoints(self): @property def version_info(self): """ - Returns a :class:`.VersionInfo` object containing detailed information: File Descritpion, File Version, Product Name, - Product Version, Company Name, Legal Copyright, and Original FileName + Returns a :class:`.VersionInfo` object containing detailed information: File Descritpion, File Version, + Product Name, Product Version, Company Name, Legal Copyright, and Original FileName """ return Binary.VersionInfo._make([self._attribute('file_desc', ""), self._attribute('file_version', ""), self._attribute('product_name', ""), self._attribute('product_version', ""), @@ -1802,7 +1801,6 @@ def is_executable_image(self): """ return self._attribute('is_executable_image', False) - @property def icon(self): """ @@ -1813,7 +1811,7 @@ def icon(self): icon = self._attribute('icon') if not icon: icon = '' - except: + except Exception: pass return base64.b64decode(icon) @@ -1917,7 +1915,7 @@ def parse_netconn(self, seq, netconn): timestamp = convert_event_time(parts[0]) try: new_conn['remote_ip'] = socket.inet_ntoa(struct.pack('>i', int(parts[1]))) - except: + except Exception: new_conn['remote_ip'] = '0.0.0.0' new_conn['remote_port'] = int(parts[2]) new_conn['proto'] = protocols[int(parts[3])] @@ -2021,7 +2019,7 @@ def _lookup_privilege(privilege_code): if parts[8] == 'true': new_crossproc['is_target'] = True - if new_crossproc['is_target'] == True: + if new_crossproc['is_target']: new_crossproc['target_procguid'] = self.process_model.id new_crossproc['target_md5'] = self.process_model.process_md5 new_crossproc['target_path'] = self.process_model.path @@ -2056,7 +2054,7 @@ def parse_netconn(self, seq, netconn): for ipfield in ('remote_ip', 'local_ip', 'proxy_ip'): try: new_conn[ipfield] = socket.inet_ntoa(struct.pack('>i', int(netconn.get(ipfield, 0)))) - except: + except Exception: new_conn[ipfield] = netconn.get(ipfield, '0.0.0.0') for portfield in ('remote_port', 'local_port', 'proxy_port'): @@ -2134,7 +2132,8 @@ def _query_implementation(cls, cb): @classmethod def new_object(cls, cb, item, max_children=15): # 'id' did not exist in some process documents from a 5.2 -> 6.1 upgrade - return cb.select(Process, item['id'] or item['unique_id'], long(item['segment_id']), max_children, initial_data=item) + return cb.select(Process, item['id'] or item['unique_id'], long(item['segment_id']), + max_children, initial_data=item) def parse_guid(self, procguid): try: @@ -2149,7 +2148,8 @@ def parse_guid(self, procguid): else: return None, None - def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=None, force_init=False, suppressed_process=False): + def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=None, force_init=False, + suppressed_process=False): self.max_children = max_children self.current_segment = segment self.suppressed_process = suppressed_process @@ -2255,7 +2255,7 @@ def _attribute(self, attrname, default=None): # Relaxing this a bit to allow for cases where the information is there if attrname in ['parent_unique_id', 'parent_name'] and not self._full_init: - if attrname in self._info and self._info[attrname] != None and self._info[attrname] != "": + if attrname in self._info and self._info[attrname] is not None and self._info[attrname] != "": return self._info[attrname] else: self._retrieve_cb_info() @@ -2300,7 +2300,7 @@ def _parse(self, obj): else: log.debug("Unable to recover start time and process PID for process %s, marking invalid" % self.id) self.valid_process = False - except Exception as e: + except Exception: log.debug("Unable to parse process GUID %s, marking invalid" % self.id) self.valid_process = False @@ -2412,7 +2412,7 @@ def end(self): if self._info.get("end") is not None: return convert_from_solr(self._info.get('end', -1)) - if self.get("terminated", False) == True and self.get("last_update") is not None: + if self.get("terminated", False) and self.get("last_update") is not None: return convert_from_solr(self._attribute('last_update', -1)) def require_events(self): @@ -2529,17 +2529,18 @@ def crossprocs(self): @property def parents(self): - current_process = self - while True: - try : - parent = current_process.parent - if not(parent) or parent.get('process_pid',-1) == -1: - break - yield parent - current_process = parent - except ObjectNotFoundError: - return - return + current_process = self + while True: + try: + parent = current_process.parent + if not(parent) or parent.get('process_pid', -1) == -1: + break + yield parent + current_process = parent + except ObjectNotFoundError: + return + return + @property def children(self): """ @@ -2560,10 +2561,10 @@ def children(self): timestamp = convert_event_time(child.get("start") or "1970-01-01T00:00:00Z") yield CbChildProcEvent(self, timestamp, i, { - "procguid": child.get("unique_id",None), - "md5": child.get("process_md5",None), - "pid": child.get("process_pid",None), - "path": child.get("path",None), + "procguid": child.get("unique_id", None), + "md5": child.get("process_md5", None), + "pid": child.get("process_pid", None), + "path": child.get("path", None), "terminated": False }, is_suppressed=child.get("is_suppressed", False), @@ -2593,7 +2594,7 @@ def all_events_segment(self): :return: list of CbEvent objects """ segment_events = list(self.modloads) + list(self.netconns) + list(self.filemods) + \ - list(self.children) + list(self.regmods) + list(self.crossprocs) + list(self.children) + list(self.regmods) + list(self.crossprocs) segment_events.sort() return segment_events @@ -2665,9 +2666,10 @@ def depth(self): @property def threat_intel_hits(self): try: - hits = self._cb.get_object("/api/v1/process/{0}/{1}/threat_intel_hits".format(self.id, self.current_segment)) + hits = self._cb.get_object("/api/v1/process/{0}/{1}/threat_intel_hits".format(self.id, + self.current_segment)) return hits - except ServerError as e: + except ServerError: raise ApiError("Sharing IOCs not set up in Cb server. See {}/#/share for more information." .format(self._cb.credentials.url)) @@ -2711,7 +2713,7 @@ def comms_ip(self): """ try: ip_address = socket.inet_ntoa(struct.pack('>i', self._attribute('comms_ip', 0))) - except: + except Exception: ip_address = self._attribute('comms_ip', 0) return ip_address @@ -2724,7 +2726,7 @@ def interface_ip(self): """ try: ip_address = socket.inet_ntoa(struct.pack('>i', self._attribute('interface_ip', 0))) - except: + except Exception: ip_address = self._attribute('interface_ip', 0) return ip_address @@ -2738,13 +2740,12 @@ def process_md5(self): except StopIteration: return None - @property def path(self): # Some processes don't have a path associated with them. Try and use the first modload as the file path # otherwise, return None. (tested with Cb Response server 5.2.0.161004.1206) try: - return self._attribute("path","") or next(self.modloads).path + return self._attribute("path", "") or next(self.modloads).path except StopIteration: return None @@ -2881,10 +2882,11 @@ def require_all_events(self): return # Get all events - res = self._cb.get_object("/api/{0}/process/{1}/{2}/event".format(self._process_event_api, self.id, 0)).get("process", {}) + res = self._cb.get_object("/api/{0}/process/{1}/{2}/event".format(self._process_event_api, + self.id, 0)).get("process", {}) self._events['all'] = {} - for (event_count,event_complete) in event_types.items(): + for (event_count, event_complete) in event_types.items(): complete_events = res.get(event_complete, 0) if complete_events: self._info[event_count] = len(complete_events) @@ -2892,7 +2894,7 @@ def require_all_events(self): self._events['all'][event_count] = self._info[event_count] self._events['all'][event_complete] = self._info[event_complete] - #total_events = sum([self._info[event_count] for event_count in event_types]) + # total_events = sum([self._info[event_count] for event_count in event_types]) # Delete references in res. if not self._full_init: @@ -2907,7 +2909,6 @@ def require_all_events(self): self.all_events_loaded = True - def all_childprocs(self): if self._cb.cb_server_version < LooseVersion('6.0.0'): self.get_segments() @@ -3040,11 +3041,12 @@ def all_netconns(self): yield self._event_parser.parse_netconn(i, raw_netconn) i += 1 + def get_constants(prefix): return dict((getattr(socket, n), n) for n in dir(socket) if n.startswith(prefix) - ) + ) protocols = get_constants("IPPROTO_") @@ -3102,7 +3104,7 @@ def tamper_event(self): class CbModLoadEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data, binary_data=None): - super(CbModLoadEvent,self).__init__(parent_process, timestamp, sequence, event_data) + super(CbModLoadEvent, self).__init__(parent_process, timestamp, sequence, event_data) self.event_type = u'Cb Module Load event' self.stat_titles.extend(['md5', 'path']) @@ -3119,21 +3121,21 @@ def is_signed(self): class CbFileModEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data): - super(CbFileModEvent,self).__init__(parent_process, timestamp, sequence, event_data) + super(CbFileModEvent, self).__init__(parent_process, timestamp, sequence, event_data) self.event_type = u'Cb File Modification event' self.stat_titles.extend(['type', 'path', 'filetype', 'md5']) class CbRegModEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data): - super(CbRegModEvent,self).__init__(parent_process, timestamp, sequence, event_data) + super(CbRegModEvent, self).__init__(parent_process, timestamp, sequence, event_data) self.event_type = u'Cb Registry Modification event' self.stat_titles.extend(['type', 'path']) class CbNetConnEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data, version=1): - super(CbNetConnEvent,self).__init__(parent_process, timestamp, sequence, event_data) + super(CbNetConnEvent, self).__init__(parent_process, timestamp, sequence, event_data) self.event_type = u'Cb Network Connection event' self.stat_titles.extend(['domain', 'remote_ip', 'remote_port', 'proto', 'direction']) if version == 2: @@ -3142,7 +3144,7 @@ def __init__(self, parent_process, timestamp, sequence, event_data, version=1): class CbChildProcEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data, is_suppressed=False, proc_data=None): - super(CbChildProcEvent,self).__init__(parent_process, timestamp, sequence, event_data) + super(CbChildProcEvent, self).__init__(parent_process, timestamp, sequence, event_data) self.event_type = u'Cb Child Process event' self.stat_titles.extend(['procguid', 'pid', 'path', 'md5']) self.is_suppressed = is_suppressed @@ -3189,7 +3191,7 @@ def process(self): class CbCrossProcEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data): - super(CbCrossProcEvent,self).__init__(parent_process, timestamp, sequence, event_data) + super(CbCrossProcEvent, self).__init__(parent_process, timestamp, sequence, event_data) self.event_type = u'Cb Cross Process event' self.stat_titles.extend(['type', 'privileges', 'target_md5', 'target_path']) diff --git a/src/cbapi/response/query.py b/src/cbapi/response/query.py index 97fab3a7..8d71221d 100644 --- a/src/cbapi/response/query.py +++ b/src/cbapi/response/query.py @@ -3,7 +3,6 @@ from cbapi.six.moves import urllib from distutils.version import LooseVersion from ..errors import ApiError -from ..utils import convert_query_params import copy import logging @@ -75,7 +74,8 @@ def _clone(self): def sort(self, new_sort): """Set the sort order for this query. - :param str new_sort: New sort order - see the `Query Reference `_. + :param str new_sort: New sort order - see the + `Query Reference `_. :return: Query object :rtype: :py:class:`Query` """ @@ -96,7 +96,8 @@ def webui_link(self): def and_(self, new_query): """Add a filter to this query. Equivalent to calling :py:meth:`where` on this object. - :param str new_query: Query string - see the `Query Reference `_. + :param str new_query: Query string - see the + `Query Reference `_. :return: Query object :rtype: :py:class:`Query` """ @@ -105,7 +106,8 @@ def and_(self, new_query): def where(self, new_query): """Add a filter to this query. - :param str new_query: Query string - see the `Query Reference `_. + :param str new_query: Query string - see the + `Query Reference `_. :return: Query object :rtype: :py:class:`Query` """ @@ -162,7 +164,8 @@ def _count(self): qargs = convert_query_params(args) - self._total_results = self._cb.get_object(self._doc_class.urlobject, query_parameters=qargs).get('total_results', 0) + self._total_results = self._cb.get_object(self._doc_class.urlobject, + query_parameters=qargs).get('total_results', 0) self._count_valid = True return self._total_results diff --git a/src/cbapi/response/rest_api.py b/src/cbapi/response/rest_api.py index 6022bc7e..408bdc78 100644 --- a/src/cbapi/response/rest_api.py +++ b/src/cbapi/response/rest_api.py @@ -21,9 +21,10 @@ class CbResponseAPI(BaseAPI): Note that calling this will automatically connect to the Carbon Black server in order to verify connectivity and get the server version. - :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. Uses the - profile named 'default' when not specified. - :param str url: (optional, discouraged) Instead of using a credential profile, pass URL and API token to the constructor. + :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black + server. Uses the profile named 'default' when not specified. + :param str url: (optional, discouraged) Instead of using a credential profile, pass URL and API token to + the constructor. :param str token: (optional, discouraged) API token :param bool ssl_verify: (optional, discouraged) Enable or disable SSL certificate verification @@ -119,9 +120,9 @@ def from_ui(self, uri): :raises ApiError: if the URL does not correspond to a recognized model object """ o = urllib.parse.urlparse(uri) - if self._parsed_url.scheme != o.scheme or \ - self._parsed_url.hostname != o.hostname or \ - self._parsed_url.port != o.port: + if self._parsed_url.scheme != o.scheme \ + or self._parsed_url.hostname != o.hostname \ + or self._parsed_url.port != o.port: raise ApiError("Invalid URL provided") frag = o.fragment.lstrip('/') @@ -181,5 +182,3 @@ class CbEnterpriseResponseAPI(CbResponseAPI): Backwards compatibility for previous scripts """ pass - - diff --git a/src/cbapi/response/utils.py b/src/cbapi/response/utils.py index f6562724..6288befc 100644 --- a/src/cbapi/response/utils.py +++ b/src/cbapi/response/utils.py @@ -30,7 +30,7 @@ def parse_42_guid(guid): def parse_process_guid(guid): sensor_id, proc_pid, proc_createtime = parse_42_guid(guid) - return sensor_id, proc_pid, datetime(1601,1,1) + timedelta(microseconds=proc_createtime / 10) + return sensor_id, proc_pid, datetime(1601, 1, 1) + timedelta(microseconds=proc_createtime / 10) def convert_to_solr(dt): @@ -55,7 +55,7 @@ def convert_from_cb(s): if s is None: return dateutil.parser.parse("1970-01-01T00:00:00Z") else: - return dateutil.parser.parse(s) + return dateutil.parser.parse(s) def convert_event_time(s): @@ -64,6 +64,7 @@ def convert_event_time(s): # is included, and sometimes it isn't... so we normalize it by stripping off the TZ data (unfortunately) return convert_from_cb(s).replace(tzinfo=None) + def convert_to_cb(dt): return dt.strftime(cb_datetime_format) @@ -73,7 +74,7 @@ def get_constants(prefix): return dict((getattr(socket, n), n) for n in dir(socket) if n.startswith(prefix) - ) + ) protocols = get_constants("IPPROTO_") From 5fed59d1de02b59a23aa4f46020941b72e60daaf Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Wed, 6 Nov 2019 13:46:30 -0700 Subject: [PATCH 029/197] first bit of version-setting script as well as marker to add new section to change log file --- docs/changelog.rst | 2 ++ setversion.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100755 setversion.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 1a57eace..743a96ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,8 @@ Updates * CB ThreatHunter * Fix List object that was not callable. +.. top-of-changelog (DO NOT REMOVE THIS COMMENT) + CbAPI 1.5.4 - Released October 24, 2019 ---------------------------------------- diff --git a/setversion.py b/setversion.py new file mode 100755 index 00000000..98b9c3e3 --- /dev/null +++ b/setversion.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +import sys +import re + +def readme_rewriter(line, versionnum, ctxt): + expr = ctxt.get("readme_expr", None) + if not expr: + expr = re.compile(r"^\*\*Latest Version:") + ctxt["readme_expr"] = expr + if expr.match(line): + return "**Latest Version: {0}**\n".format(versionnum) + return None + + +def changelog_rewriter(line, versionnum, ctxt): + expr = ctxt.get("changelog_expr", None) + if not expr: + expr = re.compile(r"^\.\. top-of-changelog") + ctxt["changelog_expr"] = expr + if expr.match(line): + pass + return None + + +def rewrite_file(infilename, rewritefunc, versionnum, ctxt): + outfilename = infilename + ".new" + infile = open(infilename, "r") + outfile = open(outfilename, "w") + try: + s = infile.readline() + while s: + s2 = rewritefunc(s, versionnum, ctxt) + if s2: + outfile.write(s2) + else: + outfile.write(s) + s = infile.readline() + finally: + infile.close() + outfile.close() + +def main(): + if len(sys.argv) < 2: + print("Error: new version number not specified") + return 1 + version = sys.argv[1] + ctxt = {} + rewrite_file("README.md", readme_rewriter, version, ctxt) + return 0 + +if __name__ == "__main__": + sys.exit(main()) From 94795a23f078e0cd63055804491b84e1de34239e Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Wed, 6 Nov 2019 15:21:27 -0700 Subject: [PATCH 030/197] the basics of the script are all now in place, awaiting some refinements --- setversion.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/setversion.py b/setversion.py index 98b9c3e3..dde7bf2f 100755 --- a/setversion.py +++ b/setversion.py @@ -2,6 +2,7 @@ import sys import re +from datetime import date def readme_rewriter(line, versionnum, ctxt): expr = ctxt.get("readme_expr", None) @@ -19,7 +20,52 @@ def changelog_rewriter(line, versionnum, ctxt): expr = re.compile(r"^\.\. top-of-changelog") ctxt["changelog_expr"] = expr if expr.match(line): - pass + datestr = date.today().strftime("%B %d, %Y") + cl1 = "CbAPI {0} - Released {1}".format(versionnum, datestr) + cl2 = "".ljust(len(cl1), "-") + updates = "\n\nUpdates\n\n.. (add your updates here)\n" + return line + "\n" + cl1 + "\n" + cl2 + updates + return None + + +def doc_conf_rewriter(line, versionnum, ctxt): + vexpr = ctxt.get("doc_version_expr", None) + if not vexpr: + vexpr = re.compile(r"^version = ") + ctxt["doc_version_expr"] = vexpr + rexpr = ctxt.get("doc_release_expr", None) + if not rexpr: + rexpr = re.compile(r"^release = ") + ctxt["doc_release_expr"] = rexpr + if vexpr.match(line): + t = re.match(r"^(\d+\.\d+)\.", versionnum) + vn = versionnum + if t: + vn = t.group(1) + return "version = u'{0}'\n".format(vn) + if rexpr.match(line): + return "release = u'{0}'\n".format(versionnum) + return None + + +def setup_rewriter(line, versionnum, ctxt): + expr = ctxt.get("setup_expr", None) + if not expr: + expr = re.compile(r"^(\s*)version=") + ctxt["setup_expr"] = expr + m = expr.match(line) + if m: + return "{0}version='{1}'\n".format(m.group(1), versionnum) + return None + + +def init_rewriter(line, versionnum, ctxt): + expr = ctxt.get("init_expr", None) + if not expr: + expr = re.compile(r"^__version__ = ") + ctxt["init_expr"] = expr + if expr.match(line): + return "__version__ = '{0}'\n".format(versionnum) return None @@ -47,6 +93,10 @@ def main(): version = sys.argv[1] ctxt = {} rewrite_file("README.md", readme_rewriter, version, ctxt) + rewrite_file("docs/changelog.rst", changelog_rewriter, version, ctxt) + rewrite_file("docs/conf.py", doc_conf_rewriter, version, ctxt) + rewrite_file("setup.py", setup_rewriter, version, ctxt) + rewrite_file("src/cbapi/__init__.py", init_rewriter, version, ctxt) return 0 if __name__ == "__main__": From 9e936397a7a01012e91eb611bc2a3a8415f1fe9b Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Thu, 7 Nov 2019 12:32:13 -0700 Subject: [PATCH 031/197] smoothed out the design, added the actual file overwriting, added argparse --- setversion.py | 55 +++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/setversion.py b/setversion.py index dde7bf2f..ffcfb78b 100755 --- a/setversion.py +++ b/setversion.py @@ -1,34 +1,36 @@ #!/usr/bin/env python import sys +import os import re +import argparse from datetime import date -def readme_rewriter(line, versionnum, ctxt): +def readme_rewriter(line, ctxt): expr = ctxt.get("readme_expr", None) if not expr: expr = re.compile(r"^\*\*Latest Version:") ctxt["readme_expr"] = expr if expr.match(line): - return "**Latest Version: {0}**\n".format(versionnum) + return "**Latest Version: {0}**\n".format(ctxt["version"]) return None -def changelog_rewriter(line, versionnum, ctxt): +def changelog_rewriter(line, ctxt): expr = ctxt.get("changelog_expr", None) if not expr: expr = re.compile(r"^\.\. top-of-changelog") ctxt["changelog_expr"] = expr if expr.match(line): datestr = date.today().strftime("%B %d, %Y") - cl1 = "CbAPI {0} - Released {1}".format(versionnum, datestr) + cl1 = "CbAPI {0} - Released {1}".format(ctxt["version"], datestr) cl2 = "".ljust(len(cl1), "-") updates = "\n\nUpdates\n\n.. (add your updates here)\n" return line + "\n" + cl1 + "\n" + cl2 + updates return None -def doc_conf_rewriter(line, versionnum, ctxt): +def doc_conf_rewriter(line, ctxt): vexpr = ctxt.get("doc_version_expr", None) if not vexpr: vexpr = re.compile(r"^version = ") @@ -38,45 +40,45 @@ def doc_conf_rewriter(line, versionnum, ctxt): rexpr = re.compile(r"^release = ") ctxt["doc_release_expr"] = rexpr if vexpr.match(line): - t = re.match(r"^(\d+\.\d+)\.", versionnum) - vn = versionnum + t = re.match(r"^(\d+\.\d+)\.", ctxt["version"]) + vn = ctxt["version"] if t: vn = t.group(1) return "version = u'{0}'\n".format(vn) if rexpr.match(line): - return "release = u'{0}'\n".format(versionnum) + return "release = u'{0}'\n".format(ctxt["version"]) return None -def setup_rewriter(line, versionnum, ctxt): +def setup_rewriter(line, ctxt): expr = ctxt.get("setup_expr", None) if not expr: expr = re.compile(r"^(\s*)version=") ctxt["setup_expr"] = expr m = expr.match(line) if m: - return "{0}version='{1}'\n".format(m.group(1), versionnum) + return "{0}version='{1}'\n".format(m.group(1), ctxt["version"]) return None -def init_rewriter(line, versionnum, ctxt): +def init_rewriter(line, ctxt): expr = ctxt.get("init_expr", None) if not expr: expr = re.compile(r"^__version__ = ") ctxt["init_expr"] = expr if expr.match(line): - return "__version__ = '{0}'\n".format(versionnum) + return "__version__ = '{0}'\n".format(ctxt["version"]) return None -def rewrite_file(infilename, rewritefunc, versionnum, ctxt): +def rewrite_file(infilename, rewritefunc, ctxt): outfilename = infilename + ".new" infile = open(infilename, "r") outfile = open(outfilename, "w") try: s = infile.readline() while s: - s2 = rewritefunc(s, versionnum, ctxt) + s2 = rewritefunc(s, ctxt) if s2: outfile.write(s2) else: @@ -85,18 +87,23 @@ def rewrite_file(infilename, rewritefunc, versionnum, ctxt): finally: infile.close() outfile.close() + if not ctxt.get("nodelete", False): + os.remove(infilename) + os.rename(outfilename, infilename) + def main(): - if len(sys.argv) < 2: - print("Error: new version number not specified") - return 1 - version = sys.argv[1] - ctxt = {} - rewrite_file("README.md", readme_rewriter, version, ctxt) - rewrite_file("docs/changelog.rst", changelog_rewriter, version, ctxt) - rewrite_file("docs/conf.py", doc_conf_rewriter, version, ctxt) - rewrite_file("setup.py", setup_rewriter, version, ctxt) - rewrite_file("src/cbapi/__init__.py", init_rewriter, version, ctxt) + parser = argparse.ArgumentParser(description="Set the version number in CbAPI source and documentation") + parser.add_argument("version", help="New version number to add") + parser.add_argument("-n", "--nodelete", action="store_true", help="Do not delete existing files, leave new files with .new extension") + + args = parser.parse_args() + ctxt = {"version": args.version, "nodelete": args.nodelete} + rewrite_file("README.md", readme_rewriter, ctxt) + rewrite_file("docs/changelog.rst", changelog_rewriter, ctxt) + rewrite_file("docs/conf.py", doc_conf_rewriter, ctxt) + rewrite_file("setup.py", setup_rewriter, ctxt) + rewrite_file("src/cbapi/__init__.py", init_rewriter, ctxt) return 0 if __name__ == "__main__": From d9037237efec32bf85bb3d54b4416bfe9a19e99d Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Thu, 7 Nov 2019 12:41:42 -0700 Subject: [PATCH 032/197] filled in some more documentation and minor behavior changes --- setversion.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/setversion.py b/setversion.py index ffcfb78b..6201103d 100755 --- a/setversion.py +++ b/setversion.py @@ -1,5 +1,8 @@ #!/usr/bin/env python +# CbAPI Project Version Number Setting Script +# AGRB 11/7/2019 + import sys import os import re @@ -87,13 +90,15 @@ def rewrite_file(infilename, rewritefunc, ctxt): finally: infile.close() outfile.close() - if not ctxt.get("nodelete", False): + if not ctxt["nodelete"]: os.remove(infilename) os.rename(outfilename, infilename) def main(): - parser = argparse.ArgumentParser(description="Set the version number in CbAPI source and documentation") + parser = argparse.ArgumentParser(description="Set the version number in CbAPI source and documentation.\n" + "Execute this on a release or hotfix branch to update the version numbers in the source.", + epilog="After running, edit docs/changelog.rst and add the new changelog information under the new heading.") parser.add_argument("version", help="New version number to add") parser.add_argument("-n", "--nodelete", action="store_true", help="Do not delete existing files, leave new files with .new extension") From 1d0c8ddf777f7a5f81f6cf62e41347fa800f9286 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Thu, 17 Oct 2019 14:11:57 -0600 Subject: [PATCH 033/197] initial cut of the models for Alerts V6 --- src/cbapi/psc/models.py | 65 ++++- src/cbapi/psc/models/base_alert.yaml | 139 +++++++++++ ...iss_alert_by_watchlist_search_request.yaml | 228 ++++++++++++++++++ .../psc/models/dismiss_status_response.yaml | 56 +++++ src/cbapi/psc/models/facet_dto.yaml | 15 ++ src/cbapi/psc/models/workflow.yaml | 23 ++ 6 files changed, 525 insertions(+), 1 deletion(-) create mode 100755 src/cbapi/psc/models/base_alert.yaml create mode 100755 src/cbapi/psc/models/dismiss_alert_by_watchlist_search_request.yaml create mode 100755 src/cbapi/psc/models/dismiss_status_response.yaml create mode 100755 src/cbapi/psc/models/facet_dto.yaml create mode 100755 src/cbapi/psc/models/workflow.yaml diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 9bb941ab..62a7c5b9 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -1,4 +1,4 @@ -from cbapi.models import MutableBaseModel +from cbapi.models import MutableBaseModel, UnrefreshableModel from cbapi.errors import ServerError from cbapi.psc.query import DeviceSearchQuery @@ -188,3 +188,66 @@ def update_sensor_version(self, sensor_version): :param dict sensor_version: New version properties for the sensor. """ return self._cb.device_update_sensor_version([self._model_unique_id], sensor_version) + + +class Workflow(UnrefreshableModel): + swagger_meta_file = "psc/models/workflow.yaml" + + def __init__(self, cb, initial_data=None): + super(Workflow, self).__init__(cb, model_unique_id=None, initial_data=initial_data) + + +class BaseAlert(PSCMutableModel): + urlobject = "/appservices/v6/orgs/{0}/alerts" + urlobject_single = "/appservices/v6/orgs/{0}/alerts/{1}" + primary_key = "id" + swagger_meta_file = "psc/models/base_alert.yaml" + + def __init__(self, cb, model_unique_id, initial_data=None): + super(BaseAlert, self).__init__(cb, model_unique_id, initial_data) + self._workflow = Workflow(cb, initial_data.get("workflow", None)) + + @classmethod + def _query_implementation(cls, cb): + pass + + def _refresh(self): + url = self.urlobject_single.format(self._cb.credentials.org_key, self._model_unique_id) + resp = self._cb.get_object(url) + self._info = resp + self._workflow = Workflow(self._cb, resp.get("workflow", None)) + self._last_refresh_time = time.time() + return True + + @property + def workflow(self): + return self._workflow + + +class DismissStatusResponse(UnrefreshableModel): + primary_key = "id" + swagger_meta_file = "psc/models/dismiss_status_response.yaml" + + def __init__(self, cb, model_unique_id, initial_data=None): + super(DismissStatusResponse, self).__init__(cb, model_unique_id, initial_data) + self._workflow = Workflow(cb, initial_data.get("workflow", None)) + + @property + def workflow(self): + return self._workflow + + +class FacetDTO(UnrefreshableModel): + primary_key = "id" + swagger_meta_file = "psc/models/facet_dto.yaml" + + def __init__(self, cb, model_unique_id, initial_data=None): + super(FacetDTO, self).__init__(cb, model_unique_id, initial_data) + + +class DismissAlertByWatchlistSearchRequest(UnrefreshableModel): + swagger_meta_file = "psc/models/dismiss_alert_by_watchlist_search_request.yaml" + + def __init__(self, cb, initial_data=None): + super(DismissAlertByWatchlistSearchRequest, self).__init__(cb, model_unique_id=None, initial_data=initial_data) + \ No newline at end of file diff --git a/src/cbapi/psc/models/base_alert.yaml b/src/cbapi/psc/models/base_alert.yaml new file mode 100755 index 00000000..ffc0b4e0 --- /dev/null +++ b/src/cbapi/psc/models/base_alert.yaml @@ -0,0 +1,139 @@ +type: object +properties: + category: + type: string + description: Alert category - Monitored vs Threat + enum: + - THREAT + - MONITORED + - INFO + - MINOR + - SERIOUS + - CRITICAL + create_time: + type: string + format: date-time + description: Time the alert was created + device_id: + type: integer + format: int64 + description: ID of the device + device_name: + type: string + description: Device name + device_os: + type: string + description: Device OS + enum: + - WINDOWS + - ANDROID + - MAC + - IOS + - LINUX + - OTHER + device_os_version: + type: string + example: Windows 10 x64 + description: Device OS Version + device_username: + type: string + description: Logged on user during the alert. This is filled on a best-effort + approach. If the user is not available it may be populated with the device + owner + first_event_time: + type: string + format: date-time + description: Time of the first event in an alert + group_details: + description: Group details for when alert grouping is on + type: object + properties: + count: + type: integer + format: int64 + description: Number of times the event has occurred + total_devices: + type: integer + format: int64 + description: The number of devices that have seen this alert + id: + type: string + description: Unique ID for this alert + last_event_time: + type: string + format: date-time + description: Time of the last event in an alert + last_update_time: + type: string + format: date-time + description: Time the alert was last updated + legacy_alert_id: + type: string + description: Unique short ID for this alert. This is deprecated and only available + on alerts stored in the old schema. + notes_present: + type: boolean + description: Are notes present for this threatId + org_key: + type: string + example: ABCD1234 + description: Unique identifier for the organization to which the alert belongs + policy_id: + type: integer + format: int64 + description: ID of the policy the device was in at the time of the alert + policy_name: + type: string + description: Name of the policy the device was in at the time of the alert + severity: + type: integer + format: int32 + description: Threat ranking + tags: + type: array + description: Tags for the alert + items: + type: string + target_value: + type: string + description: Device priority as assigned via the policy + enum: + - LOW + - MEDIUM + - HIGH + - MISSION_CRITICAL + threat_id: + type: string + description: ID of the threat to which this alert belongs. Threats are comprised + of a combination of factors that can be repeated across devices. + type: + type: string + description: Type of the alert + enum: + - CB_ANALYTICS + - VMWARE + - WATCHLIST + workflow: + description: User-updatable status of the alert + type: object + properties: + changed_by: + type: string + description: Username of the user who changed the workflow + comment: + type: string + description: Comment when updating the workflow + last_update_time: + type: string + format: date-time + description: When the workflow was last updated + remediation: + type: string + description: Alert remediation code. Indicates the result of the investigation + into the alert + state: + type: string + description: State of the workflow + enum: + - OPEN + - DISMISSED diff --git a/src/cbapi/psc/models/dismiss_alert_by_watchlist_search_request.yaml b/src/cbapi/psc/models/dismiss_alert_by_watchlist_search_request.yaml new file mode 100755 index 00000000..cb618891 --- /dev/null +++ b/src/cbapi/psc/models/dismiss_alert_by_watchlist_search_request.yaml @@ -0,0 +1,228 @@ +type: object +description: Combined alert dismiss by Watchlist search request +required: +- state +properties: + comment: + type: string + description: Comment for dismissal + criteria: + description: Watchlist Alert search criteria + type: object + properties: + category: + type: array + description: Alert categories + items: + type: string + enum: + - THREAT + - MONITORED + - INFO + - MINOR + - SERIOUS + - CRITICAL + create_time: + description: Star time and end time + type: object + properties: + all_time: + type: boolean + end: + type: string + format: date-time + description: End of the time range for a time filter (newer timestamp) + range: + type: string + description: "Relative time window for the time filter. Specified as `all` + to retrieve results from all time, or `-[quantity][units]`, where quantity + is any integer and units is one of the allowed time units:\n* `y` years + \n* `w` weeks\n* `d` days\n* `h` hours\n* `m` minutes\n* `s` seconds" + start: + type: string + format: date-time + description: Beginning of the time range for a time filter (older timestamp) + device_id: + type: array + example: + - 324552 + - 12344 + - 997745 + description: IDs of devices + items: + type: integer + format: int64 + device_name: + type: array + example: + - hostmachine + - device.local + - DOMAIN\DEVICE + description: Device names + items: + type: string + device_os: + type: array + description: Device Operating Systems + items: + type: string + enum: + - WINDOWS + - ANDROID + - MAC + - IOS + - LINUX + - OTHER + device_os_version: + type: array + description: Device Operating System Versions + items: + type: string + device_username: + type: array + description: Users or device owners of alerts + items: + type: string + group_results: + type: boolean + description: Used to turn alert grouping on + id: + type: array + description: Unique IDs of alerts + items: + type: string + legacy_alert_id: + type: array + example: + - CTAS5XKG + - TJFY5ZBW + description: Unique short IDs of alerts. This field is deprecated and only + available on alerts stored in the old schema. + items: + type: string + minimum_severity: + type: integer + format: int32 + example: 5 + description: Minimum threat ranking of returned alerts + policy_id: + type: array + example: + - 1 + - 525 + - 644 + description: IDs of policies the device was in at the time of the alert + items: + type: integer + format: int64 + policy_name: + type: array + example: + - Default + - Advanced + - Monitored + description: Names of the policies the device was in at the time of the alert + items: + type: string + process_name: + type: array + example: + - explorer.exe + - chrome.app + - setup.py + description: Process names of an alert + items: + type: string + process_sha256: + type: array + example: + - 131f95c51cc819465fa1797f6ccacf9d494aaaff46fa3eac73ae63ffbdfd8267 + description: SHA256 values of alerts + items: + type: string + report_id: + type: array + description: Report IDs that contained the IOC that caused a hit + items: + type: string + report_name: + type: array + description: Names of reports that contained the IOC that caused a hit + items: + type: string + reputation: + type: array + description: reputation of the actor hash + items: + type: string + enum: + - KNOWN_MALWARE + - SUSPECT_MALWARE + - PUP + - NOT_LISTED + - ADAPTIVE_WHITE_LIST + - COMMON_WHITE_LIST + - TRUSTED_WHITE_LIST + - COMPANY_BLACK_LIST + tag: + type: array + description: Tags for an alert + items: + type: string + target_value: + type: array + description: Device priorities as assigned via the alert policy + items: + type: string + enum: + - LOW + - MEDIUM + - HIGH + - MISSION_CRITICAL + threat_id: + type: array + example: + - 03ea43268c536a0bde8b765bca1696e9 + - 41edc35062138af3f1fea4b3bf7046a5 + description: IDs of threats + items: + type: string + type: + type: array + description: Types of alerts + items: + type: string + enum: + - CB_ANALYTICS + - VMWARE + - WATCHLIST + watchlist_id: + type: array + description: Watchlist ID + items: + type: string + watchlist_name: + type: array + description: Watchlist name + items: + type: string + workflow: + type: array + description: User-updatable statuses of an alert + items: + type: string + enum: + - OPEN + - DISMISSED + query: + type: string + description: Full-text search query string + remediation_state: + type: string + description: Remediation state + state: + type: string + description: Dismiss or undismiss + enum: + - OPEN + - DISMISSED diff --git a/src/cbapi/psc/models/dismiss_status_response.yaml b/src/cbapi/psc/models/dismiss_status_response.yaml new file mode 100755 index 00000000..202e8cb5 --- /dev/null +++ b/src/cbapi/psc/models/dismiss_status_response.yaml @@ -0,0 +1,56 @@ +type: object +description: Dismiss status response for async calls +properties: + errors: + type: array + description: Errors for dismiss alerts or threats, if no errors it won't be + included in response + items: + type: string + failed_ids: + type: array + description: Failed ids + items: + type: string + id: + type: string + description: Time based id for async job, it's not unique across the orgs + num_hits: + type: integer + format: int64 + description: Total number of alerts to be operated on + num_success: + type: integer + format: int64 + description: Successfully operated number of alerts + status: + type: string + description: Status for the async progress + enum: + - QUEUED + - IN_PROGRESS + - FINISHED + workflow: + description: Requested workflow change + type: object + properties: + changed_by: + type: string + description: Username of the user who changed the workflow + comment: + type: string + description: Comment when updating the workflow + last_update_time: + type: string + format: date-time + description: When the workflow was last updated + remediation: + type: string + description: Alert remediation code. Indicates the result of the investigation + into the alert + state: + type: string + description: State of the workflow + enum: + - OPEN + - DISMISSED diff --git a/src/cbapi/psc/models/facet_dto.yaml b/src/cbapi/psc/models/facet_dto.yaml new file mode 100755 index 00000000..3e0993e1 --- /dev/null +++ b/src/cbapi/psc/models/facet_dto.yaml @@ -0,0 +1,15 @@ +type: object +properties: + id: + type: string + description: 'Key value of the item in filter, for example : device_id, sha256_hash, + policy_id... ' + name: + type: string + description: 'Description of the item in filter, for example : device_name, + application_name, policy_name... This is an optional field, when the value + is null, it won''t be serialized' + total: + type: integer + format: int64 + description: Total number of results within this aggregation diff --git a/src/cbapi/psc/models/workflow.yaml b/src/cbapi/psc/models/workflow.yaml new file mode 100755 index 00000000..8807a69f --- /dev/null +++ b/src/cbapi/psc/models/workflow.yaml @@ -0,0 +1,23 @@ +type: object +description: Tracking system for alerts as they are triaged and resolved +properties: + changed_by: + type: string + description: Username of the user who changed the workflow + comment: + type: string + description: Comment when updating the workflow + last_update_time: + type: string + format: date-time + description: When the workflow was last updated + remediation: + type: string + description: Alert remediation code. Indicates the result of the investigation + into the alert + state: + type: string + description: State of the workflow + enum: + - OPEN + - DISMISSED From b3243d451ccd5c4b30a92961d55b81a4125ab269 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 22 Oct 2019 15:58:26 -0400 Subject: [PATCH 034/197] implemented more (all but one) of the APIs designated for the Alerts v6 ticket --- src/cbapi/psc/models.py | 87 +++- ...iss_alert_by_watchlist_search_request.yaml | 228 --------- src/cbapi/psc/query.py | 474 ++++++++++++++++++ src/cbapi/psc/rest_api.py | 23 +- 4 files changed, 570 insertions(+), 242 deletions(-) delete mode 100755 src/cbapi/psc/models/dismiss_alert_by_watchlist_search_request.yaml diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 62a7c5b9..7632794f 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -1,6 +1,6 @@ from cbapi.models import MutableBaseModel, UnrefreshableModel from cbapi.errors import ServerError -from cbapi.psc.query import DeviceSearchQuery +from cbapi.psc.query import DeviceSearchQuery, BaseAlertSearchQuery, WatchlistAlertSearchQuery from copy import deepcopy import logging @@ -114,6 +114,8 @@ class Device(PSCMutableModel): def __init__(self, cb, model_unique_id, initial_data=None): super(Device, self).__init__(cb, model_unique_id, initial_data) + if model_unique_id is not None and initial_data is None: + self._refresh() @classmethod def _query_implementation(cls, cb): @@ -125,7 +127,7 @@ def _refresh(self): self._info = resp self._last_refresh_time = time.time() return True - + def lr_session(self): """ Retrieve a Live Response session object for this Device. @@ -206,10 +208,12 @@ class BaseAlert(PSCMutableModel): def __init__(self, cb, model_unique_id, initial_data=None): super(BaseAlert, self).__init__(cb, model_unique_id, initial_data) self._workflow = Workflow(cb, initial_data.get("workflow", None)) + if model_unique_id is not None and initial_data is None: + self._refresh() @classmethod def _query_implementation(cls, cb): - pass + return BaseAlertSearchQuery(cls, cb) def _refresh(self): url = self.urlobject_single.format(self._cb.credentials.org_key, self._model_unique_id) @@ -223,19 +227,88 @@ def _refresh(self): def workflow(self): return self._workflow + def _update_workflow_status(self, state, remediation, comment): + request = {"state" : state} + if remediation: + request["remediation_state"] = remediation + if comment: + request["comment"] = comment + url = self.urlobject_single.format(self._cb.credentials.org_key, + self._model_unique_id) + "/workflow" + resp = self._cb.post_object(url, request) + self._workflow = Workflow(self._cb, resp) + self._last_refresh_time = time.time() + + def dismiss(self, remediation=None, comment=None): + self._update_workflow_status("DISMISSED", remediation, comment) + + def undismiss(self, remediation=None, comment=None): + self._update_workflow_status("OPEN", remediation, comment) + + def _update_threat_workflow_status(self, state, remediation, comment): + request = {"state" : state} + if remediation: + request["remediation_state"] = remediation + if comment: + request["comment"] = comment + url = "/appservices/v6/orgs/{0}/threat/{1}/workflow".format(self._cb.credentials.org_key, + self.threat_id) + resp = self._cb.post_object(url, request) + return Workflow(self._cb, resp) + + def dismiss_threat(self, remediation=None, comment=None): + return self._update_threat_workflow_status("DISMISSED", remediation, comment) + + def undismiss_threat(self, remediation=None, comment=None): + return self._update_threat_workflow_status("OPEN", remediation, comment) + + +class WatchlistAlert(BaseAlert): + urlobject = "/appservices/v6/orgs/{0}/alerts/watchlist" + + @classmethod + def _query_implementation(cls, cb): + return WatchlistAlertSearchQuery(cls, cb) + class DismissStatusResponse(UnrefreshableModel): + urlobject_single = "/appservices/v6/orgs/{0}/workflow/status/{1}" primary_key = "id" swagger_meta_file = "psc/models/dismiss_status_response.yaml" def __init__(self, cb, model_unique_id, initial_data=None): super(DismissStatusResponse, self).__init__(cb, model_unique_id, initial_data) self._workflow = Workflow(cb, initial_data.get("workflow", None)) + if model_unique_id is not None and initial_data is None: + self._refresh() + + def _refresh(self): + url = self.urlobject_single.format(self._cb.credentials.org_key, self._model_unique_id) + resp = self._cb.get_object(url) + self._info = resp + self._workflow = Workflow(self._cb, resp.get("workflow", None)) + self._last_refresh_time = time.time() + return True @property def workflow(self): return self._workflow + @property + def queued(self): + self._refresh() + return self._info.get("status", "") == "QUEUED" + + @property + def in_progress(self): + self._refresh() + return self._info.get("status", "") == "IN_PROGRESS" + + @property + def finished(self): + self._refresh() + return self._info.get("status", "") == "FINISHED" + class FacetDTO(UnrefreshableModel): primary_key = "id" @@ -243,11 +316,3 @@ class FacetDTO(UnrefreshableModel): def __init__(self, cb, model_unique_id, initial_data=None): super(FacetDTO, self).__init__(cb, model_unique_id, initial_data) - - -class DismissAlertByWatchlistSearchRequest(UnrefreshableModel): - swagger_meta_file = "psc/models/dismiss_alert_by_watchlist_search_request.yaml" - - def __init__(self, cb, initial_data=None): - super(DismissAlertByWatchlistSearchRequest, self).__init__(cb, model_unique_id=None, initial_data=initial_data) - \ No newline at end of file diff --git a/src/cbapi/psc/models/dismiss_alert_by_watchlist_search_request.yaml b/src/cbapi/psc/models/dismiss_alert_by_watchlist_search_request.yaml deleted file mode 100755 index cb618891..00000000 --- a/src/cbapi/psc/models/dismiss_alert_by_watchlist_search_request.yaml +++ /dev/null @@ -1,228 +0,0 @@ -type: object -description: Combined alert dismiss by Watchlist search request -required: -- state -properties: - comment: - type: string - description: Comment for dismissal - criteria: - description: Watchlist Alert search criteria - type: object - properties: - category: - type: array - description: Alert categories - items: - type: string - enum: - - THREAT - - MONITORED - - INFO - - MINOR - - SERIOUS - - CRITICAL - create_time: - description: Star time and end time - type: object - properties: - all_time: - type: boolean - end: - type: string - format: date-time - description: End of the time range for a time filter (newer timestamp) - range: - type: string - description: "Relative time window for the time filter. Specified as `all` - to retrieve results from all time, or `-[quantity][units]`, where quantity - is any integer and units is one of the allowed time units:\n* `y` years - \n* `w` weeks\n* `d` days\n* `h` hours\n* `m` minutes\n* `s` seconds" - start: - type: string - format: date-time - description: Beginning of the time range for a time filter (older timestamp) - device_id: - type: array - example: - - 324552 - - 12344 - - 997745 - description: IDs of devices - items: - type: integer - format: int64 - device_name: - type: array - example: - - hostmachine - - device.local - - DOMAIN\DEVICE - description: Device names - items: - type: string - device_os: - type: array - description: Device Operating Systems - items: - type: string - enum: - - WINDOWS - - ANDROID - - MAC - - IOS - - LINUX - - OTHER - device_os_version: - type: array - description: Device Operating System Versions - items: - type: string - device_username: - type: array - description: Users or device owners of alerts - items: - type: string - group_results: - type: boolean - description: Used to turn alert grouping on - id: - type: array - description: Unique IDs of alerts - items: - type: string - legacy_alert_id: - type: array - example: - - CTAS5XKG - - TJFY5ZBW - description: Unique short IDs of alerts. This field is deprecated and only - available on alerts stored in the old schema. - items: - type: string - minimum_severity: - type: integer - format: int32 - example: 5 - description: Minimum threat ranking of returned alerts - policy_id: - type: array - example: - - 1 - - 525 - - 644 - description: IDs of policies the device was in at the time of the alert - items: - type: integer - format: int64 - policy_name: - type: array - example: - - Default - - Advanced - - Monitored - description: Names of the policies the device was in at the time of the alert - items: - type: string - process_name: - type: array - example: - - explorer.exe - - chrome.app - - setup.py - description: Process names of an alert - items: - type: string - process_sha256: - type: array - example: - - 131f95c51cc819465fa1797f6ccacf9d494aaaff46fa3eac73ae63ffbdfd8267 - description: SHA256 values of alerts - items: - type: string - report_id: - type: array - description: Report IDs that contained the IOC that caused a hit - items: - type: string - report_name: - type: array - description: Names of reports that contained the IOC that caused a hit - items: - type: string - reputation: - type: array - description: reputation of the actor hash - items: - type: string - enum: - - KNOWN_MALWARE - - SUSPECT_MALWARE - - PUP - - NOT_LISTED - - ADAPTIVE_WHITE_LIST - - COMMON_WHITE_LIST - - TRUSTED_WHITE_LIST - - COMPANY_BLACK_LIST - tag: - type: array - description: Tags for an alert - items: - type: string - target_value: - type: array - description: Device priorities as assigned via the alert policy - items: - type: string - enum: - - LOW - - MEDIUM - - HIGH - - MISSION_CRITICAL - threat_id: - type: array - example: - - 03ea43268c536a0bde8b765bca1696e9 - - 41edc35062138af3f1fea4b3bf7046a5 - description: IDs of threats - items: - type: string - type: - type: array - description: Types of alerts - items: - type: string - enum: - - CB_ANALYTICS - - VMWARE - - WATCHLIST - watchlist_id: - type: array - description: Watchlist ID - items: - type: string - watchlist_name: - type: array - description: Watchlist name - items: - type: string - workflow: - type: array - description: User-updatable statuses of an alert - items: - type: string - enum: - - OPEN - - DISMISSED - query: - type: string - description: Full-text search query string - remediation_state: - type: string - description: Remediation state - state: - type: string - description: Dismiss or undismiss - enum: - - OPEN - - DISMISSED diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index d99af06e..42481b35 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -1,4 +1,5 @@ from cbapi.errors import ApiError, MoreThanOneResultError +from cbapi.psc.models import DismissStatusResponse import logging import functools from six import string_types @@ -571,3 +572,476 @@ def update_sensor_version(self, sensor_version): """ return self._bulk_device_action("UPDATE_SENSOR_VERSION", {"sensor_version": sensor_version}) + + +class AlertRequestCriteriaBuilder: + valid_categories = ["THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", "CRITICAL"] + valid_reputations = ["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", "COMPANY_BLACK_LIST"] + valid_alerttypes = ["CB_ANALYTICS", "VMWARE", "WATCHLIST"] + valid_workflow_vals = ["OPEN", "DISMISSED"] + + def __init__(self): + self._criteria = {} + self._time_filter = {} + + def _update_criteria(self, key, newlist): + oldlist = self._criteria.get(key, []) + self._criteria[key] = oldlist + newlist + + def categories(self, cats): + if not all((c in AlertRequestCriteriaBuilder.valid_categories) for c in cats): + raise ApiError("One or more invalid category values") + self._update_criteria("category", cats) + return self + + def create_time(self, *args, **kwargs): + """ + Restricts the devices that this query is performed on to the specified + last contact time (either specified as a start and end point or as a + range). + + :return: This instance + """ + if kwargs.get("start", None) and kwargs.get("end", None): + if kwargs.get("range", None): + raise ApiError("cannot specify range= in addition to start= and end=") + stime = kwargs["start"] + if not isinstance(stime, str): + stime = stime.isoformat() + etime = kwargs["end"] + if not isinstance(etime, str): + etime = etime.isoformat() + self._time_filter = {"start": stime, "end": etime} + elif kwargs.get("range", None): + if kwargs.get("start", None) or kwargs.get("end", None): + raise ApiError("cannot specify start= or end= in addition to range=") + self._time_filter = {"range": kwargs["range"]} + else: + raise ApiError("must specify either start= and end= or range=") + return self + + def device_ids(self, device_ids): + """ + Restricts the devices that this query is performed on to the specified + device IDs. + + :param device_ids: list of ints + :return: This instance + """ + if not all(isinstance(device_id, int) for device_id in device_ids): + raise ApiError("One or more invalid device IDs") + self._update_criteria("device_id", device_ids) + return self + + def device_names(self, device_names): + if not all(isinstance(n, str) for n in device_names): + raise ApiError("One or more invalid device names") + self._update_criteria("device_name", device_names) + return self + + def device_os(self, device_os): + if not all((osval in DeviceSearchQuery.valid_os) for osval in device_os): + raise ApiError("One or more invalid operating systems") + self._update_criteria("device_os", device_os) + return self + + def device_os_version(self, device_os_versions): + if not all(isinstance(n, str) for n in device_os_versions): + raise ApiError("One or more invalid device OS versions") + self._update_criteria("device_os_version", device_os_versions) + return self + + def device_username(self, users): + if not all(isinstance(u, str) for u in users): + raise ApiError("One or more invalid user names") + self._update_criteria("device_username", users) + return self + + def group_results(self, flag): + self._criteria["group_results"] = True if flag else False + return self + + def alert_ids(self, alert_ids): + if not all(isinstance(v, str) for v in alert_ids): + raise ApiError("One or more invalid alert ID values") + self._update_criteria("id", alert_ids) + return self + + def legacy_alert_ids(self, alert_ids): + if not all(isinstance(v, str) for v in alert_ids): + raise ApiError("One or more invalid alert ID values") + self._update_criteria("legacy_alert_id", alert_ids) + return self + + def minimum_severity(self, severity): + self._criteria["minimum_severity"] = severity + return self + + def policy_ids(self, policy_ids): + """ + Restricts the devices that this query is performed on to the specified + policy IDs. + + :param policy_ids: list of ints + :return: This instance + """ + if not all(isinstance(policy_id, int) for policy_id in policy_ids): + raise ApiError("One or more invalid policy IDs") + self._update_criteria("policy_id", policy_ids) + return self + + def policy_names(self, policy_names): + if not all(isinstance(n, str) for n in policy_names): + raise ApiError("One or more invalid policy names") + self._update_criteria("policy_name", policy_names) + return self + + def process_names(self, process_names): + if not all(isinstance(n, str) for n in process_names): + raise ApiError("One or more invalid process names") + self._update_criteria("process_name", process_names) + return self + + def process_sha256(self, shas): + if not all(isinstance(n, str) for n in shas): + raise ApiError("One or more invalid SHA256 values") + self._update_criteria("process_sha256", shas) + return self + + def reputations(self, reps): + if not all((r in AlertRequestCriteriaBuilder.valid_reputations) for r in reps): + raise ApiError("One or more invalid reputation values") + self._update_criteria("reputation", reps) + return self + + def tags(self, tags): + if not all(isinstance(tag, str) for tag in tags): + raise ApiError("One or more invalid tags") + self._update_criteria("tag", tags) + return self + + def target_priorities(self, priorities): + if not all((prio in DeviceSearchQuery.valid_priorities) for prio in priorities): + raise ApiError("One or more invalid priority values") + self._update_criteria("target_value", priorities) + return self + + def threat_ids(self, threats): + if not all(isinstance(t, str) for t in threats): + raise ApiError("One or more invalid threat ID values") + self._update_criteria("threat_id", threats) + return self + + def types(self, alerttypes): + if not all((t in AlertRequestCriteriaBuilder.valid_alerttypes) for t in alerttypes): + raise ApiError("One or more invalid alert type values") + self._update_criteria("type", alerttypes) + return self + + def workflows(self, workflow_vals): + if not all((t in AlertRequestCriteriaBuilder.valid_workflow_vals) for t in workflow_vals): + raise ApiError("One or more invalid workflow status values") + self._update_criteria("workflow", workflow_vals) + return self + + def build(self): + mycrit = self._criteria + if self._time_filter: + mycrit["create_time"] = self._time_filter + return mycrit + + +class WatchlistAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): + def __init__(self): + super().__init__() + + def watchlist_ids(self, ids): + if not all(isinstance(t, str) for t in ids): + raise ApiError("One or more invalid watchlist IDs") + self._update_criteria("watchlist_id", ids) + return self + + def watchlist_names(self, names): + if not all(isinstance(name, str) for name in names): + raise ApiError("One or more invalid watchlist names") + self._update_criteria("watchlist_name", names) + return self + + +class AlertCriteriaBuilderMixin: + def categories(self, cats): + self._criteria_builder.categories(cats) + return self + + def create_time(self, *args, **kwargs): + """ + Restricts the devices that this query is performed on to the specified + last contact time (either specified as a start and end point or as a + range). + + :return: This instance + """ + self._criteria_builder.create_time(*args, **kwargs) + return self + + def device_ids(self, device_ids): + """ + Restricts the devices that this query is performed on to the specified + device IDs. + + :param device_ids: list of ints + :return: This instance + """ + self._criteria_builder.device_ids(device_ids) + return self + + def device_names(self, device_names): + self._criteria_builder.device_names(device_names) + return self + + def device_os(self, device_os): + self._criteria_builder.device_os(device_os) + return self + + def device_os_version(self, device_os_versions): + self._criteria_builder.device_os_versions(device_os_versions) + return self + + def device_username(self, users): + self._criteria_builder.device_username(users) + return self + + def group_results(self, flag): + self._criteria_builder.group_results(flag) + return self + + def alert_ids(self, alert_ids): + self._criteria_builder.alert_ids(alert_ids) + return self + + def legacy_alert_ids(self, alert_ids): + self._criteria_builder.legacy_alert_ids(alert_ids) + return self + + def minimum_severity(self, severity): + self._criteria_builder.minimum_severity(severity) + return self + + def policy_ids(self, policy_ids): + """ + Restricts the devices that this query is performed on to the specified + policy IDs. + + :param policy_ids: list of ints + :return: This instance + """ + self._criteria_builder.policy_ids(policy_ids) + return self + + def policy_names(self, policy_names): + self._criteria_builder.policy_names(policy_names) + return self + + def process_names(self, process_names): + self._criteria_builder.process_names(process_names) + return self + + def process_sha256(self, shas): + self._criteria_builder.process_sha256(shas) + return self + + def reputations(self, reps): + self._criteria_builder.reputations(reps) + return self + + def tags(self, tags): + self._criteria_builder.tags(tags) + return self + + def target_priorities(self, priorities): + self._criteria_builder.target_priorities(priorities) + return self + + def threat_ids(self, threats): + self._criteria_builder.threat_ids(threats) + return self + + def types(self, alerttypes): + self._criteria_builder.types(alerttypes) + return self + + def workflows(self, workflow_vals): + self._criteria_builder.workflows(workflow_vals) + return self + + +class WatchlistAlertCriteriaBuilderMixin(AlertCriteriaBuilderMixin): + def watchlist_ids(self, ids): + self._criteria_builder.watchlist_ids(ids) + return self + + def watchlist_names(self, names): + self._criteria_builder.watchlist_names(names) + return self + + +class BaseAlertSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, AlertCriteriaBuilderMixin, + IterableQueryMixin): + """ + Represents a query that is used to locate BaseAlert objects. + """ + def __init__(self, doc_class, cb): + super().__init__(doc_class, cb) + self._query_builder = QueryBuilder() + self._criteria_builder = AlertRequestCriteriaBuilder() + self._sortcriteria = {} + + def sort_by(self, key, direction="ASC"): + """Sets the sorting behavior on a query's results. + + Example:: + + >>> cb.select(BaseAlert).sort_by("name") + + :param key: the key in the schema to sort by + :param direction: the sort order, either "ASC" or "DESC" + :rtype: :py:class:`BaseAlertSearchQuery` + """ + if direction not in DeviceSearchQuery.valid_directions: + raise ApiError("invalid sort direction specified") + self._sortcriteria = {"field": key, "order": direction} + return self + + def _build_request(self, from_row, max_rows): + request = {"criteria": self._criteria_builder.build()} + request["query"] = self._query_builder._collapse() + if from_row > 0: + request["start"] = from_row + if max_rows >= 0: + request["rows"] = max_rows + if self._sortcriteria != {}: + request["sort"] = [self._sortcriteria] + return request + + def _build_url(self, tail_end): + url = self._doc_class.urlobject.format(self._cb.credentials.org_key) + tail_end + return url + + def _count(self): + if self._count_valid: + return self._total_results + + url = self._build_url("/_search") + request = self._build_request(0, -1) + resp = self._cb.post_object(url, body=request) + result = resp.json() + + self._total_results = result["num_found"] + self._count_valid = True + + return self._total_results + + def _perform_query(self, from_row=0, max_rows=-1): + url = self._build_url("/_search") + current = from_row + numrows = 0 + still_querying = True + while still_querying: + request = self._build_request(current, max_rows) + resp = self._cb.post_object(url, body=request) + result = resp.json() + + self._total_results = result["num_found"] + self._count_valid = True + + results = result.get("results", []) + for item in results: + yield self._doc_class(self._cb, item["id"], item) + current += 1 + numrows += 1 + + if max_rows > 0 and numrows == max_rows: + still_querying = False + break + + from_row = current + if current >= self._total_results: + still_querying = False + break + + +class WatchlistAlertSearchQuery(BaseAlertSearchQuery, WatchlistAlertCriteriaBuilderMixin): + def __init__(self, doc_class, cb): + super().__init__(doc_class, cb) + self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() + + +class BulkUpdateAlertsBase: + def __init__(self, cb, state): + self._cb = cb + self._state = state + self._additional_fields = {} + + def remediation(self, remediation): + self._additional_fields["remediation_state"] = remediation + return self + + def comment(self, comment): + self._additional_fields["comment"] = comment + return self + + def _url(self): + raise ApiError("invalid abstract URL for the operation") + + def _build_request(self): + request = self._additional_fields + request["state"] = self._state + return request + + def run(self): + resp = self._cb.post_object(self._url(), body=self._build_request()) + return DismissStatusResponse(self._cb, resp["request_id"]) + + +class BulkUpdateAlerts(BulkUpdateAlertsBase, AlertCriteriaBuilderMixin, QueryBuilderSupportMixin): + def __init__(self, cb, state): + super().__init__(cb, state) + self._criteria_builder = AlertRequestCriteriaBuilder() + self._query_builder = QueryBuilder() + + def _url(self): + return "/v6/orgs/{0}/alerts/workflow/_criteria".format(self._cb.credentials.org_key) + + def _build_request(self): + request = super().build_request() + request["criteria"] = self._criteria_builder.build() + request["query"] = self._query_builder._collapse() + return request + + +class BulkUpdateWatchlistAlerts(BulkUpdateAlerts): + def __init__(self, cb, state): + super().__init__(cb, state) + + def _url(self): + return "/v6/orgs/{0}/alerts/watchlist/workflow/_criteria".format(self._cb.credentials.org_key) + + +class BulkUpdateThreatAlerts(BulkUpdateAlertsBase): + def __init__(self, cb, state): + super().__init__(cb, state) + self._threat_ids = [] + + def threat_ids(self, ids): + self._threat_ids = self._threat_ids + ids + return self + + def _url(self): + return "/v6/orgs/{0}/threat/workflow/_criteria".format(self._cb.credentials.org_key) + + def _build_request(self): + request = super()._build_request() + request["threat_id"] = self._threat_ids + return request + \ No newline at end of file diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index cf7b7aed..4a19d2bc 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -2,6 +2,7 @@ from cbapi.errors import ApiError, ServerError from .cblr import LiveResponseSessionManager from .models import Device +from .query import BulkUpdateAlerts, BulkUpdateWatchlistAlerts, BulkUpdateThreatAlerts import logging log = logging.getLogger(__name__) @@ -18,6 +19,9 @@ class CbPSCBaseAPI(BaseAPI): >>> from cbapi import CbPSCBaseAPI >>> cb = CbPSCBaseAPI(profile="production") """ + alert_update_queries = {"ALERT": BulkUpdateAlerts, "WATCHLIST": BulkUpdateWatchlistAlerts, + "THREAT": BulkUpdateThreatAlerts} + def __init__(self, *args, **kwargs): super(CbPSCBaseAPI, self).__init__(product_name="psc", *args, **kwargs) self._lr_scheduler = None @@ -48,9 +52,7 @@ def get_device(self, device_id): :param int device_id: The ID of the device to look up. :return: The new device object. """ - rc = Device(self, device_id) - rc.refresh() - return rc + return Device(self, device_id) def _raw_device_action(self, request): url = "/appservices/v6/orgs/{0}/device_actions".format(self.credentials.org_key) @@ -134,3 +136,18 @@ def device_update_sensor_version(self, device_ids, sensor_version): :param dict sensor_version: New version properties for the sensor. """ return self._device_action(device_ids, "UPDATE_SENSOR_VERSION", {"sensor_version": sensor_version}) + + # ---- Alerts API + + def _bulk_alert_update_query(self, state, querytype): + cls = CbPSCBaseAPI.alert_update_queries.get(querytype, None) + if cls is None: + raise ApiError("unknown query type for bulk alert update") + return cls(self, state) + + def bulk_alert_dismiss(self, querytype): + return self._bulk_alert_update_query("DISMISSED", querytype) + + def bulk_alert_undismiss(self, querytype): + return self._bulk_alert_update_query("OPEN", querytype) + \ No newline at end of file From e0c3c2c792b3eb97a08fca294efb89c5927327b5 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Wed, 23 Oct 2019 11:23:07 -0400 Subject: [PATCH 035/197] finished adding the API implementations for Alerts v6 and started on filling in all the docstrings for the new classes and methods --- src/cbapi/psc/models.py | 30 ++- src/cbapi/psc/models/facet_field_dto.yaml | 24 ++ src/cbapi/psc/query.py | 280 +++++++++++++++++++++- src/cbapi/psc/rest_api.py | 12 + 4 files changed, 332 insertions(+), 14 deletions(-) create mode 100755 src/cbapi/psc/models/facet_field_dto.yaml diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 7632794f..c3c0f6bc 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -1,6 +1,7 @@ from cbapi.models import MutableBaseModel, UnrefreshableModel from cbapi.errors import ServerError -from cbapi.psc.query import DeviceSearchQuery, BaseAlertSearchQuery, WatchlistAlertSearchQuery +from cbapi.psc.query import DeviceSearchQuery, BaseAlertSearchQuery, WatchlistAlertSearchQuery, \ + WatchlistFacetQuery from copy import deepcopy import logging @@ -313,6 +314,33 @@ def finished(self): class FacetDTO(UnrefreshableModel): primary_key = "id" swagger_meta_file = "psc/models/facet_dto.yaml" + urlobject = "/appservices/v6/orgs/{0}/alerts/watchlist/_facet" def __init__(self, cb, model_unique_id, initial_data=None): super(FacetDTO, self).__init__(cb, model_unique_id, initial_data) + + @classmethod + def _query_implementation(cls, cb): + return WatchlistFacetQuery(cls, cb) + + +class FacetFieldDTO(UnrefreshableModel): + swagger_meta_file = "psc/models/facet_field_dto.yaml" + urlobject = "/appservices/v6/orgs/{0}/alerts/watchlist/_facet" + + def __init__(self, cb, model_unique_id, initial_data=None): + super(FacetFieldDTO, self).__init__(cb, model_unique_id, initial_data) + self._items = [] + if initial_data is not None: + raw_items = initial_data.get("values", []) + for raw_item in raw_items: + self._items.append(FacetDTO(cb, raw_item.get("id", ""), raw_item)) + + @classmethod + def _query_implementation(cls, cb): + return WatchlistFacetQuery(cls, cb) + + @property + def values(self): + return self._items + \ No newline at end of file diff --git a/src/cbapi/psc/models/facet_field_dto.yaml b/src/cbapi/psc/models/facet_field_dto.yaml new file mode 100755 index 00000000..769a6fd3 --- /dev/null +++ b/src/cbapi/psc/models/facet_field_dto.yaml @@ -0,0 +1,24 @@ +type: object +properties: + field: + type: object + description: Total number of results within this aggregation + values: + type: array + description: A list of facet dto items of the same field + items: + type: object + properties: + id: + type: string + description: 'Key value of the item in filter, for example : device_id, sha256_hash, + policy_id... ' + name: + type: string + description: 'Description of the item in filter, for example : device_name, + application_name, policy_name... This is an optional field, when the value + is null, it won''t be serialized' + total: + type: integer + format: int64 + description: Total number of results within this aggregation diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 42481b35..4d9109d3 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -1,5 +1,5 @@ from cbapi.errors import ApiError, MoreThanOneResultError -from cbapi.psc.models import DismissStatusResponse +from cbapi.psc.models import DismissStatusResponse, FacetFieldDTO import logging import functools from six import string_types @@ -575,6 +575,9 @@ def update_sensor_version(self, sensor_version): class AlertRequestCriteriaBuilder: + """ + Auxiliary object that builds the criteria for alert request searches. + """ valid_categories = ["THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", "CRITICAL"] valid_reputations = ["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", "NOT_LISTED", "ADAPTIVE_WHITE_LIST", "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", "COMPANY_BLACK_LIST"] @@ -590,6 +593,13 @@ def _update_criteria(self, key, newlist): self._criteria[key] = oldlist + newlist def categories(self, cats): + """ + Restricts the alerts that this query is performed on to the specified categories. + + :param cats list: List of categories to be restricted to. Valid categories are + "THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", and "CRITICAL." + :return: This instance + """ if not all((c in AlertRequestCriteriaBuilder.valid_categories) for c in cats): raise ApiError("One or more invalid category values") self._update_criteria("category", cats) @@ -597,8 +607,8 @@ def categories(self, cats): def create_time(self, *args, **kwargs): """ - Restricts the devices that this query is performed on to the specified - last contact time (either specified as a start and end point or as a + Restricts the alerts that this query is performed on to the specified + creation time (either specified as a start and end point or as a range). :return: This instance @@ -623,10 +633,10 @@ def create_time(self, *args, **kwargs): def device_ids(self, device_ids): """ - Restricts the devices that this query is performed on to the specified + Restricts the alerts that this query is performed on to the specified device IDs. - :param device_ids: list of ints + :param device_ids list: list of integer device IDs :return: This instance """ if not all(isinstance(device_id, int) for device_id in device_ids): @@ -635,55 +645,111 @@ def device_ids(self, device_ids): return self def device_names(self, device_names): + """ + Restricts the alerts that this query is performed on to the specified + device names. + + :param device_names list: list of string device names + :return: This instance + """ if not all(isinstance(n, str) for n in device_names): raise ApiError("One or more invalid device names") self._update_criteria("device_name", device_names) return self def device_os(self, device_os): + """ + Restricts the alerts that this query is performed on to the specified + device operating systems. + + :param device_os list: List of string operating systems. Valid values are + "WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", and "OTHER." + :return: This instance + """ if not all((osval in DeviceSearchQuery.valid_os) for osval in device_os): raise ApiError("One or more invalid operating systems") self._update_criteria("device_os", device_os) return self def device_os_version(self, device_os_versions): + """ + Restricts the alerts that this query is performed on to the specified + device operating system versions. + + :param device_os_versions list: List of string operating system versions. + :return: This instance + """ if not all(isinstance(n, str) for n in device_os_versions): raise ApiError("One or more invalid device OS versions") self._update_criteria("device_os_version", device_os_versions) return self def device_username(self, users): + """ + Restricts the alerts that this query is performed on to the specified + user names. + + :param users list: List of string user names. + :return: This instance + """ if not all(isinstance(u, str) for u in users): raise ApiError("One or more invalid user names") self._update_criteria("device_username", users) return self def group_results(self, flag): + """ + Specifies whether or not to group the results of the query. + + :param flag boolean: True to group the results, False to not do so. + :return: This instance + """ self._criteria["group_results"] = True if flag else False return self def alert_ids(self, alert_ids): + """ + Restricts the alerts that this query is performed on to the specified + alert IDs. + + :param alert_ids list: List of string alert IDs. + :return: This instance + """ if not all(isinstance(v, str) for v in alert_ids): raise ApiError("One or more invalid alert ID values") self._update_criteria("id", alert_ids) return self def legacy_alert_ids(self, alert_ids): + """ + Restricts the alerts that this query is performed on to the specified + legacy alert IDs. + + :param alert_ids list: List of string legacy alert IDs. + :return: This instance + """ if not all(isinstance(v, str) for v in alert_ids): raise ApiError("One or more invalid alert ID values") self._update_criteria("legacy_alert_id", alert_ids) return self def minimum_severity(self, severity): + """ + Restricts the alerts that this query is performed on to the specified + minimum severity level. + + :param severity int: The minimum severity level for alerts. + :return: This instance + """ self._criteria["minimum_severity"] = severity return self def policy_ids(self, policy_ids): """ - Restricts the devices that this query is performed on to the specified + Restricts the alerts that this query is performed on to the specified policy IDs. - :param policy_ids: list of ints + :param policy_ids list: list of integer policy IDs :return: This instance """ if not all(isinstance(policy_id, int) for policy_id in policy_ids): @@ -692,30 +758,68 @@ def policy_ids(self, policy_ids): return self def policy_names(self, policy_names): + """ + Restricts the alerts that this query is performed on to the specified + policy names. + + :param policy_names list: list of string policy names + :return: This instance + """ if not all(isinstance(n, str) for n in policy_names): raise ApiError("One or more invalid policy names") self._update_criteria("policy_name", policy_names) return self def process_names(self, process_names): + """ + Restricts the alerts that this query is performed on to the specified + process names. + + :param process_names list: list of string process names + :return: This instance + """ if not all(isinstance(n, str) for n in process_names): raise ApiError("One or more invalid process names") self._update_criteria("process_name", process_names) return self def process_sha256(self, shas): + """ + Restricts the alerts that this query is performed on to the specified + process SHA-256 hash values. + + :param shas list: list of string process SHA-256 hash values + :return: This instance + """ if not all(isinstance(n, str) for n in shas): raise ApiError("One or more invalid SHA256 values") self._update_criteria("process_sha256", shas) return self def reputations(self, reps): + """ + Restricts the alerts that this query is performed on to the specified + reputation values. + + :param reps list: List of string reputation values. Valid values are + "KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", "NOT_LISTED", + "ADAPTIVE_WHITE_LIST", "COMMON_WHITE_LIST", + "TRUSTED_WHITE_LIST", and "COMPANY_BLACK_LIST". + :return: This instance + """ if not all((r in AlertRequestCriteriaBuilder.valid_reputations) for r in reps): raise ApiError("One or more invalid reputation values") self._update_criteria("reputation", reps) return self def tags(self, tags): + """ + Restricts the alerts that this query is performed on to the specified + tag values. + + :param tags list: list of string tag values + :return: This instance + """ if not all(isinstance(tag, str) for tag in tags): raise ApiError("One or more invalid tags") self._update_criteria("tag", tags) @@ -771,13 +875,20 @@ def watchlist_names(self, names): class AlertCriteriaBuilderMixin: def categories(self, cats): + """ + Restricts the alerts that this query is performed on to the specified categories. + + :param cats list: List of categories to be restricted to. Valid categories are + "THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", and "CRITICAL." + :return: This instance + """ self._criteria_builder.categories(cats) return self def create_time(self, *args, **kwargs): """ - Restricts the devices that this query is performed on to the specified - last contact time (either specified as a start and end point or as a + Restricts the alerts that this query is performed on to the specified + creation time (either specified as a start and end point or as a range). :return: This instance @@ -787,75 +898,169 @@ def create_time(self, *args, **kwargs): def device_ids(self, device_ids): """ - Restricts the devices that this query is performed on to the specified + Restricts the alerts that this query is performed on to the specified device IDs. - :param device_ids: list of ints + :param device_ids list: list of integer device IDs :return: This instance """ self._criteria_builder.device_ids(device_ids) return self def device_names(self, device_names): + """ + Restricts the alerts that this query is performed on to the specified + device names. + + :param device_names list: list of string device names + :return: This instance + """ self._criteria_builder.device_names(device_names) return self def device_os(self, device_os): + """ + Restricts the alerts that this query is performed on to the specified + device operating systems. + + :param device_os list: List of string operating systems. Valid values are + "WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", and "OTHER." + :return: This instance + """ self._criteria_builder.device_os(device_os) return self def device_os_version(self, device_os_versions): + """ + Restricts the alerts that this query is performed on to the specified + device operating system versions. + + :param device_os_versions list: List of string operating system versions. + :return: This instance + """ self._criteria_builder.device_os_versions(device_os_versions) return self def device_username(self, users): + """ + Restricts the alerts that this query is performed on to the specified + user names. + + :param users list: List of string user names. + :return: This instance + """ self._criteria_builder.device_username(users) return self def group_results(self, flag): + """ + Specifies whether or not to group the results of the query. + + :param flag boolean: True to group the results, False to not do so. + :return: This instance + """ self._criteria_builder.group_results(flag) return self def alert_ids(self, alert_ids): + """ + Restricts the alerts that this query is performed on to the specified + alert IDs. + + :param alert_ids list: List of string alert IDs. + :return: This instance + """ self._criteria_builder.alert_ids(alert_ids) return self def legacy_alert_ids(self, alert_ids): + """ + Restricts the alerts that this query is performed on to the specified + legacy alert IDs. + + :param alert_ids list: List of string legacy alert IDs. + :return: This instance + """ self._criteria_builder.legacy_alert_ids(alert_ids) return self def minimum_severity(self, severity): + """ + Restricts the alerts that this query is performed on to the specified + minimum severity level. + + :param severity int: The minimum severity level for alerts. + :return: This instance + """ self._criteria_builder.minimum_severity(severity) return self def policy_ids(self, policy_ids): """ - Restricts the devices that this query is performed on to the specified + Restricts the alerts that this query is performed on to the specified policy IDs. - :param policy_ids: list of ints + :param policy_ids list: list of integer policy IDs :return: This instance """ self._criteria_builder.policy_ids(policy_ids) return self def policy_names(self, policy_names): + """ + Restricts the alerts that this query is performed on to the specified + policy names. + + :param policy_names list: list of string policy names + :return: This instance + """ self._criteria_builder.policy_names(policy_names) return self def process_names(self, process_names): + """ + Restricts the alerts that this query is performed on to the specified + process names. + + :param process_names list: list of string process names + :return: This instance + """ self._criteria_builder.process_names(process_names) return self def process_sha256(self, shas): + """ + Restricts the alerts that this query is performed on to the specified + process SHA-256 hash values. + + :param shas list: list of string process SHA-256 hash values + :return: This instance + """ self._criteria_builder.process_sha256(shas) return self def reputations(self, reps): + """ + Restricts the alerts that this query is performed on to the specified + reputation values. + + :param reps list: List of string reputation values. Valid values are + "KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", "NOT_LISTED", + "ADAPTIVE_WHITE_LIST", "COMMON_WHITE_LIST", + "TRUSTED_WHITE_LIST", and "COMPANY_BLACK_LIST". + :return: This instance + """ self._criteria_builder.reputations(reps) return self def tags(self, tags): + """ + Restricts the alerts that this query is performed on to the specified + tag values. + + :param tags list: list of string tag values + :return: This instance + """ self._criteria_builder.tags(tags) return self @@ -975,7 +1180,56 @@ class WatchlistAlertSearchQuery(BaseAlertSearchQuery, WatchlistAlertCriteriaBuil def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() + + +class WatchlistFacetQuery(PSCQueryBase, QueryBuilderSupportMixin, WatchlistAlertCriteriaBuilderMixin, + IterableQueryMixin): + valid_facet_fields = ["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", + "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", + "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", + "POLICY_APPLIED", "SENSOR_ACTION"] + + def __init__(self, doc_class, cb): + super().__init__(doc_class, cb) + self._query_builder = QueryBuilder() + self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() + self._fields = [] + + def terms(self, fieldlist): + if not all((field in WatchlistFacetQuery.valid_facet_fields) for field in fieldlist): + raise ApiError("One or more invalid term field names") + self._fields = self._fields + fieldlist + return self + def _build_request(self, max_rows): + request = {"criteria": self._criteria_builder.build()} + request["query"] = self._query_builder._collapse() + request["terms"] = {"fields": self._fields, "rows": max_rows} + return request + + def _count(self): + if self._count_valid: + return self._total_results + url = self._doc_class.urlobject.format(self._cb.credentials.org_key) + request = self._build_request(0) + resp = self._cb.post_object(url, body=request) + rawresult = resp.json() + result_list = rawresult.get("results", []) + self._total_results = len(result_list) + self._count_valid = True + return self._total_results + + def _perform_query(self, max_rows=0): + url = self._doc_class.urlobject.format(self._cb.credentials.org_key) + request = self._build_request(max_rows) + resp = self._cb.post_object(url, body=request) + rawresult = resp.json() + result_list = rawresult.get("results", []) + self._total_results = len(result_list) + self._count_valid = True + for result in result_list: + yield FacetFieldDTO(self._cb, result.get("field", {}), result) + class BulkUpdateAlertsBase: def __init__(self, cb, state): diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 4a19d2bc..24ff6575 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -146,8 +146,20 @@ def _bulk_alert_update_query(self, state, querytype): return cls(self, state) def bulk_alert_dismiss(self, querytype): + """ + Start a query to dismiss multiple alerts. + + :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", or "THREAT" + :return: The new query. + """ return self._bulk_alert_update_query("DISMISSED", querytype) def bulk_alert_undismiss(self, querytype): + """ + Start a query to un-dismiss multiple alerts. + + :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", or "THREAT" + :return: The new query. + """ return self._bulk_alert_update_query("OPEN", querytype) \ No newline at end of file From 0351c7e9edf8d7be8079b932f652f20601490316 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Fri, 25 Oct 2019 11:20:55 -0600 Subject: [PATCH 036/197] finished adding docstrings for everything new --- src/cbapi/psc/models.py | 24 ++++++ src/cbapi/psc/query.py | 161 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 183 insertions(+), 2 deletions(-) diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index c3c0f6bc..06be5c32 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -241,9 +241,21 @@ def _update_workflow_status(self, state, remediation, comment): self._last_refresh_time = time.time() def dismiss(self, remediation=None, comment=None): + """ + Dismiss this alert. + + :param remediation str: The remediation status to set for the alert. + :param comment str: The comment to set for the alert. + """ self._update_workflow_status("DISMISSED", remediation, comment) def undismiss(self, remediation=None, comment=None): + """ + Un-dismiss this alert. + + :param remediation str: The remediation status to set for the alert. + :param comment str: The comment to set for the alert. + """ self._update_workflow_status("OPEN", remediation, comment) def _update_threat_workflow_status(self, state, remediation, comment): @@ -258,9 +270,21 @@ def _update_threat_workflow_status(self, state, remediation, comment): return Workflow(self._cb, resp) def dismiss_threat(self, remediation=None, comment=None): + """ + Dismiss alerts for this threat. + + :param remediation str: The remediation status to set for the alert. + :param comment str: The comment to set for the alert. + """ return self._update_threat_workflow_status("DISMISSED", remediation, comment) def undismiss_threat(self, remediation=None, comment=None): + """ + Un-dismiss alerts for this threat. + + :param remediation str: The remediation status to set for the alert. + :param comment str: The comment to set for the alert. + """ return self._update_threat_workflow_status("OPEN", remediation, comment) diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 4d9109d3..b9724c59 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -826,30 +826,66 @@ def tags(self, tags): return self def target_priorities(self, priorities): + """ + Restricts the alerts that this query is performed on to the specified + target priority values. + + :param priorities list: List of string target priority values. Valid values are + "LOW", "MEDIUM", "HIGH", and "MISSION_CRITICAL". + :return: This instance + """ if not all((prio in DeviceSearchQuery.valid_priorities) for prio in priorities): raise ApiError("One or more invalid priority values") self._update_criteria("target_value", priorities) return self def threat_ids(self, threats): + """ + Restricts the alerts that this query is performed on to the specified + threat ID values. + + :param threats list: list of string threat ID values + :return: This instance + """ if not all(isinstance(t, str) for t in threats): raise ApiError("One or more invalid threat ID values") self._update_criteria("threat_id", threats) return self def types(self, alerttypes): + """ + Restricts the alerts that this query is performed on to the specified + alert type values. + + :param alerttypes list: List of string alert type values. Valid values are + "CB_ANALYTICS", "VMWARE", and "WATCHLIST". + :return: This instance + """ if not all((t in AlertRequestCriteriaBuilder.valid_alerttypes) for t in alerttypes): raise ApiError("One or more invalid alert type values") self._update_criteria("type", alerttypes) return self def workflows(self, workflow_vals): + """ + Restricts the alerts that this query is performed on to the specified + workflow status values. + + :param workflow_vals list: List of string alert type values. Valid values are + "OPEN" and "DISMISSED". + :return: This instance + """ if not all((t in AlertRequestCriteriaBuilder.valid_workflow_vals) for t in workflow_vals): raise ApiError("One or more invalid workflow status values") self._update_criteria("workflow", workflow_vals) return self def build(self): + """ + Builds the criteria object for use in a query. + + :return: The criteria object. + """ mycrit = self._criteria if self._time_filter: mycrit["create_time"] = self._time_filter @@ -857,16 +893,33 @@ def build(self): class WatchlistAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): + """ + Auxiliary object that builds the criteria for watchlist alert request searches. + """ def __init__(self): super().__init__() def watchlist_ids(self, ids): + """ + Restricts the alerts that this query is performed on to the specified + watchlist ID values. + + :param ids list: list of string watchlist ID values + :return: This instance + """ if not all(isinstance(t, str) for t in ids): raise ApiError("One or more invalid watchlist IDs") self._update_criteria("watchlist_id", ids) return self def watchlist_names(self, names): + """ + Restricts the alerts that this query is performed on to the specified + watchlist name values. + + :param names list: list of string watchlist name values + :return: This instance + """ if not all(isinstance(name, str) for name in names): raise ApiError("One or more invalid watchlist names") self._update_criteria("watchlist_name", names) @@ -874,6 +927,9 @@ def watchlist_names(self, names): class AlertCriteriaBuilderMixin: + """ + Added to query classes to allow them to manipulate alert criteria for queries. + """ def categories(self, cats): """ Restricts the alerts that this query is performed on to the specified categories. @@ -1065,28 +1121,76 @@ def tags(self, tags): return self def target_priorities(self, priorities): + """ + Restricts the alerts that this query is performed on to the specified + target priority values. + + :param reps list: List of string target priority values. Valid values are + "LOW", "MEDIUM", "HIGH", and "MISSION_CRITICAL". + :return: This instance + """ self._criteria_builder.target_priorities(priorities) return self def threat_ids(self, threats): + """ + Restricts the alerts that this query is performed on to the specified + threat ID values. + + :param threats list: list of string threat ID values + :return: This instance + """ self._criteria_builder.threat_ids(threats) return self def types(self, alerttypes): + """ + Restricts the alerts that this query is performed on to the specified + alert type values. + + :param alerttypes list: List of string alert type values. Valid values are + "CB_ANALYTICS", "VMWARE", and "WATCHLIST". + :return: This instance + """ self._criteria_builder.types(alerttypes) return self def workflows(self, workflow_vals): + """ + Restricts the alerts that this query is performed on to the specified + workflow status values. + + :param workflow_vals list: List of string alert type values. Valid values are + "OPEN" and "DISMISSED". + :return: This instance + """ self._criteria_builder.workflows(workflow_vals) return self class WatchlistAlertCriteriaBuilderMixin(AlertCriteriaBuilderMixin): + """ + Added to query classes to allow them to manipulate watchlist alert criteria for queries. + """ def watchlist_ids(self, ids): + """ + Restricts the alerts that this query is performed on to the specified + watchlist ID values. + + :param ids list: list of string watchlist ID values + :return: This instance + """ self._criteria_builder.watchlist_ids(ids) return self def watchlist_names(self, names): + """ + Restricts the alerts that this query is performed on to the specified + watchlist name values. + + :param names list: list of string watchlist name values + :return: This instance + """ self._criteria_builder.watchlist_names(names) return self @@ -1177,6 +1281,9 @@ def _perform_query(self, from_row=0, max_rows=-1): class WatchlistAlertSearchQuery(BaseAlertSearchQuery, WatchlistAlertCriteriaBuilderMixin): + """ + Represents a query that is used to locate WatchlistAlert objects. + """ def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() @@ -1184,6 +1291,9 @@ def __init__(self, doc_class, cb): class WatchlistFacetQuery(PSCQueryBase, QueryBuilderSupportMixin, WatchlistAlertCriteriaBuilderMixin, IterableQueryMixin): + """ + Represents a query that is used to locate FacetFieldDTO and FacetDTO objects. + """ valid_facet_fields = ["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", @@ -1196,6 +1306,16 @@ def __init__(self, doc_class, cb): self._fields = [] def terms(self, fieldlist): + """ + Specifies the facet field names that are to be retrieved. + + :param fieldlist list: List of facet field names. Valid names are + "ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", + "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", + "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", + "POLICY_APPLIED", and "SENSOR_ACTION". + :return: This instance. + """ if not all((field in WatchlistFacetQuery.valid_facet_fields) for field in fieldlist): raise ApiError("One or more invalid term field names") self._fields = self._fields + fieldlist @@ -1232,16 +1352,30 @@ def _perform_query(self, max_rows=0): class BulkUpdateAlertsBase: + """ + Base query for doing bulk updates on alerts, where the result of a search is used to set + the states of multiple alerts. + """ def __init__(self, cb, state): self._cb = cb self._state = state self._additional_fields = {} def remediation(self, remediation): + """ + Sets the remediation state message to be applied to all selected alerts. + + :param remediation str: The remediation state message. + """ self._additional_fields["remediation_state"] = remediation return self def comment(self, comment): + """ + Sets the comment to be applied to all selected alerts. + + :param comment str: The comment to be used. + """ self._additional_fields["comment"] = comment return self @@ -1254,11 +1388,20 @@ def _build_request(self): return request def run(self): + """ + Executes the search query and alert state change operation. + + :return: A DismissStatusResponse object that can be used for monitoring the progress + of the operation. + """ resp = self._cb.post_object(self._url(), body=self._build_request()) return DismissStatusResponse(self._cb, resp["request_id"]) class BulkUpdateAlerts(BulkUpdateAlertsBase, AlertCriteriaBuilderMixin, QueryBuilderSupportMixin): + """ + Query for bulk update of base-level alerts. + """ def __init__(self, cb, state): super().__init__(cb, state) self._criteria_builder = AlertRequestCriteriaBuilder() @@ -1275,6 +1418,9 @@ def _build_request(self): class BulkUpdateWatchlistAlerts(BulkUpdateAlerts): + """ + Query for bulk update of watchlist alerts. + """ def __init__(self, cb, state): super().__init__(cb, state) @@ -1283,12 +1429,23 @@ def _url(self): class BulkUpdateThreatAlerts(BulkUpdateAlertsBase): + """ + Query for bulk update of threat alerts. + """ def __init__(self, cb, state): super().__init__(cb, state) self._threat_ids = [] - def threat_ids(self, ids): - self._threat_ids = self._threat_ids + ids + def threat_ids(self, threats): + """ + Specifies the threat IDs to set the status of alerts for. + + :param threats list: The list of string threat identifiers. + :return: This instance. + """ + if not all(isinstance(t, str) for t in threats): + raise ApiError("One or more invalid threat ID values") + self._threat_ids = self._threat_ids + threats return self def _url(self): From 074a3849faff04cdfc06a7c99a6be237ede4baf9 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Fri, 25 Oct 2019 15:59:45 -0600 Subject: [PATCH 037/197] elimination of the separate facet objects and implementation of the remainder of the Alerts v6 API calls (mostly extensions of existing functionality) --- src/cbapi/psc/models.py | 53 +-- src/cbapi/psc/models/facet_dto.yaml | 15 - src/cbapi/psc/models/facet_field_dto.yaml | 24 -- src/cbapi/psc/query.py | 422 +++++++++++++++++++--- src/cbapi/psc/rest_api.py | 23 +- 5 files changed, 398 insertions(+), 139 deletions(-) delete mode 100755 src/cbapi/psc/models/facet_dto.yaml delete mode 100755 src/cbapi/psc/models/facet_field_dto.yaml diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 06be5c32..2e711bef 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -1,7 +1,7 @@ from cbapi.models import MutableBaseModel, UnrefreshableModel from cbapi.errors import ServerError from cbapi.psc.query import DeviceSearchQuery, BaseAlertSearchQuery, WatchlistAlertSearchQuery, \ - WatchlistFacetQuery + CBAnalyticsAlertSearchQuery, VMwareAlertSearchQuery from copy import deepcopy import logging @@ -294,6 +294,22 @@ class WatchlistAlert(BaseAlert): @classmethod def _query_implementation(cls, cb): return WatchlistAlertSearchQuery(cls, cb) + + +class CBAnalyticsAlert(BaseAlert): + urlobject = "/appservices/v6/orgs/{0}/alerts/cbanalytics" + + @classmethod + def _query_implementation(cls, cb): + return CBAnalyticsAlertSearchQuery(cls, cb) + + +class VMwareAlert(BaseAlert): + urlobject = "/appservices/v6/orgs/{0}/alerts/vmware" + + @classmethod + def _query_implementation(cls, cb): + return VMwareAlertSearchQuery(cls, cb) class DismissStatusResponse(UnrefreshableModel): @@ -333,38 +349,3 @@ def in_progress(self): def finished(self): self._refresh() return self._info.get("status", "") == "FINISHED" - - -class FacetDTO(UnrefreshableModel): - primary_key = "id" - swagger_meta_file = "psc/models/facet_dto.yaml" - urlobject = "/appservices/v6/orgs/{0}/alerts/watchlist/_facet" - - def __init__(self, cb, model_unique_id, initial_data=None): - super(FacetDTO, self).__init__(cb, model_unique_id, initial_data) - - @classmethod - def _query_implementation(cls, cb): - return WatchlistFacetQuery(cls, cb) - - -class FacetFieldDTO(UnrefreshableModel): - swagger_meta_file = "psc/models/facet_field_dto.yaml" - urlobject = "/appservices/v6/orgs/{0}/alerts/watchlist/_facet" - - def __init__(self, cb, model_unique_id, initial_data=None): - super(FacetFieldDTO, self).__init__(cb, model_unique_id, initial_data) - self._items = [] - if initial_data is not None: - raw_items = initial_data.get("values", []) - for raw_item in raw_items: - self._items.append(FacetDTO(cb, raw_item.get("id", ""), raw_item)) - - @classmethod - def _query_implementation(cls, cb): - return WatchlistFacetQuery(cls, cb) - - @property - def values(self): - return self._items - \ No newline at end of file diff --git a/src/cbapi/psc/models/facet_dto.yaml b/src/cbapi/psc/models/facet_dto.yaml deleted file mode 100755 index 3e0993e1..00000000 --- a/src/cbapi/psc/models/facet_dto.yaml +++ /dev/null @@ -1,15 +0,0 @@ -type: object -properties: - id: - type: string - description: 'Key value of the item in filter, for example : device_id, sha256_hash, - policy_id... ' - name: - type: string - description: 'Description of the item in filter, for example : device_name, - application_name, policy_name... This is an optional field, when the value - is null, it won''t be serialized' - total: - type: integer - format: int64 - description: Total number of results within this aggregation diff --git a/src/cbapi/psc/models/facet_field_dto.yaml b/src/cbapi/psc/models/facet_field_dto.yaml deleted file mode 100755 index 769a6fd3..00000000 --- a/src/cbapi/psc/models/facet_field_dto.yaml +++ /dev/null @@ -1,24 +0,0 @@ -type: object -properties: - field: - type: object - description: Total number of results within this aggregation - values: - type: array - description: A list of facet dto items of the same field - items: - type: object - properties: - id: - type: string - description: 'Key value of the item in filter, for example : device_id, sha256_hash, - policy_id... ' - name: - type: string - description: 'Description of the item in filter, for example : device_name, - application_name, policy_name... This is an optional field, when the value - is null, it won''t be serialized' - total: - type: integer - format: int64 - description: Total number of results within this aggregation diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index b9724c59..475ed398 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -892,6 +892,174 @@ def build(self): return mycrit +class CBAnalyticsAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): + """ + Auxiliary object that builds the criteria for CB Analytics alert request searches. + """ + valid_threat_categories = ["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", "RISKY_PROGRAM"] + valid_locations = ["ONSITE", "OFFSITE", "UNKNOWN"] + valid_kill_chain_statuses = ["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", "INSTALL_RUN", + "COMMAND_AND_CONTROL", "EXECUTE_GOAL", "BREACH"] + valid_policy_applied = ["APPLIED", "NOT_APPLIED"] + valid_run_states = ["DID_NOT_RUN", "RAN", "UNKNOWN"] + valid_sensor_actions = ["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", "TERMINATE", "DENY"] + valid_threat_cause_vectors = ["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", + "REMOVABLE_MEDIA", "UNKNOWN", "APP_STORE", "THIRD_PARTY"] + def __init__(self): + super().__init__() + + def blocked_threat_categories(self, categories): + """ + Restricts the alerts that this query is performed on to the specified + threat categories that were blocked. + + :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", + "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". + :return: This instance. + """ + if not all((category in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_categories) \ + for category in categories): + raise ApiError("One or more invalid threat categories") + self._update_criteria("blocked_threat_category", categories) + return self + + def device_locations(self, locations): + """ + Restricts the alerts that this query is performed on to the specified + device locations. + + :param locations list: List of device locations to look for. Valid values are "ONSITE", "OFFSITE", + and "UNKNOWN". + :return: This instance. + """ + if not all((location in CBAnalyticsAlertRequestCriteriaBuilder.valid_locations) \ + for location in locations): + raise ApiError("One or more invalid device locations") + self._update_criteria("device_location", locations) + return self + + def kill_chain_statuses(self, statuses): + """ + Restricts the alerts that this query is performed on to the specified + kill chain statuses. + + :param statuses list: List of kill chain statuses to look for. Valid values are "RECONNAISSANCE", + "WEAPONIZE", "DELIVER_EXPLOIT", "INSTALL_RUN","COMMAND_AND_CONTROL", + "EXECUTE_GOAL", and "BREACH". + :return: This instance. + """ + if not all((status in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_categories) \ + for status in statuses): + raise ApiError("One or more invalid kill chain status values") + self._update_criteria("kill_chain_status", statuses) + return self + + def not_blocked_threat_categories(self, categories): + """ + Restricts the alerts that this query is performed on to the specified + threat categories that were NOT blocked. + + :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", + "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". + :return: This instance. + """ + if not all((category in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_categories) \ + for category in categories): + raise ApiError("One or more invalid threat categories") + self._update_criteria("not_blocked_threat_category", categories) + return self + + def policy_applied(self, applied_statuses): + """ + Restricts the alerts that this query is performed on to the specified + status values showing whether policies were applied. + + :param applied_statuses list: List of status values to look for. Valid values are + "APPLIED" and "NOT_APPLIED". + :return: This instance. + """ + if not all((s in CBAnalyticsAlertRequestCriteriaBuilder.valid_policy_applied) \ + for s in applied_statuses): + raise ApiError("One or more invalid policy-applied values") + self._update_criteria("policy_applied", applied_statuses) + return self + + def reason_code(self, reason): + """ + Restricts the alerts that this query is performed on to the specified + reason code (enum value). + + :param reason str: The reason code to look for. + :return: This instance. + """ + self._criteria["reason_code"] = reason + return self + + def run_states(self, states): + """ + Restricts the alerts that this query is performed on to the specified run states. + + :param states list: List of run states to look for. Valid values are "DID_NOT_RUN", "RAN", + and "UNKNOWN". + :return: This instance. + """ + if not all((s in CBAnalyticsAlertRequestCriteriaBuilder.valid_run_states) \ + for s in states): + raise ApiError("One or more invalid run states") + self._update_criteria("run_state", states) + return self + + def sensor_actions(self, actions): + """ + Restricts the alerts that this query is performed on to the specified sensor actions. + + :param actions list: List of sensor actions to look for. Valid values are "POLICY_NOT_APPLIED", + "ALLOW", "ALLOW_AND_LOG", "TERMINATE", and "DENY". + :return: This instance. + """ + if not all((action in CBAnalyticsAlertRequestCriteriaBuilder.valid_sensor_actions) \ + for action in actions): + raise ApiError("One or more invalid sensor actions") + self._update_criteria("sensor_action", actions) + return self + + def threat_cause_vectors(self, vectors): + """ + Restricts the alerts that this query is performed on to the specified threat cause vectors. + + :param vectors list: List of threat cause vectors to look for. Valid values are "EMAIL", "WEB", + "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", "REMOVABLE_MEDIA", + "UNKNOWN", "APP_STORE", and "THIRD_PARTY". + :return: This instance. + """ + if not all((vector in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_cause_vectors) \ + for vector in vectors): + raise ApiError("One or more invalid threat cause vectors") + self._update_criteria("threat_cause_vector", vectors) + return self + + +class VMwareAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): + """ + Auxiliary object that builds the criteria for VMware alert request searches. + """ + def __init__(self): + super().__init__() + + def group_ids(self, groupids): + """ + Restricts the alerts that this query is performed on to the specified + AppDefense-assigned alarm group IDs. + + :param groupids list: List of (integer) AppDefense-assigned alarm group IDs. + :return: This instance. + """ + if not all(isinstance(groupid, int) for groupid in groupids): + raise ApiError("One or more invalid alarm group IDs") + self._update_criteria("group_id", groupids) + return self + + class WatchlistAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): """ Auxiliary object that builds the criteria for watchlist alert request searches. @@ -1166,8 +1334,135 @@ def workflows(self, workflow_vals): """ self._criteria_builder.workflows(workflow_vals) return self + + +class CBAnalyticsAlertCriteriaBuilderMixin(AlertCriteriaBuilderMixin): + """ + Added to query classes to allow them to manipulate CB Analytics alert criteria for queries. + """ + def blocked_threat_categories(self, categories): + """ + Restricts the alerts that this query is performed on to the specified + threat categories that were blocked. + + :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", + "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". + :return: This instance. + """ + self._criteria_builder.blocked_threat_categories(categories) + return self + + def device_locations(self, locations): + """ + Restricts the alerts that this query is performed on to the specified + device locations. + + :param locations list: List of device locations to look for. Valid values are "ONSITE", "OFFSITE", + and "UNKNOWN". + :return: This instance. + """ + self._criteria_builder.device_locations(locations) + return self + + def kill_chain_statuses(self, statuses): + """ + Restricts the alerts that this query is performed on to the specified + kill chain statuses. + + :param statuses list: List of kill chain statuses to look for. Valid values are "RECONNAISSANCE", + "WEAPONIZE", "DELIVER_EXPLOIT", "INSTALL_RUN","COMMAND_AND_CONTROL", + "EXECUTE_GOAL", and "BREACH". + :return: This instance. + """ + self._criteria_builder.kill_chain_statuses(statuses) + return self + + def not_blocked_threat_categories(self, categories): + """ + Restricts the alerts that this query is performed on to the specified + threat categories that were NOT blocked. + + :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", + "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". + :return: This instance. + """ + self._criteria_builder.not_blocked_threat_categories(categories) + return self + + def policy_applied(self, applied_statuses): + """ + Restricts the alerts that this query is performed on to the specified + status values showing whether policies were applied. + + :param applied_statuses list: List of status values to look for. Valid values are + "APPLIED" and "NOT_APPLIED". + :return: This instance. + """ + self._criteria_builder.policy_applied(applied_statuses) + return self + + def reason_code(self, reason): + """ + Restricts the alerts that this query is performed on to the specified + reason code (enum value). + + :param reason str: The reason code to look for. + :return: This instance. + """ + self._criteria_builder.reason_code(reason) + return self + + def run_states(self, states): + """ + Restricts the alerts that this query is performed on to the specified run states. + + :param states list: List of run states to look for. Valid values are "DID_NOT_RUN", "RAN", + and "UNKNOWN". + :return: This instance. + """ + self._criteria_builder.run_states(states) + return self + + def sensor_actions(self, actions): + """ + Restricts the alerts that this query is performed on to the specified sensor actions. + + :param actions list: List of sensor actions to look for. Valid values are "POLICY_NOT_APPLIED", + "ALLOW", "ALLOW_AND_LOG", "TERMINATE", and "DENY". + :return: This instance. + """ + self._criteria_builder.sensor_actions(actions) + return self + + def threat_cause_vectors(self, vectors): + """ + Restricts the alerts that this query is performed on to the specified threat cause vectors. + + :param vectors list: List of threat cause vectors to look for. Valid values are "EMAIL", "WEB", + "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", "REMOVABLE_MEDIA", + "UNKNOWN", "APP_STORE", and "THIRD_PARTY". + :return: This instance. + """ + self._criteria_builder.threat_cause_vectors(vectors) + return self +class VMwareAlertCriteriaBuilderMixin(AlertCriteriaBuilderMixin): + """ + Added to query classes to allow them to manipulate VMware alert criteria for queries. + """ + def group_ids(self, groupids): + """ + Restricts the alerts that this query is performed on to the specified + AppDefense-assigned alarm group IDs. + + :param groupids list: List of (integer) AppDefense-assigned alarm group IDs. + :return: This instance. + """ + self._criteria_builder.group_ids(groupids) + return self + + class WatchlistAlertCriteriaBuilderMixin(AlertCriteriaBuilderMixin): """ Added to query classes to allow them to manipulate watchlist alert criteria for queries. @@ -1200,6 +1495,11 @@ class BaseAlertSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, AlertCriteria """ Represents a query that is used to locate BaseAlert objects. """ + valid_facet_fields = ["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", + "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", + "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", + "POLICY_APPLIED", "SENSOR_ACTION"] + def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._query_builder = QueryBuilder() @@ -1222,14 +1522,14 @@ def sort_by(self, key, direction="ASC"): self._sortcriteria = {"field": key, "order": direction} return self - def _build_request(self, from_row, max_rows): + def _build_request(self, from_row, max_rows, add_sort=True): request = {"criteria": self._criteria_builder.build()} request["query"] = self._query_builder._collapse() if from_row > 0: request["start"] = from_row if max_rows >= 0: request["rows"] = max_rows - if self._sortcriteria != {}: + if add_sort and self._sortcriteria != {}: request["sort"] = [self._sortcriteria] return request @@ -1278,6 +1578,27 @@ def _perform_query(self, from_row=0, max_rows=-1): if current >= self._total_results: still_querying = False break + + def facets(self, fieldlist, max_rows=0): + """ + Return information about the facets for this alert by search, using the defined criteria. + + :param fieldlist list: List of facet field names. Valid names are + "ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", + "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", + "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", + "POLICY_APPLIED", and "SENSOR_ACTION". + :param max_rows int: The maximum number of rows to return. 0 means return all rows. + :return: A list of facet information specified as dicts. + """ + if not all((field in BaseAlertSearchQuery.valid_facet_fields) for field in fieldlist): + raise ApiError("One or more invalid term field names") + request = self._build_request(0, -1, False) + request["terms"] = {"fields": fieldlist, "rows": max_rows} + url = self._build_url("/_facet") + resp = self._cb.post_object(url, body=request) + result = resp.json() + return result.get("results", []) class WatchlistAlertSearchQuery(BaseAlertSearchQuery, WatchlistAlertCriteriaBuilderMixin): @@ -1289,68 +1610,24 @@ def __init__(self, doc_class, cb): self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() -class WatchlistFacetQuery(PSCQueryBase, QueryBuilderSupportMixin, WatchlistAlertCriteriaBuilderMixin, - IterableQueryMixin): +class CBAnalyticsAlertSearchQuery(BaseAlertSearchQuery, CBAnalyticsAlertCriteriaBuilderMixin): """ - Represents a query that is used to locate FacetFieldDTO and FacetDTO objects. + Represents a query that is used to locate CBAnalyticsAlert objects. """ - valid_facet_fields = ["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", - "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", - "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", - "POLICY_APPLIED", "SENSOR_ACTION"] + def __init__(self, doc_class, cb): + super().__init__(doc_class, cb) + self._criteria_builder = CBAnalyticsAlertRequestCriteriaBuilder() + +class VMwareAlertSearchQuery(BaseAlertSearchQuery, CBAnalyticsAlertCriteriaBuilderMixin): + """ + Represents a query that is used to locate VMwareAlert objects. + """ def __init__(self, doc_class, cb): super().__init__(doc_class, cb) - self._query_builder = QueryBuilder() - self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() - self._fields = [] - - def terms(self, fieldlist): - """ - Specifies the facet field names that are to be retrieved. - - :param fieldlist list: List of facet field names. Valid names are - "ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", - "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", - "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", - "POLICY_APPLIED", and "SENSOR_ACTION". - :return: This instance. - """ - if not all((field in WatchlistFacetQuery.valid_facet_fields) for field in fieldlist): - raise ApiError("One or more invalid term field names") - self._fields = self._fields + fieldlist - return self - - def _build_request(self, max_rows): - request = {"criteria": self._criteria_builder.build()} - request["query"] = self._query_builder._collapse() - request["terms"] = {"fields": self._fields, "rows": max_rows} - return request - - def _count(self): - if self._count_valid: - return self._total_results - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) - request = self._build_request(0) - resp = self._cb.post_object(url, body=request) - rawresult = resp.json() - result_list = rawresult.get("results", []) - self._total_results = len(result_list) - self._count_valid = True - return self._total_results - - def _perform_query(self, max_rows=0): - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) - request = self._build_request(max_rows) - resp = self._cb.post_object(url, body=request) - rawresult = resp.json() - result_list = rawresult.get("results", []) - self._total_results = len(result_list) - self._count_valid = True - for result in result_list: - yield FacetFieldDTO(self._cb, result.get("field", {}), result) - - + self._criteria_builder = VMwareAlertRequestCriteriaBuilder() + + class BulkUpdateAlertsBase: """ Base query for doing bulk updates on alerts, where the result of a search is used to set @@ -1417,12 +1694,37 @@ def _build_request(self): return request -class BulkUpdateWatchlistAlerts(BulkUpdateAlerts): +class BulkUpdateCBAnalyticsAlerts(BulkUpdateAlerts, CBAnalyticsAlertCriteriaBuilderMixin): + """ + Query for bulk update of CB Analytics alerts. + """ + def __init__(self, cb, state): + super().__init__(cb, state) + self._criteria_builder = CBAnalyticsAlertRequestCriteriaBuilder() + + def _url(self): + return "/v6/orgs/{0}/alerts/cbanalytics/workflow/_criteria".format(self._cb.credentials.org_key) + + +class BulkUpdateVMwareAlerts(BulkUpdateAlerts, VMwareAlertCriteriaBuilderMixin): + """ + Query for bulk update of VMware alerts. + """ + def __init__(self, cb, state): + super().__init__(cb, state) + self._criteria_builder = VMwareAlertRequestCriteriaBuilder() + + def _url(self): + return "/v6/orgs/{0}/alerts/vmware/workflow/_criteria".format(self._cb.credentials.org_key) + + +class BulkUpdateWatchlistAlerts(BulkUpdateAlerts, WatchlistAlertCriteriaBuilderMixin): """ Query for bulk update of watchlist alerts. """ def __init__(self, cb, state): super().__init__(cb, state) + self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() def _url(self): return "/v6/orgs/{0}/alerts/watchlist/workflow/_criteria".format(self._cb.credentials.org_key) diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 24ff6575..88deb148 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -2,7 +2,8 @@ from cbapi.errors import ApiError, ServerError from .cblr import LiveResponseSessionManager from .models import Device -from .query import BulkUpdateAlerts, BulkUpdateWatchlistAlerts, BulkUpdateThreatAlerts +from .query import BulkUpdateAlerts, BulkUpdateWatchlistAlerts, BulkUpdateThreatAlerts, \ + BulkUpdateCBAnalyticsAlerts, BulkUpdateVMwareAlerts import logging log = logging.getLogger(__name__) @@ -20,7 +21,8 @@ class CbPSCBaseAPI(BaseAPI): >>> cb = CbPSCBaseAPI(profile="production") """ alert_update_queries = {"ALERT": BulkUpdateAlerts, "WATCHLIST": BulkUpdateWatchlistAlerts, - "THREAT": BulkUpdateThreatAlerts} + "THREAT": BulkUpdateThreatAlerts, "CBANALYTICS": BulkUpdateCBAnalyticsAlerts, + "VMWARE": BulkUpdateVMwareAlerts} def __init__(self, *args, **kwargs): super(CbPSCBaseAPI, self).__init__(product_name="psc", *args, **kwargs) @@ -139,6 +141,17 @@ def device_update_sensor_version(self, device_ids, sensor_version): # ---- Alerts API + def alert_search_suggestions(self, query): + """ + Returns suggestions for keys and field values that can be used in a search. + + :param query str: A search query to use. + :return: A list of search suggestions expressed as dict objects. + """ + query_params = {"suggest.q": query} + url = "/appservices/v6/orgs/{0}/alerts/search_suggestions".format(self.credentials.org_key) + return self.get_object(url, query_params) + def _bulk_alert_update_query(self, state, querytype): cls = CbPSCBaseAPI.alert_update_queries.get(querytype, None) if cls is None: @@ -149,7 +162,8 @@ def bulk_alert_dismiss(self, querytype): """ Start a query to dismiss multiple alerts. - :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", or "THREAT" + :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", "THREAT", + "CBANALYTICS", or "VMWARE". :return: The new query. """ return self._bulk_alert_update_query("DISMISSED", querytype) @@ -158,7 +172,8 @@ def bulk_alert_undismiss(self, querytype): """ Start a query to un-dismiss multiple alerts. - :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", or "THREAT" + :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", "THREAT", + "CBANALYTICS", or "VMWARE". :return: The new query. """ return self._bulk_alert_update_query("OPEN", querytype) From fa48139d4f7fdecfd681e9b5f5e0ae0ff7b608f0 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Fri, 25 Oct 2019 16:02:39 -0600 Subject: [PATCH 038/197] discard an excess import --- src/cbapi/psc/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 475ed398..71d898c6 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -1,5 +1,5 @@ from cbapi.errors import ApiError, MoreThanOneResultError -from cbapi.psc.models import DismissStatusResponse, FacetFieldDTO +from cbapi.psc.models import DismissStatusResponse import logging import functools from six import string_types From ee3cf944e7e554765706ddcb9deaf1b8919f7f28 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 28 Oct 2019 14:35:30 -0600 Subject: [PATCH 039/197] renamed DismissStatusResponse to WorkflowStatus (as per Alex) --- src/cbapi/psc/models.py | 6 +++--- .../{dismiss_status_response.yaml => workflow_status.yaml} | 0 src/cbapi/psc/query.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) rename src/cbapi/psc/models/{dismiss_status_response.yaml => workflow_status.yaml} (100%) diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 2e711bef..684b56bc 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -312,13 +312,13 @@ def _query_implementation(cls, cb): return VMwareAlertSearchQuery(cls, cb) -class DismissStatusResponse(UnrefreshableModel): +class WorkflowStatus(UnrefreshableModel): urlobject_single = "/appservices/v6/orgs/{0}/workflow/status/{1}" primary_key = "id" - swagger_meta_file = "psc/models/dismiss_status_response.yaml" + swagger_meta_file = "psc/models/workflow_status.yaml" def __init__(self, cb, model_unique_id, initial_data=None): - super(DismissStatusResponse, self).__init__(cb, model_unique_id, initial_data) + super(WorkflowStatus, self).__init__(cb, model_unique_id, initial_data) self._workflow = Workflow(cb, initial_data.get("workflow", None)) if model_unique_id is not None and initial_data is None: self._refresh() diff --git a/src/cbapi/psc/models/dismiss_status_response.yaml b/src/cbapi/psc/models/workflow_status.yaml similarity index 100% rename from src/cbapi/psc/models/dismiss_status_response.yaml rename to src/cbapi/psc/models/workflow_status.yaml diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 71d898c6..71873cce 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -1,5 +1,5 @@ from cbapi.errors import ApiError, MoreThanOneResultError -from cbapi.psc.models import DismissStatusResponse +from cbapi.psc.models import WorkflowStatus import logging import functools from six import string_types @@ -1668,11 +1668,11 @@ def run(self): """ Executes the search query and alert state change operation. - :return: A DismissStatusResponse object that can be used for monitoring the progress + :return: A WorkflowStatus object that can be used for monitoring the progress of the operation. """ resp = self._cb.post_object(self._url(), body=self._build_request()) - return DismissStatusResponse(self._cb, resp["request_id"]) + return WorkflowStatus(self._cb, resp["request_id"]) class BulkUpdateAlerts(BulkUpdateAlertsBase, AlertCriteriaBuilderMixin, QueryBuilderSupportMixin): From c32bcc09c1cc0fcc9978b9dbfa891b5e28339fd6 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 28 Oct 2019 15:30:03 -0600 Subject: [PATCH 040/197] inserted a "shim" method to try and get around a multiple-imports problem --- src/cbapi/psc/query.py | 3 +-- src/cbapi/psc/rest_api.py | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 71873cce..dd1caa58 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -1,5 +1,4 @@ from cbapi.errors import ApiError, MoreThanOneResultError -from cbapi.psc.models import WorkflowStatus import logging import functools from six import string_types @@ -1672,7 +1671,7 @@ def run(self): of the operation. """ resp = self._cb.post_object(self._url(), body=self._build_request()) - return WorkflowStatus(self._cb, resp["request_id"]) + return self._cb._new_workflow_status(resp["request_id"]) class BulkUpdateAlerts(BulkUpdateAlertsBase, AlertCriteriaBuilderMixin, QueryBuilderSupportMixin): diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 88deb148..100fd47b 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -1,7 +1,7 @@ from cbapi.connection import BaseAPI from cbapi.errors import ApiError, ServerError from .cblr import LiveResponseSessionManager -from .models import Device +from .models import Device, WorkflowStatus from .query import BulkUpdateAlerts, BulkUpdateWatchlistAlerts, BulkUpdateThreatAlerts, \ BulkUpdateCBAnalyticsAlerts, BulkUpdateVMwareAlerts import logging @@ -152,6 +152,9 @@ def alert_search_suggestions(self, query): url = "/appservices/v6/orgs/{0}/alerts/search_suggestions".format(self.credentials.org_key) return self.get_object(url, query_params) + def _new_workflow_status(self, requestid): + return WorkflowStatus(self, requestid) + def _bulk_alert_update_query(self, state, querytype): cls = CbPSCBaseAPI.alert_update_queries.get(querytype, None) if cls is None: From 769e1aabbd89816b42cbce0fec81e94174669f13 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Wed, 30 Oct 2019 13:11:01 -0600 Subject: [PATCH 041/197] rework device tests to ensure that we start from a correct standpoint --- test/cbapi/psc/test_models.py | 85 ++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 16 deletions(-) diff --git a/test/cbapi/psc/test_models.py b/test/cbapi/psc/test_models.py index af8b930f..df1b1aa4 100755 --- a/test/cbapi/psc/test_models.py +++ b/test/cbapi/psc/test_models.py @@ -13,19 +13,36 @@ def request_session(self, sensor_id): self.was_called = True return { "itworks": True } -def test_Device_lr_session(): +def test_Device_lr_session(monkeypatch): + _device_data = {"id": 6023} + + def mock_get_object(url, parms=None, default=None): + nonlocal _device_data + assert url == "/appservices/v6/orgs/Z100/devices/6023" + return _device_data + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) sked = MockScheduler(6023) api._lr_scheduler = sked - dev = Device(api, 6023) + monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev = Device(api, 6023, _device_data) sess = dev.lr_session() assert sess["itworks"] assert sked.was_called def test_Device_background_scan(monkeypatch): + _device_data = {"id": 6023} _was_called = False + def mock_get_object(url, parms=None, default=None): + nonlocal _device_data + assert url == "/appservices/v6/orgs/Z100/devices/6023" + return _device_data + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -38,17 +55,23 @@ def mock_post_object(url, body, **kwargs): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - dev = Device(api, 6023) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "get_object", mock_get_object) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev = Device(api, 6023, _device_data) dev.background_scan(True) assert _was_called def test_Device_bypass(monkeypatch): + _device_data = {"id": 6023} _was_called = False + def mock_get_object(url, parms=None, default=None): + nonlocal _device_data + assert url == "/appservices/v6/orgs/Z100/devices/6023" + return _device_data + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -61,17 +84,23 @@ def mock_post_object(url, body, **kwargs): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - dev = Device(api, 6023) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "get_object", mock_get_object) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev = Device(api, 6023, _device_data) dev.bypass(False) assert _was_called def test_Device_delete_sensor(monkeypatch): + _device_data = {"id": 6023} _was_called = False + def mock_get_object(url, parms=None, default=None): + nonlocal _device_data + assert url == "/appservices/v6/orgs/Z100/devices/6023" + return _device_data + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -82,17 +111,23 @@ def mock_post_object(url, body, **kwargs): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - dev = Device(api, 6023) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "get_object", mock_get_object) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev = Device(api, 6023, _device_data) dev.delete_sensor() assert _was_called def test_Device_uninstall_sensor(monkeypatch): + _device_data = {"id": 6023} _was_called = False + def mock_get_object(url, parms=None, default=None): + nonlocal _device_data + assert url == "/appservices/v6/orgs/Z100/devices/6023" + return _device_data + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -103,17 +138,23 @@ def mock_post_object(url, body, **kwargs): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - dev = Device(api, 6023) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "get_object", mock_get_object) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev = Device(api, 6023, _device_data) dev.uninstall_sensor() assert _was_called def test_Device_quarantine(monkeypatch): + _device_data = {"id": 6023} _was_called = False + def mock_get_object(url, parms=None, default=None): + nonlocal _device_data + assert url == "/appservices/v6/orgs/Z100/devices/6023" + return _device_data + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -126,17 +167,23 @@ def mock_post_object(url, body, **kwargs): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - dev = Device(api, 6023) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "get_object", mock_get_object) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev = Device(api, 6023, _device_data) dev.quarantine(True) assert _was_called def test_Device_update_policy(monkeypatch): + _device_data = {"id": 6023} _was_called = False + def mock_get_object(url, parms=None, default=None): + nonlocal _device_data + assert url == "/appservices/v6/orgs/Z100/devices/6023" + return _device_data + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -149,17 +196,23 @@ def mock_post_object(url, body, **kwargs): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - dev = Device(api, 6023) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "get_object", mock_get_object) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev = Device(api, 6023, _device_data) dev.update_policy(8675309) assert _was_called def test_Device_update_sensor_version(monkeypatch): + _device_data = {"id": 6023} _was_called = False + def mock_get_object(url, parms=None, default=None): + nonlocal _device_data + assert url == "/appservices/v6/orgs/Z100/devices/6023" + return _device_data + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -173,11 +226,11 @@ def mock_post_object(url, body, **kwargs): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - dev = Device(api, 6023) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "get_object", mock_get_object) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + dev = Device(api, 6023, _device_data) dev.update_sensor_version({"RHEL": "2.3.4.5"}) assert _was_called \ No newline at end of file From 2930774183d177bce5b153b1a77e19461adc4921 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Thu, 31 Oct 2019 14:32:04 -0600 Subject: [PATCH 042/197] implemented the Alerts v6 unit tests and worked some kinks out of the actual code --- src/cbapi/psc/models.py | 18 +- src/cbapi/psc/query.py | 27 +- src/cbapi/psc/rest_api.py | 11 +- test/cbapi/psc/test_models.py | 160 +++++- test/cbapi/psc/test_rest_api.py | 904 +++++++++++++++++++++++++++++++- 5 files changed, 1086 insertions(+), 34 deletions(-) diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 684b56bc..6455f11e 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -225,7 +225,7 @@ def _refresh(self): return True @property - def workflow(self): + def workflow_(self): return self._workflow def _update_workflow_status(self, state, remediation, comment): @@ -237,7 +237,7 @@ def _update_workflow_status(self, state, remediation, comment): url = self.urlobject_single.format(self._cb.credentials.org_key, self._model_unique_id) + "/workflow" resp = self._cb.post_object(url, request) - self._workflow = Workflow(self._cb, resp) + self._workflow = Workflow(self._cb, resp.json()) self._last_refresh_time = time.time() def dismiss(self, remediation=None, comment=None): @@ -267,7 +267,7 @@ def _update_threat_workflow_status(self, state, remediation, comment): url = "/appservices/v6/orgs/{0}/threat/{1}/workflow".format(self._cb.credentials.org_key, self.threat_id) resp = self._cb.post_object(url, request) - return Workflow(self._cb, resp) + return Workflow(self._cb, resp.json()) def dismiss_threat(self, remediation=None, comment=None): """ @@ -312,15 +312,15 @@ def _query_implementation(cls, cb): return VMwareAlertSearchQuery(cls, cb) -class WorkflowStatus(UnrefreshableModel): +class WorkflowStatus(PSCMutableModel): urlobject_single = "/appservices/v6/orgs/{0}/workflow/status/{1}" primary_key = "id" swagger_meta_file = "psc/models/workflow_status.yaml" def __init__(self, cb, model_unique_id, initial_data=None): super(WorkflowStatus, self).__init__(cb, model_unique_id, initial_data) - self._workflow = Workflow(cb, initial_data.get("workflow", None)) - if model_unique_id is not None and initial_data is None: + self._workflow = None + if model_unique_id is not None: self._refresh() def _refresh(self): @@ -332,7 +332,11 @@ def _refresh(self): return True @property - def workflow(self): + def id_(self): + return self._model_unique_id + + @property + def workflow_(self): return self._workflow @property diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index dd1caa58..532df9fc 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -670,7 +670,7 @@ def device_os(self, device_os): self._update_criteria("device_os", device_os) return self - def device_os_version(self, device_os_versions): + def device_os_versions(self, device_os_versions): """ Restricts the alerts that this query is performed on to the specified device operating system versions. @@ -947,7 +947,7 @@ def kill_chain_statuses(self, statuses): "EXECUTE_GOAL", and "BREACH". :return: This instance. """ - if not all((status in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_categories) \ + if not all((status in CBAnalyticsAlertRequestCriteriaBuilder.valid_kill_chain_statuses) \ for status in statuses): raise ApiError("One or more invalid kill chain status values") self._update_criteria("kill_chain_status", statuses) @@ -991,7 +991,9 @@ def reason_code(self, reason): :param reason str: The reason code to look for. :return: This instance. """ - self._criteria["reason_code"] = reason + if not all(isinstance(t, str) for t in reason): + raise ApiError("One or more invalid reason code values") + self._update_criteria("reason_code", reason) return self def run_states(self, states): @@ -1153,7 +1155,7 @@ def device_os(self, device_os): self._criteria_builder.device_os(device_os) return self - def device_os_version(self, device_os_versions): + def device_os_versions(self, device_os_versions): """ Restricts the alerts that this query is performed on to the specified device operating system versions. @@ -1618,7 +1620,7 @@ def __init__(self, doc_class, cb): self._criteria_builder = CBAnalyticsAlertRequestCriteriaBuilder() -class VMwareAlertSearchQuery(BaseAlertSearchQuery, CBAnalyticsAlertCriteriaBuilderMixin): +class VMwareAlertSearchQuery(BaseAlertSearchQuery, VMwareAlertCriteriaBuilderMixin): """ Represents a query that is used to locate VMwareAlert objects. """ @@ -1671,7 +1673,8 @@ def run(self): of the operation. """ resp = self._cb.post_object(self._url(), body=self._build_request()) - return self._cb._new_workflow_status(resp["request_id"]) + output = resp.json() + return self._cb._new_workflow_status(output["request_id"]) class BulkUpdateAlerts(BulkUpdateAlertsBase, AlertCriteriaBuilderMixin, QueryBuilderSupportMixin): @@ -1684,10 +1687,10 @@ def __init__(self, cb, state): self._query_builder = QueryBuilder() def _url(self): - return "/v6/orgs/{0}/alerts/workflow/_criteria".format(self._cb.credentials.org_key) + return "/appservices/v6/orgs/{0}/alerts/workflow/_criteria".format(self._cb.credentials.org_key) def _build_request(self): - request = super().build_request() + request = super()._build_request() request["criteria"] = self._criteria_builder.build() request["query"] = self._query_builder._collapse() return request @@ -1702,7 +1705,7 @@ def __init__(self, cb, state): self._criteria_builder = CBAnalyticsAlertRequestCriteriaBuilder() def _url(self): - return "/v6/orgs/{0}/alerts/cbanalytics/workflow/_criteria".format(self._cb.credentials.org_key) + return "/appservices/v6/orgs/{0}/alerts/cbanalytics/workflow/_criteria".format(self._cb.credentials.org_key) class BulkUpdateVMwareAlerts(BulkUpdateAlerts, VMwareAlertCriteriaBuilderMixin): @@ -1714,7 +1717,7 @@ def __init__(self, cb, state): self._criteria_builder = VMwareAlertRequestCriteriaBuilder() def _url(self): - return "/v6/orgs/{0}/alerts/vmware/workflow/_criteria".format(self._cb.credentials.org_key) + return "/appservices/v6/orgs/{0}/alerts/vmware/workflow/_criteria".format(self._cb.credentials.org_key) class BulkUpdateWatchlistAlerts(BulkUpdateAlerts, WatchlistAlertCriteriaBuilderMixin): @@ -1726,7 +1729,7 @@ def __init__(self, cb, state): self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() def _url(self): - return "/v6/orgs/{0}/alerts/watchlist/workflow/_criteria".format(self._cb.credentials.org_key) + return "/appservices/v6/orgs/{0}/alerts/watchlist/workflow/_criteria".format(self._cb.credentials.org_key) class BulkUpdateThreatAlerts(BulkUpdateAlertsBase): @@ -1750,7 +1753,7 @@ def threat_ids(self, threats): return self def _url(self): - return "/v6/orgs/{0}/threat/workflow/_criteria".format(self._cb.credentials.org_key) + return "/appservices/v6/orgs/{0}/threat/workflow/_criteria".format(self._cb.credentials.org_key) def _build_request(self): request = super()._build_request() diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 100fd47b..914f0ede 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -1,7 +1,7 @@ from cbapi.connection import BaseAPI from cbapi.errors import ApiError, ServerError from .cblr import LiveResponseSessionManager -from .models import Device, WorkflowStatus +from .models import WorkflowStatus from .query import BulkUpdateAlerts, BulkUpdateWatchlistAlerts, BulkUpdateThreatAlerts, \ BulkUpdateCBAnalyticsAlerts, BulkUpdateVMwareAlerts import logging @@ -47,15 +47,6 @@ def _request_lr_session(self, sensor_id): # ---- Device API - def get_device(self, device_id): - """ - Locate a device with the specified device ID. - - :param int device_id: The ID of the device to look up. - :return: The new device object. - """ - return Device(self, device_id) - def _raw_device_action(self, request): url = "/appservices/v6/orgs/{0}/device_actions".format(self.credentials.org_key) resp = self.post_object(url, body=request) diff --git a/test/cbapi/psc/test_models.py b/test/cbapi/psc/test_models.py index df1b1aa4..d54d4a63 100755 --- a/test/cbapi/psc/test_models.py +++ b/test/cbapi/psc/test_models.py @@ -1,5 +1,5 @@ import pytest -from cbapi.psc.models import Device +from cbapi.psc.models import Device, BaseAlert, WorkflowStatus from cbapi.psc.rest_api import CbPSCBaseAPI from test.mocks import MockResponse, ConnectionMocks @@ -233,4 +233,162 @@ def mock_post_object(url, body, **kwargs): dev = Device(api, 6023, _device_data) dev.update_sensor_version({"RHEL": "2.3.4.5"}) assert _was_called + +def test_BaseAlert_dismiss(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/ESD14U2C/workflow" + assert body["state"] == "DISMISSED" + assert body["remediation_state"] == "Fixed" + assert body["comment"] == "Yessir" + _was_called = True + return MockResponse({"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "workflow":{"state": "OPEN"}}) + alert.dismiss("Fixed", "Yessir") + assert _was_called + assert alert.workflow_.changed_by == "Robocop" + assert alert.workflow_.state == "DISMISSED" + assert alert.workflow_.remediation == "Fixed" + assert alert.workflow_.comment == "Yessir" + assert alert.workflow_.last_update_time == "2019-10-31T16:03:13.951Z" + +def test_BaseAlert_undismiss(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/ESD14U2C/workflow" + assert body["state"] == "OPEN" + assert body["remediation_state"] == "Fixed" + assert body["comment"] == "NoSir" + _was_called = True + return MockResponse({"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "workflow":{"state": "DISMISS"}}) + alert.undismiss("Fixed", "NoSir") + assert _was_called + assert alert.workflow_.changed_by == "Robocop" + assert alert.workflow_.state == "OPEN" + assert alert.workflow_.remediation == "Fixed" + assert alert.workflow_.comment == "NoSir" + assert alert.workflow_.last_update_time == "2019-10-31T16:03:13.951Z" + +def test_BaseAlert_dismiss_threat(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/threat/B0RG/workflow" + assert body["state"] == "DISMISSED" + assert body["remediation_state"] == "Fixed" + assert body["comment"] == "Yessir" + _was_called = True + return MockResponse({"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "threat_id": "B0RG", "workflow":{"state": "OPEN"}}) + wf = alert.dismiss_threat("Fixed", "Yessir") + assert _was_called + assert wf.changed_by == "Robocop" + assert wf.state == "DISMISSED" + assert wf.remediation == "Fixed" + assert wf.comment == "Yessir" + assert wf.last_update_time == "2019-10-31T16:03:13.951Z" + +def test_BaseAlert_undismiss_threat(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/threat/B0RG/workflow" + assert body["state"] == "OPEN" + assert body["remediation_state"] == "Fixed" + assert body["comment"] == "NoSir" + _was_called = True + return MockResponse({"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "threat_id": "B0RG", "workflow":{"state": "OPEN"}}) + wf = alert.undismiss_threat("Fixed", "NoSir") + assert _was_called + assert wf.changed_by == "Robocop" + assert wf.state == "OPEN" + assert wf.remediation == "Fixed" + assert wf.comment == "NoSir" + assert wf.last_update_time == "2019-10-31T16:03:13.951Z" + +def test_WorkflowStatus(monkeypatch): + _times_called = 0 + + def mock_get_object(url, parms=None, default=None): + nonlocal _times_called + assert url == "/appservices/v6/orgs/Z100/workflow/status/W00K13" + if _times_called >= 0 and _times_called <= 3: + _stat = "QUEUED" + elif _times_called >= 4 and _times_called <= 6: + _stat = "IN_PROGRESS" + elif _times_called >= 7 and _times_called <= 9: + _stat = "FINISHED" + else: + pytest.fail("mock_get_object called too many times") + resp = {"errors": [], "failed_ids": [], "id": "W00K13", "num_hits": 0, "num_success": 0, "status": _stat} + resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} + _times_called = _times_called + 1 + return resp + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + wfstat = WorkflowStatus(api, "W00K13") + assert wfstat.workflow_.changed_by == "Robocop" + assert wfstat.workflow_.state == "DISMISSED" + assert wfstat.workflow_.remediation == "Fixed" + assert wfstat.workflow_.comment == "Yessir" + assert wfstat.workflow_.last_update_time == "2019-10-31T16:03:13.951Z" + assert _times_called == 1 + assert wfstat.queued + assert not wfstat.in_progress + assert not wfstat.finished + assert _times_called == 4 + assert not wfstat.queued + assert wfstat.in_progress + assert not wfstat.finished + assert _times_called == 7 + assert not wfstat.queued + assert not wfstat.in_progress + assert wfstat.finished + assert _times_called == 10 \ No newline at end of file diff --git a/test/cbapi/psc/test_rest_api.py b/test/cbapi/psc/test_rest_api.py index 016371fb..0a8e062a 100755 --- a/test/cbapi/psc/test_rest_api.py +++ b/test/cbapi/psc/test_rest_api.py @@ -1,6 +1,8 @@ import pytest from cbapi.errors import ApiError -from cbapi.psc.models import Device +from cbapi.psc.models import Device, BaseAlert, CBAnalyticsAlert, VMwareAlert, WatchlistAlert +from cbapi.psc.query import BulkUpdateAlerts, BulkUpdateWatchlistAlerts, BulkUpdateThreatAlerts, \ + BulkUpdateCBAnalyticsAlerts, BulkUpdateVMwareAlerts from cbapi.psc.rest_api import CbPSCBaseAPI from test.mocks import ConnectionMocks, MockResponse @@ -20,7 +22,7 @@ def mock_get_object(url): monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - rc = api.get_device(6023) + rc = api.select(Device, 6023) assert _was_called assert isinstance(rc, Device) assert rc.device_id == 6023 @@ -225,7 +227,7 @@ def mock_post_object(url, body, **kwargs): assert d.organization_name == "thistestworks" -def test_query_device_wth_last_contact_time_as_start_end(monkeypatch): +def test_query_device_with_last_contact_time_as_start_end(monkeypatch): _was_called = False def mock_post_object(url, body, **kwargs): @@ -556,4 +558,898 @@ def mock_post_object(url, body, **kwargs): api.select(Device).where("foobar").update_sensor_version({ "RHEL": "2.3.4.5"}) assert _was_called - \ No newline at end of file + +def test_bulk_query_types_return_ok(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + bquery = api.bulk_alert_dismiss("ALERT") + assert isinstance(bquery, BulkUpdateAlerts) + bquery = api.bulk_alert_dismiss("WATCHLIST") + assert isinstance(bquery, BulkUpdateWatchlistAlerts) + bquery = api.bulk_alert_dismiss("THREAT") + assert isinstance(bquery, BulkUpdateThreatAlerts) + bquery = api.bulk_alert_dismiss("CBANALYTICS") + assert isinstance(bquery, BulkUpdateCBAnalyticsAlerts) + bquery = api.bulk_alert_dismiss("VMWARE") + assert isinstance(bquery, BulkUpdateVMwareAlerts) + with pytest.raises(ApiError): + api.bulk_alert_dismiss("CRIMSON") + + +def test_query_basealert_with_all_bells_and_whistles(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/_search" + assert body["query"] == "Blort" + t = body["criteria"] + assert t["category"] == ["SERIOUS", "CRITICAL"] + assert t["device_id"] == [6023] + assert t["device_name"] == ["HAL"] + assert t["device_os"] == ["LINUX"] + assert t["device_os_version"] == ["0.1.2"] + assert t["device_username"] == ["JRN"] + assert t.get("group_results", False) + assert t["id"] == ["S0L0"] + assert t["legacy_alert_id"] == ["S0L0_1"] + assert t.get("minimum_severity", -1) == 6 + assert t["policy_id"] == [8675309] + assert t["policy_name"] == ["Strict"] + assert t["process_name"] == ["IEXPLORE.EXE"] + assert t["process_sha256"] == ["0123456789ABCDEF0123456789ABCDEF"] + assert t["reputation"] == ["SUSPECT_MALWARE"] + assert t["tag"] == ["Frood"] + assert t["target_value"] == ["HIGH"] + assert t["threat_id"] == ["B0RG"] + assert t["type"] == ["WATCHLIST"] + assert t["workflow"] == ["OPEN"] + t = body["sort"] + t2 = t[0] + assert t2["field"] == "name" + assert t2["order"] == "DESC" + _was_called = True + body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} + envelope = { "results": [ body ], "num_found": 1 } + return MockResponse(envelope) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(BaseAlert).where("Blort").categories(["SERIOUS", "CRITICAL"]).device_ids([6023]) \ + .device_names(["HAL"]).device_os(["LINUX"]).device_os_versions(["0.1.2"]).device_username(["JRN"]) \ + .group_results(True).alert_ids(["S0L0"]).legacy_alert_ids(["S0L0_1"]).minimum_severity(6) \ + .policy_ids([8675309]).policy_names(["Strict"]).process_names(["IEXPLORE.EXE"]) \ + .process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).reputations(["SUSPECT_MALWARE"]) \ + .tags(["Frood"]).target_priorities(["HIGH"]).threat_ids(["B0RG"]).types(["WATCHLIST"]) \ + .workflows(["OPEN"]).sort_by("name", "DESC") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_basealert_with_create_time_as_start_end(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/_search" + assert body["query"] == "Blort" + t = body["criteria"] + t2 = t.get("create_time", {}) + assert t2["start"] == "2019-09-30T12:34:56" + assert t2["end"] == "2019-10-01T12:00:12" + _was_called = True + body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} + envelope = { "results": [ body ], "num_found": 1 } + return MockResponse(envelope) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(BaseAlert).where("Blort") \ + .create_time(start="2019-09-30T12:34:56", end="2019-10-01T12:00:12") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_basealert_with_create_time_as_range(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/_search" + assert body["query"] == "Blort" + t = body["criteria"] + t2 = t.get("create_time", {}) + assert t2["range"] == "-3w" + _was_called = True + body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} + envelope = { "results": [ body ], "num_found": 1 } + return MockResponse(envelope) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(BaseAlert).where("Blort").create_time(range="-3w") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_basealert_facets(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/_facet" + assert body["query"] == "Blort" + t = body["criteria"] + assert t["workflow"] == ["OPEN"] + t = body["terms"] + assert t["rows"] == 0 + assert t["fields"] == ["REPUTATION", "STATUS"] + _was_called = True + dto1 = {"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]} + dto2 = {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]} + return MockResponse({"results": [dto1, dto2]}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(BaseAlert).where("Blort").workflows(["OPEN"]) + f = query.facets(["REPUTATION", "STATUS"]) + assert _was_called + t = f[0] + assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] + t = f[1] + assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] + + +def test_query_basealert_invalid_category(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).categories(["DOUBLE_DARE"]) + + +def test_query_basealert_create_time_no_params_ok(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).create_time() + + +def test_query_basealert_create_time_range_specified_bad(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).create_time(start="2019-09-30T12:34:56", \ + end="2019-10-01T12:00:12", range="-3w") + + +def test_query_basealert_create_time_start_specified_bad(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).create_time(start="2019-09-30T12:34:56", range="-3w") + + +def test_query_basealert_create_time_end_specified_bad(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).create_time(end="2019-10-01T12:00:12", range="-3w") + + +def test_query_basealert_invalid_device_ids(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).device_ids(["Bogus"]) + + +def test_query_basealert_invalid_device_names(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).device_names([42]) + + +def test_query_basealert_invalid_device_os(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).device_os(["TI994A"]) + + +def test_query_basealert_invalid_device_os_versions(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).device_os_versions([8808]) + + +def test_query_basealert_invalid_device_username(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).device_username([-1]) + + +def test_query_basealert_invalid_alert_ids(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).alert_ids([9001]) + + +def test_query_basealert_invalid_legacy_alert_ids(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).legacy_alert_ids([9001]) + + +def test_query_basealert_invalid_policy_ids(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).policy_ids(["Bogus"]) + + +def test_query_basealert_invalid_policy_names(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).policy_names([323]) + + +def test_query_basealert_invalid_process_names(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).process_names([7071]) + + +def test_query_basealert_invalid_process_sha256(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).process_sha256([123456789]) + + +def test_query_basealert_invalid_reputations(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).reputations(["MICROSOFT_FUDWARE"]) + + +def test_query_basealert_invalid_tags(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).tags([990909]) + + +def test_query_basealert_invalid_target_priorities(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).target_priorities(["DOGWASH"]) + + +def test_query_basealert_invalid_threat_ids(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).threat_ids([4096]) + + +def test_query_basealert_invalid_types(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).types(["ERBOSOFT"]) + + +def test_query_basealert_invalid_workflows(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).workflows(["IN_LIMBO"]) + + +def test_query_cbanalyticsalert_with_all_bells_and_whistles(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/_search" + assert body["query"] == "Blort" + t = body["criteria"] + assert t["category"] == ["SERIOUS", "CRITICAL"] + assert t["device_id"] == [6023] + assert t["device_name"] == ["HAL"] + assert t["device_os"] == ["LINUX"] + assert t["device_os_version"] == ["0.1.2"] + assert t["device_username"] == ["JRN"] + assert t.get("group_results", False) + assert t["id"] == ["S0L0"] + assert t["legacy_alert_id"] == ["S0L0_1"] + assert t.get("minimum_severity", -1) == 6 + assert t["policy_id"] == [8675309] + assert t["policy_name"] == ["Strict"] + assert t["process_name"] == ["IEXPLORE.EXE"] + assert t["process_sha256"] == ["0123456789ABCDEF0123456789ABCDEF"] + assert t["reputation"] == ["SUSPECT_MALWARE"] + assert t["tag"] == ["Frood"] + assert t["target_value"] == ["HIGH"] + assert t["threat_id"] == ["B0RG"] + assert t["type"] == ["WATCHLIST"] + assert t["workflow"] == ["OPEN"] + assert t["blocked_threat_category"] == ["RISKY_PROGRAM"] + assert t["device_location"] == ["ONSITE"] + assert t["kill_chain_status"] == ["EXECUTE_GOAL"] + assert t["not_blocked_threat_category"] == ["NEW_MALWARE"] + assert t["policy_applied"] == ["APPLIED"] + assert t["reason_code"] == ["ATTACK_VECTOR"] + assert t["run_state"] == ["RAN"] + assert t["sensor_action"] == ["DENY"] + assert t["threat_cause_vector"] == ["WEB"] + + t = body["sort"] + t2 = t[0] + assert t2["field"] == "name" + assert t2["order"] == "DESC" + _was_called = True + body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} + envelope = { "results": [ body ], "num_found": 1 } + return MockResponse(envelope) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(CBAnalyticsAlert).where("Blort").categories(["SERIOUS", "CRITICAL"]).device_ids([6023]) \ + .device_names(["HAL"]).device_os(["LINUX"]).device_os_versions(["0.1.2"]).device_username(["JRN"]) \ + .group_results(True).alert_ids(["S0L0"]).legacy_alert_ids(["S0L0_1"]).minimum_severity(6) \ + .policy_ids([8675309]).policy_names(["Strict"]).process_names(["IEXPLORE.EXE"]) \ + .process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).reputations(["SUSPECT_MALWARE"]) \ + .tags(["Frood"]).target_priorities(["HIGH"]).threat_ids(["B0RG"]).types(["WATCHLIST"]) \ + .workflows(["OPEN"]).blocked_threat_categories(["RISKY_PROGRAM"]).device_locations(["ONSITE"]) \ + .kill_chain_statuses(["EXECUTE_GOAL"]).not_blocked_threat_categories(["NEW_MALWARE"]) \ + .policy_applied(["APPLIED"]).reason_code(["ATTACK_VECTOR"]).run_states(["RAN"]) \ + .sensor_actions(["DENY"]).threat_cause_vectors(["WEB"]).sort_by("name", "DESC") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_cbanalyticsalert_facets(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/_facet" + assert body["query"] == "Blort" + t = body["criteria"] + assert t["workflow"] == ["OPEN"] + t = body["terms"] + assert t["rows"] == 0 + assert t["fields"] == ["REPUTATION", "STATUS"] + _was_called = True + dto1 = {"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]} + dto2 = {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]} + return MockResponse({"results": [dto1, dto2]}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(CBAnalyticsAlert).where("Blort").workflows(["OPEN"]) + f = query.facets(["REPUTATION", "STATUS"]) + assert _was_called + t = f[0] + assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] + t = f[1] + assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] + + +def test_query_cbanalyticsalert_invalid_blocked_threat_categories(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(CBAnalyticsAlert).blocked_threat_categories(["MINOR"]) + + +def test_query_cbanalyticsalert_invalid_device_locations(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(CBAnalyticsAlert).device_locations(["NARNIA"]) + + +def test_query_cbanalyticsalert_invalid_kill_chain_statuses(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(CBAnalyticsAlert).kill_chain_statuses(["SPAWN_COPIES"]) + + +def test_query_cbanalyticsalert_invalid_not_blocked_threat_categories(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(CBAnalyticsAlert).not_blocked_threat_categories(["MINOR"]) + + +def test_query_cbanalyticsalert_invalid_policy_applied(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(CBAnalyticsAlert).policy_applied(["MAYBE"]) + + +def test_query_cbanalyticsalert_invalid_reason_code(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(CBAnalyticsAlert).reason_code([55]) + + +def test_query_cbanalyticsalert_invalid_run_states(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(CBAnalyticsAlert).run_states(["MIGHT_HAVE"]) + + +def test_query_cbanalyticsalert_invalid_sensor_actions(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(CBAnalyticsAlert).sensor_actions(["FLIP_A_COIN"]) + + +def test_query_cbanalyticsalert_invalid_threat_cause_vectors(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(CBAnalyticsAlert).threat_cause_vectors(["NETWORK"]) + + +def test_query_vmwarealert_with_all_bells_and_whistles(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/vmware/_search" + assert body["query"] == "Blort" + t = body["criteria"] + assert t["category"] == ["SERIOUS", "CRITICAL"] + assert t["device_id"] == [6023] + assert t["device_name"] == ["HAL"] + assert t["device_os"] == ["LINUX"] + assert t["device_os_version"] == ["0.1.2"] + assert t["device_username"] == ["JRN"] + assert t.get("group_results", False) + assert t["id"] == ["S0L0"] + assert t["legacy_alert_id"] == ["S0L0_1"] + assert t.get("minimum_severity", -1) == 6 + assert t["policy_id"] == [8675309] + assert t["policy_name"] == ["Strict"] + assert t["process_name"] == ["IEXPLORE.EXE"] + assert t["process_sha256"] == ["0123456789ABCDEF0123456789ABCDEF"] + assert t["reputation"] == ["SUSPECT_MALWARE"] + assert t["tag"] == ["Frood"] + assert t["target_value"] == ["HIGH"] + assert t["threat_id"] == ["B0RG"] + assert t["type"] == ["WATCHLIST"] + assert t["workflow"] == ["OPEN"] + assert t["group_id"] == [14] + t = body["sort"] + t2 = t[0] + assert t2["field"] == "name" + assert t2["order"] == "DESC" + _was_called = True + body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} + envelope = { "results": [ body ], "num_found": 1 } + return MockResponse(envelope) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(VMwareAlert).where("Blort").categories(["SERIOUS", "CRITICAL"]).device_ids([6023]) \ + .device_names(["HAL"]).device_os(["LINUX"]).device_os_versions(["0.1.2"]).device_username(["JRN"]) \ + .group_results(True).alert_ids(["S0L0"]).legacy_alert_ids(["S0L0_1"]).minimum_severity(6) \ + .policy_ids([8675309]).policy_names(["Strict"]).process_names(["IEXPLORE.EXE"]) \ + .process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).reputations(["SUSPECT_MALWARE"]) \ + .tags(["Frood"]).target_priorities(["HIGH"]).threat_ids(["B0RG"]).types(["WATCHLIST"]) \ + .workflows(["OPEN"]).group_ids([14]).sort_by("name", "DESC") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_vmwarealert_facets(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/vmware/_facet" + assert body["query"] == "Blort" + t = body["criteria"] + assert t["workflow"] == ["OPEN"] + t = body["terms"] + assert t["rows"] == 0 + assert t["fields"] == ["REPUTATION", "STATUS"] + _was_called = True + dto1 = {"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]} + dto2 = {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]} + return MockResponse({"results": [dto1, dto2]}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(VMwareAlert).where("Blort").workflows(["OPEN"]) + f = query.facets(["REPUTATION", "STATUS"]) + assert _was_called + t = f[0] + assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] + t = f[1] + assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] + + +def test_query_vmwarealert_invalid_group_ids(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(VMwareAlert).group_ids(["Bogus"]) + + +def test_query_watchlistalert_with_all_bells_and_whistles(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/_search" + assert body["query"] == "Blort" + t = body["criteria"] + assert t["category"] == ["SERIOUS", "CRITICAL"] + assert t["device_id"] == [6023] + assert t["device_name"] == ["HAL"] + assert t["device_os"] == ["LINUX"] + assert t["device_os_version"] == ["0.1.2"] + assert t["device_username"] == ["JRN"] + assert t.get("group_results", False) + assert t["id"] == ["S0L0"] + assert t["legacy_alert_id"] == ["S0L0_1"] + assert t.get("minimum_severity", -1) == 6 + assert t["policy_id"] == [8675309] + assert t["policy_name"] == ["Strict"] + assert t["process_name"] == ["IEXPLORE.EXE"] + assert t["process_sha256"] == ["0123456789ABCDEF0123456789ABCDEF"] + assert t["reputation"] == ["SUSPECT_MALWARE"] + assert t["tag"] == ["Frood"] + assert t["target_value"] == ["HIGH"] + assert t["threat_id"] == ["B0RG"] + assert t["type"] == ["WATCHLIST"] + assert t["workflow"] == ["OPEN"] + assert t["watchlist_id"] == ["100"] + assert t["watchlist_name"] == ["Gandalf"] + t = body["sort"] + t2 = t[0] + assert t2["field"] == "name" + assert t2["order"] == "DESC" + _was_called = True + body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} + envelope = { "results": [ body ], "num_found": 1 } + return MockResponse(envelope) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(WatchlistAlert).where("Blort").categories(["SERIOUS", "CRITICAL"]).device_ids([6023]) \ + .device_names(["HAL"]).device_os(["LINUX"]).device_os_versions(["0.1.2"]).device_username(["JRN"]) \ + .group_results(True).alert_ids(["S0L0"]).legacy_alert_ids(["S0L0_1"]).minimum_severity(6) \ + .policy_ids([8675309]).policy_names(["Strict"]).process_names(["IEXPLORE.EXE"]) \ + .process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).reputations(["SUSPECT_MALWARE"]) \ + .tags(["Frood"]).target_priorities(["HIGH"]).threat_ids(["B0RG"]).types(["WATCHLIST"]) \ + .workflows(["OPEN"]).watchlist_ids(["100"]).watchlist_names(["Gandalf"]).sort_by("name", "DESC") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_watchlistalert_facets(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/_facet" + assert body["query"] == "Blort" + t = body["criteria"] + assert t["workflow"] == ["OPEN"] + t = body["terms"] + assert t["rows"] == 0 + assert t["fields"] == ["REPUTATION", "STATUS"] + _was_called = True + dto1 = {"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]} + dto2 = {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]} + return MockResponse({"results": [dto1, dto2]}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + query = api.select(WatchlistAlert).where("Blort").workflows(["OPEN"]) + f = query.facets(["REPUTATION", "STATUS"]) + assert _was_called + t = f[0] + assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] + t = f[1] + assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] + + +def test_query_watchlistalert_invalid_watchlist_ids(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(WatchlistAlert).watchlist_ids([888]) + + +def test_query_watchlistalert_invalid_watchlist_names(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(WatchlistAlert).watchlist_names([69]) + + +def test_alerts_bulk_dismiss(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/workflow/_criteria" + assert body["query"] == "Blort" + assert body["state"] == "DISMISSED" + assert body["remediation_state"] == "Fixed" + assert body["comment"] == "Yessir" + t = body["criteria"] + assert t["device_name"] == ["HAL9000"] + _was_called = True + return MockResponse({"request_id": "497ABX"}) + + def mock_get_object(url, parms=None, default=None): + assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" + resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} + resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} + return resp + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + q = api.bulk_alert_dismiss("ALERT").remediation("Fixed").comment("Yessir") + wstat = q.where("Blort").device_names(["HAL9000"]).run() + assert _was_called + assert wstat.id_ == "497ABX" + + +def test_alerts_bulk_undismiss(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/workflow/_criteria" + assert body["query"] == "Blort" + assert body["state"] == "OPEN" + assert body["remediation_state"] == "Fixed" + assert body["comment"] == "NoSir" + t = body["criteria"] + assert t["device_name"] == ["HAL9000"] + _was_called = True + return MockResponse({"request_id": "497ABX"}) + + def mock_get_object(url, parms=None, default=None): + assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" + resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} + resp["workflow"] = {"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} + return resp + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + q = api.bulk_alert_undismiss("ALERT").remediation("Fixed").comment("NoSir") + wstat = q.where("Blort").device_names(["HAL9000"]).run() + assert _was_called + assert wstat.id_ == "497ABX" + + +def test_alerts_bulk_dismiss_watchlist(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/workflow/_criteria" + assert body["query"] == "Blort" + assert body["state"] == "DISMISSED" + assert body["remediation_state"] == "Fixed" + assert body["comment"] == "Yessir" + t = body["criteria"] + assert t["device_name"] == ["HAL9000"] + _was_called = True + return MockResponse({"request_id": "497ABX"}) + + def mock_get_object(url, parms=None, default=None): + assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" + resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} + resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} + return resp + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + q = api.bulk_alert_dismiss("WATCHLIST").remediation("Fixed").comment("Yessir") + wstat = q.where("Blort").device_names(["HAL9000"]).run() + assert _was_called + assert wstat.id_ == "497ABX" + + +def test_alerts_bulk_dismiss_cbanalytics(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/workflow/_criteria" + assert body["query"] == "Blort" + assert body["state"] == "DISMISSED" + assert body["remediation_state"] == "Fixed" + assert body["comment"] == "Yessir" + t = body["criteria"] + assert t["device_name"] == ["HAL9000"] + _was_called = True + return MockResponse({"request_id": "497ABX"}) + + def mock_get_object(url, parms=None, default=None): + assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" + resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} + resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} + return resp + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + q = api.bulk_alert_dismiss("CBANALYTICS").remediation("Fixed").comment("Yessir") + wstat = q.where("Blort").device_names(["HAL9000"]).run() + assert _was_called + assert wstat.id_ == "497ABX" + + +def test_alerts_bulk_dismiss_vmware(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/vmware/workflow/_criteria" + assert body["query"] == "Blort" + assert body["state"] == "DISMISSED" + assert body["remediation_state"] == "Fixed" + assert body["comment"] == "Yessir" + t = body["criteria"] + assert t["device_name"] == ["HAL9000"] + _was_called = True + return MockResponse({"request_id": "497ABX"}) + + def mock_get_object(url, parms=None, default=None): + assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" + resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} + resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} + return resp + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + q = api.bulk_alert_dismiss("VMWARE").remediation("Fixed").comment("Yessir") + wstat = q.where("Blort").device_names(["HAL9000"]).run() + assert _was_called + assert wstat.id_ == "497ABX" + + +def test_alerts_bulk_dismiss_threat(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/threat/workflow/_criteria" + assert body["threat_id"] == ["B0RG", "F3R3NG1"] + assert body["state"] == "DISMISSED" + assert body["remediation_state"] == "Fixed" + assert body["comment"] == "Yessir" + _was_called = True + return MockResponse({"request_id": "497ABX"}) + + def mock_get_object(url, parms=None, default=None): + assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" + resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} + resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} + return resp + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + q = api.bulk_alert_dismiss("THREAT").remediation("Fixed").comment("Yessir") + wstat = q.threat_ids(["B0RG", "F3R3NG1"]).run() + assert _was_called + assert wstat.id_ == "497ABX" From f1f573dc800a8cb2fae9b32990a4b862b659741c Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Thu, 31 Oct 2019 14:51:05 -0600 Subject: [PATCH 043/197] de-flake8'ed all the new code that was added --- src/cbapi/psc/models.py | 46 +++--- src/cbapi/psc/query.py | 318 +++++++++++++++++++------------------- src/cbapi/psc/rest_api.py | 25 ++- 3 files changed, 194 insertions(+), 195 deletions(-) diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 6455f11e..1c04c608 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -128,7 +128,7 @@ def _refresh(self): self._info = resp self._last_refresh_time = time.time() return True - + def lr_session(self): """ Retrieve a Live Response session object for this Device. @@ -211,11 +211,11 @@ def __init__(self, cb, model_unique_id, initial_data=None): self._workflow = Workflow(cb, initial_data.get("workflow", None)) if model_unique_id is not None and initial_data is None: self._refresh() - + @classmethod def _query_implementation(cls, cb): return BaseAlertSearchQuery(cls, cb) - + def _refresh(self): url = self.urlobject_single.format(self._cb.credentials.org_key, self._model_unique_id) resp = self._cb.get_object(url) @@ -227,9 +227,9 @@ def _refresh(self): @property def workflow_(self): return self._workflow - + def _update_workflow_status(self, state, remediation, comment): - request = {"state" : state} + request = {"state": state} if remediation: request["remediation_state"] = remediation if comment: @@ -239,11 +239,11 @@ def _update_workflow_status(self, state, remediation, comment): resp = self._cb.post_object(url, request) self._workflow = Workflow(self._cb, resp.json()) self._last_refresh_time = time.time() - + def dismiss(self, remediation=None, comment=None): """ Dismiss this alert. - + :param remediation str: The remediation status to set for the alert. :param comment str: The comment to set for the alert. """ @@ -252,14 +252,14 @@ def dismiss(self, remediation=None, comment=None): def undismiss(self, remediation=None, comment=None): """ Un-dismiss this alert. - + :param remediation str: The remediation status to set for the alert. :param comment str: The comment to set for the alert. """ self._update_workflow_status("OPEN", remediation, comment) - + def _update_threat_workflow_status(self, state, remediation, comment): - request = {"state" : state} + request = {"state": state} if remediation: request["remediation_state"] = remediation if comment: @@ -268,11 +268,11 @@ def _update_threat_workflow_status(self, state, remediation, comment): self.threat_id) resp = self._cb.post_object(url, request) return Workflow(self._cb, resp.json()) - + def dismiss_threat(self, remediation=None, comment=None): """ Dismiss alerts for this threat. - + :param remediation str: The remediation status to set for the alert. :param comment str: The comment to set for the alert. """ @@ -281,12 +281,12 @@ def dismiss_threat(self, remediation=None, comment=None): def undismiss_threat(self, remediation=None, comment=None): """ Un-dismiss alerts for this threat. - + :param remediation str: The remediation status to set for the alert. :param comment str: The comment to set for the alert. """ return self._update_threat_workflow_status("OPEN", remediation, comment) - + class WatchlistAlert(BaseAlert): urlobject = "/appservices/v6/orgs/{0}/alerts/watchlist" @@ -302,16 +302,16 @@ class CBAnalyticsAlert(BaseAlert): @classmethod def _query_implementation(cls, cb): return CBAnalyticsAlertSearchQuery(cls, cb) - - + + class VMwareAlert(BaseAlert): urlobject = "/appservices/v6/orgs/{0}/alerts/vmware" @classmethod def _query_implementation(cls, cb): return VMwareAlertSearchQuery(cls, cb) - - + + class WorkflowStatus(PSCMutableModel): urlobject_single = "/appservices/v6/orgs/{0}/workflow/status/{1}" primary_key = "id" @@ -322,7 +322,7 @@ def __init__(self, cb, model_unique_id, initial_data=None): self._workflow = None if model_unique_id is not None: self._refresh() - + def _refresh(self): url = self.urlobject_single.format(self._cb.credentials.org_key, self._model_unique_id) resp = self._cb.get_object(url) @@ -330,11 +330,11 @@ def _refresh(self): self._workflow = Workflow(self._cb, resp.get("workflow", None)) self._last_refresh_time = time.time() return True - + @property def id_(self): return self._model_unique_id - + @property def workflow_(self): return self._workflow @@ -343,12 +343,12 @@ def workflow_(self): def queued(self): self._refresh() return self._info.get("status", "") == "QUEUED" - + @property def in_progress(self): self._refresh() return self._info.get("status", "") == "IN_PROGRESS" - + @property def finished(self): self._refresh() diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 532df9fc..771b50da 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -571,7 +571,7 @@ def update_sensor_version(self, sensor_version): """ return self._bulk_device_action("UPDATE_SENSOR_VERSION", {"sensor_version": sensor_version}) - + class AlertRequestCriteriaBuilder: """ @@ -582,19 +582,19 @@ class AlertRequestCriteriaBuilder: "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", "COMPANY_BLACK_LIST"] valid_alerttypes = ["CB_ANALYTICS", "VMWARE", "WATCHLIST"] valid_workflow_vals = ["OPEN", "DISMISSED"] - + def __init__(self): self._criteria = {} self._time_filter = {} - + def _update_criteria(self, key, newlist): oldlist = self._criteria.get(key, []) self._criteria[key] = oldlist + newlist - + def categories(self, cats): """ Restricts the alerts that this query is performed on to the specified categories. - + :param cats list: List of categories to be restricted to. Valid categories are "THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", and "CRITICAL." :return: This instance @@ -603,7 +603,7 @@ def categories(self, cats): raise ApiError("One or more invalid category values") self._update_criteria("category", cats) return self - + def create_time(self, *args, **kwargs): """ Restricts the alerts that this query is performed on to the specified @@ -655,8 +655,8 @@ def device_names(self, device_names): raise ApiError("One or more invalid device names") self._update_criteria("device_name", device_names) return self - - def device_os(self, device_os): + + def device_os(self, device_os): """ Restricts the alerts that this query is performed on to the specified device operating systems. @@ -669,7 +669,7 @@ def device_os(self, device_os): raise ApiError("One or more invalid operating systems") self._update_criteria("device_os", device_os) return self - + def device_os_versions(self, device_os_versions): """ Restricts the alerts that this query is performed on to the specified @@ -682,7 +682,7 @@ def device_os_versions(self, device_os_versions): raise ApiError("One or more invalid device OS versions") self._update_criteria("device_os_version", device_os_versions) return self - + def device_username(self, users): """ Restricts the alerts that this query is performed on to the specified @@ -695,17 +695,17 @@ def device_username(self, users): raise ApiError("One or more invalid user names") self._update_criteria("device_username", users) return self - + def group_results(self, flag): """ Specifies whether or not to group the results of the query. - + :param flag boolean: True to group the results, False to not do so. :return: This instance """ self._criteria["group_results"] = True if flag else False return self - + def alert_ids(self, alert_ids): """ Restricts the alerts that this query is performed on to the specified @@ -718,7 +718,7 @@ def alert_ids(self, alert_ids): raise ApiError("One or more invalid alert ID values") self._update_criteria("id", alert_ids) return self - + def legacy_alert_ids(self, alert_ids): """ Restricts the alerts that this query is performed on to the specified @@ -731,18 +731,18 @@ def legacy_alert_ids(self, alert_ids): raise ApiError("One or more invalid alert ID values") self._update_criteria("legacy_alert_id", alert_ids) return self - + def minimum_severity(self, severity): """ Restricts the alerts that this query is performed on to the specified minimum severity level. - + :param severity int: The minimum severity level for alerts. :return: This instance """ self._criteria["minimum_severity"] = severity return self - + def policy_ids(self, policy_ids): """ Restricts the alerts that this query is performed on to the specified @@ -768,7 +768,7 @@ def policy_names(self, policy_names): raise ApiError("One or more invalid policy names") self._update_criteria("policy_name", policy_names) return self - + def process_names(self, process_names): """ Restricts the alerts that this query is performed on to the specified @@ -781,7 +781,7 @@ def process_names(self, process_names): raise ApiError("One or more invalid process names") self._update_criteria("process_name", process_names) return self - + def process_sha256(self, shas): """ Restricts the alerts that this query is performed on to the specified @@ -794,7 +794,7 @@ def process_sha256(self, shas): raise ApiError("One or more invalid SHA256 values") self._update_criteria("process_sha256", shas) return self - + def reputations(self, reps): """ Restricts the alerts that this query is performed on to the specified @@ -810,7 +810,7 @@ def reputations(self, reps): raise ApiError("One or more invalid reputation values") self._update_criteria("reputation", reps) return self - + def tags(self, tags): """ Restricts the alerts that this query is performed on to the specified @@ -823,7 +823,7 @@ def tags(self, tags): raise ApiError("One or more invalid tags") self._update_criteria("tag", tags) return self - + def target_priorities(self, priorities): """ Restricts the alerts that this query is performed on to the specified @@ -837,7 +837,7 @@ def target_priorities(self, priorities): raise ApiError("One or more invalid priority values") self._update_criteria("target_value", priorities) return self - + def threat_ids(self, threats): """ Restricts the alerts that this query is performed on to the specified @@ -850,7 +850,7 @@ def threat_ids(self, threats): raise ApiError("One or more invalid threat ID values") self._update_criteria("threat_id", threats) return self - + def types(self, alerttypes): """ Restricts the alerts that this query is performed on to the specified @@ -864,7 +864,7 @@ def types(self, alerttypes): raise ApiError("One or more invalid alert type values") self._update_criteria("type", alerttypes) return self - + def workflows(self, workflow_vals): """ Restricts the alerts that this query is performed on to the specified @@ -878,20 +878,20 @@ def workflows(self, workflow_vals): raise ApiError("One or more invalid workflow status values") self._update_criteria("workflow", workflow_vals) return self - + def build(self): """ Builds the criteria object for use in a query. - + :return: The criteria object. """ mycrit = self._criteria if self._time_filter: mycrit["create_time"] = self._time_filter return mycrit - - -class CBAnalyticsAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): + + +class CBAnalyticsAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): """ Auxiliary object that builds the criteria for CB Analytics alert request searches. """ @@ -904,90 +904,91 @@ class CBAnalyticsAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): valid_sensor_actions = ["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", "TERMINATE", "DENY"] valid_threat_cause_vectors = ["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", "APP_STORE", "THIRD_PARTY"] + def __init__(self): super().__init__() - + def blocked_threat_categories(self, categories): """ Restricts the alerts that this query is performed on to the specified threat categories that were blocked. - + :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". :return: This instance. """ - if not all((category in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_categories) \ + if not all((category in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_categories) for category in categories): raise ApiError("One or more invalid threat categories") self._update_criteria("blocked_threat_category", categories) return self - + def device_locations(self, locations): """ Restricts the alerts that this query is performed on to the specified device locations. - + :param locations list: List of device locations to look for. Valid values are "ONSITE", "OFFSITE", - and "UNKNOWN". + and "UNKNOWN". :return: This instance. """ - if not all((location in CBAnalyticsAlertRequestCriteriaBuilder.valid_locations) \ + if not all((location in CBAnalyticsAlertRequestCriteriaBuilder.valid_locations) for location in locations): raise ApiError("One or more invalid device locations") self._update_criteria("device_location", locations) return self - + def kill_chain_statuses(self, statuses): """ Restricts the alerts that this query is performed on to the specified kill chain statuses. - + :param statuses list: List of kill chain statuses to look for. Valid values are "RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", "INSTALL_RUN","COMMAND_AND_CONTROL", - "EXECUTE_GOAL", and "BREACH". + "EXECUTE_GOAL", and "BREACH". :return: This instance. """ - if not all((status in CBAnalyticsAlertRequestCriteriaBuilder.valid_kill_chain_statuses) \ + if not all((status in CBAnalyticsAlertRequestCriteriaBuilder.valid_kill_chain_statuses) for status in statuses): raise ApiError("One or more invalid kill chain status values") self._update_criteria("kill_chain_status", statuses) return self - + def not_blocked_threat_categories(self, categories): """ Restricts the alerts that this query is performed on to the specified threat categories that were NOT blocked. - + :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". :return: This instance. """ - if not all((category in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_categories) \ + if not all((category in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_categories) for category in categories): raise ApiError("One or more invalid threat categories") self._update_criteria("not_blocked_threat_category", categories) return self - + def policy_applied(self, applied_statuses): """ Restricts the alerts that this query is performed on to the specified status values showing whether policies were applied. - + :param applied_statuses list: List of status values to look for. Valid values are - "APPLIED" and "NOT_APPLIED". + "APPLIED" and "NOT_APPLIED". :return: This instance. """ - if not all((s in CBAnalyticsAlertRequestCriteriaBuilder.valid_policy_applied) \ + if not all((s in CBAnalyticsAlertRequestCriteriaBuilder.valid_policy_applied) for s in applied_statuses): raise ApiError("One or more invalid policy-applied values") self._update_criteria("policy_applied", applied_statuses) return self - + def reason_code(self, reason): """ Restricts the alerts that this query is performed on to the specified reason code (enum value). - + :param reason str: The reason code to look for. :return: This instance. """ @@ -995,63 +996,63 @@ def reason_code(self, reason): raise ApiError("One or more invalid reason code values") self._update_criteria("reason_code", reason) return self - + def run_states(self, states): """ Restricts the alerts that this query is performed on to the specified run states. - + :param states list: List of run states to look for. Valid values are "DID_NOT_RUN", "RAN", - and "UNKNOWN". + and "UNKNOWN". :return: This instance. """ - if not all((s in CBAnalyticsAlertRequestCriteriaBuilder.valid_run_states) \ + if not all((s in CBAnalyticsAlertRequestCriteriaBuilder.valid_run_states) for s in states): raise ApiError("One or more invalid run states") self._update_criteria("run_state", states) return self - + def sensor_actions(self, actions): """ Restricts the alerts that this query is performed on to the specified sensor actions. - + :param actions list: List of sensor actions to look for. Valid values are "POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", "TERMINATE", and "DENY". :return: This instance. """ - if not all((action in CBAnalyticsAlertRequestCriteriaBuilder.valid_sensor_actions) \ + if not all((action in CBAnalyticsAlertRequestCriteriaBuilder.valid_sensor_actions) for action in actions): raise ApiError("One or more invalid sensor actions") self._update_criteria("sensor_action", actions) return self - + def threat_cause_vectors(self, vectors): """ Restricts the alerts that this query is performed on to the specified threat cause vectors. - + :param vectors list: List of threat cause vectors to look for. Valid values are "EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", "APP_STORE", and "THIRD_PARTY". :return: This instance. """ - if not all((vector in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_cause_vectors) \ + if not all((vector in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_cause_vectors) for vector in vectors): raise ApiError("One or more invalid threat cause vectors") self._update_criteria("threat_cause_vector", vectors) return self - - -class VMwareAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): + + +class VMwareAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): """ Auxiliary object that builds the criteria for VMware alert request searches. """ def __init__(self): super().__init__() - + def group_ids(self, groupids): """ Restricts the alerts that this query is performed on to the specified AppDefense-assigned alarm group IDs. - + :param groupids list: List of (integer) AppDefense-assigned alarm group IDs. :return: This instance. """ @@ -1059,15 +1060,15 @@ def group_ids(self, groupids): raise ApiError("One or more invalid alarm group IDs") self._update_criteria("group_id", groupids) return self - - + + class WatchlistAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): """ Auxiliary object that builds the criteria for watchlist alert request searches. """ def __init__(self): super().__init__() - + def watchlist_ids(self, ids): """ Restricts the alerts that this query is performed on to the specified @@ -1080,7 +1081,7 @@ def watchlist_ids(self, ids): raise ApiError("One or more invalid watchlist IDs") self._update_criteria("watchlist_id", ids) return self - + def watchlist_names(self, names): """ Restricts the alerts that this query is performed on to the specified @@ -1102,14 +1103,14 @@ class AlertCriteriaBuilderMixin: def categories(self, cats): """ Restricts the alerts that this query is performed on to the specified categories. - + :param cats list: List of categories to be restricted to. Valid categories are "THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", and "CRITICAL." :return: This instance """ self._criteria_builder.categories(cats) return self - + def create_time(self, *args, **kwargs): """ Restricts the alerts that this query is performed on to the specified @@ -1142,8 +1143,8 @@ def device_names(self, device_names): """ self._criteria_builder.device_names(device_names) return self - - def device_os(self, device_os): + + def device_os(self, device_os): """ Restricts the alerts that this query is performed on to the specified device operating systems. @@ -1154,7 +1155,7 @@ def device_os(self, device_os): """ self._criteria_builder.device_os(device_os) return self - + def device_os_versions(self, device_os_versions): """ Restricts the alerts that this query is performed on to the specified @@ -1165,7 +1166,7 @@ def device_os_versions(self, device_os_versions): """ self._criteria_builder.device_os_versions(device_os_versions) return self - + def device_username(self, users): """ Restricts the alerts that this query is performed on to the specified @@ -1176,17 +1177,17 @@ def device_username(self, users): """ self._criteria_builder.device_username(users) return self - + def group_results(self, flag): """ Specifies whether or not to group the results of the query. - + :param flag boolean: True to group the results, False to not do so. :return: This instance """ self._criteria_builder.group_results(flag) return self - + def alert_ids(self, alert_ids): """ Restricts the alerts that this query is performed on to the specified @@ -1197,7 +1198,7 @@ def alert_ids(self, alert_ids): """ self._criteria_builder.alert_ids(alert_ids) return self - + def legacy_alert_ids(self, alert_ids): """ Restricts the alerts that this query is performed on to the specified @@ -1208,18 +1209,18 @@ def legacy_alert_ids(self, alert_ids): """ self._criteria_builder.legacy_alert_ids(alert_ids) return self - + def minimum_severity(self, severity): """ Restricts the alerts that this query is performed on to the specified minimum severity level. - + :param severity int: The minimum severity level for alerts. :return: This instance """ self._criteria_builder.minimum_severity(severity) return self - + def policy_ids(self, policy_ids): """ Restricts the alerts that this query is performed on to the specified @@ -1241,7 +1242,7 @@ def policy_names(self, policy_names): """ self._criteria_builder.policy_names(policy_names) return self - + def process_names(self, process_names): """ Restricts the alerts that this query is performed on to the specified @@ -1252,7 +1253,7 @@ def process_names(self, process_names): """ self._criteria_builder.process_names(process_names) return self - + def process_sha256(self, shas): """ Restricts the alerts that this query is performed on to the specified @@ -1263,7 +1264,7 @@ def process_sha256(self, shas): """ self._criteria_builder.process_sha256(shas) return self - + def reputations(self, reps): """ Restricts the alerts that this query is performed on to the specified @@ -1277,7 +1278,7 @@ def reputations(self, reps): """ self._criteria_builder.reputations(reps) return self - + def tags(self, tags): """ Restricts the alerts that this query is performed on to the specified @@ -1288,7 +1289,7 @@ def tags(self, tags): """ self._criteria_builder.tags(tags) return self - + def target_priorities(self, priorities): """ Restricts the alerts that this query is performed on to the specified @@ -1300,7 +1301,7 @@ def target_priorities(self, priorities): """ self._criteria_builder.target_priorities(priorities) return self - + def threat_ids(self, threats): """ Restricts the alerts that this query is performed on to the specified @@ -1311,7 +1312,7 @@ def threat_ids(self, threats): """ self._criteria_builder.threat_ids(threats) return self - + def types(self, alerttypes): """ Restricts the alerts that this query is performed on to the specified @@ -1323,7 +1324,7 @@ def types(self, alerttypes): """ self._criteria_builder.types(alerttypes) return self - + def workflows(self, workflow_vals): """ Restricts the alerts that this query is performed on to the specified @@ -1345,100 +1346,100 @@ def blocked_threat_categories(self, categories): """ Restricts the alerts that this query is performed on to the specified threat categories that were blocked. - + :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". :return: This instance. """ self._criteria_builder.blocked_threat_categories(categories) return self - + def device_locations(self, locations): """ Restricts the alerts that this query is performed on to the specified device locations. - + :param locations list: List of device locations to look for. Valid values are "ONSITE", "OFFSITE", - and "UNKNOWN". + and "UNKNOWN". :return: This instance. """ self._criteria_builder.device_locations(locations) return self - + def kill_chain_statuses(self, statuses): """ Restricts the alerts that this query is performed on to the specified kill chain statuses. - + :param statuses list: List of kill chain statuses to look for. Valid values are "RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", "INSTALL_RUN","COMMAND_AND_CONTROL", - "EXECUTE_GOAL", and "BREACH". + "EXECUTE_GOAL", and "BREACH". :return: This instance. """ self._criteria_builder.kill_chain_statuses(statuses) return self - + def not_blocked_threat_categories(self, categories): """ Restricts the alerts that this query is performed on to the specified threat categories that were NOT blocked. - + :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". :return: This instance. """ self._criteria_builder.not_blocked_threat_categories(categories) return self - + def policy_applied(self, applied_statuses): """ Restricts the alerts that this query is performed on to the specified status values showing whether policies were applied. - + :param applied_statuses list: List of status values to look for. Valid values are - "APPLIED" and "NOT_APPLIED". + "APPLIED" and "NOT_APPLIED". :return: This instance. """ self._criteria_builder.policy_applied(applied_statuses) return self - + def reason_code(self, reason): """ Restricts the alerts that this query is performed on to the specified reason code (enum value). - + :param reason str: The reason code to look for. :return: This instance. """ self._criteria_builder.reason_code(reason) return self - + def run_states(self, states): """ Restricts the alerts that this query is performed on to the specified run states. - + :param states list: List of run states to look for. Valid values are "DID_NOT_RUN", "RAN", - and "UNKNOWN". + and "UNKNOWN". :return: This instance. """ self._criteria_builder.run_states(states) return self - + def sensor_actions(self, actions): """ Restricts the alerts that this query is performed on to the specified sensor actions. - + :param actions list: List of sensor actions to look for. Valid values are "POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", "TERMINATE", and "DENY". :return: This instance. """ self._criteria_builder.sensor_actions(actions) return self - + def threat_cause_vectors(self, vectors): """ Restricts the alerts that this query is performed on to the specified threat cause vectors. - + :param vectors list: List of threat cause vectors to look for. Valid values are "EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", "APP_STORE", and "THIRD_PARTY". @@ -1446,7 +1447,7 @@ def threat_cause_vectors(self, vectors): """ self._criteria_builder.threat_cause_vectors(vectors) return self - + class VMwareAlertCriteriaBuilderMixin(AlertCriteriaBuilderMixin): """ @@ -1456,7 +1457,7 @@ def group_ids(self, groupids): """ Restricts the alerts that this query is performed on to the specified AppDefense-assigned alarm group IDs. - + :param groupids list: List of (integer) AppDefense-assigned alarm group IDs. :return: This instance. """ @@ -1478,7 +1479,7 @@ def watchlist_ids(self, ids): """ self._criteria_builder.watchlist_ids(ids) return self - + def watchlist_names(self, names): """ Restricts the alerts that this query is performed on to the specified @@ -1489,8 +1490,8 @@ def watchlist_names(self, names): """ self._criteria_builder.watchlist_names(names) return self - - + + class BaseAlertSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, AlertCriteriaBuilderMixin, IterableQueryMixin): """ @@ -1500,13 +1501,13 @@ class BaseAlertSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, AlertCriteria "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", "POLICY_APPLIED", "SENSOR_ACTION"] - + def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._query_builder = QueryBuilder() self._criteria_builder = AlertRequestCriteriaBuilder() self._sortcriteria = {} - + def sort_by(self, key, direction="ASC"): """Sets the sorting behavior on a query's results. @@ -1524,7 +1525,7 @@ def sort_by(self, key, direction="ASC"): return self def _build_request(self, from_row, max_rows, add_sort=True): - request = {"criteria": self._criteria_builder.build()} + request = {"criteria": self._criteria_builder.build()} request["query"] = self._query_builder._collapse() if from_row > 0: request["start"] = from_row @@ -1533,7 +1534,7 @@ def _build_request(self, from_row, max_rows, add_sort=True): if add_sort and self._sortcriteria != {}: request["sort"] = [self._sortcriteria] return request - + def _build_url(self, tail_end): url = self._doc_class.urlobject.format(self._cb.credentials.org_key) + tail_end return url @@ -1583,7 +1584,7 @@ def _perform_query(self, from_row=0, max_rows=-1): def facets(self, fieldlist, max_rows=0): """ Return information about the facets for this alert by search, using the defined criteria. - + :param fieldlist list: List of facet field names. Valid names are "ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", @@ -1600,8 +1601,8 @@ def facets(self, fieldlist, max_rows=0): resp = self._cb.post_object(url, body=request) result = resp.json() return result.get("results", []) - - + + class WatchlistAlertSearchQuery(BaseAlertSearchQuery, WatchlistAlertCriteriaBuilderMixin): """ Represents a query that is used to locate WatchlistAlert objects. @@ -1609,8 +1610,8 @@ class WatchlistAlertSearchQuery(BaseAlertSearchQuery, WatchlistAlertCriteriaBuil def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() - - + + class CBAnalyticsAlertSearchQuery(BaseAlertSearchQuery, CBAnalyticsAlertCriteriaBuilderMixin): """ Represents a query that is used to locate CBAnalyticsAlert objects. @@ -1618,8 +1619,8 @@ class CBAnalyticsAlertSearchQuery(BaseAlertSearchQuery, CBAnalyticsAlertCriteria def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._criteria_builder = CBAnalyticsAlertRequestCriteriaBuilder() - - + + class VMwareAlertSearchQuery(BaseAlertSearchQuery, VMwareAlertCriteriaBuilderMixin): """ Represents a query that is used to locate VMwareAlert objects. @@ -1627,8 +1628,8 @@ class VMwareAlertSearchQuery(BaseAlertSearchQuery, VMwareAlertCriteriaBuilderMix def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._criteria_builder = VMwareAlertRequestCriteriaBuilder() - - + + class BulkUpdateAlertsBase: """ Base query for doing bulk updates on alerts, where the result of a search is used to set @@ -1638,54 +1639,54 @@ def __init__(self, cb, state): self._cb = cb self._state = state self._additional_fields = {} - + def remediation(self, remediation): """ Sets the remediation state message to be applied to all selected alerts. - + :param remediation str: The remediation state message. """ self._additional_fields["remediation_state"] = remediation return self - + def comment(self, comment): """ Sets the comment to be applied to all selected alerts. - + :param comment str: The comment to be used. """ self._additional_fields["comment"] = comment return self - + def _url(self): raise ApiError("invalid abstract URL for the operation") - + def _build_request(self): request = self._additional_fields request["state"] = self._state return request - + def run(self): """ Executes the search query and alert state change operation. - + :return: A WorkflowStatus object that can be used for monitoring the progress of the operation. """ resp = self._cb.post_object(self._url(), body=self._build_request()) output = resp.json() return self._cb._new_workflow_status(output["request_id"]) - - + + class BulkUpdateAlerts(BulkUpdateAlertsBase, AlertCriteriaBuilderMixin, QueryBuilderSupportMixin): """ Query for bulk update of base-level alerts. """ def __init__(self, cb, state): - super().__init__(cb, state) + super().__init__(cb, state) self._criteria_builder = AlertRequestCriteriaBuilder() self._query_builder = QueryBuilder() - + def _url(self): return "/appservices/v6/orgs/{0}/alerts/workflow/_criteria".format(self._cb.credentials.org_key) @@ -1694,56 +1695,56 @@ def _build_request(self): request["criteria"] = self._criteria_builder.build() request["query"] = self._query_builder._collapse() return request - - -class BulkUpdateCBAnalyticsAlerts(BulkUpdateAlerts, CBAnalyticsAlertCriteriaBuilderMixin): + + +class BulkUpdateCBAnalyticsAlerts(BulkUpdateAlerts, CBAnalyticsAlertCriteriaBuilderMixin): """ Query for bulk update of CB Analytics alerts. """ def __init__(self, cb, state): - super().__init__(cb, state) + super().__init__(cb, state) self._criteria_builder = CBAnalyticsAlertRequestCriteriaBuilder() def _url(self): return "/appservices/v6/orgs/{0}/alerts/cbanalytics/workflow/_criteria".format(self._cb.credentials.org_key) - - + + class BulkUpdateVMwareAlerts(BulkUpdateAlerts, VMwareAlertCriteriaBuilderMixin): """ Query for bulk update of VMware alerts. """ def __init__(self, cb, state): - super().__init__(cb, state) + super().__init__(cb, state) self._criteria_builder = VMwareAlertRequestCriteriaBuilder() def _url(self): return "/appservices/v6/orgs/{0}/alerts/vmware/workflow/_criteria".format(self._cb.credentials.org_key) - - + + class BulkUpdateWatchlistAlerts(BulkUpdateAlerts, WatchlistAlertCriteriaBuilderMixin): """ Query for bulk update of watchlist alerts. """ def __init__(self, cb, state): - super().__init__(cb, state) + super().__init__(cb, state) self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() - + def _url(self): return "/appservices/v6/orgs/{0}/alerts/watchlist/workflow/_criteria".format(self._cb.credentials.org_key) - - + + class BulkUpdateThreatAlerts(BulkUpdateAlertsBase): """ Query for bulk update of threat alerts. """ def __init__(self, cb, state): - super().__init__(cb, state) + super().__init__(cb, state) self._threat_ids = [] - + def threat_ids(self, threats): """ Specifies the threat IDs to set the status of alerts for. - + :param threats list: The list of string threat identifiers. :return: This instance. """ @@ -1754,9 +1755,8 @@ def threat_ids(self, threats): def _url(self): return "/appservices/v6/orgs/{0}/threat/workflow/_criteria".format(self._cb.credentials.org_key) - + def _build_request(self): request = super()._build_request() request["threat_id"] = self._threat_ids return request - \ No newline at end of file diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 914f0ede..4dc2a666 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -23,7 +23,7 @@ class CbPSCBaseAPI(BaseAPI): alert_update_queries = {"ALERT": BulkUpdateAlerts, "WATCHLIST": BulkUpdateWatchlistAlerts, "THREAT": BulkUpdateThreatAlerts, "CBANALYTICS": BulkUpdateCBAnalyticsAlerts, "VMWARE": BulkUpdateVMwareAlerts} - + def __init__(self, *args, **kwargs): super(CbPSCBaseAPI, self).__init__(product_name="psc", *args, **kwargs) self._lr_scheduler = None @@ -129,46 +129,45 @@ def device_update_sensor_version(self, device_ids, sensor_version): :param dict sensor_version: New version properties for the sensor. """ return self._device_action(device_ids, "UPDATE_SENSOR_VERSION", {"sensor_version": sensor_version}) - + # ---- Alerts API - + def alert_search_suggestions(self, query): """ Returns suggestions for keys and field values that can be used in a search. - + :param query str: A search query to use. :return: A list of search suggestions expressed as dict objects. """ query_params = {"suggest.q": query} url = "/appservices/v6/orgs/{0}/alerts/search_suggestions".format(self.credentials.org_key) return self.get_object(url, query_params) - + def _new_workflow_status(self, requestid): return WorkflowStatus(self, requestid) - + def _bulk_alert_update_query(self, state, querytype): cls = CbPSCBaseAPI.alert_update_queries.get(querytype, None) if cls is None: raise ApiError("unknown query type for bulk alert update") return cls(self, state) - + def bulk_alert_dismiss(self, querytype): """ Start a query to dismiss multiple alerts. - - :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", "THREAT", + + :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", "THREAT", "CBANALYTICS", or "VMWARE". :return: The new query. """ return self._bulk_alert_update_query("DISMISSED", querytype) - + def bulk_alert_undismiss(self, querytype): """ Start a query to un-dismiss multiple alerts. - - :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", "THREAT", + + :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", "THREAT", "CBANALYTICS", or "VMWARE". :return: The new query. """ return self._bulk_alert_update_query("OPEN", querytype) - \ No newline at end of file From 1285ec029c0989df698a1cbcb540bf456e8c030b Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Thu, 31 Oct 2019 15:45:57 -0600 Subject: [PATCH 044/197] fixed documentation for reason_code method --- src/cbapi/psc/query.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 771b50da..98187748 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -987,9 +987,9 @@ def policy_applied(self, applied_statuses): def reason_code(self, reason): """ Restricts the alerts that this query is performed on to the specified - reason code (enum value). + reason codes (enum values). - :param reason str: The reason code to look for. + :param reason list: List of string reason codes to look for. :return: This instance. """ if not all(isinstance(t, str) for t in reason): @@ -1406,9 +1406,9 @@ def policy_applied(self, applied_statuses): def reason_code(self, reason): """ Restricts the alerts that this query is performed on to the specified - reason code (enum value). + reason codes (enum values). - :param reason str: The reason code to look for. + :param reason list: List of string reason codes to look for. :return: This instance. """ self._criteria_builder.reason_code(reason) From 2b3e33f541f00b24a9896c102a7e81c3d5e2b922 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 4 Nov 2019 15:57:12 -0700 Subject: [PATCH 045/197] added Alerts v6 example scripts, tweaked some of the CBAPI code to work better in live situations, rechecked unit tests and flake8 status --- examples/psc/alert_search_suggestions.py | 22 +++ examples/psc/bulk_update_alerts.py | 124 +++++++++++++ .../psc/bulk_update_cbanalytics_alerts.py | 169 ++++++++++++++++++ examples/psc/bulk_update_threat_alerts.py | 52 ++++++ examples/psc/bulk_update_vmware_alerts.py | 131 ++++++++++++++ examples/psc/bulk_update_watchlist_alerts.py | 133 ++++++++++++++ examples/psc/device_control.py | 2 +- examples/psc/list_alert_facets.py | 103 +++++++++++ examples/psc/list_alerts.py | 103 +++++++++++ examples/psc/list_cbanalytics_alert_facets.py | 145 +++++++++++++++ examples/psc/list_cbanalytics_alerts.py | 145 +++++++++++++++ examples/psc/list_vmware_alert_facets.py | 107 +++++++++++ examples/psc/list_vmware_alerts.py | 107 +++++++++++ examples/psc/list_watchlist_alert_facets.py | 109 +++++++++++ examples/psc/list_watchlist_alerts.py | 109 +++++++++++ src/cbapi/psc/models.py | 5 +- src/cbapi/psc/rest_api.py | 3 +- 17 files changed, 1565 insertions(+), 4 deletions(-) create mode 100755 examples/psc/alert_search_suggestions.py create mode 100755 examples/psc/bulk_update_alerts.py create mode 100755 examples/psc/bulk_update_cbanalytics_alerts.py create mode 100755 examples/psc/bulk_update_threat_alerts.py create mode 100755 examples/psc/bulk_update_vmware_alerts.py create mode 100755 examples/psc/bulk_update_watchlist_alerts.py create mode 100755 examples/psc/list_alert_facets.py create mode 100755 examples/psc/list_alerts.py create mode 100755 examples/psc/list_cbanalytics_alert_facets.py create mode 100755 examples/psc/list_cbanalytics_alerts.py create mode 100755 examples/psc/list_vmware_alert_facets.py create mode 100755 examples/psc/list_vmware_alerts.py create mode 100755 examples/psc/list_watchlist_alert_facets.py create mode 100755 examples/psc/list_watchlist_alerts.py diff --git a/examples/psc/alert_search_suggestions.py b/examples/psc/alert_search_suggestions.py new file mode 100755 index 00000000..49395a2a --- /dev/null +++ b/examples/psc/alert_search_suggestions.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object + +def main(): + parser = build_cli_parser("Get suggestions for searching alerts") + parser.add_argument("-q", "--query", default="", help="Query string for looking for alerts") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + suggestions = cb.alert_search_suggestions(args.query) + for suggestion in suggestions: + print("Search term: '{0}'".format(suggestion["term"])) + print("\tWeight: {0}".format(suggestion["weight"])) + print("\tAvailable with products: {0}".format(", ".join(suggestion["required_skus_some"]))) + + +if __name__ == "__main__": + sys.exit(main()) + \ No newline at end of file diff --git a/examples/psc/bulk_update_alerts.py b/examples/psc/bulk_update_alerts.py new file mode 100755 index 00000000..e0ef2ce2 --- /dev/null +++ b/examples/psc/bulk_update_alerts.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +import sys +from time import sleep +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import BaseAlert + +def main(): + parser = build_cli_parser("Bulk update the status of alerts") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") + parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") + operation = parser.add_mutually_exclusive_group(required=True) + operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") + operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + if args.dismiss: + query = cb.bulk_alert_dismiss("ALERT") + elif args.undismiss: + query = cb.bulk_alert_undismiss("ALERT") + else: + raise NotImplemented("one of --dismiss or --undismiss must be specified") + + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + + if args.remediation: + query = query.remediation(args.remediation) + if args.comment: + query = query.comment(args.comment) + statobj = query.run() + print("Submitted query with ID {0}".format(statobj.id_)) + while not statobj.finished: + print("Waiting...") + sleep(1) + if statobj.errors: + print("Errors encountered:") + for err in statobj.errors: + print("\t{0}".format(err)) + if statobj.failed_ids: + print("Failed alert IDs:") + for i in statobj.failed_ids: + print("\t{0}".format(err)) + print("{0} total alert(s) found, of which {1} were successfully changed" \ + .format(statobj.num_hits, statobj.num_success)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/psc/bulk_update_cbanalytics_alerts.py b/examples/psc/bulk_update_cbanalytics_alerts.py new file mode 100755 index 00000000..18b1709a --- /dev/null +++ b/examples/psc/bulk_update_cbanalytics_alerts.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python + +import sys +from time import sleep +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import CBAnalyticsAlert + +def main(): + parser = build_cli_parser("Bulk update the status of CB Analytics alerts") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("--blockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", + "KNOWN_MALWARE", "RISKY_PROGRAM"], + help="Restrict search to the specified threat categories that were blocked") + parser.add_argument("--location", action="append", choices=["ONSITE", "OFFSITE", "UNKNOWN"], + help="Restrict search to the specified device locations") + parser.add_argument("--killchain", action="append", choices=["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", + "INSTALL_RUN", "COMMAND_AND_CONTROL", "EXECUTE_GOAL", + "BREACH"], + help="Restrict search to the specified kill chain status values") + parser.add_argument("--notblockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", + "KNOWN_MALWARE", "RISKY_PROGRAM"], + help="Restrict search to the specified threat categories that were NOT blocked") + parser.add_argument("--policyapplied", action="append", choices=["APPLIED", "NOT_APPLIED"], + help="Restrict search to the specified policy-application status values") + parser.add_argument("--reason", action="append", type=str, help="Restrict search to the specified reason codes") + parser.add_argument("--runstate", action="append", choices=["DID_NOT_RUN", "RAN", "UNKNOWN"], + help="Restrict search to the specified run states") + parser.add_argument("--sensoraction", action="append", choices=["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", + "TERMINATE", "DENY"], + help="Restrict search to the specified sensor actions") + parser.add_argument("--vector", action="append", choices=["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", + "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", + "APP_STORE", "THIRD_PARTY"], + help="Restrict search to the specified threat cause vectors") + parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") + parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") + operation = parser.add_mutually_exclusive_group(required=True) + operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") + operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + if args.dismiss: + query = cb.bulk_alert_dismiss("CBANALYTICS") + elif args.undismiss: + query = cb.bulk_alert_undismiss("CBANALYTICS") + else: + raise NotImplemented("one of --dismiss or --undismiss must be specified") + + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + if args.blockedthreat: + query = query.blocked_threat_categories(args.blockedthreat) + if args.location: + query = query.device_locations(args.location) + if args.killchain: + query = query.kill_chain_statuses(args.killchain) + if args.notblockedthreat: + query = query.not_blocked_threat_categories(args.notblockedthreat) + if args.policyapplied: + query = query.policy_applied(args.policyapplied) + if args.reason: + query = query.reason_code(args.reason) + if args.runstate: + query = query.run_states(args.runstate) + if args.sensoraction: + query = query.sensor_actions(args.sensoraction) + if args.vector: + query = query.threat_cause_vectors(args.vector) + if args.sort_by: + direction = "DESC" if args.reverse else "ASC" + query = query.sort_by(args.sort_by, direction) + + if args.remediation: + query = query.remediation(args.remediation) + if args.comment: + query = query.comment(args.comment) + statobj = query.run() + print("Submitted query with ID {0}".format(statobj.id_)) + while not statobj.finished: + print("Waiting...") + sleep(1) + if statobj.errors: + print("Errors encountered:") + for err in statobj.errors: + print("\t{0}".format(err)) + if statobj.failed_ids: + print("Failed alert IDs:") + for i in statobj.failed_ids: + print("\t{0}".format(err)) + print("{0} total alert(s) found, of which {1} were successfully changed" \ + .format(statobj.num_hits, statobj.num_success)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/psc/bulk_update_threat_alerts.py b/examples/psc/bulk_update_threat_alerts.py new file mode 100755 index 00000000..616a2369 --- /dev/null +++ b/examples/psc/bulk_update_threat_alerts.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +import sys +from time import sleep +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import BaseAlert + +def main(): + parser = build_cli_parser("Bulk update the status of alerts by threat ID") + parser.add_argument("-T", "--threatid", action="append", type=str, required=True, + help="Threat IDs to update the alerts for") + parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") + parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") + operation = parser.add_mutually_exclusive_group(required=True) + operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") + operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + if args.dismiss: + query = cb.bulk_alert_dismiss("THREAT") + elif args.undismiss: + query = cb.bulk_alert_undismiss("THREAT") + else: + raise NotImplemented("one of --dismiss or --undismiss must be specified") + + query.threat_ids(args.threatid) + if args.remediation: + query = query.remediation(args.remediation) + if args.comment: + query = query.comment(args.comment) + statobj = query.run() + print("Submitted query with ID {0}".format(statobj.id_)) + while not statobj.finished: + print("Waiting...") + sleep(1) + if statobj.errors: + print("Errors encountered:") + for err in statobj.errors: + print("\t{0}".format(err)) + if statobj.failed_ids: + print("Failed alert IDs:") + for i in statobj.failed_ids: + print("\t{0}".format(err)) + print("{0} total alert(s) found, of which {1} were successfully changed" \ + .format(statobj.num_hits, statobj.num_success)) + + +if __name__ == "__main__": + sys.exit(main()) + \ No newline at end of file diff --git a/examples/psc/bulk_update_vmware_alerts.py b/examples/psc/bulk_update_vmware_alerts.py new file mode 100755 index 00000000..8440fb34 --- /dev/null +++ b/examples/psc/bulk_update_vmware_alerts.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +import sys +from time import sleep +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import VMwareAlert + +def main(): + parser = build_cli_parser("Bulk update the status of VMware alerts") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("--groupid", action="append", type=int, + help="Restrict search to the specified AppDefense alarm group IDs") + parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") + parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") + operation = parser.add_mutually_exclusive_group(required=True) + operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") + operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + if args.dismiss: + query = cb.bulk_alert_dismiss("VMWARE") + elif args.undismiss: + query = cb.bulk_alert_undismiss("VMWARE") + else: + raise NotImplemented("one of --dismiss or --undismiss must be specified") + + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + if args.groupid: + query = query.group_ids(args.groupid) + if args.sort_by: + direction = "DESC" if args.reverse else "ASC" + query = query.sort_by(args.sort_by, direction) + + if args.remediation: + query = query.remediation(args.remediation) + if args.comment: + query = query.comment(args.comment) + statobj = query.run() + print("Submitted query with ID {0}".format(statobj.id_)) + while not statobj.finished: + print("Waiting...") + sleep(1) + if statobj.errors: + print("Errors encountered:") + for err in statobj.errors: + print("\t{0}".format(err)) + if statobj.failed_ids: + print("Failed alert IDs:") + for i in statobj.failed_ids: + print("\t{0}".format(err)) + print("{0} total alert(s) found, of which {1} were successfully changed" \ + .format(statobj.num_hits, statobj.num_success)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/psc/bulk_update_watchlist_alerts.py b/examples/psc/bulk_update_watchlist_alerts.py new file mode 100755 index 00000000..0f11ae6b --- /dev/null +++ b/examples/psc/bulk_update_watchlist_alerts.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +import sys +from time import sleep +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import WatchlistAlert + +def main(): + parser = build_cli_parser("Bulk update the status of watchlist alerts") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("--watchlistid", action="append", type=str, help="Restrict search to the specified watchlists by ID") + parser.add_argument("--watchlistname", action="append", type=str, help="Restrict search to the specified watchlists by name") + parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") + parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") + operation = parser.add_mutually_exclusive_group(required=True) + operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") + operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + if args.dismiss: + query = cb.bulk_alert_dismiss("WATCHLIST") + elif args.undismiss: + query = cb.bulk_alert_undismiss("WATCHLIST") + else: + raise NotImplemented("one of --dismiss or --undismiss must be specified") + + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + if args.watchlistid: + query = query.watchlist_ids(args.watchlistid) + if args.watchlistname: + query = query.watchlist_names(args.watchlistname) + if args.sort_by: + direction = "DESC" if args.reverse else "ASC" + query = query.sort_by(args.sort_by, direction) + + if args.remediation: + query = query.remediation(args.remediation) + if args.comment: + query = query.comment(args.comment) + statobj = query.run() + print("Submitted query with ID {0}".format(statobj.id_)) + while not statobj.finished: + print("Waiting...") + sleep(1) + if statobj.errors: + print("Errors encountered:") + for err in statobj.errors: + print("\t{0}".format(err)) + if statobj.failed_ids: + print("Failed alert IDs:") + for i in statobj.failed_ids: + print("\t{0}".format(err)) + print("{0} total alert(s) found, of which {1} were successfully changed" \ + .format(statobj.num_hits, statobj.num_success)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/psc/device_control.py b/examples/psc/device_control.py index 497de7b5..d89dfd1d 100755 --- a/examples/psc/device_control.py +++ b/examples/psc/device_control.py @@ -43,7 +43,7 @@ def main(): args = parser.parse_args() cb = get_cb_psc_object(args) - dev = cb.get_device(args.device_id) + dev = cb.select(Device, args.device_id) if args.command: if args.command == "background_scan": diff --git a/examples/psc/list_alert_facets.py b/examples/psc/list_alert_facets.py new file mode 100755 index 00000000..134dbaa3 --- /dev/null +++ b/examples/psc/list_alert_facets.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import BaseAlert + +def main(): + parser = build_cli_parser("List alert facets") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", + "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", + "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", + "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", + "POLICY_APPLIED", "SENSOR_ACTION"], + required=True, help="Retrieve these fields as facet information") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + query = cb.select(BaseAlert) + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + + facetinfos = query.facets(args.facet) + for facetinfo in facetinfos: + print("For field '{0}':".format(facetinfo["field"])) + for facetval in facetinfo["values"]: + print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/psc/list_alerts.py b/examples/psc/list_alerts.py new file mode 100755 index 00000000..3dabf915 --- /dev/null +++ b/examples/psc/list_alerts.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import BaseAlert + +def main(): + parser = build_cli_parser("List alerts") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("-S", "--sort_by", help="Field to sort the output by") + parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + query = cb.select(BaseAlert) + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + if args.sort_by: + direction = "DESC" if args.reverse else "ASC" + query = query.sort_by(args.sort_by, direction) + + alerts = list(query) + print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) + for alert in alerts: + print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", \ + alert.threat_id or "Unknown", \ + alert.last_update_time)) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/psc/list_cbanalytics_alert_facets.py b/examples/psc/list_cbanalytics_alert_facets.py new file mode 100755 index 00000000..06a0e202 --- /dev/null +++ b/examples/psc/list_cbanalytics_alert_facets.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import CBAnalyticsAlert + +def main(): + parser = build_cli_parser("List CB Analytics alert facets") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("--blockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", + "KNOWN_MALWARE", "RISKY_PROGRAM"], + help="Restrict search to the specified threat categories that were blocked") + parser.add_argument("--location", action="append", choices=["ONSITE", "OFFSITE", "UNKNOWN"], + help="Restrict search to the specified device locations") + parser.add_argument("--killchain", action="append", choices=["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", + "INSTALL_RUN", "COMMAND_AND_CONTROL", "EXECUTE_GOAL", + "BREACH"], + help="Restrict search to the specified kill chain status values") + parser.add_argument("--notblockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", + "KNOWN_MALWARE", "RISKY_PROGRAM"], + help="Restrict search to the specified threat categories that were NOT blocked") + parser.add_argument("--policyapplied", action="append", choices=["APPLIED", "NOT_APPLIED"], + help="Restrict search to the specified policy-application status values") + parser.add_argument("--reason", action="append", type=str, help="Restrict search to the specified reason codes") + parser.add_argument("--runstate", action="append", choices=["DID_NOT_RUN", "RAN", "UNKNOWN"], + help="Restrict search to the specified run states") + parser.add_argument("--sensoraction", action="append", choices=["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", + "TERMINATE", "DENY"], + help="Restrict search to the specified sensor actions") + parser.add_argument("--vector", action="append", choices=["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", + "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", + "APP_STORE", "THIRD_PARTY"], + help="Restrict search to the specified threat cause vectors") + parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", + "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", + "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", + "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", + "POLICY_APPLIED", "SENSOR_ACTION"], + required=True, help="Retrieve these fields as facet information") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + query = cb.select(CBAnalyticsAlert) + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + if args.blockedthreat: + query = query.blocked_threat_categories(args.blockedthreat) + if args.location: + query = query.device_locations(args.location) + if args.killchain: + query = query.kill_chain_statuses(args.killchain) + if args.notblockedthreat: + query = query.not_blocked_threat_categories(args.notblockedthreat) + if args.policyapplied: + query = query.policy_applied(args.policyapplied) + if args.reason: + query = query.reason_code(args.reason) + if args.runstate: + query = query.run_states(args.runstate) + if args.sensoraction: + query = query.sensor_actions(args.sensoraction) + if args.vector: + query = query.threat_cause_vectors(args.vector) + + facetinfos = query.facets(args.facet) + for facetinfo in facetinfos: + print("For field '{0}':".format(facetinfo["field"])) + for facetval in facetinfo["values"]: + print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/psc/list_cbanalytics_alerts.py b/examples/psc/list_cbanalytics_alerts.py new file mode 100755 index 00000000..e4cba7fc --- /dev/null +++ b/examples/psc/list_cbanalytics_alerts.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import CBAnalyticsAlert + +def main(): + parser = build_cli_parser("List CB Analytics alerts") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("--blockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", + "KNOWN_MALWARE", "RISKY_PROGRAM"], + help="Restrict search to the specified threat categories that were blocked") + parser.add_argument("--location", action="append", choices=["ONSITE", "OFFSITE", "UNKNOWN"], + help="Restrict search to the specified device locations") + parser.add_argument("--killchain", action="append", choices=["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", + "INSTALL_RUN", "COMMAND_AND_CONTROL", "EXECUTE_GOAL", + "BREACH"], + help="Restrict search to the specified kill chain status values") + parser.add_argument("--notblockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", + "KNOWN_MALWARE", "RISKY_PROGRAM"], + help="Restrict search to the specified threat categories that were NOT blocked") + parser.add_argument("--policyapplied", action="append", choices=["APPLIED", "NOT_APPLIED"], + help="Restrict search to the specified policy-application status values") + parser.add_argument("--reason", action="append", type=str, help="Restrict search to the specified reason codes") + parser.add_argument("--runstate", action="append", choices=["DID_NOT_RUN", "RAN", "UNKNOWN"], + help="Restrict search to the specified run states") + parser.add_argument("--sensoraction", action="append", choices=["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", + "TERMINATE", "DENY"], + help="Restrict search to the specified sensor actions") + parser.add_argument("--vector", action="append", choices=["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", + "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", + "APP_STORE", "THIRD_PARTY"], + help="Restrict search to the specified threat cause vectors") + parser.add_argument("-S", "--sort_by", help="Field to sort the output by") + parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + query = cb.select(CBAnalyticsAlert) + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + if args.blockedthreat: + query = query.blocked_threat_categories(args.blockedthreat) + if args.location: + query = query.device_locations(args.location) + if args.killchain: + query = query.kill_chain_statuses(args.killchain) + if args.notblockedthreat: + query = query.not_blocked_threat_categories(args.notblockedthreat) + if args.policyapplied: + query = query.policy_applied(args.policyapplied) + if args.reason: + query = query.reason_code(args.reason) + if args.runstate: + query = query.run_states(args.runstate) + if args.sensoraction: + query = query.sensor_actions(args.sensoraction) + if args.vector: + query = query.threat_cause_vectors(args.vector) + if args.sort_by: + direction = "DESC" if args.reverse else "ASC" + query = query.sort_by(args.sort_by, direction) + + alerts = list(query) + print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) + for alert in alerts: + print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", \ + alert.threat_id or "Unknown", \ + alert.last_update_time)) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/psc/list_vmware_alert_facets.py b/examples/psc/list_vmware_alert_facets.py new file mode 100755 index 00000000..4a8dbe9d --- /dev/null +++ b/examples/psc/list_vmware_alert_facets.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import VMwareAlert + +def main(): + parser = build_cli_parser("List VMware alert facets") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("--groupid", action="append", type=int, + help="Restrict search to the specified AppDefense alarm group IDs") + parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", + "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", + "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", + "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", + "POLICY_APPLIED", "SENSOR_ACTION"], + required=True, help="Retrieve these fields as facet information") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + query = cb.select(VMwareAlert) + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + if args.groupid: + query = query.group_ids(args.groupid) + + facetinfos = query.facets(args.facet) + for facetinfo in facetinfos: + print("For field '{0}':".format(facetinfo["field"])) + for facetval in facetinfo["values"]: + print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/psc/list_vmware_alerts.py b/examples/psc/list_vmware_alerts.py new file mode 100755 index 00000000..6ef3b315 --- /dev/null +++ b/examples/psc/list_vmware_alerts.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import VMwareAlert + +def main(): + parser = build_cli_parser("List VMware alerts") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("--groupid", action="append", type=int, + help="Restrict search to the specified AppDefense alarm group IDs") + parser.add_argument("-S", "--sort_by", help="Field to sort the output by") + parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + query = cb.select(VMwareAlert) + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + if args.groupid: + query = query.group_ids(args.groupid) + if args.sort_by: + direction = "DESC" if args.reverse else "ASC" + query = query.sort_by(args.sort_by, direction) + + alerts = list(query) + print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) + for alert in alerts: + print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", \ + alert.threat_id or "Unknown", \ + alert.last_update_time)) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/psc/list_watchlist_alert_facets.py b/examples/psc/list_watchlist_alert_facets.py new file mode 100755 index 00000000..3d4f182c --- /dev/null +++ b/examples/psc/list_watchlist_alert_facets.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import WatchlistAlert + +def main(): + parser = build_cli_parser("List watchlist alert facets") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("--watchlistid", action="append", type=str, help="Restrict search to the specified watchlists by ID") + parser.add_argument("--watchlistname", action="append", type=str, help="Restrict search to the specified watchlists by name") + parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", + "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", + "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", + "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", + "POLICY_APPLIED", "SENSOR_ACTION"], + required=True, help="Retrieve these fields as facet information") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + query = cb.select(WatchlistAlert) + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + if args.watchlistid: + query = query.watchlist_ids(args.watchlistid) + if args.watchlistname: + query = query.watchlist_names(args.watchlistname) + + facetinfos = query.facets(args.facet) + for facetinfo in facetinfos: + print("For field '{0}':".format(facetinfo["field"])) + for facetval in facetinfo["values"]: + print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/psc/list_watchlist_alerts.py b/examples/psc/list_watchlist_alerts.py new file mode 100755 index 00000000..43caa2cc --- /dev/null +++ b/examples/psc/list_watchlist_alerts.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python + +import sys +from cbapi.example_helpers import build_cli_parser, get_cb_psc_object +from cbapi.psc.models import WatchlistAlert + +def main(): + parser = build_cli_parser("List watchlist alerts") + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + parser.add_argument("--watchlistid", action="append", type=str, help="Restrict search to the specified watchlists by ID") + parser.add_argument("--watchlistname", action="append", type=str, help="Restrict search to the specified watchlists by name") + parser.add_argument("-S", "--sort_by", help="Field to sort the output by") + parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") + + args = parser.parse_args() + cb = get_cb_psc_object(args) + + query = cb.select(WatchlistAlert) + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + if args.watchlistid: + query = query.watchlist_ids(args.watchlistid) + if args.watchlistname: + query = query.watchlist_names(args.watchlistname) + if args.sort_by: + direction = "DESC" if args.reverse else "ASC" + query = query.sort_by(args.sort_by, direction) + + alerts = list(query) + print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) + for alert in alerts: + print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", \ + alert.threat_id or "Unknown", \ + alert.last_update_time)) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 1c04c608..8e839112 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -319,12 +319,13 @@ class WorkflowStatus(PSCMutableModel): def __init__(self, cb, model_unique_id, initial_data=None): super(WorkflowStatus, self).__init__(cb, model_unique_id, initial_data) + self._request_id = model_unique_id self._workflow = None if model_unique_id is not None: self._refresh() def _refresh(self): - url = self.urlobject_single.format(self._cb.credentials.org_key, self._model_unique_id) + url = self.urlobject_single.format(self._cb.credentials.org_key, self._request_id) resp = self._cb.get_object(url) self._info = resp self._workflow = Workflow(self._cb, resp.get("workflow", None)) @@ -333,7 +334,7 @@ def _refresh(self): @property def id_(self): - return self._model_unique_id + return self._request_id @property def workflow_(self): diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 4dc2a666..15c37d9e 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -141,7 +141,8 @@ def alert_search_suggestions(self, query): """ query_params = {"suggest.q": query} url = "/appservices/v6/orgs/{0}/alerts/search_suggestions".format(self.credentials.org_key) - return self.get_object(url, query_params) + output = self.get_object(url, query_params) + return output["suggestions"] def _new_workflow_status(self, requestid): return WorkflowStatus(self, requestid) From 371b6d427f809074388970c7e646f9066f930e5c Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 5 Nov 2019 11:32:00 -0700 Subject: [PATCH 046/197] refactored examples to move a lot of the tedious argument parsing and loading to common functions --- examples/psc/alertsv6common.py | 158 ++++++++++++++++++ examples/psc/bulk_update_alerts.py | 78 +-------- .../psc/bulk_update_cbanalytics_alerts.py | 123 +------------- examples/psc/bulk_update_vmware_alerts.py | 85 +--------- examples/psc/bulk_update_watchlist_alerts.py | 87 +--------- examples/psc/list_alert_facets.py | 78 +-------- examples/psc/list_alerts.py | 78 +-------- examples/psc/list_cbanalytics_alert_facets.py | 120 +------------ examples/psc/list_cbanalytics_alerts.py | 120 +------------ examples/psc/list_vmware_alert_facets.py | 82 +-------- examples/psc/list_vmware_alerts.py | 82 +-------- examples/psc/list_watchlist_alert_facets.py | 84 +--------- examples/psc/list_watchlist_alerts.py | 84 +--------- 13 files changed, 194 insertions(+), 1065 deletions(-) create mode 100755 examples/psc/alertsv6common.py diff --git a/examples/psc/alertsv6common.py b/examples/psc/alertsv6common.py new file mode 100755 index 00000000..812fe2b4 --- /dev/null +++ b/examples/psc/alertsv6common.py @@ -0,0 +1,158 @@ +import sys + +def setup_parser_with_basic_criteria(parser): + parser.add_argument("-q", "--query", help="Query string for looking for alerts") + parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", + "MINOR", "SERIOUS", "CRITICAL"], + help="Restrict search to the specified categories") + parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") + parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") + parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], + help="Restrict search to the specified device operating systems") + parser.add_argument("--osversion", action="append", type=str, + help="Restrict search to the specified device operating system versions") + parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") + parser.add_argument("--group", action="store_true", help="Group results") + parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") + parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") + parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") + parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processhash", action="append", type=str, + help="Restrict search to the specified process SHA-256 hash values") + parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", + "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", + "COMPANY_BLACK_LIST"], + help="Restrict search to the specified reputation values") + parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") + parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], + help="Restrict search to the specified priority values") + parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") + parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], + help="Restrict search to the specified alert types") + parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], + help="Restrict search to the specified workflow statuses") + + +def setup_parser_with_cbanalytics_criteria(parser): + setup_parser_with_basic_criteria(parser) + parser.add_argument("--blockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", + "KNOWN_MALWARE", "RISKY_PROGRAM"], + help="Restrict search to the specified threat categories that were blocked") + parser.add_argument("--location", action="append", choices=["ONSITE", "OFFSITE", "UNKNOWN"], + help="Restrict search to the specified device locations") + parser.add_argument("--killchain", action="append", choices=["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", + "INSTALL_RUN", "COMMAND_AND_CONTROL", "EXECUTE_GOAL", + "BREACH"], + help="Restrict search to the specified kill chain status values") + parser.add_argument("--notblockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", + "KNOWN_MALWARE", "RISKY_PROGRAM"], + help="Restrict search to the specified threat categories that were NOT blocked") + parser.add_argument("--policyapplied", action="append", choices=["APPLIED", "NOT_APPLIED"], + help="Restrict search to the specified policy-application status values") + parser.add_argument("--reason", action="append", type=str, help="Restrict search to the specified reason codes") + parser.add_argument("--runstate", action="append", choices=["DID_NOT_RUN", "RAN", "UNKNOWN"], + help="Restrict search to the specified run states") + parser.add_argument("--sensoraction", action="append", choices=["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", + "TERMINATE", "DENY"], + help="Restrict search to the specified sensor actions") + parser.add_argument("--vector", action="append", choices=["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", + "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", + "APP_STORE", "THIRD_PARTY"], + help="Restrict search to the specified threat cause vectors") + + +def setup_parser_with_vmware_criteria(parser): + setup_parser_with_basic_criteria(parser) + parser.add_argument("--groupid", action="append", type=int, + help="Restrict search to the specified AppDefense alarm group IDs") + + +def setup_parser_with_watchlist_criteria(parser): + setup_parser_with_basic_criteria(parser) + parser.add_argument("--watchlistid", action="append", type=str, help="Restrict search to the specified watchlists by ID") + parser.add_argument("--watchlistname", action="append", type=str, help="Restrict search to the specified watchlists by name") + + +def load_basic_criteria(query, args): + if args.query: + query = query.where(args.query) + if args.category: + query = query.categories(args.category) + if args.deviceid: + query = query.device_ids(args.deviceid) + if args.devicename: + query = query.device_names(args.devicename) + if args.os: + query = query.device_os(args.os) + if args.osversion: + query = query.device_os_version(args.osversion) + if args.username: + query = query.device_username(args.username) + if args.group: + query = query.group_results(True) + if args.alertid: + query = query.alert_ids(args.alertid) + if args.legacyalertid: + query = query.legacy_alert_ids(args.legacyalertid) + if args.severity: + query = query.minimum_severity(args.severity) + if args.policyid: + query = query.policy_ids(args.policyid) + if args.policyname: + query = query.policy_names(args.policyname) + if args.processname: + query = query.process_names(args.processname) + if args.processhash: + query = query.process_sha256(args.processhash) + if args.reputation: + query = query.reputations(args.reputation) + if args.tag: + query = query.tags(args.tag) + if args.priority: + query = query.target_priorities(args.priority) + if args.threatid: + query = query.threat_ids(args.threatid) + if args.type: + query = query.types(args.type) + if args.workflow: + query = query.workflows(args.workflow) + + +def load_cbanalytics_criteria(query, args): + load_basic_criteria(query, args) + if args.blockedthreat: + query = query.blocked_threat_categories(args.blockedthreat) + if args.location: + query = query.device_locations(args.location) + if args.killchain: + query = query.kill_chain_statuses(args.killchain) + if args.notblockedthreat: + query = query.not_blocked_threat_categories(args.notblockedthreat) + if args.policyapplied: + query = query.policy_applied(args.policyapplied) + if args.reason: + query = query.reason_code(args.reason) + if args.runstate: + query = query.run_states(args.runstate) + if args.sensoraction: + query = query.sensor_actions(args.sensoraction) + if args.vector: + query = query.threat_cause_vectors(args.vector) + + +def load_vmware_criteria(query, args): + load_basic_criteria(query, args) + if args.groupid: + query = query.group_ids(args.groupid) + + +def load_watchlist_criteria(query, args): + load_basic_criteria(query, args) + if args.watchlistid: + query = query.watchlist_ids(args.watchlistid) + if args.watchlistname: + query = query.watchlist_names(args.watchlistname) + \ No newline at end of file diff --git a/examples/psc/bulk_update_alerts.py b/examples/psc/bulk_update_alerts.py index e0ef2ce2..8d0d11e3 100755 --- a/examples/psc/bulk_update_alerts.py +++ b/examples/psc/bulk_update_alerts.py @@ -4,42 +4,11 @@ from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import BaseAlert +from alertsv6common import setup_parser_with_basic_criteria, load_basic_criteria def main(): parser = build_cli_parser("Bulk update the status of alerts") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") + setup_parser_with_basic_criteria(parser) parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") operation = parser.add_mutually_exclusive_group(required=True) @@ -56,48 +25,7 @@ def main(): else: raise NotImplemented("one of --dismiss or --undismiss must be specified") - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) + load_basic_criteria(query, args) if args.remediation: query = query.remediation(args.remediation) diff --git a/examples/psc/bulk_update_cbanalytics_alerts.py b/examples/psc/bulk_update_cbanalytics_alerts.py index 18b1709a..a25bdf17 100755 --- a/examples/psc/bulk_update_cbanalytics_alerts.py +++ b/examples/psc/bulk_update_cbanalytics_alerts.py @@ -4,66 +4,11 @@ from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import CBAnalyticsAlert +from alertsv6common import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria def main(): parser = build_cli_parser("Bulk update the status of CB Analytics alerts") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") - parser.add_argument("--blockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", - "KNOWN_MALWARE", "RISKY_PROGRAM"], - help="Restrict search to the specified threat categories that were blocked") - parser.add_argument("--location", action="append", choices=["ONSITE", "OFFSITE", "UNKNOWN"], - help="Restrict search to the specified device locations") - parser.add_argument("--killchain", action="append", choices=["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", - "INSTALL_RUN", "COMMAND_AND_CONTROL", "EXECUTE_GOAL", - "BREACH"], - help="Restrict search to the specified kill chain status values") - parser.add_argument("--notblockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", - "KNOWN_MALWARE", "RISKY_PROGRAM"], - help="Restrict search to the specified threat categories that were NOT blocked") - parser.add_argument("--policyapplied", action="append", choices=["APPLIED", "NOT_APPLIED"], - help="Restrict search to the specified policy-application status values") - parser.add_argument("--reason", action="append", type=str, help="Restrict search to the specified reason codes") - parser.add_argument("--runstate", action="append", choices=["DID_NOT_RUN", "RAN", "UNKNOWN"], - help="Restrict search to the specified run states") - parser.add_argument("--sensoraction", action="append", choices=["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", - "TERMINATE", "DENY"], - help="Restrict search to the specified sensor actions") - parser.add_argument("--vector", action="append", choices=["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", - "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", - "APP_STORE", "THIRD_PARTY"], - help="Restrict search to the specified threat cause vectors") + setup_parser_with_cbanalytics_criteria(parser) parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") operation = parser.add_mutually_exclusive_group(required=True) @@ -80,69 +25,7 @@ def main(): else: raise NotImplemented("one of --dismiss or --undismiss must be specified") - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) - if args.blockedthreat: - query = query.blocked_threat_categories(args.blockedthreat) - if args.location: - query = query.device_locations(args.location) - if args.killchain: - query = query.kill_chain_statuses(args.killchain) - if args.notblockedthreat: - query = query.not_blocked_threat_categories(args.notblockedthreat) - if args.policyapplied: - query = query.policy_applied(args.policyapplied) - if args.reason: - query = query.reason_code(args.reason) - if args.runstate: - query = query.run_states(args.runstate) - if args.sensoraction: - query = query.sensor_actions(args.sensoraction) - if args.vector: - query = query.threat_cause_vectors(args.vector) - if args.sort_by: - direction = "DESC" if args.reverse else "ASC" - query = query.sort_by(args.sort_by, direction) + load_cbanalytics_criteria(query, args) if args.remediation: query = query.remediation(args.remediation) diff --git a/examples/psc/bulk_update_vmware_alerts.py b/examples/psc/bulk_update_vmware_alerts.py index 8440fb34..b747d4c0 100755 --- a/examples/psc/bulk_update_vmware_alerts.py +++ b/examples/psc/bulk_update_vmware_alerts.py @@ -4,44 +4,11 @@ from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import VMwareAlert +from alertsv6common import setup_parser_with_vmware_criteria, load_vmware_criteria def main(): parser = build_cli_parser("Bulk update the status of VMware alerts") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") - parser.add_argument("--groupid", action="append", type=int, - help="Restrict search to the specified AppDefense alarm group IDs") + setup_parser_with_vmware_criteria(parser) parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") operation = parser.add_mutually_exclusive_group(required=True) @@ -58,53 +25,7 @@ def main(): else: raise NotImplemented("one of --dismiss or --undismiss must be specified") - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) - if args.groupid: - query = query.group_ids(args.groupid) - if args.sort_by: - direction = "DESC" if args.reverse else "ASC" - query = query.sort_by(args.sort_by, direction) + load_vmware_criteria(query, args) if args.remediation: query = query.remediation(args.remediation) diff --git a/examples/psc/bulk_update_watchlist_alerts.py b/examples/psc/bulk_update_watchlist_alerts.py index 0f11ae6b..9dd1d9b0 100755 --- a/examples/psc/bulk_update_watchlist_alerts.py +++ b/examples/psc/bulk_update_watchlist_alerts.py @@ -4,44 +4,11 @@ from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import WatchlistAlert +from alertsv6common import setup_parser_with_watchlist_criteria, load_watchlist_criteria def main(): parser = build_cli_parser("Bulk update the status of watchlist alerts") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") - parser.add_argument("--watchlistid", action="append", type=str, help="Restrict search to the specified watchlists by ID") - parser.add_argument("--watchlistname", action="append", type=str, help="Restrict search to the specified watchlists by name") + setup_parser_with_watchlist_criteria(parser) parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") operation = parser.add_mutually_exclusive_group(required=True) @@ -58,55 +25,7 @@ def main(): else: raise NotImplemented("one of --dismiss or --undismiss must be specified") - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) - if args.watchlistid: - query = query.watchlist_ids(args.watchlistid) - if args.watchlistname: - query = query.watchlist_names(args.watchlistname) - if args.sort_by: - direction = "DESC" if args.reverse else "ASC" - query = query.sort_by(args.sort_by, direction) + load_watchlist_criteria(query, args) if args.remediation: query = query.remediation(args.remediation) diff --git a/examples/psc/list_alert_facets.py b/examples/psc/list_alert_facets.py index 134dbaa3..7afcdbc3 100755 --- a/examples/psc/list_alert_facets.py +++ b/examples/psc/list_alert_facets.py @@ -3,42 +3,11 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import BaseAlert +from alertsv6common import setup_parser_with_basic_criteria, load_basic_criteria def main(): parser = build_cli_parser("List alert facets") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") + setup_parser_with_basic_criteria(parser) parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", @@ -50,48 +19,7 @@ def main(): cb = get_cb_psc_object(args) query = cb.select(BaseAlert) - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) + load_basic_criteria(query, args) facetinfos = query.facets(args.facet) for facetinfo in facetinfos: diff --git a/examples/psc/list_alerts.py b/examples/psc/list_alerts.py index 3dabf915..c49e4df3 100755 --- a/examples/psc/list_alerts.py +++ b/examples/psc/list_alerts.py @@ -3,42 +3,11 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import BaseAlert +from alertsv6common import setup_parser_with_basic_criteria, load_basic_criteria def main(): parser = build_cli_parser("List alerts") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") + setup_parser_with_basic_criteria(parser) parser.add_argument("-S", "--sort_by", help="Field to sort the output by") parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") @@ -46,48 +15,7 @@ def main(): cb = get_cb_psc_object(args) query = cb.select(BaseAlert) - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) + load_basic_criteria(query, args) if args.sort_by: direction = "DESC" if args.reverse else "ASC" query = query.sort_by(args.sort_by, direction) diff --git a/examples/psc/list_cbanalytics_alert_facets.py b/examples/psc/list_cbanalytics_alert_facets.py index 06a0e202..7b4d84d3 100755 --- a/examples/psc/list_cbanalytics_alert_facets.py +++ b/examples/psc/list_cbanalytics_alert_facets.py @@ -3,66 +3,11 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import CBAnalyticsAlert +from alertsv6common import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria def main(): parser = build_cli_parser("List CB Analytics alert facets") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") - parser.add_argument("--blockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", - "KNOWN_MALWARE", "RISKY_PROGRAM"], - help="Restrict search to the specified threat categories that were blocked") - parser.add_argument("--location", action="append", choices=["ONSITE", "OFFSITE", "UNKNOWN"], - help="Restrict search to the specified device locations") - parser.add_argument("--killchain", action="append", choices=["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", - "INSTALL_RUN", "COMMAND_AND_CONTROL", "EXECUTE_GOAL", - "BREACH"], - help="Restrict search to the specified kill chain status values") - parser.add_argument("--notblockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", - "KNOWN_MALWARE", "RISKY_PROGRAM"], - help="Restrict search to the specified threat categories that were NOT blocked") - parser.add_argument("--policyapplied", action="append", choices=["APPLIED", "NOT_APPLIED"], - help="Restrict search to the specified policy-application status values") - parser.add_argument("--reason", action="append", type=str, help="Restrict search to the specified reason codes") - parser.add_argument("--runstate", action="append", choices=["DID_NOT_RUN", "RAN", "UNKNOWN"], - help="Restrict search to the specified run states") - parser.add_argument("--sensoraction", action="append", choices=["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", - "TERMINATE", "DENY"], - help="Restrict search to the specified sensor actions") - parser.add_argument("--vector", action="append", choices=["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", - "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", - "APP_STORE", "THIRD_PARTY"], - help="Restrict search to the specified threat cause vectors") + setup_parser_with_cbanalytics_criteria(parser) parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", @@ -74,66 +19,7 @@ def main(): cb = get_cb_psc_object(args) query = cb.select(CBAnalyticsAlert) - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) - if args.blockedthreat: - query = query.blocked_threat_categories(args.blockedthreat) - if args.location: - query = query.device_locations(args.location) - if args.killchain: - query = query.kill_chain_statuses(args.killchain) - if args.notblockedthreat: - query = query.not_blocked_threat_categories(args.notblockedthreat) - if args.policyapplied: - query = query.policy_applied(args.policyapplied) - if args.reason: - query = query.reason_code(args.reason) - if args.runstate: - query = query.run_states(args.runstate) - if args.sensoraction: - query = query.sensor_actions(args.sensoraction) - if args.vector: - query = query.threat_cause_vectors(args.vector) + load_cbanalytics_criteria(query, args) facetinfos = query.facets(args.facet) for facetinfo in facetinfos: diff --git a/examples/psc/list_cbanalytics_alerts.py b/examples/psc/list_cbanalytics_alerts.py index e4cba7fc..9d0ca285 100755 --- a/examples/psc/list_cbanalytics_alerts.py +++ b/examples/psc/list_cbanalytics_alerts.py @@ -3,66 +3,11 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import CBAnalyticsAlert +from alertsv6common import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria def main(): parser = build_cli_parser("List CB Analytics alerts") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") - parser.add_argument("--blockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", - "KNOWN_MALWARE", "RISKY_PROGRAM"], - help="Restrict search to the specified threat categories that were blocked") - parser.add_argument("--location", action="append", choices=["ONSITE", "OFFSITE", "UNKNOWN"], - help="Restrict search to the specified device locations") - parser.add_argument("--killchain", action="append", choices=["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", - "INSTALL_RUN", "COMMAND_AND_CONTROL", "EXECUTE_GOAL", - "BREACH"], - help="Restrict search to the specified kill chain status values") - parser.add_argument("--notblockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", - "KNOWN_MALWARE", "RISKY_PROGRAM"], - help="Restrict search to the specified threat categories that were NOT blocked") - parser.add_argument("--policyapplied", action="append", choices=["APPLIED", "NOT_APPLIED"], - help="Restrict search to the specified policy-application status values") - parser.add_argument("--reason", action="append", type=str, help="Restrict search to the specified reason codes") - parser.add_argument("--runstate", action="append", choices=["DID_NOT_RUN", "RAN", "UNKNOWN"], - help="Restrict search to the specified run states") - parser.add_argument("--sensoraction", action="append", choices=["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", - "TERMINATE", "DENY"], - help="Restrict search to the specified sensor actions") - parser.add_argument("--vector", action="append", choices=["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", - "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", - "APP_STORE", "THIRD_PARTY"], - help="Restrict search to the specified threat cause vectors") + setup_parser_with_cbanalytics_criteria(parser) parser.add_argument("-S", "--sort_by", help="Field to sort the output by") parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") @@ -70,66 +15,7 @@ def main(): cb = get_cb_psc_object(args) query = cb.select(CBAnalyticsAlert) - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) - if args.blockedthreat: - query = query.blocked_threat_categories(args.blockedthreat) - if args.location: - query = query.device_locations(args.location) - if args.killchain: - query = query.kill_chain_statuses(args.killchain) - if args.notblockedthreat: - query = query.not_blocked_threat_categories(args.notblockedthreat) - if args.policyapplied: - query = query.policy_applied(args.policyapplied) - if args.reason: - query = query.reason_code(args.reason) - if args.runstate: - query = query.run_states(args.runstate) - if args.sensoraction: - query = query.sensor_actions(args.sensoraction) - if args.vector: - query = query.threat_cause_vectors(args.vector) + load_cbanalytics_criteria(query, args) if args.sort_by: direction = "DESC" if args.reverse else "ASC" query = query.sort_by(args.sort_by, direction) diff --git a/examples/psc/list_vmware_alert_facets.py b/examples/psc/list_vmware_alert_facets.py index 4a8dbe9d..8f95ecb6 100755 --- a/examples/psc/list_vmware_alert_facets.py +++ b/examples/psc/list_vmware_alert_facets.py @@ -3,44 +3,11 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import VMwareAlert +from alertsv6common import setup_parser_with_vmware_criteria, load_vmware_criteria def main(): parser = build_cli_parser("List VMware alert facets") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") - parser.add_argument("--groupid", action="append", type=int, - help="Restrict search to the specified AppDefense alarm group IDs") + setup_parser_with_vmware_criteria(parser) parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", @@ -52,50 +19,7 @@ def main(): cb = get_cb_psc_object(args) query = cb.select(VMwareAlert) - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) - if args.groupid: - query = query.group_ids(args.groupid) + load_vmware_criteria(query, args) facetinfos = query.facets(args.facet) for facetinfo in facetinfos: diff --git a/examples/psc/list_vmware_alerts.py b/examples/psc/list_vmware_alerts.py index 6ef3b315..b5a139d8 100755 --- a/examples/psc/list_vmware_alerts.py +++ b/examples/psc/list_vmware_alerts.py @@ -3,44 +3,11 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import VMwareAlert +from alertsv6common import setup_parser_with_vmware_criteria, load_vmware_criteria def main(): parser = build_cli_parser("List VMware alerts") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") - parser.add_argument("--groupid", action="append", type=int, - help="Restrict search to the specified AppDefense alarm group IDs") + setup_parser_with_vmware_criteria(parser) parser.add_argument("-S", "--sort_by", help="Field to sort the output by") parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") @@ -48,50 +15,7 @@ def main(): cb = get_cb_psc_object(args) query = cb.select(VMwareAlert) - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) - if args.groupid: - query = query.group_ids(args.groupid) + load_vmware_criteria(query, args) if args.sort_by: direction = "DESC" if args.reverse else "ASC" query = query.sort_by(args.sort_by, direction) diff --git a/examples/psc/list_watchlist_alert_facets.py b/examples/psc/list_watchlist_alert_facets.py index 3d4f182c..b8aa2a15 100755 --- a/examples/psc/list_watchlist_alert_facets.py +++ b/examples/psc/list_watchlist_alert_facets.py @@ -3,44 +3,11 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import WatchlistAlert +from alertsv6common import setup_parser_with_watchlist_criteria, load_watchlist_criteria def main(): parser = build_cli_parser("List watchlist alert facets") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") - parser.add_argument("--watchlistid", action="append", type=str, help="Restrict search to the specified watchlists by ID") - parser.add_argument("--watchlistname", action="append", type=str, help="Restrict search to the specified watchlists by name") + setup_parser_with_watchlist_criteria(parser) parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", @@ -52,52 +19,7 @@ def main(): cb = get_cb_psc_object(args) query = cb.select(WatchlistAlert) - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) - if args.watchlistid: - query = query.watchlist_ids(args.watchlistid) - if args.watchlistname: - query = query.watchlist_names(args.watchlistname) + load_watchlist_criteria(query, args) facetinfos = query.facets(args.facet) for facetinfo in facetinfos: diff --git a/examples/psc/list_watchlist_alerts.py b/examples/psc/list_watchlist_alerts.py index 43caa2cc..89ec1562 100755 --- a/examples/psc/list_watchlist_alerts.py +++ b/examples/psc/list_watchlist_alerts.py @@ -3,44 +3,11 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import WatchlistAlert +from alertsv6common import setup_parser_with_watchlist_criteria, load_watchlist_criteria def main(): parser = build_cli_parser("List watchlist alerts") - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") - parser.add_argument("--watchlistid", action="append", type=str, help="Restrict search to the specified watchlists by ID") - parser.add_argument("--watchlistname", action="append", type=str, help="Restrict search to the specified watchlists by name") + setup_parser_with_watchlist_criteria(parser) parser.add_argument("-S", "--sort_by", help="Field to sort the output by") parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") @@ -48,52 +15,7 @@ def main(): cb = get_cb_psc_object(args) query = cb.select(WatchlistAlert) - if args.query: - query = query.where(args.query) - if args.category: - query = query.categories(args.category) - if args.deviceid: - query = query.device_ids(args.deviceid) - if args.devicename: - query = query.device_names(args.devicename) - if args.os: - query = query.device_os(args.os) - if args.osversion: - query = query.device_os_version(args.osversion) - if args.username: - query = query.device_username(args.username) - if args.group: - query = query.group_results(True) - if args.alertid: - query = query.alert_ids(args.alertid) - if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.minimum_severity(args.severity) - if args.policyid: - query = query.policy_ids(args.policyid) - if args.policyname: - query = query.policy_names(args.policyname) - if args.processname: - query = query.process_names(args.processname) - if args.processhash: - query = query.process_sha256(args.processhash) - if args.reputation: - query = query.reputations(args.reputation) - if args.tag: - query = query.tags(args.tag) - if args.priority: - query = query.target_priorities(args.priority) - if args.threatid: - query = query.threat_ids(args.threatid) - if args.type: - query = query.types(args.type) - if args.workflow: - query = query.workflows(args.workflow) - if args.watchlistid: - query = query.watchlist_ids(args.watchlistid) - if args.watchlistname: - query = query.watchlist_names(args.watchlistname) + load_watchlist_criteria(query, args) if args.sort_by: direction = "DESC" if args.reverse else "ASC" query = query.sort_by(args.sort_by, direction) From 5bbd2dda537db3c5548e0e6bff9692cf316ed9cd Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Wed, 6 Nov 2019 10:43:40 -0700 Subject: [PATCH 047/197] added documentation for Alerts v6 to the PSC API page --- docs/psc-api.rst | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/docs/psc-api.rst b/docs/psc-api.rst index 9805ceb0..27d928a0 100755 --- a/docs/psc-api.rst +++ b/docs/psc-api.rst @@ -42,3 +42,95 @@ Selects all devices running Linux from the current organization. :members: :undoc-members: +Alerts API +---------- + +Using the API, you can search for alerts within your organization, and dismiss or +undismiss them, either individually or in bulk. + +You can use the select() method on the CbPSCBaseAPI to create a query object for +BaseAlert objects, which can be used to locate a list of alerts. You can also +search for more specialized alert types: + +* CBAnalyticsAlert - Alerts from CB Analytics +* VMwareAlert - Alerts from VMware +* WatchlistAlert - Alerts from watch lists + +*Example:* + + >>> cbapi = CbPSCBaseAPI(...) + >>> alerts = cbapi.select(BaseAlert).device_os(["WINDOWS"]).process_name(["IEXPLORE.EXE"]) + +Selects all alerts on a Windows device running the Internet Explorer process. + +Individual alerts may have their status changed using the dismiss() or undismiss() +methods on the BaseAlert object. To dismiss multiple alerts at once, you can use +the bulk_alert_dismiss() or bulk_alert_undismiss() methods on the CbPSCBaseAPI +object to set up a query, which you can add criteria to just as with a search +query. When you call the run() method on the query, a WorkflowStatus object is +returned; querying this object's "finished" property will let you know when the +operation is finished. + +*Example:* + + >>> cbapi = CbPSCBaseAPI(...) + >>> query = cbapi.bulk_alert_dismiss("ALERT").process_name(["IEXPLORE.EXE"]) + >>> stat = query.remediation("Using Chrome").run() + +Dismisses all alerts which reference the Internet Explorer process. + +**Query Objects:** + +.. autoclass:: cbapi.psc.query.BaseAlertSearchQuery + :members: + +.. autoclass:: cbapi.psc.query.CBAnalyticsAlertSearchQuery + :members: + +.. autoclass:: cbapi.psc.query.VMwareAlertSearchQuery + :members: + +.. autoclass:: cbapi.psc.query.WatchlistAlertSearchQuery + :members: + +.. autoclass:: cbapi.psc.query.BulkUpdateAlerts + :members: + +.. autoclass:: cbapi.psc.query.BulkUpdateCBAnalyticsAlerts + :members: + +.. autoclass:: cbapi.psc.query.BulkUpdateVMwareAlerts + :members: + +.. autoclass:: cbapi.psc.query.BulkUpdateWatchlistAlerts + :members: + +.. autoclass:: cbapi.psc.query.BulkUpdateThreatAlerts + :members: + +**Model Objects:** + +.. autoclass:: cbapi.psc.models.Workflow + :members: + :undoc-members: + +.. autoclass:: cbapi.psc.models.BaseAlert + :members: + :undoc-members: + +.. autoclass:: cbapi.psc.models.CBAnalyticsAlert + :members: + :undoc-members: + +.. autoclass:: cbapi.psc.models.VMwareAlert + :members: + :undoc-members: + +.. autoclass:: cbapi.psc.models.WatchlistAlert + :members: + :undoc-members: + +.. autoclass:: cbapi.psc.models.WorkflowStatus + :members: + :undoc-members: + From 22f6c13bacf59401676983411677cdaab3e7d844 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 11 Nov 2019 15:28:42 -0700 Subject: [PATCH 048/197] implemented changes suggested by Alex which remove many redundant classes and combine functionality. This code is now as tight as it can possibly get. :) --- src/cbapi/psc/query.py | 919 ++++++++------------------------------ src/cbapi/psc/rest_api.py | 74 +-- 2 files changed, 232 insertions(+), 761 deletions(-) diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 98187748..663dfe35 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -573,19 +573,27 @@ def update_sensor_version(self, sensor_version): {"sensor_version": sensor_version}) -class AlertRequestCriteriaBuilder: +class BaseAlertSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): """ - Auxiliary object that builds the criteria for alert request searches. + Represents a query that is used to locate BaseAlert objects. """ valid_categories = ["THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", "CRITICAL"] valid_reputations = ["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", "NOT_LISTED", "ADAPTIVE_WHITE_LIST", "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", "COMPANY_BLACK_LIST"] valid_alerttypes = ["CB_ANALYTICS", "VMWARE", "WATCHLIST"] valid_workflow_vals = ["OPEN", "DISMISSED"] + valid_facet_fields = ["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", + "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", + "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", + "POLICY_APPLIED", "SENSOR_ACTION"] - def __init__(self): + def __init__(self, doc_class, cb): + super().__init__(doc_class, cb) + self._query_builder = QueryBuilder() self._criteria = {} self._time_filter = {} + self._sortcriteria = {} + self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/workflow/_criteria" def _update_criteria(self, key, newlist): oldlist = self._criteria.get(key, []) @@ -599,7 +607,7 @@ def categories(self, cats): "THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", and "CRITICAL." :return: This instance """ - if not all((c in AlertRequestCriteriaBuilder.valid_categories) for c in cats): + if not all((c in BaseAlertSearchQuery.valid_categories) for c in cats): raise ApiError("One or more invalid category values") self._update_criteria("category", cats) return self @@ -806,7 +814,7 @@ def reputations(self, reps): "TRUSTED_WHITE_LIST", and "COMPANY_BLACK_LIST". :return: This instance """ - if not all((r in AlertRequestCriteriaBuilder.valid_reputations) for r in reps): + if not all((r in BaseAlertSearchQuery.valid_reputations) for r in reps): raise ApiError("One or more invalid reputation values") self._update_criteria("reputation", reps) return self @@ -860,7 +868,7 @@ def types(self, alerttypes): "CB_ANALYTICS", "VMWARE", and "WATCHLIST". :return: This instance """ - if not all((t in AlertRequestCriteriaBuilder.valid_alerttypes) for t in alerttypes): + if not all((t in BaseAlertSearchQuery.valid_alerttypes) for t in alerttypes): raise ApiError("One or more invalid alert type values") self._update_criteria("type", alerttypes) return self @@ -874,12 +882,12 @@ def workflows(self, workflow_vals): "OPEN" and "DISMISSED". :return: This instance """ - if not all((t in AlertRequestCriteriaBuilder.valid_workflow_vals) for t in workflow_vals): + if not all((t in BaseAlertSearchQuery.valid_workflow_vals) for t in workflow_vals): raise ApiError("One or more invalid workflow status values") self._update_criteria("workflow", workflow_vals) return self - def build(self): + def _build_criteria(self): """ Builds the criteria object for use in a query. @@ -890,10 +898,169 @@ def build(self): mycrit["create_time"] = self._time_filter return mycrit + def sort_by(self, key, direction="ASC"): + """Sets the sorting behavior on a query's results. + + Example:: + + >>> cb.select(BaseAlert).sort_by("name") + + :param key: the key in the schema to sort by + :param direction: the sort order, either "ASC" or "DESC" + :rtype: :py:class:`BaseAlertSearchQuery` + """ + if direction not in DeviceSearchQuery.valid_directions: + raise ApiError("invalid sort direction specified") + self._sortcriteria = {"field": key, "order": direction} + return self + + def _build_request(self, from_row, max_rows, add_sort=True): + request = {"criteria": self._build_criteria()} + request["query"] = self._query_builder._collapse() + if from_row > 0: + request["start"] = from_row + if max_rows >= 0: + request["rows"] = max_rows + if add_sort and self._sortcriteria != {}: + request["sort"] = [self._sortcriteria] + return request + + def _build_url(self, tail_end): + url = self._doc_class.urlobject.format(self._cb.credentials.org_key) + tail_end + return url + + def _count(self): + if self._count_valid: + return self._total_results + + url = self._build_url("/_search") + request = self._build_request(0, -1) + resp = self._cb.post_object(url, body=request) + result = resp.json() + + self._total_results = result["num_found"] + self._count_valid = True + + return self._total_results + + def _perform_query(self, from_row=0, max_rows=-1): + url = self._build_url("/_search") + current = from_row + numrows = 0 + still_querying = True + while still_querying: + request = self._build_request(current, max_rows) + resp = self._cb.post_object(url, body=request) + result = resp.json() + + self._total_results = result["num_found"] + self._count_valid = True + + results = result.get("results", []) + for item in results: + yield self._doc_class(self._cb, item["id"], item) + current += 1 + numrows += 1 + + if max_rows > 0 and numrows == max_rows: + still_querying = False + break + + from_row = current + if current >= self._total_results: + still_querying = False + break + + def facets(self, fieldlist, max_rows=0): + """ + Return information about the facets for this alert by search, using the defined criteria. + + :param fieldlist list: List of facet field names. Valid names are + "ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", + "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", + "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", + "POLICY_APPLIED", and "SENSOR_ACTION". + :param max_rows int: The maximum number of rows to return. 0 means return all rows. + :return: A list of facet information specified as dicts. + """ + if not all((field in BaseAlertSearchQuery.valid_facet_fields) for field in fieldlist): + raise ApiError("One or more invalid term field names") + request = self._build_request(0, -1, False) + request["terms"] = {"fields": fieldlist, "rows": max_rows} + url = self._build_url("/_facet") + resp = self._cb.post_object(url, body=request) + result = resp.json() + return result.get("results", []) + + def _update_status(self, status, remediation, comment): + request = { "state": status, "criteria": self._build_criteria(), "query": self._query_builder._collapse()} + if remediation is not None: + request["remediation_state"] = remediation + if comment is not None: + request["comment"] = comment + resp = self._cb.post_object(self._bulkupdate_url.format(self._cb.credentials.org_key), body=request) + output = resp.json() + return output["request_id"] + + def update(self, remediation=None, comment=None): + """ + Update all alerts matching the given query. The alerts will be left in an OPEN state after this request. + + :param remediation str: The remediation state to set for all alerts. + :param comment str: The comment to set for all alerts. + :return: The request ID, which may be used to select a WorkflowStatus object. + """ + return self._update_status("OPEN", remediation, comment) + + def dismiss(self, remediation=None, comment=None): + """ + Dismiss all alerts matching the given query. The alerts will be left in a DISMISSED state after this request. + + :param remediation str: The remediation state to set for all alerts. + :param comment str: The comment to set for all alerts. + :return: The request ID, which may be used to select a WorkflowStatus object. + """ + return self._update_status("DISMISSED", remediation, comment) + + +class WatchlistAlertSearchQuery(BaseAlertSearchQuery): + """ + Represents a query that is used to locate WatchlistAlert objects. + """ + def __init__(self, doc_class, cb): + super().__init__(doc_class, cb) + self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/watchlist/workflow/_criteria" + + def watchlist_ids(self, ids): + """ + Restricts the alerts that this query is performed on to the specified + watchlist ID values. + + :param ids list: list of string watchlist ID values + :return: This instance + """ + if not all(isinstance(t, str) for t in ids): + raise ApiError("One or more invalid watchlist IDs") + self._update_criteria("watchlist_id", ids) + return self + + def watchlist_names(self, names): + """ + Restricts the alerts that this query is performed on to the specified + watchlist name values. + + :param names list: list of string watchlist name values + :return: This instance + """ + if not all(isinstance(name, str) for name in names): + raise ApiError("One or more invalid watchlist names") + self._update_criteria("watchlist_name", names) + return self + -class CBAnalyticsAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): +class CBAnalyticsAlertSearchQuery(BaseAlertSearchQuery): """ - Auxiliary object that builds the criteria for CB Analytics alert request searches. + Represents a query that is used to locate CBAnalyticsAlert objects. """ valid_threat_categories = ["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", "RISKY_PROGRAM"] valid_locations = ["ONSITE", "OFFSITE", "UNKNOWN"] @@ -904,9 +1071,10 @@ class CBAnalyticsAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): valid_sensor_actions = ["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", "TERMINATE", "DENY"] valid_threat_cause_vectors = ["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", "APP_STORE", "THIRD_PARTY"] - - def __init__(self): - super().__init__() + + def __init__(self, doc_class, cb): + super().__init__(doc_class, cb) + self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/cbanalytics/workflow/_criteria" def blocked_threat_categories(self, categories): """ @@ -917,7 +1085,7 @@ def blocked_threat_categories(self, categories): "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". :return: This instance. """ - if not all((category in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_categories) + if not all((category in CBAnalyticsAlertSearchQuery.valid_threat_categories) for category in categories): raise ApiError("One or more invalid threat categories") self._update_criteria("blocked_threat_category", categories) @@ -932,7 +1100,7 @@ def device_locations(self, locations): and "UNKNOWN". :return: This instance. """ - if not all((location in CBAnalyticsAlertRequestCriteriaBuilder.valid_locations) + if not all((location in CBAnalyticsAlertSearchQuery.valid_locations) for location in locations): raise ApiError("One or more invalid device locations") self._update_criteria("device_location", locations) @@ -948,7 +1116,7 @@ def kill_chain_statuses(self, statuses): "EXECUTE_GOAL", and "BREACH". :return: This instance. """ - if not all((status in CBAnalyticsAlertRequestCriteriaBuilder.valid_kill_chain_statuses) + if not all((status in CBAnalyticsAlertSearchQuery.valid_kill_chain_statuses) for status in statuses): raise ApiError("One or more invalid kill chain status values") self._update_criteria("kill_chain_status", statuses) @@ -963,7 +1131,7 @@ def not_blocked_threat_categories(self, categories): "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". :return: This instance. """ - if not all((category in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_categories) + if not all((category in CBAnalyticsAlertSearchQuery.valid_threat_categories) for category in categories): raise ApiError("One or more invalid threat categories") self._update_criteria("not_blocked_threat_category", categories) @@ -978,7 +1146,7 @@ def policy_applied(self, applied_statuses): "APPLIED" and "NOT_APPLIED". :return: This instance. """ - if not all((s in CBAnalyticsAlertRequestCriteriaBuilder.valid_policy_applied) + if not all((s in CBAnalyticsAlertSearchQuery.valid_policy_applied) for s in applied_statuses): raise ApiError("One or more invalid policy-applied values") self._update_criteria("policy_applied", applied_statuses) @@ -1005,7 +1173,7 @@ def run_states(self, states): and "UNKNOWN". :return: This instance. """ - if not all((s in CBAnalyticsAlertRequestCriteriaBuilder.valid_run_states) + if not all((s in CBAnalyticsAlertSearchQuery.valid_run_states) for s in states): raise ApiError("One or more invalid run states") self._update_criteria("run_state", states) @@ -1019,7 +1187,7 @@ def sensor_actions(self, actions): "ALLOW", "ALLOW_AND_LOG", "TERMINATE", and "DENY". :return: This instance. """ - if not all((action in CBAnalyticsAlertRequestCriteriaBuilder.valid_sensor_actions) + if not all((action in CBAnalyticsAlertSearchQuery.valid_sensor_actions) for action in actions): raise ApiError("One or more invalid sensor actions") self._update_criteria("sensor_action", actions) @@ -1034,19 +1202,20 @@ def threat_cause_vectors(self, vectors): "UNKNOWN", "APP_STORE", and "THIRD_PARTY". :return: This instance. """ - if not all((vector in CBAnalyticsAlertRequestCriteriaBuilder.valid_threat_cause_vectors) + if not all((vector in CBAnalyticsAlertSearchQuery.valid_threat_cause_vectors) for vector in vectors): raise ApiError("One or more invalid threat cause vectors") self._update_criteria("threat_cause_vector", vectors) return self -class VMwareAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): +class VMwareAlertSearchQuery(BaseAlertSearchQuery): """ - Auxiliary object that builds the criteria for VMware alert request searches. + Represents a query that is used to locate VMwareAlert objects. """ - def __init__(self): - super().__init__() + def __init__(self, doc_class, cb): + super().__init__(doc_class, cb) + self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/vmware/workflow/_criteria" def group_ids(self, groupids): """ @@ -1060,703 +1229,3 @@ def group_ids(self, groupids): raise ApiError("One or more invalid alarm group IDs") self._update_criteria("group_id", groupids) return self - - -class WatchlistAlertRequestCriteriaBuilder(AlertRequestCriteriaBuilder): - """ - Auxiliary object that builds the criteria for watchlist alert request searches. - """ - def __init__(self): - super().__init__() - - def watchlist_ids(self, ids): - """ - Restricts the alerts that this query is performed on to the specified - watchlist ID values. - - :param ids list: list of string watchlist ID values - :return: This instance - """ - if not all(isinstance(t, str) for t in ids): - raise ApiError("One or more invalid watchlist IDs") - self._update_criteria("watchlist_id", ids) - return self - - def watchlist_names(self, names): - """ - Restricts the alerts that this query is performed on to the specified - watchlist name values. - - :param names list: list of string watchlist name values - :return: This instance - """ - if not all(isinstance(name, str) for name in names): - raise ApiError("One or more invalid watchlist names") - self._update_criteria("watchlist_name", names) - return self - - -class AlertCriteriaBuilderMixin: - """ - Added to query classes to allow them to manipulate alert criteria for queries. - """ - def categories(self, cats): - """ - Restricts the alerts that this query is performed on to the specified categories. - - :param cats list: List of categories to be restricted to. Valid categories are - "THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", and "CRITICAL." - :return: This instance - """ - self._criteria_builder.categories(cats) - return self - - def create_time(self, *args, **kwargs): - """ - Restricts the alerts that this query is performed on to the specified - creation time (either specified as a start and end point or as a - range). - - :return: This instance - """ - self._criteria_builder.create_time(*args, **kwargs) - return self - - def device_ids(self, device_ids): - """ - Restricts the alerts that this query is performed on to the specified - device IDs. - - :param device_ids list: list of integer device IDs - :return: This instance - """ - self._criteria_builder.device_ids(device_ids) - return self - - def device_names(self, device_names): - """ - Restricts the alerts that this query is performed on to the specified - device names. - - :param device_names list: list of string device names - :return: This instance - """ - self._criteria_builder.device_names(device_names) - return self - - def device_os(self, device_os): - """ - Restricts the alerts that this query is performed on to the specified - device operating systems. - - :param device_os list: List of string operating systems. Valid values are - "WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", and "OTHER." - :return: This instance - """ - self._criteria_builder.device_os(device_os) - return self - - def device_os_versions(self, device_os_versions): - """ - Restricts the alerts that this query is performed on to the specified - device operating system versions. - - :param device_os_versions list: List of string operating system versions. - :return: This instance - """ - self._criteria_builder.device_os_versions(device_os_versions) - return self - - def device_username(self, users): - """ - Restricts the alerts that this query is performed on to the specified - user names. - - :param users list: List of string user names. - :return: This instance - """ - self._criteria_builder.device_username(users) - return self - - def group_results(self, flag): - """ - Specifies whether or not to group the results of the query. - - :param flag boolean: True to group the results, False to not do so. - :return: This instance - """ - self._criteria_builder.group_results(flag) - return self - - def alert_ids(self, alert_ids): - """ - Restricts the alerts that this query is performed on to the specified - alert IDs. - - :param alert_ids list: List of string alert IDs. - :return: This instance - """ - self._criteria_builder.alert_ids(alert_ids) - return self - - def legacy_alert_ids(self, alert_ids): - """ - Restricts the alerts that this query is performed on to the specified - legacy alert IDs. - - :param alert_ids list: List of string legacy alert IDs. - :return: This instance - """ - self._criteria_builder.legacy_alert_ids(alert_ids) - return self - - def minimum_severity(self, severity): - """ - Restricts the alerts that this query is performed on to the specified - minimum severity level. - - :param severity int: The minimum severity level for alerts. - :return: This instance - """ - self._criteria_builder.minimum_severity(severity) - return self - - def policy_ids(self, policy_ids): - """ - Restricts the alerts that this query is performed on to the specified - policy IDs. - - :param policy_ids list: list of integer policy IDs - :return: This instance - """ - self._criteria_builder.policy_ids(policy_ids) - return self - - def policy_names(self, policy_names): - """ - Restricts the alerts that this query is performed on to the specified - policy names. - - :param policy_names list: list of string policy names - :return: This instance - """ - self._criteria_builder.policy_names(policy_names) - return self - - def process_names(self, process_names): - """ - Restricts the alerts that this query is performed on to the specified - process names. - - :param process_names list: list of string process names - :return: This instance - """ - self._criteria_builder.process_names(process_names) - return self - - def process_sha256(self, shas): - """ - Restricts the alerts that this query is performed on to the specified - process SHA-256 hash values. - - :param shas list: list of string process SHA-256 hash values - :return: This instance - """ - self._criteria_builder.process_sha256(shas) - return self - - def reputations(self, reps): - """ - Restricts the alerts that this query is performed on to the specified - reputation values. - - :param reps list: List of string reputation values. Valid values are - "KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", "NOT_LISTED", - "ADAPTIVE_WHITE_LIST", "COMMON_WHITE_LIST", - "TRUSTED_WHITE_LIST", and "COMPANY_BLACK_LIST". - :return: This instance - """ - self._criteria_builder.reputations(reps) - return self - - def tags(self, tags): - """ - Restricts the alerts that this query is performed on to the specified - tag values. - - :param tags list: list of string tag values - :return: This instance - """ - self._criteria_builder.tags(tags) - return self - - def target_priorities(self, priorities): - """ - Restricts the alerts that this query is performed on to the specified - target priority values. - - :param reps list: List of string target priority values. Valid values are - "LOW", "MEDIUM", "HIGH", and "MISSION_CRITICAL". - :return: This instance - """ - self._criteria_builder.target_priorities(priorities) - return self - - def threat_ids(self, threats): - """ - Restricts the alerts that this query is performed on to the specified - threat ID values. - - :param threats list: list of string threat ID values - :return: This instance - """ - self._criteria_builder.threat_ids(threats) - return self - - def types(self, alerttypes): - """ - Restricts the alerts that this query is performed on to the specified - alert type values. - - :param alerttypes list: List of string alert type values. Valid values are - "CB_ANALYTICS", "VMWARE", and "WATCHLIST". - :return: This instance - """ - self._criteria_builder.types(alerttypes) - return self - - def workflows(self, workflow_vals): - """ - Restricts the alerts that this query is performed on to the specified - workflow status values. - - :param workflow_vals list: List of string alert type values. Valid values are - "OPEN" and "DISMISSED". - :return: This instance - """ - self._criteria_builder.workflows(workflow_vals) - return self - - -class CBAnalyticsAlertCriteriaBuilderMixin(AlertCriteriaBuilderMixin): - """ - Added to query classes to allow them to manipulate CB Analytics alert criteria for queries. - """ - def blocked_threat_categories(self, categories): - """ - Restricts the alerts that this query is performed on to the specified - threat categories that were blocked. - - :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", - "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". - :return: This instance. - """ - self._criteria_builder.blocked_threat_categories(categories) - return self - - def device_locations(self, locations): - """ - Restricts the alerts that this query is performed on to the specified - device locations. - - :param locations list: List of device locations to look for. Valid values are "ONSITE", "OFFSITE", - and "UNKNOWN". - :return: This instance. - """ - self._criteria_builder.device_locations(locations) - return self - - def kill_chain_statuses(self, statuses): - """ - Restricts the alerts that this query is performed on to the specified - kill chain statuses. - - :param statuses list: List of kill chain statuses to look for. Valid values are "RECONNAISSANCE", - "WEAPONIZE", "DELIVER_EXPLOIT", "INSTALL_RUN","COMMAND_AND_CONTROL", - "EXECUTE_GOAL", and "BREACH". - :return: This instance. - """ - self._criteria_builder.kill_chain_statuses(statuses) - return self - - def not_blocked_threat_categories(self, categories): - """ - Restricts the alerts that this query is performed on to the specified - threat categories that were NOT blocked. - - :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", - "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". - :return: This instance. - """ - self._criteria_builder.not_blocked_threat_categories(categories) - return self - - def policy_applied(self, applied_statuses): - """ - Restricts the alerts that this query is performed on to the specified - status values showing whether policies were applied. - - :param applied_statuses list: List of status values to look for. Valid values are - "APPLIED" and "NOT_APPLIED". - :return: This instance. - """ - self._criteria_builder.policy_applied(applied_statuses) - return self - - def reason_code(self, reason): - """ - Restricts the alerts that this query is performed on to the specified - reason codes (enum values). - - :param reason list: List of string reason codes to look for. - :return: This instance. - """ - self._criteria_builder.reason_code(reason) - return self - - def run_states(self, states): - """ - Restricts the alerts that this query is performed on to the specified run states. - - :param states list: List of run states to look for. Valid values are "DID_NOT_RUN", "RAN", - and "UNKNOWN". - :return: This instance. - """ - self._criteria_builder.run_states(states) - return self - - def sensor_actions(self, actions): - """ - Restricts the alerts that this query is performed on to the specified sensor actions. - - :param actions list: List of sensor actions to look for. Valid values are "POLICY_NOT_APPLIED", - "ALLOW", "ALLOW_AND_LOG", "TERMINATE", and "DENY". - :return: This instance. - """ - self._criteria_builder.sensor_actions(actions) - return self - - def threat_cause_vectors(self, vectors): - """ - Restricts the alerts that this query is performed on to the specified threat cause vectors. - - :param vectors list: List of threat cause vectors to look for. Valid values are "EMAIL", "WEB", - "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", "REMOVABLE_MEDIA", - "UNKNOWN", "APP_STORE", and "THIRD_PARTY". - :return: This instance. - """ - self._criteria_builder.threat_cause_vectors(vectors) - return self - - -class VMwareAlertCriteriaBuilderMixin(AlertCriteriaBuilderMixin): - """ - Added to query classes to allow them to manipulate VMware alert criteria for queries. - """ - def group_ids(self, groupids): - """ - Restricts the alerts that this query is performed on to the specified - AppDefense-assigned alarm group IDs. - - :param groupids list: List of (integer) AppDefense-assigned alarm group IDs. - :return: This instance. - """ - self._criteria_builder.group_ids(groupids) - return self - - -class WatchlistAlertCriteriaBuilderMixin(AlertCriteriaBuilderMixin): - """ - Added to query classes to allow them to manipulate watchlist alert criteria for queries. - """ - def watchlist_ids(self, ids): - """ - Restricts the alerts that this query is performed on to the specified - watchlist ID values. - - :param ids list: list of string watchlist ID values - :return: This instance - """ - self._criteria_builder.watchlist_ids(ids) - return self - - def watchlist_names(self, names): - """ - Restricts the alerts that this query is performed on to the specified - watchlist name values. - - :param names list: list of string watchlist name values - :return: This instance - """ - self._criteria_builder.watchlist_names(names) - return self - - -class BaseAlertSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, AlertCriteriaBuilderMixin, - IterableQueryMixin): - """ - Represents a query that is used to locate BaseAlert objects. - """ - valid_facet_fields = ["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", - "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", - "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", - "POLICY_APPLIED", "SENSOR_ACTION"] - - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_builder = QueryBuilder() - self._criteria_builder = AlertRequestCriteriaBuilder() - self._sortcriteria = {} - - def sort_by(self, key, direction="ASC"): - """Sets the sorting behavior on a query's results. - - Example:: - - >>> cb.select(BaseAlert).sort_by("name") - - :param key: the key in the schema to sort by - :param direction: the sort order, either "ASC" or "DESC" - :rtype: :py:class:`BaseAlertSearchQuery` - """ - if direction not in DeviceSearchQuery.valid_directions: - raise ApiError("invalid sort direction specified") - self._sortcriteria = {"field": key, "order": direction} - return self - - def _build_request(self, from_row, max_rows, add_sort=True): - request = {"criteria": self._criteria_builder.build()} - request["query"] = self._query_builder._collapse() - if from_row > 0: - request["start"] = from_row - if max_rows >= 0: - request["rows"] = max_rows - if add_sort and self._sortcriteria != {}: - request["sort"] = [self._sortcriteria] - return request - - def _build_url(self, tail_end): - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) + tail_end - return url - - def _count(self): - if self._count_valid: - return self._total_results - - url = self._build_url("/_search") - request = self._build_request(0, -1) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - return self._total_results - - def _perform_query(self, from_row=0, max_rows=-1): - url = self._build_url("/_search") - current = from_row - numrows = 0 - still_querying = True - while still_querying: - request = self._build_request(current, max_rows) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - results = result.get("results", []) - for item in results: - yield self._doc_class(self._cb, item["id"], item) - current += 1 - numrows += 1 - - if max_rows > 0 and numrows == max_rows: - still_querying = False - break - - from_row = current - if current >= self._total_results: - still_querying = False - break - - def facets(self, fieldlist, max_rows=0): - """ - Return information about the facets for this alert by search, using the defined criteria. - - :param fieldlist list: List of facet field names. Valid names are - "ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", - "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", - "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", - "POLICY_APPLIED", and "SENSOR_ACTION". - :param max_rows int: The maximum number of rows to return. 0 means return all rows. - :return: A list of facet information specified as dicts. - """ - if not all((field in BaseAlertSearchQuery.valid_facet_fields) for field in fieldlist): - raise ApiError("One or more invalid term field names") - request = self._build_request(0, -1, False) - request["terms"] = {"fields": fieldlist, "rows": max_rows} - url = self._build_url("/_facet") - resp = self._cb.post_object(url, body=request) - result = resp.json() - return result.get("results", []) - - -class WatchlistAlertSearchQuery(BaseAlertSearchQuery, WatchlistAlertCriteriaBuilderMixin): - """ - Represents a query that is used to locate WatchlistAlert objects. - """ - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() - - -class CBAnalyticsAlertSearchQuery(BaseAlertSearchQuery, CBAnalyticsAlertCriteriaBuilderMixin): - """ - Represents a query that is used to locate CBAnalyticsAlert objects. - """ - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._criteria_builder = CBAnalyticsAlertRequestCriteriaBuilder() - - -class VMwareAlertSearchQuery(BaseAlertSearchQuery, VMwareAlertCriteriaBuilderMixin): - """ - Represents a query that is used to locate VMwareAlert objects. - """ - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._criteria_builder = VMwareAlertRequestCriteriaBuilder() - - -class BulkUpdateAlertsBase: - """ - Base query for doing bulk updates on alerts, where the result of a search is used to set - the states of multiple alerts. - """ - def __init__(self, cb, state): - self._cb = cb - self._state = state - self._additional_fields = {} - - def remediation(self, remediation): - """ - Sets the remediation state message to be applied to all selected alerts. - - :param remediation str: The remediation state message. - """ - self._additional_fields["remediation_state"] = remediation - return self - - def comment(self, comment): - """ - Sets the comment to be applied to all selected alerts. - - :param comment str: The comment to be used. - """ - self._additional_fields["comment"] = comment - return self - - def _url(self): - raise ApiError("invalid abstract URL for the operation") - - def _build_request(self): - request = self._additional_fields - request["state"] = self._state - return request - - def run(self): - """ - Executes the search query and alert state change operation. - - :return: A WorkflowStatus object that can be used for monitoring the progress - of the operation. - """ - resp = self._cb.post_object(self._url(), body=self._build_request()) - output = resp.json() - return self._cb._new_workflow_status(output["request_id"]) - - -class BulkUpdateAlerts(BulkUpdateAlertsBase, AlertCriteriaBuilderMixin, QueryBuilderSupportMixin): - """ - Query for bulk update of base-level alerts. - """ - def __init__(self, cb, state): - super().__init__(cb, state) - self._criteria_builder = AlertRequestCriteriaBuilder() - self._query_builder = QueryBuilder() - - def _url(self): - return "/appservices/v6/orgs/{0}/alerts/workflow/_criteria".format(self._cb.credentials.org_key) - - def _build_request(self): - request = super()._build_request() - request["criteria"] = self._criteria_builder.build() - request["query"] = self._query_builder._collapse() - return request - - -class BulkUpdateCBAnalyticsAlerts(BulkUpdateAlerts, CBAnalyticsAlertCriteriaBuilderMixin): - """ - Query for bulk update of CB Analytics alerts. - """ - def __init__(self, cb, state): - super().__init__(cb, state) - self._criteria_builder = CBAnalyticsAlertRequestCriteriaBuilder() - - def _url(self): - return "/appservices/v6/orgs/{0}/alerts/cbanalytics/workflow/_criteria".format(self._cb.credentials.org_key) - - -class BulkUpdateVMwareAlerts(BulkUpdateAlerts, VMwareAlertCriteriaBuilderMixin): - """ - Query for bulk update of VMware alerts. - """ - def __init__(self, cb, state): - super().__init__(cb, state) - self._criteria_builder = VMwareAlertRequestCriteriaBuilder() - - def _url(self): - return "/appservices/v6/orgs/{0}/alerts/vmware/workflow/_criteria".format(self._cb.credentials.org_key) - - -class BulkUpdateWatchlistAlerts(BulkUpdateAlerts, WatchlistAlertCriteriaBuilderMixin): - """ - Query for bulk update of watchlist alerts. - """ - def __init__(self, cb, state): - super().__init__(cb, state) - self._criteria_builder = WatchlistAlertRequestCriteriaBuilder() - - def _url(self): - return "/appservices/v6/orgs/{0}/alerts/watchlist/workflow/_criteria".format(self._cb.credentials.org_key) - - -class BulkUpdateThreatAlerts(BulkUpdateAlertsBase): - """ - Query for bulk update of threat alerts. - """ - def __init__(self, cb, state): - super().__init__(cb, state) - self._threat_ids = [] - - def threat_ids(self, threats): - """ - Specifies the threat IDs to set the status of alerts for. - - :param threats list: The list of string threat identifiers. - :return: This instance. - """ - if not all(isinstance(t, str) for t in threats): - raise ApiError("One or more invalid threat ID values") - self._threat_ids = self._threat_ids + threats - return self - - def _url(self): - return "/appservices/v6/orgs/{0}/threat/workflow/_criteria".format(self._cb.credentials.org_key) - - def _build_request(self): - request = super()._build_request() - request["threat_id"] = self._threat_ids - return request diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 15c37d9e..65fcafc6 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -1,9 +1,6 @@ from cbapi.connection import BaseAPI from cbapi.errors import ApiError, ServerError from .cblr import LiveResponseSessionManager -from .models import WorkflowStatus -from .query import BulkUpdateAlerts, BulkUpdateWatchlistAlerts, BulkUpdateThreatAlerts, \ - BulkUpdateCBAnalyticsAlerts, BulkUpdateVMwareAlerts import logging log = logging.getLogger(__name__) @@ -20,10 +17,6 @@ class CbPSCBaseAPI(BaseAPI): >>> from cbapi import CbPSCBaseAPI >>> cb = CbPSCBaseAPI(profile="production") """ - alert_update_queries = {"ALERT": BulkUpdateAlerts, "WATCHLIST": BulkUpdateWatchlistAlerts, - "THREAT": BulkUpdateThreatAlerts, "CBANALYTICS": BulkUpdateCBAnalyticsAlerts, - "VMWARE": BulkUpdateVMwareAlerts} - def __init__(self, *args, **kwargs): super(CbPSCBaseAPI, self).__init__(product_name="psc", *args, **kwargs) self._lr_scheduler = None @@ -143,32 +136,41 @@ def alert_search_suggestions(self, query): url = "/appservices/v6/orgs/{0}/alerts/search_suggestions".format(self.credentials.org_key) output = self.get_object(url, query_params) return output["suggestions"] - - def _new_workflow_status(self, requestid): - return WorkflowStatus(self, requestid) - - def _bulk_alert_update_query(self, state, querytype): - cls = CbPSCBaseAPI.alert_update_queries.get(querytype, None) - if cls is None: - raise ApiError("unknown query type for bulk alert update") - return cls(self, state) - - def bulk_alert_dismiss(self, querytype): - """ - Start a query to dismiss multiple alerts. - - :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", "THREAT", - "CBANALYTICS", or "VMWARE". - :return: The new query. - """ - return self._bulk_alert_update_query("DISMISSED", querytype) - - def bulk_alert_undismiss(self, querytype): - """ - Start a query to un-dismiss multiple alerts. - - :param querytype str: The type of query to create, either "ALERT", "WATCHLIST", "THREAT", - "CBANALYTICS", or "VMWARE". - :return: The new query. - """ - return self._bulk_alert_update_query("OPEN", querytype) + + def _bulk_threat_update_status(self, threat_ids, status, remediation, comment): + if not all(isinstance(t, str) for t in threat_ids): + raise ApiError("One or more invalid threat ID values") + request = { "state": status, "threat_id": threat_ids} + if remediation is not None: + request["remediation_state"] = remediation + if comment is not None: + request["comment"] = comment + url = "/appservices/v6/orgs/{0}/threat/workflow/_criteria".format(self._cb.credentials.org_key) + resp = self._cb.post_object(url, body=request) + output = resp.json() + return output["request_id"] + + def bulk_threat_update(self, threat_ids, remediation=None, comment=None): + """ + Update the alert status of alerts associated with multiple threat IDs. + The alerts will be left in an OPEN state after this request. + + :param threat_ids list: List of string threat IDs. + :param remediation str: The remediation state to set for all alerts. + :param comment str: The comment to set for all alerts. + :return: The request ID, which may be used to select a WorkflowStatus object. + """ + return self._bulk_threat_update_status(threat_ids, "OPEN", remediation, comment) + + def bulk_threat_dismiss(self, threat_ids, remediation=None, comment=None): + """ + Dismiss the alerts associated with multiple threat IDs. + The alerts will be left in a DISMISSED state after this request. + + :param threat_ids list: List of string threat IDs. + :param remediation str: The remediation state to set for all alerts. + :param comment str: The comment to set for all alerts. + :return: The request ID, which may be used to select a WorkflowStatus object. + """ + return self._bulk_threat_update_status(threat_ids, "DISMISSED", remediation, comment) + \ No newline at end of file From 38d2bbe0d11e832d074ce94df84fecb6042c85b2 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 11 Nov 2019 15:47:54 -0700 Subject: [PATCH 049/197] updated the unit tests to match the updated code --- src/cbapi/psc/rest_api.py | 4 +- test/cbapi/psc/test_rest_api.py | 154 +++++++++++++++----------------- 2 files changed, 75 insertions(+), 83 deletions(-) diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 65fcafc6..7c9b76e2 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -145,8 +145,8 @@ def _bulk_threat_update_status(self, threat_ids, status, remediation, comment): request["remediation_state"] = remediation if comment is not None: request["comment"] = comment - url = "/appservices/v6/orgs/{0}/threat/workflow/_criteria".format(self._cb.credentials.org_key) - resp = self._cb.post_object(url, body=request) + url = "/appservices/v6/orgs/{0}/threat/workflow/_criteria".format(self.credentials.org_key) + resp = self.post_object(url, body=request) output = resp.json() return output["request_id"] diff --git a/test/cbapi/psc/test_rest_api.py b/test/cbapi/psc/test_rest_api.py index 0a8e062a..1a5ae84d 100755 --- a/test/cbapi/psc/test_rest_api.py +++ b/test/cbapi/psc/test_rest_api.py @@ -1,11 +1,12 @@ import pytest from cbapi.errors import ApiError -from cbapi.psc.models import Device, BaseAlert, CBAnalyticsAlert, VMwareAlert, WatchlistAlert -from cbapi.psc.query import BulkUpdateAlerts, BulkUpdateWatchlistAlerts, BulkUpdateThreatAlerts, \ - BulkUpdateCBAnalyticsAlerts, BulkUpdateVMwareAlerts +from cbapi.psc.models import Device, BaseAlert, CBAnalyticsAlert, VMwareAlert, WatchlistAlert, WorkflowStatus from cbapi.psc.rest_api import CbPSCBaseAPI from test.mocks import ConnectionMocks, MockResponse +# +# --- Device v6 Tests +# def test_get_device(monkeypatch): _was_called = False @@ -557,25 +558,11 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) api.select(Device).where("foobar").update_sensor_version({ "RHEL": "2.3.4.5"}) assert _was_called + +# +# --- Alerts v6 Tests +# - -def test_bulk_query_types_return_ok(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - bquery = api.bulk_alert_dismiss("ALERT") - assert isinstance(bquery, BulkUpdateAlerts) - bquery = api.bulk_alert_dismiss("WATCHLIST") - assert isinstance(bquery, BulkUpdateWatchlistAlerts) - bquery = api.bulk_alert_dismiss("THREAT") - assert isinstance(bquery, BulkUpdateThreatAlerts) - bquery = api.bulk_alert_dismiss("CBANALYTICS") - assert isinstance(bquery, BulkUpdateCBAnalyticsAlerts) - bquery = api.bulk_alert_dismiss("VMWARE") - assert isinstance(bquery, BulkUpdateVMwareAlerts) - with pytest.raises(ApiError): - api.bulk_alert_dismiss("CRIMSON") - - def test_query_basealert_with_all_bells_and_whistles(monkeypatch): _was_called = False @@ -1268,23 +1255,16 @@ def mock_post_object(url, body, **kwargs): _was_called = True return MockResponse({"request_id": "497ABX"}) - def mock_get_object(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" - resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} - resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} - return resp - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - q = api.bulk_alert_dismiss("ALERT").remediation("Fixed").comment("Yessir") - wstat = q.where("Blort").device_names(["HAL9000"]).run() + q = api.select(BaseAlert).where("Blort").device_names(["HAL9000"]) + reqid = q.dismiss("Fixed", "Yessir") assert _was_called - assert wstat.id_ == "497ABX" + assert reqid == "497ABX" def test_alerts_bulk_undismiss(monkeypatch): @@ -1302,23 +1282,16 @@ def mock_post_object(url, body, **kwargs): _was_called = True return MockResponse({"request_id": "497ABX"}) - def mock_get_object(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" - resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} - resp["workflow"] = {"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} - return resp - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - q = api.bulk_alert_undismiss("ALERT").remediation("Fixed").comment("NoSir") - wstat = q.where("Blort").device_names(["HAL9000"]).run() + q = api.select(BaseAlert).where("Blort").device_names(["HAL9000"]) + reqid = q.update("Fixed", "NoSir") assert _was_called - assert wstat.id_ == "497ABX" + assert reqid == "497ABX" def test_alerts_bulk_dismiss_watchlist(monkeypatch): @@ -1336,23 +1309,16 @@ def mock_post_object(url, body, **kwargs): _was_called = True return MockResponse({"request_id": "497ABX"}) - def mock_get_object(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" - resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} - resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} - return resp - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - q = api.bulk_alert_dismiss("WATCHLIST").remediation("Fixed").comment("Yessir") - wstat = q.where("Blort").device_names(["HAL9000"]).run() + q = api.select(WatchlistAlert).where("Blort").device_names(["HAL9000"]) + reqid = q.dismiss("Fixed", "Yessir") assert _was_called - assert wstat.id_ == "497ABX" + assert reqid == "497ABX" def test_alerts_bulk_dismiss_cbanalytics(monkeypatch): @@ -1370,23 +1336,16 @@ def mock_post_object(url, body, **kwargs): _was_called = True return MockResponse({"request_id": "497ABX"}) - def mock_get_object(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" - resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} - resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} - return resp - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - q = api.bulk_alert_dismiss("CBANALYTICS").remediation("Fixed").comment("Yessir") - wstat = q.where("Blort").device_names(["HAL9000"]).run() + q = api.select(CBAnalyticsAlert).where("Blort").device_names(["HAL9000"]) + reqid = q.dismiss("Fixed", "Yessir") assert _was_called - assert wstat.id_ == "497ABX" + assert reqid == "497ABX" def test_alerts_bulk_dismiss_vmware(monkeypatch): @@ -1404,23 +1363,16 @@ def mock_post_object(url, body, **kwargs): _was_called = True return MockResponse({"request_id": "497ABX"}) - def mock_get_object(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" - resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} - resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} - return resp - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - q = api.bulk_alert_dismiss("VMWARE").remediation("Fixed").comment("Yessir") - wstat = q.where("Blort").device_names(["HAL9000"]).run() + q = api.select(VMwareAlert).where("Blort").device_names(["HAL9000"]) + reqid = q.dismiss("Fixed", "Yessir") assert _was_called - assert wstat.id_ == "497ABX" + assert reqid == "497ABX" def test_alerts_bulk_dismiss_threat(monkeypatch): @@ -1436,20 +1388,60 @@ def mock_post_object(url, body, **kwargs): _was_called = True return MockResponse({"request_id": "497ABX"}) + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + reqid = api.bulk_threat_dismiss(["B0RG", "F3R3NG1"], "Fixed", "Yessir") + assert _was_called + assert reqid == "497ABX" + + +def test_alerts_bulk_undismiss_threat(monkeypatch): + _was_called = False + + def mock_post_object(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/threat/workflow/_criteria" + assert body["threat_id"] == ["B0RG", "F3R3NG1"] + assert body["state"] == "OPEN" + assert body["remediation_state"] == "Fixed" + assert body["comment"] == "NoSir" + _was_called = True + return MockResponse({"request_id": "497ABX"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) + monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) + monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + reqid = api.bulk_threat_update(["B0RG", "F3R3NG1"], "Fixed", "NoSir") + assert _was_called + assert reqid == "497ABX" + + +def test_load_workflow(monkeypatch): + _was_called = False + def mock_get_object(url, parms=None, default=None): + nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" + _was_called = True resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} return resp - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", mock_post_object) + monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - q = api.bulk_alert_dismiss("THREAT").remediation("Fixed").comment("Yessir") - wstat = q.threat_ids(["B0RG", "F3R3NG1"]).run() + workflow = api.select(WorkflowStatus, "497ABX") assert _was_called - assert wstat.id_ == "497ABX" + assert workflow.id_ == "497ABX" + \ No newline at end of file From 42bbda3530c6649088205a4cc783c69272acaa30 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 11 Nov 2019 15:59:29 -0700 Subject: [PATCH 050/197] updated examples and module export file --- examples/psc/bulk_update_alerts.py | 19 +++++++-------- .../psc/bulk_update_cbanalytics_alerts.py | 19 +++++++-------- examples/psc/bulk_update_threat_alerts.py | 15 ++++-------- examples/psc/bulk_update_vmware_alerts.py | 23 ++++++++----------- examples/psc/bulk_update_watchlist_alerts.py | 19 +++++++-------- src/cbapi/psc/__init__.py | 2 +- 6 files changed, 40 insertions(+), 57 deletions(-) diff --git a/examples/psc/bulk_update_alerts.py b/examples/psc/bulk_update_alerts.py index 8d0d11e3..014af7e1 100755 --- a/examples/psc/bulk_update_alerts.py +++ b/examples/psc/bulk_update_alerts.py @@ -3,7 +3,7 @@ import sys from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import BaseAlert +from cbapi.psc.models import BaseAlert, WorkflowStatus from alertsv6common import setup_parser_with_basic_criteria, load_basic_criteria def main(): @@ -18,21 +18,18 @@ def main(): args = parser.parse_args() cb = get_cb_psc_object(args) + query = cb.select(BaseAlert) + load_basic_criteria(query, args) + if args.dismiss: - query = cb.bulk_alert_dismiss("ALERT") + reqid = query.dismiss(args.remediation, args.comment) elif args.undismiss: - query = cb.bulk_alert_undismiss("ALERT") + reqid = query.update(args.remediation, args.comment) else: raise NotImplemented("one of --dismiss or --undismiss must be specified") - load_basic_criteria(query, args) - - if args.remediation: - query = query.remediation(args.remediation) - if args.comment: - query = query.comment(args.comment) - statobj = query.run() - print("Submitted query with ID {0}".format(statobj.id_)) + print("Submitted query with ID {0}".format(reqid)) + statobj = cb.select(WorkflowStatus, reqid) while not statobj.finished: print("Waiting...") sleep(1) diff --git a/examples/psc/bulk_update_cbanalytics_alerts.py b/examples/psc/bulk_update_cbanalytics_alerts.py index a25bdf17..54df2794 100755 --- a/examples/psc/bulk_update_cbanalytics_alerts.py +++ b/examples/psc/bulk_update_cbanalytics_alerts.py @@ -3,7 +3,7 @@ import sys from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import CBAnalyticsAlert +from cbapi.psc.models import CBAnalyticsAlert, WorkflowStatus from alertsv6common import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria def main(): @@ -17,22 +17,19 @@ def main(): args = parser.parse_args() cb = get_cb_psc_object(args) + + query = cb.select(CBAnalyticsAlert) + load_cbanalytics_criteria(query, args) if args.dismiss: - query = cb.bulk_alert_dismiss("CBANALYTICS") + reqid = query.dismiss(args.remediation, args.comment) elif args.undismiss: - query = cb.bulk_alert_undismiss("CBANALYTICS") + reqid = query.update(args.remediation, args.comment) else: raise NotImplemented("one of --dismiss or --undismiss must be specified") - load_cbanalytics_criteria(query, args) - - if args.remediation: - query = query.remediation(args.remediation) - if args.comment: - query = query.comment(args.comment) - statobj = query.run() - print("Submitted query with ID {0}".format(statobj.id_)) + print("Submitted query with ID {0}".format(reqid)) + statobj = cb.select(WorkflowStatus, reqid) while not statobj.finished: print("Waiting...") sleep(1) diff --git a/examples/psc/bulk_update_threat_alerts.py b/examples/psc/bulk_update_threat_alerts.py index 616a2369..57b23ae6 100755 --- a/examples/psc/bulk_update_threat_alerts.py +++ b/examples/psc/bulk_update_threat_alerts.py @@ -3,7 +3,7 @@ import sys from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import BaseAlert +from cbapi.psc.models import BaseAlert, WorkflowStatus def main(): parser = build_cli_parser("Bulk update the status of alerts by threat ID") @@ -19,19 +19,14 @@ def main(): cb = get_cb_psc_object(args) if args.dismiss: - query = cb.bulk_alert_dismiss("THREAT") + reqid = cb.bulk_threat_dismiss(args.threatid, args.remediation, args.comment) elif args.undismiss: - query = cb.bulk_alert_undismiss("THREAT") + reqid = cb.bulk_threat_update(args.threatid, args.remediation, args.comment) else: raise NotImplemented("one of --dismiss or --undismiss must be specified") - query.threat_ids(args.threatid) - if args.remediation: - query = query.remediation(args.remediation) - if args.comment: - query = query.comment(args.comment) - statobj = query.run() - print("Submitted query with ID {0}".format(statobj.id_)) + print("Submitted query with ID {0}".format(reqid)) + statobj = cb.select(WorkflowStatus, reqid) while not statobj.finished: print("Waiting...") sleep(1) diff --git a/examples/psc/bulk_update_vmware_alerts.py b/examples/psc/bulk_update_vmware_alerts.py index b747d4c0..e09dfae5 100755 --- a/examples/psc/bulk_update_vmware_alerts.py +++ b/examples/psc/bulk_update_vmware_alerts.py @@ -3,7 +3,7 @@ import sys from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import VMwareAlert +from cbapi.psc.models import VMwareAlert, WorkflowStatus from alertsv6common import setup_parser_with_vmware_criteria, load_vmware_criteria def main(): @@ -17,22 +17,19 @@ def main(): args = parser.parse_args() cb = get_cb_psc_object(args) - + + query = cb.select(CBAnalyticsAlert) + load_vmware_criteria(query, args) + if args.dismiss: - query = cb.bulk_alert_dismiss("VMWARE") + reqid = query.dismiss(args.remediation, args.comment) elif args.undismiss: - query = cb.bulk_alert_undismiss("VMWARE") + reqid = query.update(args.remediation, args.comment) else: raise NotImplemented("one of --dismiss or --undismiss must be specified") - - load_vmware_criteria(query, args) - - if args.remediation: - query = query.remediation(args.remediation) - if args.comment: - query = query.comment(args.comment) - statobj = query.run() - print("Submitted query with ID {0}".format(statobj.id_)) + + print("Submitted query with ID {0}".format(reqid)) + statobj = cb.select(WorkflowStatus, reqid) while not statobj.finished: print("Waiting...") sleep(1) diff --git a/examples/psc/bulk_update_watchlist_alerts.py b/examples/psc/bulk_update_watchlist_alerts.py index 9dd1d9b0..a2c8f5a8 100755 --- a/examples/psc/bulk_update_watchlist_alerts.py +++ b/examples/psc/bulk_update_watchlist_alerts.py @@ -3,7 +3,7 @@ import sys from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import WatchlistAlert +from cbapi.psc.models import WatchlistAlert, WorkflowStatus from alertsv6common import setup_parser_with_watchlist_criteria, load_watchlist_criteria def main(): @@ -18,21 +18,18 @@ def main(): args = parser.parse_args() cb = get_cb_psc_object(args) + query = cb.select(CBAnalyticsAlert) + load_watchlist_criteria(query, args) + if args.dismiss: - query = cb.bulk_alert_dismiss("WATCHLIST") + reqid = query.dismiss(args.remediation, args.comment) elif args.undismiss: - query = cb.bulk_alert_undismiss("WATCHLIST") + reqid = query.update(args.remediation, args.comment) else: raise NotImplemented("one of --dismiss or --undismiss must be specified") - load_watchlist_criteria(query, args) - - if args.remediation: - query = query.remediation(args.remediation) - if args.comment: - query = query.comment(args.comment) - statobj = query.run() - print("Submitted query with ID {0}".format(statobj.id_)) + print("Submitted query with ID {0}".format(reqid)) + statobj = cb.select(WorkflowStatus, reqid) while not statobj.finished: print("Waiting...") sleep(1) diff --git a/src/cbapi/psc/__init__.py b/src/cbapi/psc/__init__.py index 239b1eb6..3f3be5f0 100644 --- a/src/cbapi/psc/__init__.py +++ b/src/cbapi/psc/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import from .rest_api import CbPSCBaseAPI -from .models import Device +from .models import Device, Workflow, BaseAlert, WatchlistAlert, CBAnalyticsAlert, VMwareAlert, WorkflowStatus From bd3966834aeb95102e244c17c3d9be9c4aaa49d5 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 11 Nov 2019 16:08:09 -0700 Subject: [PATCH 051/197] updated docs and renamed model methods to get rid of "undismiss" terminology --- docs/psc-api.rst | 20 +++++++++++--------- src/cbapi/psc/models.py | 8 ++++---- test/cbapi/psc/test_models.py | 4 ++-- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/psc-api.rst b/docs/psc-api.rst index 27d928a0..1266939c 100755 --- a/docs/psc-api.rst +++ b/docs/psc-api.rst @@ -63,21 +63,23 @@ search for more specialized alert types: Selects all alerts on a Windows device running the Internet Explorer process. -Individual alerts may have their status changed using the dismiss() or undismiss() +Individual alerts may have their status changed using the dismiss() or update() methods on the BaseAlert object. To dismiss multiple alerts at once, you can use -the bulk_alert_dismiss() or bulk_alert_undismiss() methods on the CbPSCBaseAPI -object to set up a query, which you can add criteria to just as with a search -query. When you call the run() method on the query, a WorkflowStatus object is -returned; querying this object's "finished" property will let you know when the -operation is finished. +the dismiss() or update() methods on the standard query, after adding criteria to it. +This method returns a request ID, which can be used to create a WorkflowStatus object; +querying this object's "finished" property will let you know when the operation is +finished. *Example:* >>> cbapi = CbPSCBaseAPI(...) - >>> query = cbapi.bulk_alert_dismiss("ALERT").process_name(["IEXPLORE.EXE"]) - >>> stat = query.remediation("Using Chrome").run() + >>> query = cbapi.select(BaseAlert).process_name(["IEXPLORE.EXE"]) + >>> reqid = query.dismiss("Using Chrome") + >>> stat = cbapi.select(WorkflowStatus, reqid) + >>> while not stat.finished: + >>> # wait for it to finish -Dismisses all alerts which reference the Internet Explorer process. +This dismisses all alerts which reference the Internet Explorer process. **Query Objects:** diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 8e839112..958184ef 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -249,9 +249,9 @@ def dismiss(self, remediation=None, comment=None): """ self._update_workflow_status("DISMISSED", remediation, comment) - def undismiss(self, remediation=None, comment=None): + def update(self, remediation=None, comment=None): """ - Un-dismiss this alert. + Update this alert. :param remediation str: The remediation status to set for the alert. :param comment str: The comment to set for the alert. @@ -278,9 +278,9 @@ def dismiss_threat(self, remediation=None, comment=None): """ return self._update_threat_workflow_status("DISMISSED", remediation, comment) - def undismiss_threat(self, remediation=None, comment=None): + def update_threat(self, remediation=None, comment=None): """ - Un-dismiss alerts for this threat. + Update alerts for this threat. :param remediation str: The remediation status to set for the alert. :param comment str: The comment to set for the alert. diff --git a/test/cbapi/psc/test_models.py b/test/cbapi/psc/test_models.py index d54d4a63..fef23551 100755 --- a/test/cbapi/psc/test_models.py +++ b/test/cbapi/psc/test_models.py @@ -282,7 +282,7 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "workflow":{"state": "DISMISS"}}) - alert.undismiss("Fixed", "NoSir") + alert.update("Fixed", "NoSir") assert _was_called assert alert.workflow_.changed_by == "Robocop" assert alert.workflow_.state == "OPEN" @@ -338,7 +338,7 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "threat_id": "B0RG", "workflow":{"state": "OPEN"}}) - wf = alert.undismiss_threat("Fixed", "NoSir") + wf = alert.update_threat("Fixed", "NoSir") assert _was_called assert wf.changed_by == "Robocop" assert wf.state == "OPEN" From 33c1687e38d42f4304610e226ac606922ccedf75 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 11 Nov 2019 16:09:22 -0700 Subject: [PATCH 052/197] get rid of documentation references to queries that no longer exist --- docs/psc-api.rst | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/psc-api.rst b/docs/psc-api.rst index 1266939c..b122a3c1 100755 --- a/docs/psc-api.rst +++ b/docs/psc-api.rst @@ -95,21 +95,6 @@ This dismisses all alerts which reference the Internet Explorer process. .. autoclass:: cbapi.psc.query.WatchlistAlertSearchQuery :members: -.. autoclass:: cbapi.psc.query.BulkUpdateAlerts - :members: - -.. autoclass:: cbapi.psc.query.BulkUpdateCBAnalyticsAlerts - :members: - -.. autoclass:: cbapi.psc.query.BulkUpdateVMwareAlerts - :members: - -.. autoclass:: cbapi.psc.query.BulkUpdateWatchlistAlerts - :members: - -.. autoclass:: cbapi.psc.query.BulkUpdateThreatAlerts - :members: - **Model Objects:** .. autoclass:: cbapi.psc.models.Workflow From c330a01201fd9febebf61d18df3bea877e124b84 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Fri, 15 Nov 2019 14:04:06 -0700 Subject: [PATCH 053/197] re-flake8'd the source --- src/cbapi/psc/query.py | 14 +++++++------- src/cbapi/psc/rest_api.py | 13 ++++++------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/query.py index 663dfe35..6d29386e 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/query.py @@ -991,9 +991,9 @@ def facets(self, fieldlist, max_rows=0): resp = self._cb.post_object(url, body=request) result = resp.json() return result.get("results", []) - + def _update_status(self, status, remediation, comment): - request = { "state": status, "criteria": self._build_criteria(), "query": self._query_builder._collapse()} + request = {"state": status, "criteria": self._build_criteria(), "query": self._query_builder._collapse()} if remediation is not None: request["remediation_state"] = remediation if comment is not None: @@ -1001,21 +1001,21 @@ def _update_status(self, status, remediation, comment): resp = self._cb.post_object(self._bulkupdate_url.format(self._cb.credentials.org_key), body=request) output = resp.json() return output["request_id"] - + def update(self, remediation=None, comment=None): """ Update all alerts matching the given query. The alerts will be left in an OPEN state after this request. - + :param remediation str: The remediation state to set for all alerts. :param comment str: The comment to set for all alerts. :return: The request ID, which may be used to select a WorkflowStatus object. """ return self._update_status("OPEN", remediation, comment) - + def dismiss(self, remediation=None, comment=None): """ Dismiss all alerts matching the given query. The alerts will be left in a DISMISSED state after this request. - + :param remediation str: The remediation state to set for all alerts. :param comment str: The comment to set for all alerts. :return: The request ID, which may be used to select a WorkflowStatus object. @@ -1071,7 +1071,7 @@ class CBAnalyticsAlertSearchQuery(BaseAlertSearchQuery): valid_sensor_actions = ["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", "TERMINATE", "DENY"] valid_threat_cause_vectors = ["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", "APP_STORE", "THIRD_PARTY"] - + def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/cbanalytics/workflow/_criteria" diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index 7c9b76e2..ded3a9e2 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -136,11 +136,11 @@ def alert_search_suggestions(self, query): url = "/appservices/v6/orgs/{0}/alerts/search_suggestions".format(self.credentials.org_key) output = self.get_object(url, query_params) return output["suggestions"] - + def _bulk_threat_update_status(self, threat_ids, status, remediation, comment): if not all(isinstance(t, str) for t in threat_ids): raise ApiError("One or more invalid threat ID values") - request = { "state": status, "threat_id": threat_ids} + request = {"state": status, "threat_id": threat_ids} if remediation is not None: request["remediation_state"] = remediation if comment is not None: @@ -149,28 +149,27 @@ def _bulk_threat_update_status(self, threat_ids, status, remediation, comment): resp = self.post_object(url, body=request) output = resp.json() return output["request_id"] - + def bulk_threat_update(self, threat_ids, remediation=None, comment=None): """ Update the alert status of alerts associated with multiple threat IDs. The alerts will be left in an OPEN state after this request. - + :param threat_ids list: List of string threat IDs. :param remediation str: The remediation state to set for all alerts. :param comment str: The comment to set for all alerts. :return: The request ID, which may be used to select a WorkflowStatus object. """ return self._bulk_threat_update_status(threat_ids, "OPEN", remediation, comment) - + def bulk_threat_dismiss(self, threat_ids, remediation=None, comment=None): """ Dismiss the alerts associated with multiple threat IDs. The alerts will be left in a DISMISSED state after this request. - + :param threat_ids list: List of string threat IDs. :param remediation str: The remediation state to set for all alerts. :param comment str: The comment to set for all alerts. :return: The request ID, which may be used to select a WorkflowStatus object. """ return self._bulk_threat_update_status(threat_ids, "DISMISSED", remediation, comment) - \ No newline at end of file From a584cde914d3e7416afdcb0e0f95406038d68770 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 19 Nov 2019 12:02:03 -0700 Subject: [PATCH 054/197] ran all the examples through flake8 --- .gitignore | 1 + examples/defense/cblr/jobrunner.py | 5 +- examples/defense/cblr_cli.py | 2 +- examples/defense/list_devices.py | 3 +- examples/defense/list_events.py | 4 +- .../defense/list_events_with_cmdline_csv.py | 20 +- examples/defense/policy_operations.py | 5 +- examples/livequery/manage_run.py | 20 +- examples/livequery/run_device_summary.py | 6 +- examples/livequery/run_facets.py | 15 +- .../protection/empty_analysis_connector.py | 3 +- examples/protection/policy_return.py | 2 + examples/protection/remove_duplicates.py | 1 + examples/protection/virus_total_connector.py | 19 +- examples/psc/alert_search_suggestions.py | 8 +- examples/psc/alertsv6common.py | 43 ++--- examples/psc/bulk_update_alerts.py | 15 +- .../psc/bulk_update_cbanalytics_alerts.py | 11 +- examples/psc/bulk_update_threat_alerts.py | 18 +- examples/psc/bulk_update_vmware_alerts.py | 11 +- examples/psc/bulk_update_watchlist_alerts.py | 13 +- examples/psc/device_control.py | 24 +-- examples/psc/download_device_list.py | 10 +- examples/psc/list_alert_facets.py | 13 +- examples/psc/list_alerts.py | 10 +- examples/psc/list_cbanalytics_alert_facets.py | 13 +- examples/psc/list_cbanalytics_alerts.py | 10 +- examples/psc/list_devices.py | 16 +- examples/psc/list_vmware_alert_facets.py | 13 +- examples/psc/list_vmware_alerts.py | 10 +- examples/psc/list_watchlist_alert_facets.py | 13 +- examples/psc/list_watchlist_alerts.py | 10 +- examples/response/alert_search.py | 2 +- examples/response/binary_export.py | 1 + examples/response/binary_search.py | 6 +- examples/response/cmd_exe_filemods.py | 1 + examples/response/download_from_lr.py | 10 +- examples/response/dump_all_binaries.py | 19 +- examples/response/enumerate_usb_devices.py | 4 +- examples/response/event/get_reg_autoruns.py | 10 +- .../response/event/watchlist_automation.py | 12 +- examples/response/event_export.py | 3 +- examples/response/feed_operations.py | 6 +- examples/response/new_binaries_after_date.py | 9 +- .../response/new_binaries_with_netconns.py | 7 +- examples/response/partition_operations.py | 7 +- examples/response/process_netconn_rate.py | 2 +- examples/response/s3-watchlist-ban.py | 9 +- examples/response/sensor_export.py | 3 +- examples/response/sensor_group_operations.py | 14 +- examples/response/sensor_operations.py | 7 +- examples/response/tf.py | 8 +- examples/response/user_operations.py | 6 +- examples/response/walk_children.py | 4 +- examples/response/watchlist_exporter.py | 5 +- examples/response/watchlist_importer.py | 6 +- examples/response/watchlist_operations.py | 4 +- examples/threathunter/events.py | 1 + examples/threathunter/events_exporter.py | 10 +- .../threathunter/import_response_feeds.py | 173 +++++++++--------- examples/threathunter/process_exporter.py | 24 +-- examples/threathunter/process_query.py | 1 + examples/threathunter/process_tree.py | 1 + .../threathunter/process_tree_exporter.py | 3 +- examples/threathunter/search.py | 4 +- examples/threathunter/watchlist_operations.py | 9 +- 66 files changed, 399 insertions(+), 359 deletions(-) diff --git a/.gitignore b/.gitignore index db1e3353..f79adb10 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,7 @@ target/ # Eclipse/PyDev /.project /.pydevproject +/.settings # Credential files .carbonblack/ diff --git a/examples/defense/cblr/jobrunner.py b/examples/defense/cblr/jobrunner.py index e9da853b..6ec0c062 100755 --- a/examples/defense/cblr/jobrunner.py +++ b/examples/defense/cblr/jobrunner.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from cbapi.defense import * +from cbapi.defense import Device from cbapi.example_helpers import build_cli_parser, get_cb_defense_object from concurrent.futures import as_completed import sys @@ -60,12 +60,11 @@ def main(): print("Sensor {0} had error:".format(futures[f])) print(f.exception()) - still_to_do = set([s.deviceId for s in online_sensors]) - set(completed_sensors) print("The following sensors were attempted but not completed or errored out:") for sensor in still_to_do: print(" {0}".format(still_to_do)) - + if __name__ == '__main__': sys.exit(main()) diff --git a/examples/defense/cblr_cli.py b/examples/defense/cblr_cli.py index e2207e39..5e2594e7 100644 --- a/examples/defense/cblr_cli.py +++ b/examples/defense/cblr_cli.py @@ -30,7 +30,7 @@ def main(): parser.add_argument("--log", help="Log activity to a file", default='') args = parser.parse_args() cb = get_cb_defense_object(args) - + if args.log: file_handler = logging.FileHandler(args.log) file_handler.setLevel(logging.DEBUG) diff --git a/examples/defense/list_devices.py b/examples/defense/list_devices.py index 2777c4a4..1c8a5406 100644 --- a/examples/defense/list_devices.py +++ b/examples/defense/list_devices.py @@ -24,7 +24,8 @@ def main(): print("{0:9} {1:40}{2:18}{3}".format("ID", "Hostname", "IP Address", "Last Checkin Time")) for device in devices: - print("{0:9} {1:40s}{2:18s}{3}".format(device.deviceId, device.name or "None", device.lastInternalIpAddress or "Unknown", device.lastContact)) + print("{0:9} {1:40s}{2:18s}{3}".format(device.deviceId, device.name or "None", + device.lastInternalIpAddress or "Unknown", device.lastContact)) if __name__ == "__main__": diff --git a/examples/defense/list_events.py b/examples/defense/list_events.py index e6056b8f..882650c0 100644 --- a/examples/defense/list_events.py +++ b/examples/defense/list_events.py @@ -42,8 +42,8 @@ def main(): if args.hostname: events = list(cb.select(Event).where("hostNameExact:{0}".format(args.hostname))) elif args.start and args.end: - # flipped the start and end arguments around so script can be called with the start date being the earliest date. - # it's just easier on the eyes for most folks. + # flipped the start and end arguments around so script can be called with the start date being + # the earliest date. it's just easier on the eyes for most folks. events = list(cb.select(Event).where("startTime:{0}".format(args.end))) and ( cb.select(Event).where("endTime:{0}".format(args.start))) diff --git a/examples/defense/list_events_with_cmdline_csv.py b/examples/defense/list_events_with_cmdline_csv.py index e282bd00..ab0f13fa 100644 --- a/examples/defense/list_events_with_cmdline_csv.py +++ b/examples/defense/list_events_with_cmdline_csv.py @@ -7,7 +7,8 @@ # Notes on this script: -# - based on https://github.com/carbonblack/cbapi-python/blob/master/examples/defense/list_events.py with 2 primary changes +# - based on https://github.com/carbonblack/cbapi-python/blob/master/examples/defense/list_events.py +# with 2 primary changes # 1. this script outputs the command line of the main process process # 2. this script places a '|' delimiter between fields so it can be read into a spreadsheet # - can only pull up to 2 weeks of events at one time ( this is an API limitation) @@ -48,8 +49,8 @@ def main(): if args.hostname: events = list(cb.select(Event).where("hostNameExact:{0}".format(args.hostname))) elif args.start and args.end: - # flipped the start and end arguments around so script can be called with the start date being the earliest date. - # it's just easier on the eyes for most folks. + # flipped the start and end arguments around so script can be called with the start date + # being the earliest date. it's just easier on the eyes for most folks. events = list(cb.select(Event).where("startTime:{0}".format(args.end))) and ( cb.select(Event).where("endTime:{0}".format(args.start))) @@ -65,21 +66,20 @@ def main(): create_time = str(convert_time(event.eventTime)) # stripping HTML tags out of the long description - long_description = unicodedata.normalize('NFD',strip_html(event.longDescription)) + long_description = unicodedata.normalize('NFD', strip_html(event.longDescription)) if event.processDetails: # stripping out the command line arguments from the processDetails field processDetails = str(event.processDetails) start_cmdline = processDetails.find("u'commandLine'") end_cmdline = processDetails.find(", u'parentName'") - commandline = processDetails[start_cmdline + 18: end_cmdline -1] - print("{0}|{1}|{2}|{3}|{4}|{5}".format(event_time, event.eventId, create_time, event.eventType, long_description, commandline)) + commandline = processDetails[start_cmdline + 18: end_cmdline - 1] + print("{0}|{1}|{2}|{3}|{4}|{5}".format(event_time, event.eventId, create_time, event.eventType, + long_description, commandline)) else: - print("{0}|{1}|{2}|{3}|{4}".format(event_time, event.eventId, create_time, event.eventType, long_description)) + print("{0}|{1}|{2}|{3}|{4}".format(event_time, event.eventId, create_time, event.eventType, + long_description)) # format and print out the event time, Event ID, Creation time, Event type and Description - - - if __name__ == "__main__": diff --git a/examples/defense/policy_operations.py b/examples/defense/policy_operations.py index 00656cc1..6f5b2199 100644 --- a/examples/defense/policy_operations.py +++ b/examples/defense/policy_operations.py @@ -134,11 +134,12 @@ def replace_rule(cb, parser, args): print("Replaced rule id {0} from policy {1} with rule from file {2}.".format(args.ruleid, policy.name, args.rulefile)) + def main(): parser = build_cli_parser("Policy operations") commands = parser.add_subparsers(help="Policy commands", dest="command_name") - list_command = commands.add_parser("list", help="List all configured policies") + commands.add_parser("list", help="List all configured policies") import_policy_command = commands.add_parser("import", help="Import policy from JSON file") import_policy_command.add_argument("-N", "--name", help="Name of new policy", required=True) @@ -157,7 +158,7 @@ def main(): del_policy_specifier = del_command.add_mutually_exclusive_group(required=True) del_policy_specifier.add_argument("-i", "--id", type=int, help="ID of policy to delete") del_policy_specifier.add_argument("-N", "--name", help="Name of policy to delete. Specify --force to delete" - " multiple policies that have the same name") + " multiple policies that have the same name") del_command.add_argument("--force", help="If NAME matches multiple policies, delete all matching policies", action="store_true", default=False) diff --git a/examples/livequery/manage_run.py b/examples/livequery/manage_run.py index 493f4097..46e67552 100644 --- a/examples/livequery/manage_run.py +++ b/examples/livequery/manage_run.py @@ -25,8 +25,8 @@ def create_run(cb, args): def run_status(cb, args): run = cb.select(Run, args.id) print(run) - - + + def run_stop(cb, args): run = cb.select(Run, args.id) if run.stop(): @@ -34,16 +34,16 @@ def run_stop(cb, args): print(run) else: print("Unable to stop run {}".format(run.id)) - - + + def run_delete(cb, args): run = cb.select(Run, args.id) if run.delete(): print("Run {} has been deleted.".format(run.id)) else: print("Unable to delete run {}".format(run.id)) - - + + def run_history(cb, args): results = cb.query_history(args.query) if args.sort_by: @@ -51,7 +51,6 @@ def run_history(cb, args): results.sort_by(args.sort_by, direction=dir) for result in results: print(result) - def main(): @@ -99,21 +98,21 @@ def main(): status_command.add_argument( "-i", "--id", type=str, required=True, help="The run ID" ) - + stop_command = commands.add_parser( "stop", help="Stops/cancels a current run" ) stop_command.add_argument( "-i", "--id", type=str, required=True, help="The run ID" ) - + delete_command = commands.add_parser( "delete", help="Permanently delete a run" ) delete_command.add_argument( "-i", "--id", type=str, required=True, help="The run ID" ) - + history_command = commands.add_parser( "history", help="List history of all runs" ) @@ -145,5 +144,6 @@ def main(): elif args.command_name == "history": return run_history(cb, args) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/livequery/run_device_summary.py b/examples/livequery/run_device_summary.py index 12bd1a69..33ee8931 100755 --- a/examples/livequery/run_device_summary.py +++ b/examples/livequery/run_device_summary.py @@ -56,13 +56,13 @@ def main(): args = parser.parse_args() cb = get_cb_livequery_object(args) - + results = cb.select(Result).run_id(args.id) result = results.first() if result is None: print("ERROR: No results.") return 1 - + summaries = result.query_device_summaries() if args.query: summaries.where(args.query) @@ -79,7 +79,7 @@ def main(): if args.sort_by: dir = "DESC" if args.descending_results else "ASC" summaries.sort_by(args.sort_by, direction=dir) - + for summary in summaries: print(summary) diff --git a/examples/livequery/run_facets.py b/examples/livequery/run_facets.py index 8cabb8a6..14bfc6c4 100755 --- a/examples/livequery/run_facets.py +++ b/examples/livequery/run_facets.py @@ -25,7 +25,7 @@ def main(): required=False, help="Fields to be displayed in results", ) - + parser.add_argument("-q", "--query", type=str, required=False, help="Search query") parser.add_argument( "--device_ids", @@ -70,15 +70,15 @@ def main(): if args.result and args.device_summary: print("ERROR: --result and --device_summary cannot both be specified") return 1 - + cb = get_cb_livequery_object(args) - + results = cb.select(Result).run_id(args.id) result = results.first() if result is None: print("ERROR: No results.") return 1 - + if args.result: facets = result.query_result_facets() elif args.device_summary: @@ -97,11 +97,10 @@ def main(): facets.criteria(policy_name=args.policy_names) if args.statuses: facets.criteria(status=args.statuses) - + for facet in facets: print(facet) - - + + if __name__ == "__main__": sys.exit(main()) - diff --git a/examples/protection/empty_analysis_connector.py b/examples/protection/empty_analysis_connector.py index 882efa53..f4162a1e 100644 --- a/examples/protection/empty_analysis_connector.py +++ b/examples/protection/empty_analysis_connector.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from cbapi.protection import Connector, PendingAnalysis, Notification +from cbapi.protection import Connector, Notification from cbapi.example_helpers import get_cb_protection_object, build_cli_parser import logging import time @@ -48,4 +48,3 @@ def process_request(api, i): for i in vt_connector.pendingAnalyses: process_request(api, i) time.sleep(10) - diff --git a/examples/protection/policy_return.py b/examples/protection/policy_return.py index dabb79aa..3c0e13b5 100644 --- a/examples/protection/policy_return.py +++ b/examples/protection/policy_return.py @@ -2,6 +2,7 @@ from cbapi.protection.models import Computer, Policy import sys + def main(): parser = build_cli_parser("Revert computers in policy to previous policy") parser.add_argument("--policy", "-p", help="Policy name or ID", required=True) @@ -20,5 +21,6 @@ def main(): computer.save() print("%s is now in %s" % (computer.name, computer.policyName)) + if __name__ == '__main__': sys.exit(main()) diff --git a/examples/protection/remove_duplicates.py b/examples/protection/remove_duplicates.py index 06d68e5a..369b10f2 100644 --- a/examples/protection/remove_duplicates.py +++ b/examples/protection/remove_duplicates.py @@ -29,5 +29,6 @@ def main(): print("deleting from server...") p.select(Computer, computer_id["id"]).delete() + if __name__ == '__main__': sys.exit(main()) diff --git a/examples/protection/virus_total_connector.py b/examples/protection/virus_total_connector.py index 15d48d3a..1c229068 100644 --- a/examples/protection/virus_total_connector.py +++ b/examples/protection/virus_total_connector.py @@ -38,7 +38,7 @@ def __init__(self, api, vt_token=None, connector_name='VirusTotal', allow_upload raise TypeError("Missing required VT authentication token.") self.vt_token = vt_token self.vt_url = 'https://www.virustotal.com/vtapi/v2' - self.polling_frequency = 30 # seconds + self.polling_frequency = 30 # seconds self.malicious_threshold = malicious_threshold self.potential_threshold = potential_threshold @@ -83,7 +83,8 @@ def process_response(self, binary, scanResults): self.schedule_check(binary, binary.fileHash) else: binary.analysisStatus = PendingAnalysis.StatusCancelled - log.info("%s: VirusTotal has no information and we aren't allowed to upload it. Cancelling the analysis request." % binary.fileHash) + log.info("%s: VirusTotal has no information and we aren't allowed to upload it. " + "Cancelling the analysis request." % binary.fileHash) binary.save() def report_result(self, binary, scanResults): @@ -143,9 +144,7 @@ def upload_to_vt(self, binary): r = requests.post(self.vt_url + "/file/scan", files=files, params={'apikey': self.vt_token}) if r.status_code != 200: vt_upload_error = True - else: - scanId = r.json()['scan_id'] - except: + except Exception: log.exception("Could not send file %s to VirusTotal" % (binary.fileHash,)) vt_upload_error = True @@ -157,13 +156,14 @@ def upload_to_vt(self, binary): binary.save() else: - log.info("%s: VirusTotal has no information on this hash. Waiting for agent to upload it." % binary.fileHash) + log.info("%s: VirusTotal has no information on this hash. Waiting for agent to upload it." + % binary.fileHash) def schedule_check(self, binary, scanId): next_check = datetime.datetime.now() + datetime.timedelta(0, 3600) self.awaiting_results[binary.fileHash] = {'scanId': scanId, 'nextCheck': next_check} - log.info("%s: Waiting for analysis to complete. Will check back after %s." % (binary.fileHash, - next_check.strftime("%Y-%m-%d %H:%M:%S"))) + log.info("%s: Waiting for analysis to complete. Will check back after %s." + % (binary.fileHash, next_check.strftime("%Y-%m-%d %H:%M:%S"))) def process_request(self, binary): if binary.fileHash in self.awaiting_results: @@ -218,7 +218,7 @@ def main(): log.info("Configuration:") for k, v in iteritems(config): - log.info(" %-20s: %s" % (k,v)) + log.info(" %-20s: %s" % (k, v)) api = get_cb_protection_object(args) @@ -235,4 +235,3 @@ def main(): if __name__ == '__main__': sys.exit(main()) - diff --git a/examples/psc/alert_search_suggestions.py b/examples/psc/alert_search_suggestions.py index 49395a2a..cd871e19 100755 --- a/examples/psc/alert_search_suggestions.py +++ b/examples/psc/alert_search_suggestions.py @@ -3,10 +3,11 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object + def main(): parser = build_cli_parser("Get suggestions for searching alerts") parser.add_argument("-q", "--query", default="", help="Query string for looking for alerts") - + args = parser.parse_args() cb = get_cb_psc_object(args) @@ -15,8 +16,7 @@ def main(): print("Search term: '{0}'".format(suggestion["term"])) print("\tWeight: {0}".format(suggestion["weight"])) print("\tAvailable with products: {0}".format(", ".join(suggestion["required_skus_some"]))) - - + + if __name__ == "__main__": sys.exit(main()) - \ No newline at end of file diff --git a/examples/psc/alertsv6common.py b/examples/psc/alertsv6common.py index 812fe2b4..ba49228e 100755 --- a/examples/psc/alertsv6common.py +++ b/examples/psc/alertsv6common.py @@ -1,5 +1,3 @@ -import sys - def setup_parser_with_basic_criteria(parser): parser.add_argument("-q", "--query", help="Query string for looking for alerts") parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", @@ -14,11 +12,13 @@ def setup_parser_with_basic_criteria(parser): parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") parser.add_argument("--group", action="store_true", help="Group results") parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, help="Restrict search to the specified legacy alert IDs") + parser.add_argument("--legacyalertid", action="append", type=str, + help="Restrict search to the specified legacy alert IDs") parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, help="Restrict search to the specified process names") + parser.add_argument("--processname", action="append", type=str, + help="Restrict search to the specified process names") parser.add_argument("--processhash", action="append", type=str, help="Restrict search to the specified process SHA-256 hash values") parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", @@ -34,10 +34,10 @@ def setup_parser_with_basic_criteria(parser): help="Restrict search to the specified alert types") parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], help="Restrict search to the specified workflow statuses") - - + + def setup_parser_with_cbanalytics_criteria(parser): - setup_parser_with_basic_criteria(parser) + setup_parser_with_basic_criteria(parser) parser.add_argument("--blockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", "RISKY_PROGRAM"], help="Restrict search to the specified threat categories that were blocked") @@ -62,18 +62,20 @@ def setup_parser_with_cbanalytics_criteria(parser): "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", "APP_STORE", "THIRD_PARTY"], help="Restrict search to the specified threat cause vectors") - - -def setup_parser_with_vmware_criteria(parser): - setup_parser_with_basic_criteria(parser) + + +def setup_parser_with_vmware_criteria(parser): + setup_parser_with_basic_criteria(parser) parser.add_argument("--groupid", action="append", type=int, help="Restrict search to the specified AppDefense alarm group IDs") - - -def setup_parser_with_watchlist_criteria(parser): - setup_parser_with_basic_criteria(parser) - parser.add_argument("--watchlistid", action="append", type=str, help="Restrict search to the specified watchlists by ID") - parser.add_argument("--watchlistname", action="append", type=str, help="Restrict search to the specified watchlists by name") + + +def setup_parser_with_watchlist_criteria(parser): + setup_parser_with_basic_criteria(parser) + parser.add_argument("--watchlistid", action="append", type=str, + help="Restrict search to the specified watchlists by ID") + parser.add_argument("--watchlistname", action="append", type=str, + help="Restrict search to the specified watchlists by name") def load_basic_criteria(query, args): @@ -141,18 +143,17 @@ def load_cbanalytics_criteria(query, args): query = query.sensor_actions(args.sensoraction) if args.vector: query = query.threat_cause_vectors(args.vector) - + def load_vmware_criteria(query, args): load_basic_criteria(query, args) if args.groupid: query = query.group_ids(args.groupid) - - + + def load_watchlist_criteria(query, args): load_basic_criteria(query, args) if args.watchlistid: query = query.watchlist_ids(args.watchlistid) if args.watchlistname: query = query.watchlist_names(args.watchlistname) - \ No newline at end of file diff --git a/examples/psc/bulk_update_alerts.py b/examples/psc/bulk_update_alerts.py index 014af7e1..80ae7a7d 100755 --- a/examples/psc/bulk_update_alerts.py +++ b/examples/psc/bulk_update_alerts.py @@ -6,6 +6,7 @@ from cbapi.psc.models import BaseAlert, WorkflowStatus from alertsv6common import setup_parser_with_basic_criteria, load_basic_criteria + def main(): parser = build_cli_parser("Bulk update the status of alerts") setup_parser_with_basic_criteria(parser) @@ -14,10 +15,10 @@ def main(): operation = parser.add_mutually_exclusive_group(required=True) operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(BaseAlert) load_basic_criteria(query, args) @@ -26,8 +27,8 @@ def main(): elif args.undismiss: reqid = query.update(args.remediation, args.comment) else: - raise NotImplemented("one of --dismiss or --undismiss must be specified") - + raise NotImplementedError("one of --dismiss or --undismiss must be specified") + print("Submitted query with ID {0}".format(reqid)) statobj = cb.select(WorkflowStatus, reqid) while not statobj.finished: @@ -41,9 +42,9 @@ def main(): print("Failed alert IDs:") for i in statobj.failed_ids: print("\t{0}".format(err)) - print("{0} total alert(s) found, of which {1} were successfully changed" \ + print("{0} total alert(s) found, of which {1} were successfully changed" .format(statobj.num_hits, statobj.num_success)) - - + + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/psc/bulk_update_cbanalytics_alerts.py b/examples/psc/bulk_update_cbanalytics_alerts.py index 54df2794..8ac4599f 100755 --- a/examples/psc/bulk_update_cbanalytics_alerts.py +++ b/examples/psc/bulk_update_cbanalytics_alerts.py @@ -6,6 +6,7 @@ from cbapi.psc.models import CBAnalyticsAlert, WorkflowStatus from alertsv6common import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria + def main(): parser = build_cli_parser("Bulk update the status of CB Analytics alerts") setup_parser_with_cbanalytics_criteria(parser) @@ -14,10 +15,10 @@ def main(): operation = parser.add_mutually_exclusive_group(required=True) operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(CBAnalyticsAlert) load_cbanalytics_criteria(query, args) @@ -26,8 +27,8 @@ def main(): elif args.undismiss: reqid = query.update(args.remediation, args.comment) else: - raise NotImplemented("one of --dismiss or --undismiss must be specified") - + raise NotImplementedError("one of --dismiss or --undismiss must be specified") + print("Submitted query with ID {0}".format(reqid)) statobj = cb.select(WorkflowStatus, reqid) while not statobj.finished: @@ -41,7 +42,7 @@ def main(): print("Failed alert IDs:") for i in statobj.failed_ids: print("\t{0}".format(err)) - print("{0} total alert(s) found, of which {1} were successfully changed" \ + print("{0} total alert(s) found, of which {1} were successfully changed" .format(statobj.num_hits, statobj.num_success)) diff --git a/examples/psc/bulk_update_threat_alerts.py b/examples/psc/bulk_update_threat_alerts.py index 57b23ae6..b3922390 100755 --- a/examples/psc/bulk_update_threat_alerts.py +++ b/examples/psc/bulk_update_threat_alerts.py @@ -3,7 +3,8 @@ import sys from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import BaseAlert, WorkflowStatus +from cbapi.psc.models import WorkflowStatus + def main(): parser = build_cli_parser("Bulk update the status of alerts by threat ID") @@ -14,17 +15,17 @@ def main(): operation = parser.add_mutually_exclusive_group(required=True) operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + if args.dismiss: reqid = cb.bulk_threat_dismiss(args.threatid, args.remediation, args.comment) elif args.undismiss: reqid = cb.bulk_threat_update(args.threatid, args.remediation, args.comment) else: - raise NotImplemented("one of --dismiss or --undismiss must be specified") - + raise NotImplementedError("one of --dismiss or --undismiss must be specified") + print("Submitted query with ID {0}".format(reqid)) statobj = cb.select(WorkflowStatus, reqid) while not statobj.finished: @@ -38,10 +39,9 @@ def main(): print("Failed alert IDs:") for i in statobj.failed_ids: print("\t{0}".format(err)) - print("{0} total alert(s) found, of which {1} were successfully changed" \ + print("{0} total alert(s) found, of which {1} were successfully changed" .format(statobj.num_hits, statobj.num_success)) - - + + if __name__ == "__main__": sys.exit(main()) - \ No newline at end of file diff --git a/examples/psc/bulk_update_vmware_alerts.py b/examples/psc/bulk_update_vmware_alerts.py index e09dfae5..0158acf4 100755 --- a/examples/psc/bulk_update_vmware_alerts.py +++ b/examples/psc/bulk_update_vmware_alerts.py @@ -6,6 +6,7 @@ from cbapi.psc.models import VMwareAlert, WorkflowStatus from alertsv6common import setup_parser_with_vmware_criteria, load_vmware_criteria + def main(): parser = build_cli_parser("Bulk update the status of VMware alerts") setup_parser_with_vmware_criteria(parser) @@ -14,11 +15,11 @@ def main(): operation = parser.add_mutually_exclusive_group(required=True) operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") - + args = parser.parse_args() cb = get_cb_psc_object(args) - query = cb.select(CBAnalyticsAlert) + query = cb.select(VMwareAlert) load_vmware_criteria(query, args) if args.dismiss: @@ -26,8 +27,8 @@ def main(): elif args.undismiss: reqid = query.update(args.remediation, args.comment) else: - raise NotImplemented("one of --dismiss or --undismiss must be specified") - + raise NotImplementedError("one of --dismiss or --undismiss must be specified") + print("Submitted query with ID {0}".format(reqid)) statobj = cb.select(WorkflowStatus, reqid) while not statobj.finished: @@ -41,7 +42,7 @@ def main(): print("Failed alert IDs:") for i in statobj.failed_ids: print("\t{0}".format(err)) - print("{0} total alert(s) found, of which {1} were successfully changed" \ + print("{0} total alert(s) found, of which {1} were successfully changed" .format(statobj.num_hits, statobj.num_success)) diff --git a/examples/psc/bulk_update_watchlist_alerts.py b/examples/psc/bulk_update_watchlist_alerts.py index a2c8f5a8..9589a815 100755 --- a/examples/psc/bulk_update_watchlist_alerts.py +++ b/examples/psc/bulk_update_watchlist_alerts.py @@ -6,6 +6,7 @@ from cbapi.psc.models import WatchlistAlert, WorkflowStatus from alertsv6common import setup_parser_with_watchlist_criteria, load_watchlist_criteria + def main(): parser = build_cli_parser("Bulk update the status of watchlist alerts") setup_parser_with_watchlist_criteria(parser) @@ -14,11 +15,11 @@ def main(): operation = parser.add_mutually_exclusive_group(required=True) operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") - + args = parser.parse_args() cb = get_cb_psc_object(args) - - query = cb.select(CBAnalyticsAlert) + + query = cb.select(WatchlistAlert) load_watchlist_criteria(query, args) if args.dismiss: @@ -26,8 +27,8 @@ def main(): elif args.undismiss: reqid = query.update(args.remediation, args.comment) else: - raise NotImplemented("one of --dismiss or --undismiss must be specified") - + raise NotImplementedError("one of --dismiss or --undismiss must be specified") + print("Submitted query with ID {0}".format(reqid)) statobj = cb.select(WorkflowStatus, reqid) while not statobj.finished: @@ -41,7 +42,7 @@ def main(): print("Failed alert IDs:") for i in statobj.failed_ids: print("\t{0}".format(err)) - print("{0} total alert(s) found, of which {1} were successfully changed" \ + print("{0} total alert(s) found, of which {1} were successfully changed" .format(statobj.num_hits, statobj.num_success)) diff --git a/examples/psc/device_control.py b/examples/psc/device_control.py index d89dfd1d..bb07228a 100755 --- a/examples/psc/device_control.py +++ b/examples/psc/device_control.py @@ -4,6 +4,7 @@ from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc import Device + def toggle_value(args): if args.on: return True @@ -11,40 +12,41 @@ def toggle_value(args): return False raise Exception("Unknown toggle value") + def main(): parser = build_cli_parser("Send control messages to device") parser.add_argument("-d", "--device_id", type=int, required=True, help="The ID of the device to be controlled") subparsers = parser.add_subparsers(dest="command", help="Device command help") - + bgscan_p = subparsers.add_parser("background_scan", help="Set background scanning status") toggle = bgscan_p.add_mutually_exclusive_group(required=True) toggle.add_argument("--on", action="store_true", help="Turn background scanning on") - toggle.add_argument("--off", action="store_true", help="Turn background scanning off") - + toggle.add_argument("--off", action="store_true", help="Turn background scanning off") + bypass_p = subparsers.add_parser("bypass", help="Set bypass mode") toggle = bypass_p.add_mutually_exclusive_group(required=True) toggle.add_argument("--on", action="store_true", help="Enable bypass mode") toggle.add_argument("--off", action="store_true", help="Disable bypass mode") - + subparsers.add_parser("delete", help="Delete sensor") subparsers.add_parser("uninstall", help="Uninstall sensor") - + quarantine_p = subparsers.add_parser("quarantine", help="Set quarantine mode") toggle = quarantine_p.add_mutually_exclusive_group(required=True) toggle.add_argument("--on", action="store_true", help="Enable quarantine mode") toggle.add_argument("--off", action="store_true", help="Disable quarantine mode") - + policy_p = subparsers.add_parser("policy", help="Update policy for node") policy_p.add_argument("-p", "--policy_id", type=int, required=True, help="New policy ID to set for node") - + sensorv_p = subparsers.add_parser("sensor_version", help="Update sensor version for node") sensorv_p.add_argument("-o", "--os", required=True, help="Operating system for sensor") sensorv_p.add_argument("-V", "--version", required=True, help="Version number of sensor") - + args = parser.parse_args() cb = get_cb_psc_object(args) dev = cb.select(Device, args.device_id) - + if args.command: if args.command == "background_scan": dev.background_scan(toggle_value(args)) @@ -65,7 +67,7 @@ def main(): print("OK") else: print(dev) - + + if __name__ == "__main__": sys.exit(main()) - \ No newline at end of file diff --git a/examples/psc/download_device_list.py b/examples/psc/download_device_list.py index 65fec973..23bf2552 100755 --- a/examples/psc/download_device_list.py +++ b/examples/psc/download_device_list.py @@ -5,7 +5,8 @@ from cbapi.psc import Device import logging -logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(level=logging.DEBUG) + def main(): parser = build_cli_parser("Download device list in CSV format") @@ -17,10 +18,10 @@ def main(): parser.add_argument("-S", "--sort_by", help="Field to sort the output by") parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") parser.add_argument("-O", "--output", help="File to save output to (default stdout)") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(Device) if args.query: query = query.where(args.query) @@ -44,7 +45,6 @@ def main(): else: print(data) - + if __name__ == "__main__": sys.exit(main()) - \ No newline at end of file diff --git a/examples/psc/list_alert_facets.py b/examples/psc/list_alert_facets.py index 7afcdbc3..62de7fcc 100755 --- a/examples/psc/list_alert_facets.py +++ b/examples/psc/list_alert_facets.py @@ -5,19 +5,21 @@ from cbapi.psc.models import BaseAlert from alertsv6common import setup_parser_with_basic_criteria, load_basic_criteria + def main(): parser = build_cli_parser("List alert facets") setup_parser_with_basic_criteria(parser) parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", - "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", - "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", - "POLICY_APPLIED", "SENSOR_ACTION"], + "DEVICE_NAME", "APPLICATION_HASH", + "APPLICATION_NAME", "STATUS", "RUN_STATE", + "POLICY_APPLIED_STATE", "POLICY_APPLIED", + "SENSOR_ACTION"], required=True, help="Retrieve these fields as facet information") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(BaseAlert) load_basic_criteria(query, args) @@ -27,5 +29,6 @@ def main(): for facetval in facetinfo["values"]: print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/psc/list_alerts.py b/examples/psc/list_alerts.py index c49e4df3..a71ece6e 100755 --- a/examples/psc/list_alerts.py +++ b/examples/psc/list_alerts.py @@ -5,15 +5,16 @@ from cbapi.psc.models import BaseAlert from alertsv6common import setup_parser_with_basic_criteria, load_basic_criteria + def main(): parser = build_cli_parser("List alerts") setup_parser_with_basic_criteria(parser) parser.add_argument("-S", "--sort_by", help="Field to sort the output by") parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(BaseAlert) load_basic_criteria(query, args) if args.sort_by: @@ -23,9 +24,10 @@ def main(): alerts = list(query) print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) for alert in alerts: - print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", \ - alert.threat_id or "Unknown", \ + print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", + alert.threat_id or "Unknown", alert.last_update_time)) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/psc/list_cbanalytics_alert_facets.py b/examples/psc/list_cbanalytics_alert_facets.py index 7b4d84d3..688d33a1 100755 --- a/examples/psc/list_cbanalytics_alert_facets.py +++ b/examples/psc/list_cbanalytics_alert_facets.py @@ -5,19 +5,21 @@ from cbapi.psc.models import CBAnalyticsAlert from alertsv6common import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria + def main(): parser = build_cli_parser("List CB Analytics alert facets") setup_parser_with_cbanalytics_criteria(parser) parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", - "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", - "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", - "POLICY_APPLIED", "SENSOR_ACTION"], + "DEVICE_NAME", "APPLICATION_HASH", + "APPLICATION_NAME", "STATUS", "RUN_STATE", + "POLICY_APPLIED_STATE", "POLICY_APPLIED", + "SENSOR_ACTION"], required=True, help="Retrieve these fields as facet information") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(CBAnalyticsAlert) load_cbanalytics_criteria(query, args) @@ -27,5 +29,6 @@ def main(): for facetval in facetinfo["values"]: print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/psc/list_cbanalytics_alerts.py b/examples/psc/list_cbanalytics_alerts.py index 9d0ca285..29f50896 100755 --- a/examples/psc/list_cbanalytics_alerts.py +++ b/examples/psc/list_cbanalytics_alerts.py @@ -5,15 +5,16 @@ from cbapi.psc.models import CBAnalyticsAlert from alertsv6common import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria + def main(): parser = build_cli_parser("List CB Analytics alerts") setup_parser_with_cbanalytics_criteria(parser) parser.add_argument("-S", "--sort_by", help="Field to sort the output by") parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(CBAnalyticsAlert) load_cbanalytics_criteria(query, args) if args.sort_by: @@ -23,9 +24,10 @@ def main(): alerts = list(query) print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) for alert in alerts: - print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", \ - alert.threat_id or "Unknown", \ + print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", + alert.threat_id or "Unknown", alert.last_update_time)) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/psc/list_devices.py b/examples/psc/list_devices.py index 918b0bc3..49b3961c 100755 --- a/examples/psc/list_devices.py +++ b/examples/psc/list_devices.py @@ -4,6 +4,7 @@ from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc import Device + def main(): parser = build_cli_parser("List devices") parser.add_argument("-q", "--query", help="Query string for looking for devices") @@ -13,10 +14,10 @@ def main(): parser.add_argument("-P", "--priority", action="append", help="Target priority of device") parser.add_argument("-S", "--sort_by", help="Field to sort the output by") parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(Device) if args.query: query = query.where(args.query) @@ -31,15 +32,14 @@ def main(): if args.sort_by: direction = "DESC" if args.reverse else "ASC" query = query.sort_by(args.sort_by, direction) - + devices = list(query) print("{0:9} {1:40}{2:18}{3}".format("ID", "Hostname", "IP Address", "Last Checkin Time")) for device in devices: - print("{0:9} {1:40s}{2:18s}{3}".format(device.id, device.name or "None", \ - device.last_internal_ip_address or "Unknown", \ + print("{0:9} {1:40s}{2:18s}{3}".format(device.id, device.name or "None", + device.last_internal_ip_address or "Unknown", device.last_contact_time)) - - + + if __name__ == "__main__": sys.exit(main()) - \ No newline at end of file diff --git a/examples/psc/list_vmware_alert_facets.py b/examples/psc/list_vmware_alert_facets.py index 8f95ecb6..67fa4ed5 100755 --- a/examples/psc/list_vmware_alert_facets.py +++ b/examples/psc/list_vmware_alert_facets.py @@ -5,19 +5,21 @@ from cbapi.psc.models import VMwareAlert from alertsv6common import setup_parser_with_vmware_criteria, load_vmware_criteria + def main(): parser = build_cli_parser("List VMware alert facets") setup_parser_with_vmware_criteria(parser) parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", - "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", - "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", - "POLICY_APPLIED", "SENSOR_ACTION"], + "DEVICE_NAME", "APPLICATION_HASH", + "APPLICATION_NAME", "STATUS", "RUN_STATE", + "POLICY_APPLIED_STATE", "POLICY_APPLIED", + "SENSOR_ACTION"], required=True, help="Retrieve these fields as facet information") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(VMwareAlert) load_vmware_criteria(query, args) @@ -27,5 +29,6 @@ def main(): for facetval in facetinfo["values"]: print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/psc/list_vmware_alerts.py b/examples/psc/list_vmware_alerts.py index b5a139d8..986f984c 100755 --- a/examples/psc/list_vmware_alerts.py +++ b/examples/psc/list_vmware_alerts.py @@ -5,15 +5,16 @@ from cbapi.psc.models import VMwareAlert from alertsv6common import setup_parser_with_vmware_criteria, load_vmware_criteria + def main(): parser = build_cli_parser("List VMware alerts") setup_parser_with_vmware_criteria(parser) parser.add_argument("-S", "--sort_by", help="Field to sort the output by") parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(VMwareAlert) load_vmware_criteria(query, args) if args.sort_by: @@ -23,9 +24,10 @@ def main(): alerts = list(query) print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) for alert in alerts: - print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", \ - alert.threat_id or "Unknown", \ + print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", + alert.threat_id or "Unknown", alert.last_update_time)) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/psc/list_watchlist_alert_facets.py b/examples/psc/list_watchlist_alert_facets.py index b8aa2a15..f5b8b803 100755 --- a/examples/psc/list_watchlist_alert_facets.py +++ b/examples/psc/list_watchlist_alert_facets.py @@ -5,19 +5,21 @@ from cbapi.psc.models import WatchlistAlert from alertsv6common import setup_parser_with_watchlist_criteria, load_watchlist_criteria + def main(): parser = build_cli_parser("List watchlist alert facets") setup_parser_with_watchlist_criteria(parser) parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", - "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", - "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", - "POLICY_APPLIED", "SENSOR_ACTION"], + "DEVICE_NAME", "APPLICATION_HASH", + "APPLICATION_NAME", "STATUS", "RUN_STATE", + "POLICY_APPLIED_STATE", "POLICY_APPLIED", + "SENSOR_ACTION"], required=True, help="Retrieve these fields as facet information") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(WatchlistAlert) load_watchlist_criteria(query, args) @@ -27,5 +29,6 @@ def main(): for facetval in facetinfo["values"]: print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/psc/list_watchlist_alerts.py b/examples/psc/list_watchlist_alerts.py index 89ec1562..2d830645 100755 --- a/examples/psc/list_watchlist_alerts.py +++ b/examples/psc/list_watchlist_alerts.py @@ -5,15 +5,16 @@ from cbapi.psc.models import WatchlistAlert from alertsv6common import setup_parser_with_watchlist_criteria, load_watchlist_criteria + def main(): parser = build_cli_parser("List watchlist alerts") setup_parser_with_watchlist_criteria(parser) parser.add_argument("-S", "--sort_by", help="Field to sort the output by") parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") - + args = parser.parse_args() cb = get_cb_psc_object(args) - + query = cb.select(WatchlistAlert) load_watchlist_criteria(query, args) if args.sort_by: @@ -23,9 +24,10 @@ def main(): alerts = list(query) print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) for alert in alerts: - print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", \ - alert.threat_id or "Unknown", \ + print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", + alert.threat_id or "Unknown", alert.last_update_time)) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/response/alert_search.py b/examples/response/alert_search.py index 3d19a659..2248dfab 100755 --- a/examples/response/alert_search.py +++ b/examples/response/alert_search.py @@ -27,4 +27,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/examples/response/binary_export.py b/examples/response/binary_export.py index 0d5e0837..24a8eb3d 100755 --- a/examples/response/binary_export.py +++ b/examples/response/binary_export.py @@ -19,5 +19,6 @@ def main(): return 0 + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/response/binary_search.py b/examples/response/binary_search.py index ed374658..218df892 100755 --- a/examples/response/binary_search.py +++ b/examples/response/binary_search.py @@ -13,13 +13,14 @@ def main(): cb = get_cb_response_object(args) binary_query = cb.select(Binary).where(args.query) - # for each result + # for each result for binary in binary_query: print(binary.md5sum) print("-" * 80) print("%-20s : %s" % ('Size (bytes)', binary.size)) print("%-20s : %s" % ('Signature Status', binary.signed)) - print("%-20s : %s" % ('Publisher', binary.digsig_publisher) if binary.signed == True else "%-20s : %s" % ('Publisher', 'n/a')) + print("%-20s : %s" % ('Publisher', binary.digsig_publisher) if binary.signed + else "%-20s : %s" % ('Publisher', 'n/a')) print("%-20s : %s" % ('Product Version', binary.product_version)) print("%-20s : %s" % ('File Version', binary.file_version)) print("%-20s : %s" % ('64-bit (x64)', binary.is_64bit)) @@ -30,5 +31,6 @@ def main(): print('\n') + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/response/cmd_exe_filemods.py b/examples/response/cmd_exe_filemods.py index 6c9439ef..3e3afac9 100755 --- a/examples/response/cmd_exe_filemods.py +++ b/examples/response/cmd_exe_filemods.py @@ -31,5 +31,6 @@ def main(): print("%s,%s,%s,%s,%s,%s,%s,%s,%s" % (str(fm.timestamp), proc.hostname, proc.username, proc.path, fm.path, fm.type, fm.md5, signed, product_name)) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/response/download_from_lr.py b/examples/response/download_from_lr.py index 533b9f31..f9ebb125 100755 --- a/examples/response/download_from_lr.py +++ b/examples/response/download_from_lr.py @@ -2,9 +2,8 @@ # import sys -from cbapi.response import * +from cbapi.response import Sensor from cbapi.example_helpers import build_cli_parser, get_cb_response_object -from cbapi.errors import ServerError import logging log = logging.getLogger(__name__) @@ -12,7 +11,7 @@ def download_file(sensor, path, output_filename=None): with sensor.lr_session() as session: - + # get basename of the file, if output_filename is None if output_filename is None: if session.os_type == 1: # Windows uses backslashes @@ -25,7 +24,7 @@ def download_file(sensor, path, output_filename=None): print("Successfully wrote {0} to local file {1}.".format(path, output_filename)) return 0 - + def main(): parser = build_cli_parser("Download binary from endpoint through Live Response") @@ -41,7 +40,8 @@ def main(): if sensor.status == "Online": return download_file(sensor, args.path, args.output) - print("No sensors for hostname {0} are online, exiting".format(hostname)) + print("No sensors for hostname {0} are online, exiting".format(args.hostname)) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/response/dump_all_binaries.py b/examples/response/dump_all_binaries.py index 224f90c7..db34ff59 100755 --- a/examples/response/dump_all_binaries.py +++ b/examples/response/dump_all_binaries.py @@ -21,7 +21,7 @@ def get_path_for_md5(d, basepath=''): def create_directory(pathname): try: os.makedirs(os.path.dirname(pathname)) - except: + except Exception: pass @@ -35,21 +35,21 @@ def already_exists(self, pathname, item): filesize = os.path.getsize(pathname) if filesize == item.copied_size: return True - except: + except Exception: pass return False def run(self): - l = logging.getLogger(__name__) - l.setLevel(logging.INFO) + log = logging.getLogger(__name__) + log.setLevel(logging.INFO) while True: item = worker_queue.get() pathname = get_path_for_md5(item.md5sum, self.basepath) if self.already_exists(pathname, item): - l.info('already have %s' % item.md5sum) + log.info('already have %s' % item.md5sum) else: create_directory(pathname) write_progress = 0 @@ -58,15 +58,15 @@ def run(self): write_progress += 1 json.dump(item.original_document, open(pathname + '.json', 'w')) write_progress += 1 - except: + except Exception: pass if not write_progress: - l.error(u'Could not grab {0:s}'.format(item.md5sum)) + log.error(u'Could not grab {0:s}'.format(item.md5sum)) elif write_progress == 1: - l.info(u'Grabbed {0:s} binary'.format(item.md5sum)) + log.info(u'Grabbed {0:s} binary'.format(item.md5sum)) elif write_progress == 2: - l.info(u'Grabbed {0:s} binary & process document'.format(item.md5sum)) + log.info(u'Grabbed {0:s} binary & process document'.format(item.md5sum)) worker_queue.task_done() @@ -112,4 +112,3 @@ def main(): if __name__ == '__main__': sys.exit(main()) - diff --git a/examples/response/enumerate_usb_devices.py b/examples/response/enumerate_usb_devices.py index e60f9c48..f024e950 100755 --- a/examples/response/enumerate_usb_devices.py +++ b/examples/response/enumerate_usb_devices.py @@ -13,7 +13,7 @@ def main(): args = parser.parse_args() cb = get_cb_response_object(args) - query_string = r'regmod:registry\machine\system\currentcontrolset\control\deviceclasses\{53f56307-b6bf-11d0-94f2-00a0c91efb8b}\*' + query_string = r'regmod:registry\machine\system\currentcontrolset\control\deviceclasses\{53f56307-b6bf-11d0-94f2-00a0c91efb8b}\*' # noqa: E501 if args.start_time: query_string += ' start:{0:s}'.format(args.start_time) @@ -24,7 +24,7 @@ def main(): if len(pieces) < 2: print("WARN:::: {0}".format(str(pieces))) else: - device_info = pieces[1] #.split('{53f56307-b6bf-11d0-94f2-00a0c91efb8b}')[0] + device_info = pieces[1] # .split('{53f56307-b6bf-11d0-94f2-00a0c91efb8b}')[0] print(device_info) diff --git a/examples/response/event/get_reg_autoruns.py b/examples/response/event/get_reg_autoruns.py index 402d6f76..14d5f532 100644 --- a/examples/response/event/get_reg_autoruns.py +++ b/examples/response/event/get_reg_autoruns.py @@ -30,8 +30,7 @@ import sys import time - -autoruns_regex = re.compile("|".join("""\\registry\\machine\\system\\currentcontrolset\\control\\session manager\\bootexecute(.*) +autoruns = """\\registry\\machine\\system\\currentcontrolset\\control\\session manager\\bootexecute(.*) \\registry\\machine\\system\\currentcontrolset\\services(.*) \\registry\\machine\\system\\currentcontrolset\\services(.*) \\registry\\machine\\software\\microsoft\\windows\\currentversion\\runservicesonce(.*) @@ -53,8 +52,9 @@ \\registry\\user\\(.*)\\software\\microsoft\\windows\\currentversion\\run(.*) \\registry\\user\\(.*)\\software\\microsoft\\windows\\currentversion\\runonce(.*) \\registry\\user\\(.*)\\software\\microsoft\\windows\\currentversion\\policies\\explorer\\run(.*) -\\registry\\user\\(.*)\\software\\microsoft\\windows nt\\currentversion\\windows\\load(.*)""".replace("\\", "\\\\").split("\n"))) +\\registry\\user\\(.*)\\software\\microsoft\\windows nt\\currentversion\\windows\\load(.*)""" +autoruns_regex = re.compile("|".join(autoruns.replace("\\", "\\\\").split("\n"))) class GetRegistryValue(object): @@ -85,7 +85,7 @@ def process_callback(cb, event_type, event_data): def print_result(registry_job): try: timestamp, sensor_id, registry_key, registry_value = registry_job.result() - except: + except Exception: print("Error encountered when pulling registry key: {0}".format(registry_job.exception())) else: print("Got result for sensor ID {0} registry key {1}: value is {2}".format(sensor_id, registry_key, @@ -118,7 +118,5 @@ def main(): print(error["exception"]) - if __name__ == "__main__": sys.exit(main()) - diff --git a/examples/response/event/watchlist_automation.py b/examples/response/event/watchlist_automation.py index 175c9a63..f15d661c 100644 --- a/examples/response/event/watchlist_automation.py +++ b/examples/response/event/watchlist_automation.py @@ -75,9 +75,11 @@ def perform_liveresponse(lr_session): db.row_factory = sqlite3.Row cur = db.cursor() cur.execute( - "SELECT url, title, datetime(last_visit_time / 1000000 + (strftime('%s', '1601-01-01')), 'unixepoch') as last_visit_time FROM urls ORDER BY last_visit_time DESC LIMIT 10") + "SELECT url, title, " + "datetime(last_visit_time / 1000000 + (strftime('%s', '1601-01-01')), 'unixepoch') " + "as last_visit_time FROM urls ORDER BY last_visit_time DESC LIMIT 10") urls = [dict(u) for u in cur.fetchall()] - except: + except Exception: pass else: results[user] = urls @@ -90,7 +92,7 @@ def perform_liveresponse(lr_session): def print_result(lr_job): try: sensor_id, running_services, urls = lr_job.result() - except: + except Exception: print("Error encountered when pulling Live Response data: {0}".format(lr_job.exception())) else: print("Running services for sensor ID {0}:".format(sensor_id)) @@ -140,7 +142,8 @@ def process_callback(cb, event_type, event_data): def main(): - parser = build_cli_parser("Event-driven example to BAN hashes, ISOLATE sensors, or LOCK (both ban & isolate) based on watchlist hits") + parser = build_cli_parser("Event-driven example to BAN hashes, ISOLATE sensors, or LOCK " + "(both ban & isolate) based on watchlist hits") args = parser.parse_args() cb = get_cb_response_object(args) @@ -164,4 +167,3 @@ def main(): if __name__ == "__main__": sys.exit(main()) - diff --git a/examples/response/event_export.py b/examples/response/event_export.py index d718f520..b0fcc38c 100755 --- a/examples/response/event_export.py +++ b/examples/response/event_export.py @@ -8,7 +8,8 @@ from cbapi.example_helpers import build_cli_parser, get_cb_response_object import csv from cbapi.six import PY3 -from cbapi.response.models import CbChildProcEvent, CbFileModEvent, CbNetConnEvent, CbRegModEvent, CbModLoadEvent, CbCrossProcEvent +from cbapi.response.models import CbChildProcEvent, CbFileModEvent, CbNetConnEvent, CbRegModEvent, \ + CbModLoadEvent, CbCrossProcEvent # UnicodeWriter class from http://python3porting.com/problems.html diff --git a/examples/response/feed_operations.py b/examples/response/feed_operations.py index d135c466..2fe30fd9 100755 --- a/examples/response/feed_operations.py +++ b/examples/response/feed_operations.py @@ -2,7 +2,7 @@ # import sys -from cbapi.response.models import Feed, FeedAction +from cbapi.response.models import Feed from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id from cbapi.errors import ServerError import logging @@ -145,7 +145,7 @@ def main(): parser = build_cli_parser() commands = parser.add_subparsers(help="Feed commands", dest="command_name") - list_command = commands.add_parser("list", help="List all configured feeds") + commands.add_parser("list", help="List all configured feeds") list_actions_command = commands.add_parser("list-actions", help="List actions associated with a feed") list_actions_specifier = list_actions_command.add_mutually_exclusive_group(required=True) @@ -195,7 +195,7 @@ def main(): elif args.command_name == "delete": return delete_feed(cb, parser, args) elif args.command_name in ("disable", "enable"): - return toggle_feed(cb, args.feedname, enable=args.command_name=="enable") + return toggle_feed(cb, args.feedname, enable=(args.command_name == "enable")) if __name__ == "__main__": diff --git a/examples/response/new_binaries_after_date.py b/examples/response/new_binaries_after_date.py index fc405e90..597b05dc 100755 --- a/examples/response/new_binaries_after_date.py +++ b/examples/response/new_binaries_after_date.py @@ -27,7 +27,7 @@ def main(): # parser = build_cli_parser("System Check After Specified Date") parser.add_argument("-d", "--date-to-query", action="store", dest="date", - help="New since DATE, format YYYY-MM-DD") + help="New since DATE, format YYYY-MM-DD") parser.add_argument("-f", "--output-file", action="store", dest="output_file", help="output file in csv format") @@ -122,11 +122,10 @@ def main(): binary.host_count, binary_timestamp, number_of_times_executed)) - except: - print binary + except Exception: + print(binary) pbar.finish() + if __name__ == "__main__": sys.exit(main()) - - diff --git a/examples/response/new_binaries_with_netconns.py b/examples/response/new_binaries_with_netconns.py index d81de5f0..402637ba 100755 --- a/examples/response/new_binaries_with_netconns.py +++ b/examples/response/new_binaries_with_netconns.py @@ -29,7 +29,7 @@ def main(): # parser = build_cli_parser("New Binaries with Netconns") parser.add_argument("-d", "--date-to-query", action="store", dest="date", - help="New since DATE, format YYYY-MM-DD") + help="New since DATE, format YYYY-MM-DD") parser.add_argument("-f", "--output-file", action="store", dest="output_file", help="output file in csv format") @@ -111,10 +111,11 @@ def main(): binary.server_added_timestamp, binary.host_count, binary_timestamp)) - except: - print binary + except Exception: + print(binary) pbar.finish() + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/response/partition_operations.py b/examples/response/partition_operations.py index c19628e1..07a87bad 100755 --- a/examples/response/partition_operations.py +++ b/examples/response/partition_operations.py @@ -2,8 +2,7 @@ from distutils.version import LooseVersion from cbapi.response.models import StoragePartition -from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id -from cbapi.errors import ServerError +from cbapi.example_helpers import build_cli_parser, get_cb_response_object import logging log = logging.getLogger(__name__) @@ -58,9 +57,9 @@ def main(): parser = build_cli_parser() commands = parser.add_subparsers(help="Storage Partition commands", dest="command_name") - list_command = commands.add_parser("list", help="List all storage partitions") + commands.add_parser("list", help="List all storage partitions") - create_command = commands.add_parser("create", help="Create new active writer partition") + commands.add_parser("create", help="Create new active writer partition") del_command = commands.add_parser("delete", help="Delete partition") del_command.add_argument("-N", "--name", help="Name of partition to delete.", required=True) diff --git a/examples/response/process_netconn_rate.py b/examples/response/process_netconn_rate.py index 866ec571..572114c4 100755 --- a/examples/response/process_netconn_rate.py +++ b/examples/response/process_netconn_rate.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -#The MIT License (MIT) +# The MIT License (MIT) ## # Copyright (c) 2015 Bit9 + Carbon Black # diff --git a/examples/response/s3-watchlist-ban.py b/examples/response/s3-watchlist-ban.py index 4457e644..e9286d69 100755 --- a/examples/response/s3-watchlist-ban.py +++ b/examples/response/s3-watchlist-ban.py @@ -1,7 +1,5 @@ import boto3 import json -import pprint -import traceback import time from cbapi.response.models import BannedHash from cbapi.example_helpers import build_cli_parser, get_cb_response_object @@ -47,7 +45,6 @@ def process_events(data): print(e.message) - def save_progress(processed_list): # # Save our progress in a log file @@ -55,6 +52,7 @@ def save_progress(processed_list): with open('script_progress.log', 'wb') as hfile: hfile.write(json.dumps(list(processed_list))) + def listen_mode(bucket): print("[+]: Listen Mode") @@ -74,7 +72,7 @@ def listen_mode(bucket): for obj in bucket.objects.all(): key = obj.key if key not in current_list: - print "[+]: New File: {}".format(key) + print("[+]: New File: {}".format(key)) # # We have not processed this file. # @@ -142,7 +140,7 @@ def listen_mode(bucket): with open('script_progress.log', 'rb') as hfile: for item in json.loads(hfile.read()): processed_list.add(item) - except: + except Exception: print("[?]: No previous progress file found: script_progress.log") processed_list = set() @@ -170,4 +168,3 @@ def listen_mode(bucket): print("[+]: saving progess to file script_progress.log") save_progress(processed_list) - diff --git a/examples/response/sensor_export.py b/examples/response/sensor_export.py index 383b37c9..0a5077ea 100755 --- a/examples/response/sensor_export.py +++ b/examples/response/sensor_export.py @@ -34,7 +34,8 @@ def main(): parser = build_cli_parser(description="Export CbR Sensors from your environment as CSV") parser.add_argument("--output", "-o", dest="exportfile", help="The file to export to", required=True) parser.add_argument("--fields", "-f", dest="exportfields", help="The fields to export", - default="id,hostname,group_id,network_interfaces,os_environment_display_string,build_version_string,network_isolation_enabled,last_checkin_time", + default="id,hostname,group_id,network_interfaces,os_environment_display_string," + "build_version_string,network_isolation_enabled,last_checkin_time", required=False) parser.add_argument("--query", "-q", dest="query", help="optional query to filter exported sensors", required=False) args = parser.parse_args() diff --git a/examples/response/sensor_group_operations.py b/examples/response/sensor_group_operations.py index b11e8200..46798e53 100755 --- a/examples/response/sensor_group_operations.py +++ b/examples/response/sensor_group_operations.py @@ -76,8 +76,8 @@ def delete_sensor_group(cb, parser, args): num_matching_sensor_groups = len(groups) if num_matching_sensor_groups > 1 and not args.force: - print("{0:d} sensor groups match {1:s} and --force not specified. No action taken.".format(num_matching_sensor_groups, - attempted_to_find)) + print("{0:d} sensor groups match {1:s} and --force not specified. No action taken." + .format(num_matching_sensor_groups, attempted_to_find)) return for g in groups: @@ -93,7 +93,7 @@ def main(): parser = build_cli_parser() commands = parser.add_subparsers(help="Sensor Group commands", dest="command_name") - list_command = commands.add_parser("list", help="List all configured sensor groups") + commands.add_parser("list", help="List all configured sensor groups") add_command = commands.add_parser("add", help="Add new sensor group") add_command.add_argument("-n", "--name", action="store", help="Sensor group name", required=True, @@ -105,9 +105,11 @@ def main(): del_command = commands.add_parser("delete", help="Delete sensor groups") del_sensor_group_specifier = del_command.add_mutually_exclusive_group(required=True) del_sensor_group_specifier.add_argument("-i", "--id", type=int, help="ID of sensor group to delete") - del_sensor_group_specifier.add_argument("-n", "--name", help="Name of sensor group to delete. Specify --force to delete" - " multiple sensor groups that have the same name", dest="groupname") - del_command.add_argument("--force", help="If NAME matches multiple sensor groups, delete all matching sensor groups", + del_sensor_group_specifier.add_argument("-n", "--name", + help="Name of sensor group to delete. Specify --force to delete" + " multiple sensor groups that have the same name", dest="groupname") + del_command.add_argument("--force", + help="If NAME matches multiple sensor groups, delete all matching sensor groups", action="store_true", default=False) list_sensors_command = commands.add_parser("list-sensors", help="List all sensors in a sensor group") diff --git a/examples/response/sensor_operations.py b/examples/response/sensor_operations.py index d48f53ae..55a11111 100755 --- a/examples/response/sensor_operations.py +++ b/examples/response/sensor_operations.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -#ZE 2018 AD +# ZE 2018 AD import sys from cbapi.response.models import Alert @@ -11,9 +11,10 @@ ''' This is a utility designed to use watchlists to perform operations on affected sensors: -supported operations: memory dump, isolation and process termination. +supported operations: memory dump, isolation and process termination. ''' + def sensor_operations(cb, watchlists, operation, dryrun=False): print("Trying to {0} based on watchlists: {1}".format(operation, watchlists)) where_clause = " or ".join(("watchlist_name:" + wl for wl in watchlists.split(","))) @@ -34,7 +35,7 @@ def sensor_operations(cb, watchlists, operation, dryrun=False): lr.close() else: print("DRYRUN: would have {0} sensor {1}".format(sensor, operation)) - except Exception as e: + except Exception: print(traceback.format_exc(0)) print("Sensor operations finished") diff --git a/examples/response/tf.py b/examples/response/tf.py index 7b5f1a05..e7b0b565 100755 --- a/examples/response/tf.py +++ b/examples/response/tf.py @@ -4,7 +4,7 @@ import sys from cbapi.response.models import Process from cbapi.example_helpers import build_cli_parser, get_cb_response_object -from cbapi.errors import ServerError, ObjectNotFoundError +from cbapi.errors import ObjectNotFoundError import logging log = logging.getLogger(__name__) @@ -68,13 +68,13 @@ def process_hit(cb, parent_proc_name, proc_name, value, ratio, child_behavior): def main(): parser = build_cli_parser("Term Frequency Analysis") parser.add_argument("-p", "--percentage", action="store", default="2", dest="percentless", - help="Max Percentage of Term Frequency e.g., 2 ") + help="Max Percentage of Term Frequency e.g., 2 ") process_selection = parser.add_mutually_exclusive_group(required=True) process_selection.add_argument("-t", "--term", action="store", default=None, dest="procname", - help="Comma separated list of parent processes to get term frequency") + help="Comma separated list of parent processes to get term frequency") process_selection.add_argument("-f", "--termfile", action="store", default=None, dest="procnamefile", - help="Text file new line separated list of parent processes to get term frequency") + help="Text file new line separated list of parent processes to get term frequency") output_selection = parser.add_mutually_exclusive_group(required=False) output_selection.add_argument("--count", action="store_true", diff --git a/examples/response/user_operations.py b/examples/response/user_operations.py index e2dedccb..0231da3f 100755 --- a/examples/response/user_operations.py +++ b/examples/response/user_operations.py @@ -3,7 +3,7 @@ import sys from cbapi.response.models import User, Team, SensorGroup -from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id +from cbapi.example_helpers import build_cli_parser, get_cb_response_object from cbapi.errors import ServerError import logging import getpass @@ -118,8 +118,8 @@ def main(): parser = build_cli_parser() commands = parser.add_subparsers(help="User commands", dest="command_name") - list_command = commands.add_parser("list", help="List all configured users") - list_teams_command = commands.add_parser("list-teams", help="List all configured user teams") + commands.add_parser("list", help="List all configured users") + commands.add_parser("list-teams", help="List all configured user teams") add_command = commands.add_parser("add", help="Add new user") add_command.add_argument("-u", "--username", help="New user's username", required=True) diff --git a/examples/response/walk_children.py b/examples/response/walk_children.py index 54d9771a..a59263bb 100755 --- a/examples/response/walk_children.py +++ b/examples/response/walk_children.py @@ -30,7 +30,7 @@ def main(): if args.process: try: procs = [c.select(Process, args.process, max_children=args.children, force_init=True)] - except ObjectNotFoundError as e: + except ObjectNotFoundError: print("Could not find process {0:s}".format(args.procss)) return 1 except ApiError as e: @@ -53,7 +53,7 @@ def main(): duration = str(root_proc.end - root_proc.start) print("Process {0:s} on {1:s} executed by {2:s}:".format(root_proc.cmdline, root_proc.hostname, - root_proc.username)) + root_proc.username)) print("started at {0} ({1})".format(str(root_proc.start), duration)) print("Cb Response console link: {0}".format(root_proc.webui_link)) root_proc.walk_children(visitor) diff --git a/examples/response/watchlist_exporter.py b/examples/response/watchlist_exporter.py index f7ca3675..ae521789 100755 --- a/examples/response/watchlist_exporter.py +++ b/examples/response/watchlist_exporter.py @@ -4,8 +4,7 @@ import sys import cbapi.six as six from cbapi.response.models import Watchlist -from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id -from cbapi.errors import ServerError +from cbapi.example_helpers import build_cli_parser, get_cb_response_object import logging from datetime import datetime import json @@ -16,7 +15,7 @@ if six.PY3: confirm_input = input else: - confirm_input = raw_input + confirm_input = raw_input # noqa: F821 def confirm(watch_list): diff --git a/examples/response/watchlist_importer.py b/examples/response/watchlist_importer.py index 6fa7e75a..e5cd5514 100755 --- a/examples/response/watchlist_importer.py +++ b/examples/response/watchlist_importer.py @@ -4,10 +4,8 @@ import sys import cbapi.six as six from cbapi.response.models import Watchlist -from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id -from cbapi.errors import ServerError +from cbapi.example_helpers import build_cli_parser, get_cb_response_object import logging -from datetime import datetime import json log = logging.getLogger(__name__) @@ -16,7 +14,7 @@ if six.PY3: confirm_input = input else: - confirm_input = raw_input + confirm_input = raw_input # noqa: F821 def confirm(watch_list): diff --git a/examples/response/watchlist_operations.py b/examples/response/watchlist_operations.py index 655e2647..b6e69989 100755 --- a/examples/response/watchlist_operations.py +++ b/examples/response/watchlist_operations.py @@ -50,7 +50,7 @@ def delete_watchlist(cb, parser, args): num_matching_watchlists = len(watchlists) if num_matching_watchlists > 1 and not args.force: print("{0:d} watchlists match {1:s} and --force not specified. No action taken.".format(num_matching_watchlists, - attempted_to_find)) + attempted_to_find)) return for f in watchlists: @@ -78,7 +78,7 @@ def main(): parser = build_cli_parser() commands = parser.add_subparsers(help="Watchlist commands", dest="command_name") - list_command = commands.add_parser("list", help="List all configured watchlists") + commands.add_parser("list", help="List all configured watchlists") list_actions_command = commands.add_parser("list-actions", help="List actions associated with a watchlist") list_actions_specifier = list_actions_command.add_mutually_exclusive_group(required=True) diff --git a/examples/threathunter/events.py b/examples/threathunter/events.py index d0fe23d0..244fd7cd 100644 --- a/examples/threathunter/events.py +++ b/examples/threathunter/events.py @@ -28,5 +28,6 @@ def main(): print("\tEvent GUID: {}".format(event.event_guid)) print("\tEvent Timestamp: {}".format(event.event_timestamp)) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/threathunter/events_exporter.py b/examples/threathunter/events_exporter.py index f9fe8d59..2f31c9f4 100644 --- a/examples/threathunter/events_exporter.py +++ b/examples/threathunter/events_exporter.py @@ -5,16 +5,16 @@ from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object from cbapi.psc.threathunter import Event import json -import csv +import csv -def main(): +def main(): parser = build_cli_parser("Query processes") parser.add_argument("-p", type=str, help="process guid", default=None) - parser.add_argument("-s",type=bool, help="silent mode",default=False) + parser.add_argument("-s", type=bool, help="silent mode", default=False) parser.add_argument("-n", type=int, help="only output N events", default=None) - parser.add_argument("-f", type=str, help="output file name",default=None) - parser.add_argument("-of", type=str,help="output file format: csv or json",default="json") + parser.add_argument("-f", type=str, help="output file name", default=None) + parser.add_argument("-of", type=str, help="output file format: csv or json", default="json") args = parser.parse_args() cb = get_cb_threathunter_object(args) diff --git a/examples/threathunter/import_response_feeds.py b/examples/threathunter/import_response_feeds.py index 3bd48279..85146ea3 100644 --- a/examples/threathunter/import_response_feeds.py +++ b/examples/threathunter/import_response_feeds.py @@ -3,18 +3,19 @@ import sys from cbapi.psc.threathunter import CbThreatHunterAPI from cbapi.psc.threathunter.models import Feed as FeedTH -from cbapi.response.models import Feed, ThreatReport -from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id +from cbapi.response.models import Feed +from cbapi.example_helpers import build_cli_parser, get_cb_response_object from cbapi.errors import ServerError from urllib.parse import unquote import logging log = logging.getLogger(__name__) -""" -Lists the feeds in CB Response -""" + def list_feeds(cb, parser, args): + """ + Lists the feeds in CB Response + """ for f in cb.select(Feed): for fieldname in ["id", "category", "display_name", "enabled", "provider_url", "summary", "tech_data", "feed_url", "use_proxy", "validate_server_cert"]: @@ -30,93 +31,96 @@ def list_feeds(cb, parser, args): print("\n") -""" -Lists the reports in a feed from CB Response -:param: id - The ID of a feed -""" + def list_reports(cb, parser, args): - feed = cb.select(Feed, args.id, force_init=True) - for report in feed.reports: - print (report) - print("\n") - -""" -Converts and copies a feed from CB Response to CB Threat Hunter -:param: id - The ID of a feed from CB Response - -Requires a credentials profile for both CB Response and CB Threat Hunter - Ensure that your credentials for CB Threat Hunter have permissions to the Feed Manager APIs -""" + """ + Lists the reports in a feed from CB Response + :param: id - The ID of a feed + """ + feed = cb.select(Feed, args.id, force_init=True) + for report in feed.reports: + print(report) + print("\n") + + def convert_feed(cb, cb_th, parser, args): - th_feed = { "feedinfo": {}, "reports": []} - # Fetches the CB Response feed - feed = cb.select(Feed, args.id, force_init=True) - - th_feed["feedinfo"]["name"] = feed.name - th_feed["feedinfo"]["provider_url"] = feed.provider_url - th_feed["feedinfo"]["summary"] = feed.summary - th_feed["feedinfo"]["category"] = feed.category - th_feed["feedinfo"]["access"] = "private" - - # Temporary values until refresh - th_feed["feedinfo"]["owner"] = "org_key" - th_feed["feedinfo"]["id"] = "id" - - # Iterates the reports in the CB Response feed - for report in feed.reports: - th_report = {} - th_report["id"] = report.id - th_report["timestamp"] = report.timestamp - th_report["title"] = report.title - th_report["severity"] = (report.score % 10) + 1 - if hasattr(report, "description"): - th_report["description"] = report.description - else: - th_report["description"] = "" - if hasattr(report, "link"): - th_report["link"] = report.link - th_report["iocs"] = {} - if report.iocs: - if "md5" in report.iocs: - th_report["iocs"]["md5"] = report.iocs["md5"] - if "ipv4" in report.iocs: - th_report["iocs"]["ipv4"] = report.iocs["ipv4"] - if "ipv6" in report.iocs: - th_report["iocs"]["ipv6"] = report.iocs["ipv6"] - if "dns" in report.iocs: - th_report["iocs"]["dns"] = report.iocs["dns"] - - if "query" in report.iocs: - th_report["iocs"]["query"] = [] - for query in report.iocs.get("query", []): - try: - search = query.get('search_query', "") - if "q=" in search: - params = search.split('&') - for p in params: - if "q=" in p: - search = unquote(p[2:]) - # Converts the CB Response query to CB Threat Hunter - th_query = cb_th.convert_query(search) - if th_query: - query["search_query"] = th_query - th_report["iocs"]["query"].append(query) - except ServerError as e: - print ('Invalid query {}'.format(query.get('search_query', ""))) - - th_feed["reports"].append(th_report) - - # Pushes the new feed to CB Threat Hunter - new_feed = cb_th.create(FeedTH, th_feed) - new_feed.save() - print ("{}\n".format(new_feed)) + """ + Converts and copies a feed from CB Response to CB Threat Hunter + :param: id - The ID of a feed from CB Response + + Requires a credentials profile for both CB Response and CB Threat Hunter + Ensure that your credentials for CB Threat Hunter have permissions to the Feed Manager APIs + """ + th_feed = {"feedinfo": {}, "reports": []} + # Fetches the CB Response feed + feed = cb.select(Feed, args.id, force_init=True) + + th_feed["feedinfo"]["name"] = feed.name + th_feed["feedinfo"]["provider_url"] = feed.provider_url + th_feed["feedinfo"]["summary"] = feed.summary + th_feed["feedinfo"]["category"] = feed.category + th_feed["feedinfo"]["access"] = "private" + + # Temporary values until refresh + th_feed["feedinfo"]["owner"] = "org_key" + th_feed["feedinfo"]["id"] = "id" + + # Iterates the reports in the CB Response feed + for report in feed.reports: + th_report = {} + th_report["id"] = report.id + th_report["timestamp"] = report.timestamp + th_report["title"] = report.title + th_report["severity"] = (report.score % 10) + 1 + if hasattr(report, "description"): + th_report["description"] = report.description + else: + th_report["description"] = "" + if hasattr(report, "link"): + th_report["link"] = report.link + th_report["iocs"] = {} + if report.iocs: + if "md5" in report.iocs: + th_report["iocs"]["md5"] = report.iocs["md5"] + if "ipv4" in report.iocs: + th_report["iocs"]["ipv4"] = report.iocs["ipv4"] + if "ipv6" in report.iocs: + th_report["iocs"]["ipv6"] = report.iocs["ipv6"] + if "dns" in report.iocs: + th_report["iocs"]["dns"] = report.iocs["dns"] + + if "query" in report.iocs: + th_report["iocs"]["query"] = [] + for query in report.iocs.get("query", []): + try: + search = query.get('search_query', "") + if "q=" in search: + params = search.split('&') + for p in params: + if "q=" in p: + search = unquote(p[2:]) + # Converts the CB Response query to CB Threat Hunter + th_query = cb_th.convert_query(search) + if th_query: + query["search_query"] = th_query + th_report["iocs"]["query"].append(query) + except ServerError: + print('Invalid query {}'.format(query.get('search_query', ""))) + + th_feed["reports"].append(th_report) + + # Pushes the new feed to CB Threat Hunter + new_feed = cb_th.create(FeedTH, th_feed) + new_feed.save() + print("{}\n".format(new_feed)) + def main(): parser = build_cli_parser() parser.add_argument("-thp", "--threatprofile", help="Threat Hunter profile", default="default") commands = parser.add_subparsers(help="Feed commands", dest="command_name") - list_command = commands.add_parser("list", help="List all configured feeds") + commands.add_parser("list", help="List all configured feeds") list_reports_command = commands.add_parser("list-reports", help="List all configured reports for a feed") list_reports_command.add_argument("-i", "--id", type=str, help="Feed ID") @@ -135,5 +139,6 @@ def main(): if args.command_name == "convert": return convert_feed(cb, cb_th, parser, args) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/threathunter/process_exporter.py b/examples/threathunter/process_exporter.py index 562c83c8..e86b113f 100644 --- a/examples/threathunter/process_exporter.py +++ b/examples/threathunter/process_exporter.py @@ -5,17 +5,17 @@ from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object from cbapi.psc.threathunter import Process import json -import csv +import csv -def main(): +def main(): parser = build_cli_parser("Query processes") parser.add_argument("-p", type=str, help="process guid", default=None) - parser.add_argument("-q",type=str,help="query string",default=None) - parser.add_argument("-s",type=bool, help="silent mode",default=False) + parser.add_argument("-q", type=str, help="query string", default=None) + parser.add_argument("-s", type=bool, help="silent mode", default=False) parser.add_argument("-n", type=int, help="only output N events", default=None) - parser.add_argument("-f", type=str, help="output file name",default=None) - parser.add_argument("-of", type=str,help="output file format: csv or json",default="json") + parser.add_argument("-f", type=str, help="output file name", default=None) + parser.add_argument("-of", type=str, help="output file format: csv or json", default="json") args = parser.parse_args() cb = get_cb_threathunter_object(args) @@ -30,15 +30,15 @@ def main(): processes = cb.select(Process).where(process_guid=args.p) if args.n: - processes = [ p for p in processes[0:args.n]] + processes = [p for p in processes[0:args.n]] if not args.s: for process in processes: - print("Process: {}".format(process.process_name)) - print("\tPIDs: {}".format(process.process_pids)) - print("\tSHA256: {}".format(process.process_sha256)) - print("\tGUID: {}".format(process.process_guid)) - + print("Process: {}".format(process.process_name)) + print("\tPIDs: {}".format(process.process_pids)) + print("\tSHA256: {}".format(process.process_sha256)) + print("\tGUID: {}".format(process.process_guid)) + if args.f is not None: if args.of == "json": with open(args.f, 'w') as outfile: diff --git a/examples/threathunter/process_query.py b/examples/threathunter/process_query.py index d1c2845a..1b3bfbd8 100644 --- a/examples/threathunter/process_query.py +++ b/examples/threathunter/process_query.py @@ -35,5 +35,6 @@ def main(): except ObjectNotFoundError: print("No binary found for process with hash: {}".format(process.process_sha256)) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/threathunter/process_tree.py b/examples/threathunter/process_tree.py index c6e1cb3d..df2e64f2 100644 --- a/examples/threathunter/process_tree.py +++ b/examples/threathunter/process_tree.py @@ -24,5 +24,6 @@ def main(): print("\tGUID: {}".format(child.process_guid)) print("\tNumber of children: {}".format(len(child.children))) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/threathunter/process_tree_exporter.py b/examples/threathunter/process_tree_exporter.py index 63321b0a..2eef7326 100644 --- a/examples/threathunter/process_tree_exporter.py +++ b/examples/threathunter/process_tree_exporter.py @@ -7,6 +7,7 @@ import csv import json + def main(): parser = build_cli_parser("Query processes") parser.add_argument("-p", type=str, help="process guid", default=None) @@ -35,7 +36,7 @@ def main(): else: with open(args.f, 'w') as outfile: csvwriter = csv.writer(outfile) - for idx,child in enumerate(tree.children): + for idx, child in enumerate(tree.children): csvwriter.writerow(child.original_document) diff --git a/examples/threathunter/search.py b/examples/threathunter/search.py index 067c045b..ee05c2f9 100644 --- a/examples/threathunter/search.py +++ b/examples/threathunter/search.py @@ -3,8 +3,7 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Process, Event, Tree -from solrq import Range, Value +from cbapi.psc.threathunter import Process def main(): @@ -66,7 +65,6 @@ def main(): else: print("\t{}: {}".format(parent.process_name, parent.process_sha256)) - if args.t: print("=========== tree =============") tree = process.tree() diff --git a/examples/threathunter/watchlist_operations.py b/examples/threathunter/watchlist_operations.py index 0fe95149..096690f1 100644 --- a/examples/threathunter/watchlist_operations.py +++ b/examples/threathunter/watchlist_operations.py @@ -243,7 +243,8 @@ def main(): commands = parser.add_subparsers(help="Feed commands", dest="command_name") list_command = commands.add_parser("list", help="List all configured watchlists") - list_command.add_argument("-r", "--reports", action="store_true", help="List reports for each watchlist", default=False) + list_command.add_argument("-r", "--reports", action="store_true", help="List reports for each watchlist", + default=False) subscribe_command = commands.add_parser("subscribe", help="Create a watchlist with a feed") subscribe_command.add_argument("-i", "--feed_id", type=str, help="The Feed ID", required=True) @@ -252,7 +253,8 @@ def main(): subscribe_command.add_argument("-t", "--tags", action="store_true", help="Enable tags", default=False) subscribe_command.add_argument("-a", "--alerts", action="store_true", help="Enable alerts", default=False) subscribe_command.add_argument("-T", "--timestamp", type=int, help="Creation timestamp", default=int(time.time())) - subscribe_command.add_argument("-U", "--last_update", type=int, help="Last update timestamp", default=int(time.time())) + subscribe_command.add_argument("-U", "--last_update", type=int, help="Last update timestamp", + default=int(time.time())) create_command = commands.add_parser("create", help="Create a watchlist with a report") create_command.add_argument("-w", "--watchlist_name", type=str, help="Watchlist name", required=True) @@ -271,7 +273,8 @@ def main(): create_command.add_argument("--rep_visibility", type=str, help="Report visibility") delete_command = commands.add_parser("delete", help="Delete a watchlist") - delete_command.add_argument("-R", "--reports", action="store_true", help="Delete all associated reports too", default=False) + delete_command.add_argument("-R", "--reports", action="store_true", help="Delete all associated reports too", + default=False) specifier = delete_command.add_mutually_exclusive_group(required=True) specifier.add_argument("-i", "--watchlist_id", type=str, help="The watchlist ID") specifier.add_argument("-w", "--watchlist_name", type=str, help="The watchlist name") From cd213c13f519e8a0b384f71956ae74b5aa781599 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 19 Nov 2019 12:51:28 -0700 Subject: [PATCH 055/197] ran all the test code through flake8 --- test/cbapi/psc/livequery/test_models.py | 188 ++++----- test/cbapi/psc/livequery/test_rest_api.py | 98 ++--- test/cbapi/psc/test_models.py | 103 +++-- test/cbapi/psc/test_rest_api.py | 479 +++++++++++----------- test/mocks.py | 23 +- 5 files changed, 456 insertions(+), 435 deletions(-) diff --git a/test/cbapi/psc/livequery/test_models.py b/test/cbapi/psc/livequery/test_models.py index bbda0539..c172a87f 100755 --- a/test/cbapi/psc/livequery/test_models.py +++ b/test/cbapi/psc/livequery/test_models.py @@ -8,87 +8,87 @@ def test_run_refresh(monkeypatch): _was_called = False - + def mock_get_object(url, parms=None, default=None): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" assert parms is None assert default is None _was_called = True - return {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"COMPLETE"} - + return {"org_key": "Z100", "name": "FoobieBletch", + "id": "abcdefg", "status": "COMPLETE"} + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"ACTIVE"}) + run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", + "id": "abcdefg", "status": "ACTIVE"}) monkeypatch.setattr(api, "get_object", mock_get_object) monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) rc = run.refresh() assert _was_called - assert rc == True + assert rc assert run.org_key == "Z100" assert run.name == "FoobieBletch" assert run.id == "abcdefg" assert run.status == "COMPLETE" - - + + def test_run_stop(monkeypatch): _was_called = False - + def mock_put_object(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/status" assert body["status"] == "CANCELLED" _was_called = True - return MockResponse({"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"CANCELLED"}) - + return MockResponse({"org_key": "Z100", "name": "FoobieBletch", + "id": "abcdefg", "status": "CANCELLED"}) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"ACTIVE"}) + run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", + "id": "abcdefg", "status": "ACTIVE"}) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) monkeypatch.setattr(api, "put_object", mock_put_object) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) rc = run.stop() assert _was_called - assert rc == True + assert rc assert run.org_key == "Z100" assert run.name == "FoobieBletch" assert run.id == "abcdefg" assert run.status == "CANCELLED" - + def test_run_stop_failed(monkeypatch): _was_called = False - + def mock_put_object(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/status" assert body["status"] == "CANCELLED" _was_called = True - return MockResponse({"error_message":"The query is not presently running."}, 409) - + return MockResponse({"error_message": "The query is not presently running."}, 409) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"CANCELLED"}) + run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", + "id": "abcdefg", "status": "CANCELLED"}) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) monkeypatch.setattr(api, "put_object", mock_put_object) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) rc = run.stop() assert _was_called - assert rc == False - - + assert not rc + + def test_run_delete(monkeypatch): _was_called = False - + def mock_delete_object(url): nonlocal _was_called if _was_called: @@ -96,18 +96,18 @@ def mock_delete_object(url): assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" _was_called = True return MockResponse(None) - + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"ACTIVE"}) + run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", + "id": "abcdefg", "status": "ACTIVE"}) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", mock_delete_object) rc = run.delete() assert _was_called - assert rc == True + assert rc assert run._is_deleted # Now ensure that certain operations that don't make sense on a deleted object raise ApiError with pytest.raises(ApiError): @@ -116,31 +116,31 @@ def mock_delete_object(url): run.stop() # And make sure that deleting a deleted object returns True immediately rc = run.delete() - assert rc == True - - + assert rc + + def test_run_delete_failed(monkeypatch): _was_called = False - + def mock_delete_object(url): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" _was_called = True return MockResponse(None, 403) - + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"ACTIVE"}) + run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", + "id": "abcdefg", "status": "ACTIVE"}) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", mock_delete_object) rc = run.delete() assert _was_called - assert rc == False + assert not rc assert not run._is_deleted - + def test_result_device_summaries(monkeypatch): _was_called = False @@ -155,22 +155,22 @@ def mock_post_object(url, body, **kwargs): assert t["field"] == "device_name" assert t["order"] == "ASC" _was_called = True - metrics = [{"key":"aaa", "value":0.0}, {"key":"bbb", "value":0.0}] - res1 = {"id":"ghijklm", "total_results":2, "device_id":314159, "metrics":metrics} - res2 = {"id":"mnopqrs", "total_results":3, "device_id":271828, "metrics":metrics} - return MockResponse({"org_key":"Z100", "num_found":2, "results":[res1, res2]}) - + metrics = [{"key": "aaa", "value": 0.0}, {"key": "bbb", "value": 0.0}] + res1 = {"id": "ghijklm", "total_results": 2, "device_id": 314159, "metrics": metrics} + res2 = {"id": "mnopqrs", "total_results": 3, "device_id": 271828, "metrics": metrics} + return MockResponse({"org_key": "Z100", "num_found": 2, "results": [res1, res2]}) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - tmp_id = {"id":"abcdefg"} - result = Result(api, {"id": "abcdefg", "device":tmp_id, "fields":{}, "metrics":{}}) + tmp_id = {"id": "abcdefg"} + result = Result(api, {"id": "abcdefg", "device": tmp_id, "fields": {}, "metrics": {}}) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) query = result.query_device_summaries().where("foo").criteria(device_name=["AxCx", "A7X"]) query = query.sort_by("device_name") - assert isinstance(query, ResultQuery) + assert isinstance(query, ResultQuery) count = 0 for item in query.all(): if item.id == "ghijklm": @@ -184,8 +184,8 @@ def mock_post_object(url, body, **kwargs): count = count + 1 assert _was_called assert count == 2 - - + + def test_result_query_result_facets(monkeypatch): _was_called = False @@ -198,51 +198,51 @@ def mock_post_object(url, body, **kwargs): t = body["terms"] assert t["fields"] == ["alpha", "bravo", "charlie"] _was_called = True - v1 = {"total":1, "id":"alpha1", "name":"alpha1"} - v2 = {"total":2, "id":"alpha2", "name":"alpha2"} - term1 = {"field":"alpha", "values":[v1, v2]} - v1 = {"total":1, "id":"bravo1", "name":"bravo1"} - v2 = {"total":2, "id":"bravo2", "name":"bravo2"} - term2 = {"field":"bravo", "values":[v1, v2]} - v1 = {"total":1, "id":"charlie1", "name":"charlie1"} - v2 = {"total":2, "id":"charlie2", "name":"charlie2"} - term3 = {"field":"charlie", "values":[v1, v2]} - return MockResponse({"terms":[term1, term2, term3]}) - + v1 = {"total": 1, "id": "alpha1", "name": "alpha1"} + v2 = {"total": 2, "id": "alpha2", "name": "alpha2"} + term1 = {"field": "alpha", "values": [v1, v2]} + v1 = {"total": 1, "id": "bravo1", "name": "bravo1"} + v2 = {"total": 2, "id": "bravo2", "name": "bravo2"} + term2 = {"field": "bravo", "values": [v1, v2]} + v1 = {"total": 1, "id": "charlie1", "name": "charlie1"} + v2 = {"total": 2, "id": "charlie2", "name": "charlie2"} + term3 = {"field": "charlie", "values": [v1, v2]} + return MockResponse({"terms": [term1, term2, term3]}) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - tmp_id = {"id":"abcdefg"} - result = Result(api, {"id": "abcdefg", "device":tmp_id, "fields":{}, "metrics":{}}) + tmp_id = {"id": "abcdefg"} + result = Result(api, {"id": "abcdefg", "device": tmp_id, "fields": {}, "metrics": {}}) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) query = result.query_result_facets().where("xyzzy") query = query.facet_field("alpha").facet_field(["bravo", "charlie"]) - query = query.criteria(device_name=["AxCx", "A7X"]) + query = query.criteria(device_name=["AxCx", "A7X"]) assert isinstance(query, FacetQuery) count = 0 for item in query.all(): vals = item.values if item.field == "alpha": - assert vals[0]["id"] == "alpha1" - assert vals[1]["id"] == "alpha2" + assert vals[0]["id"] == "alpha1" + assert vals[1]["id"] == "alpha2" elif item.field == "bravo": - assert vals[0]["id"] == "bravo1" - assert vals[1]["id"] == "bravo2" + assert vals[0]["id"] == "bravo1" + assert vals[1]["id"] == "bravo2" elif item.field == "charlie": - assert vals[0]["id"] == "charlie1" - assert vals[1]["id"] == "charlie2" + assert vals[0]["id"] == "charlie1" + assert vals[1]["id"] == "charlie2" else: - pytest.fail("Unknown field name %s seen" % item.field) + pytest.fail("Unknown field name %s seen" % item.field) count = count + 1 assert _was_called assert count == 3 - - + + def test_result_query_device_summary_facets(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/results/device_summaries/_facet" @@ -252,43 +252,43 @@ def mock_post_object(url, body, **kwargs): t = body["terms"] assert t["fields"] == ["alpha", "bravo", "charlie"] _was_called = True - v1 = {"total":1, "id":"alpha1", "name":"alpha1"} - v2 = {"total":2, "id":"alpha2", "name":"alpha2"} - term1 = {"field":"alpha", "values":[v1, v2]} - v1 = {"total":1, "id":"bravo1", "name":"bravo1"} - v2 = {"total":2, "id":"bravo2", "name":"bravo2"} - term2 = {"field":"bravo", "values":[v1, v2]} - v1 = {"total":1, "id":"charlie1", "name":"charlie1"} - v2 = {"total":2, "id":"charlie2", "name":"charlie2"} - term3 = {"field":"charlie", "values":[v1, v2]} - return MockResponse({"terms":[term1, term2, term3]}) - + v1 = {"total": 1, "id": "alpha1", "name": "alpha1"} + v2 = {"total": 2, "id": "alpha2", "name": "alpha2"} + term1 = {"field": "alpha", "values": [v1, v2]} + v1 = {"total": 1, "id": "bravo1", "name": "bravo1"} + v2 = {"total": 2, "id": "bravo2", "name": "bravo2"} + term2 = {"field": "bravo", "values": [v1, v2]} + v1 = {"total": 1, "id": "charlie1", "name": "charlie1"} + v2 = {"total": 2, "id": "charlie2", "name": "charlie2"} + term3 = {"field": "charlie", "values": [v1, v2]} + return MockResponse({"terms": [term1, term2, term3]}) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - tmp_id = {"id":"abcdefg"} - result = Result(api, {"id": "abcdefg", "device":tmp_id, "fields":{}, "metrics":{}}) + tmp_id = {"id": "abcdefg"} + result = Result(api, {"id": "abcdefg", "device": tmp_id, "fields": {}, "metrics": {}}) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) query = result.query_device_summary_facets().where("xyzzy") query = query.facet_field("alpha").facet_field(["bravo", "charlie"]) - query = query.criteria(device_name=["AxCx", "A7X"]) + query = query.criteria(device_name=["AxCx", "A7X"]) assert isinstance(query, FacetQuery) count = 0 for item in query.all(): vals = item.values if item.field == "alpha": - assert vals[0]["id"] == "alpha1" - assert vals[1]["id"] == "alpha2" + assert vals[0]["id"] == "alpha1" + assert vals[1]["id"] == "alpha2" elif item.field == "bravo": - assert vals[0]["id"] == "bravo1" - assert vals[1]["id"] == "bravo2" + assert vals[0]["id"] == "bravo1" + assert vals[1]["id"] == "bravo2" elif item.field == "charlie": - assert vals[0]["id"] == "charlie1" - assert vals[1]["id"] == "charlie2" + assert vals[0]["id"] == "charlie1" + assert vals[1]["id"] == "charlie2" else: - pytest.fail("Unknown field name %s seen" % item.field) + pytest.fail("Unknown field name %s seen" % item.field) count = count + 1 assert _was_called assert count == 3 diff --git a/test/cbapi/psc/livequery/test_rest_api.py b/test/cbapi/psc/livequery/test_rest_api.py index 97a60b5b..05458e3b 100755 --- a/test/cbapi/psc/livequery/test_rest_api.py +++ b/test/cbapi/psc/livequery/test_rest_api.py @@ -5,23 +5,24 @@ from cbapi.errors import ApiError, CredentialError from test.mocks import MockResponse, ConnectionMocks + def test_no_org_key(): with pytest.raises(CredentialError): - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - ssl_verify=True) # note: no org_key - - + CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", + ssl_verify=True) # note: no org_key + + def test_simple_get(monkeypatch): _was_called = False - + def mock_get_object(url, parms=None, default=None): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" assert parms is None assert default is None _was_called = True - return {"org_key":"Z100", "name":"FoobieBletch", "id":"abcdefg"} - + return {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"} + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", mock_get_object) @@ -33,33 +34,33 @@ def mock_get_object(url, parms=None, default=None): assert run.org_key == "Z100" assert run.name == "FoobieBletch" assert run.id == "abcdefg" - - + + def test_query(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs" assert body["sql"] == "select * from whatever;" _was_called = True - return MockResponse({"org_key":"Z100", "name":"FoobieBletch", "id":"abcdefg"}) - + return MockResponse({"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.query("select * from whatever;"); + query = api.query("select * from whatever;") assert isinstance(query, RunQuery) run = query.submit() assert _was_called assert run.org_key == "Z100" assert run.name == "FoobieBletch" assert run.id == "abcdefg" - - + + def test_query_with_everything(monkeypatch): _was_called = False @@ -68,14 +69,14 @@ def mock_post_object(url, body, **kwargs): assert url == "/livequery/v1/orgs/Z100/runs" assert body["sql"] == "select * from whatever;" assert body["name"] == "AmyWasHere" - assert body["notify_on_finish"] == True + assert body["notify_on_finish"] df = body["device_filter"] assert df["device_ids"] == [1, 2, 3] assert df["device_types"] == ["Alpha", "Bravo", "Charlie"] assert df["policy_ids"] == [16, 27, 38] _was_called = True - return MockResponse({"org_key":"Z100", "name":"FoobieBletch", "id":"abcdefg"}) - + return MockResponse({"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -83,7 +84,7 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) query = api.query("select * from whatever;").device_ids([1, 2, 3]) - query = query.device_types(["Alpha", "Bravo", "Charlie"]); + query = query.device_types(["Alpha", "Bravo", "Charlie"]) query = query.policy_ids([16, 27, 38]) query = query.name("AmyWasHere").notify_on_finish() assert isinstance(query, RunQuery) @@ -92,45 +93,45 @@ def mock_post_object(url, body, **kwargs): assert run.org_key == "Z100" assert run.name == "FoobieBletch" assert run.id == "abcdefg" - - + + def test_query_device_ids_broken(): api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - query = api.query("select * from whatever;"); + query = api.query("select * from whatever;") with pytest.raises(ApiError): query = query.device_ids(["Bogus"]) - - + + def test_query_device_types_broken(): api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - query = api.query("select * from whatever;"); + query = api.query("select * from whatever;") with pytest.raises(ApiError): - query = query.device_types([420]); - - + query = query.device_types([420]) + + def test_query_policy_ids_broken(): api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - query = api.query("select * from whatever;"); + query = api.query("select * from whatever;") with pytest.raises(ApiError): query = query.policy_ids(["Bogus"]) - - + + def test_query_history(monkeypatch): - _was_called = False - + _was_called = False + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/_search" assert body["query"] == "xyzzy" - _was_called = True - run1 = {"org_key":"Z100", "name":"FoobieBletch", "id":"abcdefg"} - run2 = {"org_key":"Z100", "name":"Aoxomoxoa", "id":"cdefghi"} - run3 = {"org_key":"Z100", "name":"Read_Me", "id":"efghijk"} - return MockResponse({"org_key":"Z100", "num_found":3, "results":[run1, run2, run3]}) - + _was_called = True + run1 = {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"} + run2 = {"org_key": "Z100", "name": "Aoxomoxoa", "id": "cdefghi"} + run3 = {"org_key": "Z100", "name": "Read_Me", "id": "efghijk"} + return MockResponse({"org_key": "Z100", "num_found": 3, "results": [run1, run2, run3]}) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -153,10 +154,10 @@ def mock_post_object(url, body, **kwargs): count = count + 1 assert _was_called assert count == 3 - - + + def test_query_history_with_everything(monkeypatch): - _was_called = False + _was_called = False def mock_post_object(url, body, **kwargs): nonlocal _was_called @@ -165,12 +166,12 @@ def mock_post_object(url, body, **kwargs): t = body["sort"][0] assert t["field"] == "id" assert t["order"] == "ASC" - _was_called = True - run1 = {"org_key":"Z100", "name":"FoobieBletch", "id":"abcdefg"} - run2 = {"org_key":"Z100", "name":"Aoxomoxoa", "id":"cdefghi"} - run3 = {"org_key":"Z100", "name":"Read_Me", "id":"efghijk"} - return MockResponse({"org_key":"Z100", "num_found":3, "results":[run1, run2, run3]}) - + _was_called = True + run1 = {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"} + run2 = {"org_key": "Z100", "name": "Aoxomoxoa", "id": "cdefghi"} + run3 = {"org_key": "Z100", "name": "Read_Me", "id": "efghijk"} + return MockResponse({"org_key": "Z100", "num_found": 3, "results": [run1, run2, run3]}) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -193,4 +194,3 @@ def mock_post_object(url, body, **kwargs): count = count + 1 assert _was_called assert count == 3 - \ No newline at end of file diff --git a/test/cbapi/psc/test_models.py b/test/cbapi/psc/test_models.py index fef23551..99cdfb4e 100755 --- a/test/cbapi/psc/test_models.py +++ b/test/cbapi/psc/test_models.py @@ -3,15 +3,17 @@ from cbapi.psc.rest_api import CbPSCBaseAPI from test.mocks import MockResponse, ConnectionMocks + class MockScheduler: def __init__(self, expected_id): self.expected_id = expected_id self.was_called = False - + def request_session(self, sensor_id): assert sensor_id == self.expected_id self.was_called = True - return { "itworks": True } + return {"itworks": True} + def test_Device_lr_session(monkeypatch): _device_data = {"id": 6023} @@ -33,26 +35,27 @@ def mock_get_object(url, parms=None, default=None): sess = dev.lr_session() assert sess["itworks"] assert sked.was_called - + + def test_Device_background_scan(monkeypatch): _device_data = {"id": 6023} _was_called = False - + def mock_get_object(url, parms=None, default=None): nonlocal _device_data assert url == "/appservices/v6/orgs/Z100/devices/6023" return _device_data - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "BACKGROUND_SCAN" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] t = body["options"] assert t["toggle"] == "ON" _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", mock_get_object) @@ -62,21 +65,22 @@ def mock_post_object(url, body, **kwargs): dev = Device(api, 6023, _device_data) dev.background_scan(True) assert _was_called - + + def test_Device_bypass(monkeypatch): _device_data = {"id": 6023} _was_called = False - + def mock_get_object(url, parms=None, default=None): nonlocal _device_data assert url == "/appservices/v6/orgs/Z100/devices/6023" return _device_data - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "BYPASS" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] t = body["options"] assert t["toggle"] == "OFF" _was_called = True @@ -91,21 +95,22 @@ def mock_post_object(url, body, **kwargs): dev = Device(api, 6023, _device_data) dev.bypass(False) assert _was_called - + + def test_Device_delete_sensor(monkeypatch): _device_data = {"id": 6023} _was_called = False - + def mock_get_object(url, parms=None, default=None): nonlocal _device_data assert url == "/appservices/v6/orgs/Z100/devices/6023" return _device_data - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "DELETE_SENSOR" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] _was_called = True return MockResponse(None, 204) @@ -119,20 +124,21 @@ def mock_post_object(url, body, **kwargs): dev.delete_sensor() assert _was_called + def test_Device_uninstall_sensor(monkeypatch): _device_data = {"id": 6023} _was_called = False - + def mock_get_object(url, parms=None, default=None): nonlocal _device_data assert url == "/appservices/v6/orgs/Z100/devices/6023" return _device_data - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "UNINSTALL_SENSOR" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] _was_called = True return MockResponse(None, 204) @@ -145,26 +151,27 @@ def mock_post_object(url, body, **kwargs): dev = Device(api, 6023, _device_data) dev.uninstall_sensor() assert _was_called - + + def test_Device_quarantine(monkeypatch): _device_data = {"id": 6023} _was_called = False - + def mock_get_object(url, parms=None, default=None): nonlocal _device_data assert url == "/appservices/v6/orgs/Z100/devices/6023" return _device_data - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "QUARANTINE" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] t = body["options"] assert t["toggle"] == "ON" _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", mock_get_object) @@ -175,25 +182,26 @@ def mock_post_object(url, body, **kwargs): dev.quarantine(True) assert _was_called + def test_Device_update_policy(monkeypatch): _device_data = {"id": 6023} _was_called = False - + def mock_get_object(url, parms=None, default=None): nonlocal _device_data assert url == "/appservices/v6/orgs/Z100/devices/6023" return _device_data - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "UPDATE_POLICY" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] t = body["options"] assert t["policy_id"] == 8675309 _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", mock_get_object) @@ -204,26 +212,27 @@ def mock_post_object(url, body, **kwargs): dev.update_policy(8675309) assert _was_called + def test_Device_update_sensor_version(monkeypatch): _device_data = {"id": 6023} _was_called = False - + def mock_get_object(url, parms=None, default=None): nonlocal _device_data assert url == "/appservices/v6/orgs/Z100/devices/6023" return _device_data - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "UPDATE_SENSOR_VERSION" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] t = body["options"] t2 = t["sensor_version"] assert t2["RHEL"] == "2.3.4.5" _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", mock_get_object) @@ -233,7 +242,8 @@ def mock_post_object(url, body, **kwargs): dev = Device(api, 6023, _device_data) dev.update_sensor_version({"RHEL": "2.3.4.5"}) assert _was_called - + + def test_BaseAlert_dismiss(monkeypatch): _was_called = False @@ -246,14 +256,14 @@ def mock_post_object(url, body, **kwargs): _was_called = True return MockResponse({"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "workflow":{"state": "OPEN"}}) + alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "workflow": {"state": "OPEN"}}) alert.dismiss("Fixed", "Yessir") assert _was_called assert alert.workflow_.changed_by == "Robocop" @@ -262,6 +272,7 @@ def mock_post_object(url, body, **kwargs): assert alert.workflow_.comment == "Yessir" assert alert.workflow_.last_update_time == "2019-10-31T16:03:13.951Z" + def test_BaseAlert_undismiss(monkeypatch): _was_called = False @@ -274,14 +285,14 @@ def mock_post_object(url, body, **kwargs): _was_called = True return MockResponse({"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "workflow":{"state": "DISMISS"}}) + alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "workflow": {"state": "DISMISS"}}) alert.update("Fixed", "NoSir") assert _was_called assert alert.workflow_.changed_by == "Robocop" @@ -290,9 +301,10 @@ def mock_post_object(url, body, **kwargs): assert alert.workflow_.comment == "NoSir" assert alert.workflow_.last_update_time == "2019-10-31T16:03:13.951Z" + def test_BaseAlert_dismiss_threat(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/threat/B0RG/workflow" @@ -302,14 +314,14 @@ def mock_post_object(url, body, **kwargs): _was_called = True return MockResponse({"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "threat_id": "B0RG", "workflow":{"state": "OPEN"}}) + alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "threat_id": "B0RG", "workflow": {"state": "OPEN"}}) wf = alert.dismiss_threat("Fixed", "Yessir") assert _was_called assert wf.changed_by == "Robocop" @@ -318,6 +330,7 @@ def mock_post_object(url, body, **kwargs): assert wf.comment == "Yessir" assert wf.last_update_time == "2019-10-31T16:03:13.951Z" + def test_BaseAlert_undismiss_threat(monkeypatch): _was_called = False @@ -330,14 +343,14 @@ def mock_post_object(url, body, **kwargs): _was_called = True return MockResponse({"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "threat_id": "B0RG", "workflow":{"state": "OPEN"}}) + alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "threat_id": "B0RG", "workflow": {"state": "OPEN"}}) wf = alert.update_threat("Fixed", "NoSir") assert _was_called assert wf.changed_by == "Robocop" @@ -345,10 +358,11 @@ def mock_post_object(url, body, **kwargs): assert wf.remediation == "Fixed" assert wf.comment == "NoSir" assert wf.last_update_time == "2019-10-31T16:03:13.951Z" - + + def test_WorkflowStatus(monkeypatch): _times_called = 0 - + def mock_get_object(url, parms=None, default=None): nonlocal _times_called assert url == "/appservices/v6/orgs/Z100/workflow/status/W00K13" @@ -365,7 +379,7 @@ def mock_get_object(url, parms=None, default=None): "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} _times_called = _times_called + 1 return resp - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", mock_get_object) @@ -391,4 +405,3 @@ def mock_get_object(url, parms=None, default=None): assert not wfstat.in_progress assert wfstat.finished assert _times_called == 10 - \ No newline at end of file diff --git a/test/cbapi/psc/test_rest_api.py b/test/cbapi/psc/test_rest_api.py index 1a5ae84d..40b50bf6 100755 --- a/test/cbapi/psc/test_rest_api.py +++ b/test/cbapi/psc/test_rest_api.py @@ -8,15 +8,16 @@ # --- Device v6 Tests # + def test_get_device(monkeypatch): _was_called = False - + def mock_get_object(url): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/devices/6023" _was_called = True - return { "device_id": 6023, "organization_name": "thistestworks" } - + return {"device_id": 6023, "organization_name": "thistestworks"} + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", mock_get_object) @@ -28,39 +29,39 @@ def mock_get_object(url): assert isinstance(rc, Device) assert rc.device_id == 6023 assert rc.organization_name == "thistestworks" - - + + def test_device_background_scan(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "BACKGROUND_SCAN" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] t = body["options"] assert t["toggle"] == "ON" _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_background_scan([ 6023 ], True) + api.device_background_scan([6023], True) assert _was_called - - + + def test_device_bypass(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "BYPASS" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] t = body["options"] assert t["toggle"] == "OFF" _was_called = True @@ -72,18 +73,18 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_bypass([ 6023 ], False) + api.device_bypass([6023], False) assert _was_called - - + + def test_device_delete_sensor(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "DELETE_SENSOR" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] _was_called = True return MockResponse(None, 204) @@ -93,18 +94,18 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_delete_sensor([ 6023 ]) + api.device_delete_sensor([6023]) assert _was_called def test_device_uninstall_sensor(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "UNINSTALL_SENSOR" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] _was_called = True return MockResponse(None, 204) @@ -114,123 +115,123 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_uninstall_sensor([ 6023 ]) + api.device_uninstall_sensor([6023]) assert _was_called - - + + def test_device_quarantine(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "QUARANTINE" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] t = body["options"] assert t["toggle"] == "ON" _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_quarantine([ 6023 ], True) + api.device_quarantine([6023], True) assert _was_called - - + + def test_device_update_policy(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "UPDATE_POLICY" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] t = body["options"] assert t["policy_id"] == 8675309 _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_update_policy([ 6023 ], 8675309) + api.device_update_policy([6023], 8675309) assert _was_called - - + + def test_device_update_sensor_version(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" assert body["action_type"] == "UPDATE_SENSOR_VERSION" - assert body["device_id"] == [ 6023 ] + assert body["device_id"] == [6023] t = body["options"] t2 = t["sensor_version"] assert t2["RHEL"] == "2.3.4.5" _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_update_sensor_version([ 6023 ], { "RHEL": "2.3.4.5"}) + api.device_update_sensor_version([6023], {"RHEL": "2.3.4.5"}) assert _was_called - - + + def test_query_device_with_all_bells_and_whistles(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/devices/_search" assert body["query"] == "foobar" t = body.get("criteria", {}) - assert t["ad_group_id"] == [ 14, 25 ] - assert t["os"] == [ "LINUX" ] - assert t["policy_id"] == [ 8675309 ] - assert t["status"] == [ "ALL" ] - assert t["target_priority"] == [ "HIGH" ] + assert t["ad_group_id"] == [14, 25] + assert t["os"] == ["LINUX"] + assert t["policy_id"] == [8675309] + assert t["status"] == ["ALL"] + assert t["target_priority"] == ["HIGH"] t = body.get("exclusions", {}) - assert t["sensor_version"] == [ "0.1" ] + assert t["sensor_version"] == ["0.1"] t = body.get("sort", []) t2 = t[0] assert t2["field"] == "name" assert t2["order"] == "DESC" _was_called = True - body = { "id": 6023, "organization_name": "thistestworks" } - envelope = { "results": [ body ], "num_found": 1 } + body = {"id": 6023, "organization_name": "thistestworks"} + envelope = {"results": [body], "num_found": 1} return MockResponse(envelope) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(Device).where("foobar").ad_group_ids([ 14, 25 ]) \ - .os([ "LINUX" ]).policy_ids([ 8675309 ]).status([ "ALL" ]) \ + query = api.select(Device).where("foobar").ad_group_ids([14, 25]) \ + .os(["LINUX"]).policy_ids([8675309]).status(["ALL"]) \ .target_priorities(["HIGH"]).exclude_sensor_versions(["0.1"]) \ .sort_by("name", "DESC") d = query.one() assert _was_called assert d.id == 6023 assert d.organization_name == "thistestworks" - - + + def test_query_device_with_last_contact_time_as_start_end(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/devices/_search" @@ -240,10 +241,10 @@ def mock_post_object(url, body, **kwargs): assert t2["start"] == "2019-09-30T12:34:56" assert t2["end"] == "2019-10-01T12:00:12" _was_called = True - body = { "id": 6023, "organization_name": "thistestworks" } - envelope = { "results": [ body ], "num_found": 1 } + body = {"id": 6023, "organization_name": "thistestworks"} + envelope = {"results": [body], "num_found": 1} return MockResponse(envelope) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -260,7 +261,7 @@ def mock_post_object(url, body, **kwargs): def test_query_device_with_last_contact_time_as_range(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/devices/_search" @@ -269,10 +270,10 @@ def mock_post_object(url, body, **kwargs): t2 = t.get("last_contact_time", {}) assert t2["range"] == "-3w" _was_called = True - body = { "id": 6023, "organization_name": "thistestworks" } - envelope = { "results": [ body ], "num_found": 1 } + body = {"id": 6023, "organization_name": "thistestworks"} + envelope = {"results": [body], "num_found": 1} return MockResponse(envelope) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -285,33 +286,33 @@ def mock_post_object(url, body, **kwargs): assert d.id == 6023 assert d.organization_name == "thistestworks" - + def test_query_device_invalid_ad_group_ids(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): - api.select(Device).ad_group_ids([ "Bogus" ]) - - + api.select(Device).ad_group_ids(["Bogus"]) + + def test_query_device_invalid_policy_ids(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): - api.select(Device).policy_ids([ "Bogus" ]) - - + api.select(Device).policy_ids(["Bogus"]) + + def test_query_device_last_contact_time_no_params_ok(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(Device).last_contact_time() - + def test_query_device_last_contact_time_range_specified_bad(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): - api.select(Device).last_contact_time(start="2019-09-30T12:34:56", \ + api.select(Device).last_contact_time(start="2019-09-30T12:34:56", end="2019-10-01T12:00:12", range="-3w") @@ -319,7 +320,7 @@ def test_query_device_last_contact_time_start_specified_bad(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): - api.select(Device).last_contact_time(start="2019-09-30T12:34:56", \ + api.select(Device).last_contact_time(start="2019-09-30T12:34:56", range="-3w") @@ -334,40 +335,40 @@ def test_query_device_invalid_os(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): - api.select(Device).os([ "COMMODORE_64" ]) - - + api.select(Device).os(["COMMODORE_64"]) + + def test_query_device_invalid_status(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): - api.select(Device).status([ "Bogus" ]) - - + api.select(Device).status(["Bogus"]) + + def test_query_device_invalid_priority(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): - api.select(Device).target_priorities([ "Bogus" ]) - - + api.select(Device).target_priorities(["Bogus"]) + + def test_query_device_invalid_exclude_sensor_version(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): - api.select(Device).exclude_sensor_versions([ 12703 ]) - - + api.select(Device).exclude_sensor_versions([12703]) + + def test_query_device_invalid_sort_direction(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(Device).sort_by("policy_name", "BOGUS") - - + + def test_query_device_download(monkeypatch): _was_called = False - + def mock_get_raw_data(url, query_params, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/devices/_search/download" @@ -380,7 +381,7 @@ def mock_get_raw_data(url, query_params, **kwargs): assert query_params["sort_order"] == "DESC" _was_called = True return "123456789,123456789,123456789" - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_raw_data", mock_get_raw_data) @@ -388,16 +389,16 @@ def mock_get_raw_data(url, query_params, **kwargs): monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - rc = api.select(Device).where("foobar").ad_group_ids([ 14, 25 ]) \ - .policy_ids([ 8675309 ]).status([ "ALL" ]).target_priorities(["HIGH"]) \ + rc = api.select(Device).where("foobar").ad_group_ids([14, 25]) \ + .policy_ids([8675309]).status(["ALL"]).target_priorities(["HIGH"]) \ .sort_by("name", "DESC").download() assert _was_called assert rc == "123456789,123456789,123456789" - - + + def test_query_device_do_background_scan(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -408,7 +409,7 @@ def mock_post_object(url, body, **kwargs): assert t["toggle"] == "ON" _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -417,11 +418,11 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) api.select(Device).where("foobar").background_scan(True) assert _was_called - - + + def test_query_device_do_bypass(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -445,7 +446,7 @@ def mock_post_object(url, body, **kwargs): def test_query_device_do_delete_sensor(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -467,7 +468,7 @@ def mock_post_object(url, body, **kwargs): def test_query_device_do_uninstall_sensor(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -485,11 +486,11 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) api.select(Device).where("foobar").uninstall_sensor() assert _was_called - - + + def test_query_device_do_quarantine(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -500,7 +501,7 @@ def mock_post_object(url, body, **kwargs): assert t["toggle"] == "ON" _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -510,10 +511,10 @@ def mock_post_object(url, body, **kwargs): api.select(Device).where("foobar").quarantine(True) assert _was_called - + def test_query_device_do_update_policy(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -524,7 +525,7 @@ def mock_post_object(url, body, **kwargs): assert t["policy_id"] == 8675309 _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -533,11 +534,11 @@ def mock_post_object(url, body, **kwargs): monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) api.select(Device).where("foobar").update_policy(8675309) assert _was_called - - + + def test_query_device_do_update_sensor_version(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" @@ -549,23 +550,24 @@ def mock_post_object(url, body, **kwargs): assert t2["RHEL"] == "2.3.4.5" _was_called = True return MockResponse(None, 204) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) monkeypatch.setattr(api, "post_object", mock_post_object) monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.select(Device).where("foobar").update_sensor_version({ "RHEL": "2.3.4.5"}) + api.select(Device).where("foobar").update_sensor_version({"RHEL": "2.3.4.5"}) assert _was_called - + # # --- Alerts v6 Tests # + def test_query_basealert_with_all_bells_and_whistles(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/_search" @@ -597,9 +599,9 @@ def mock_post_object(url, body, **kwargs): assert t2["order"] == "DESC" _was_called = True body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = { "results": [ body ], "num_found": 1 } + envelope = {"results": [body], "num_found": 1} return MockResponse(envelope) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -623,7 +625,7 @@ def mock_post_object(url, body, **kwargs): def test_query_basealert_with_create_time_as_start_end(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/_search" @@ -634,7 +636,7 @@ def mock_post_object(url, body, **kwargs): assert t2["end"] == "2019-10-01T12:00:12" _was_called = True body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = { "results": [ body ], "num_found": 1 } + envelope = {"results": [body], "num_found": 1} return MockResponse(envelope) api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", @@ -655,7 +657,7 @@ def mock_post_object(url, body, **kwargs): def test_query_basealert_with_create_time_as_range(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/_search" @@ -665,9 +667,9 @@ def mock_post_object(url, body, **kwargs): assert t2["range"] == "-3w" _was_called = True body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = { "results": [ body ], "num_found": 1 } + envelope = {"results": [body], "num_found": 1} return MockResponse(envelope) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -681,11 +683,11 @@ def mock_post_object(url, body, **kwargs): assert a.org_key == "Z100" assert a.threat_id == "B0RG" assert a.workflow_.state == "OPEN" - - -def test_query_basealert_facets(monkeypatch): + + +def test_query_basealert_facets(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/_facet" @@ -713,16 +715,16 @@ def mock_post_object(url, body, **kwargs): assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] t = f[1] assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] - + def test_query_basealert_invalid_category(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).categories(["DOUBLE_DARE"]) - - -def test_query_basealert_create_time_no_params_ok(): + + +def test_query_basealert_create_time_no_params_ok(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): @@ -733,18 +735,18 @@ def test_query_basealert_create_time_range_specified_bad(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): - api.select(BaseAlert).create_time(start="2019-09-30T12:34:56", \ + api.select(BaseAlert).create_time(start="2019-09-30T12:34:56", end="2019-10-01T12:00:12", range="-3w") - - -def test_query_basealert_create_time_start_specified_bad(): + + +def test_query_basealert_create_time_start_specified_bad(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).create_time(start="2019-09-30T12:34:56", range="-3w") - - -def test_query_basealert_create_time_end_specified_bad(): + + +def test_query_basealert_create_time_end_specified_bad(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): @@ -756,123 +758,123 @@ def test_query_basealert_invalid_device_ids(): org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).device_ids(["Bogus"]) - - + + def test_query_basealert_invalid_device_names(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).device_names([42]) - - -def test_query_basealert_invalid_device_os(): + + +def test_query_basealert_invalid_device_os(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).device_os(["TI994A"]) - - + + def test_query_basealert_invalid_device_os_versions(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).device_os_versions([8808]) - - -def test_query_basealert_invalid_device_username(): + + +def test_query_basealert_invalid_device_username(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).device_username([-1]) - - -def test_query_basealert_invalid_alert_ids(): + + +def test_query_basealert_invalid_alert_ids(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).alert_ids([9001]) - - -def test_query_basealert_invalid_legacy_alert_ids(): + + +def test_query_basealert_invalid_legacy_alert_ids(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).legacy_alert_ids([9001]) - + def test_query_basealert_invalid_policy_ids(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).policy_ids(["Bogus"]) - - + + def test_query_basealert_invalid_policy_names(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).policy_names([323]) - - + + def test_query_basealert_invalid_process_names(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).process_names([7071]) - - -def test_query_basealert_invalid_process_sha256(): + + +def test_query_basealert_invalid_process_sha256(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).process_sha256([123456789]) - - + + def test_query_basealert_invalid_reputations(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).reputations(["MICROSOFT_FUDWARE"]) - - + + def test_query_basealert_invalid_tags(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).tags([990909]) - - + + def test_query_basealert_invalid_target_priorities(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).target_priorities(["DOGWASH"]) - - + + def test_query_basealert_invalid_threat_ids(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).threat_ids([4096]) - - + + def test_query_basealert_invalid_types(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).types(["ERBOSOFT"]) - - + + def test_query_basealert_invalid_workflows(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(BaseAlert).workflows(["IN_LIMBO"]) - + def test_query_cbanalyticsalert_with_all_bells_and_whistles(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/_search" @@ -907,16 +909,16 @@ def mock_post_object(url, body, **kwargs): assert t["run_state"] == ["RAN"] assert t["sensor_action"] == ["DENY"] assert t["threat_cause_vector"] == ["WEB"] - + t = body["sort"] t2 = t[0] assert t2["field"] == "name" assert t2["order"] == "DESC" _was_called = True body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = { "results": [ body ], "num_found": 1 } + envelope = {"results": [body], "num_found": 1} return MockResponse(envelope) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -940,10 +942,10 @@ def mock_post_object(url, body, **kwargs): assert a.threat_id == "B0RG" assert a.workflow_.state == "OPEN" - -def test_query_cbanalyticsalert_facets(monkeypatch): + +def test_query_cbanalyticsalert_facets(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/_facet" @@ -971,22 +973,22 @@ def mock_post_object(url, body, **kwargs): assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] t = f[1] assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] - + def test_query_cbanalyticsalert_invalid_blocked_threat_categories(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(CBAnalyticsAlert).blocked_threat_categories(["MINOR"]) - + def test_query_cbanalyticsalert_invalid_device_locations(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(CBAnalyticsAlert).device_locations(["NARNIA"]) - - + + def test_query_cbanalyticsalert_invalid_kill_chain_statuses(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) @@ -999,29 +1001,29 @@ def test_query_cbanalyticsalert_invalid_not_blocked_threat_categories(): org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(CBAnalyticsAlert).not_blocked_threat_categories(["MINOR"]) - - + + def test_query_cbanalyticsalert_invalid_policy_applied(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(CBAnalyticsAlert).policy_applied(["MAYBE"]) - + def test_query_cbanalyticsalert_invalid_reason_code(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(CBAnalyticsAlert).reason_code([55]) - - -def test_query_cbanalyticsalert_invalid_run_states(): + + +def test_query_cbanalyticsalert_invalid_run_states(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(CBAnalyticsAlert).run_states(["MIGHT_HAVE"]) - - + + def test_query_cbanalyticsalert_invalid_sensor_actions(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) @@ -1034,11 +1036,11 @@ def test_query_cbanalyticsalert_invalid_threat_cause_vectors(): org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(CBAnalyticsAlert).threat_cause_vectors(["NETWORK"]) - - + + def test_query_vmwarealert_with_all_bells_and_whistles(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/vmware/_search" @@ -1071,9 +1073,9 @@ def mock_post_object(url, body, **kwargs): assert t2["order"] == "DESC" _was_called = True body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = { "results": [ body ], "num_found": 1 } + envelope = {"results": [body], "num_found": 1} return MockResponse(envelope) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -1093,11 +1095,11 @@ def mock_post_object(url, body, **kwargs): assert a.org_key == "Z100" assert a.threat_id == "B0RG" assert a.workflow_.state == "OPEN" - - -def test_query_vmwarealert_facets(monkeypatch): + + +def test_query_vmwarealert_facets(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/vmware/_facet" @@ -1125,18 +1127,18 @@ def mock_post_object(url, body, **kwargs): assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] t = f[1] assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] - + def test_query_vmwarealert_invalid_group_ids(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(VMwareAlert).group_ids(["Bogus"]) - - + + def test_query_watchlistalert_with_all_bells_and_whistles(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/_search" @@ -1170,9 +1172,9 @@ def mock_post_object(url, body, **kwargs): assert t2["order"] == "DESC" _was_called = True body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = { "results": [ body ], "num_found": 1 } + envelope = {"results": [body], "num_found": 1} return MockResponse(envelope) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -1192,11 +1194,11 @@ def mock_post_object(url, body, **kwargs): assert a.org_key == "Z100" assert a.threat_id == "B0RG" assert a.workflow_.state == "OPEN" - - -def test_query_watchlistalert_facets(monkeypatch): + + +def test_query_watchlistalert_facets(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/_facet" @@ -1224,25 +1226,25 @@ def mock_post_object(url, body, **kwargs): assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] t = f[1] assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] - + def test_query_watchlistalert_invalid_watchlist_ids(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(WatchlistAlert).watchlist_ids([888]) - - -def test_query_watchlistalert_invalid_watchlist_names(): + + +def test_query_watchlistalert_invalid_watchlist_names(): api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) with pytest.raises(ApiError): api.select(WatchlistAlert).watchlist_names([69]) - - + + def test_alerts_bulk_dismiss(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/workflow/_criteria" @@ -1254,7 +1256,7 @@ def mock_post_object(url, body, **kwargs): assert t["device_name"] == ["HAL9000"] _was_called = True return MockResponse({"request_id": "497ABX"}) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -1269,7 +1271,7 @@ def mock_post_object(url, body, **kwargs): def test_alerts_bulk_undismiss(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/workflow/_criteria" @@ -1281,7 +1283,7 @@ def mock_post_object(url, body, **kwargs): assert t["device_name"] == ["HAL9000"] _was_called = True return MockResponse({"request_id": "497ABX"}) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -1292,11 +1294,11 @@ def mock_post_object(url, body, **kwargs): reqid = q.update("Fixed", "NoSir") assert _was_called assert reqid == "497ABX" - - + + def test_alerts_bulk_dismiss_watchlist(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/workflow/_criteria" @@ -1308,7 +1310,7 @@ def mock_post_object(url, body, **kwargs): assert t["device_name"] == ["HAL9000"] _was_called = True return MockResponse({"request_id": "497ABX"}) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -1323,7 +1325,7 @@ def mock_post_object(url, body, **kwargs): def test_alerts_bulk_dismiss_cbanalytics(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/workflow/_criteria" @@ -1335,7 +1337,7 @@ def mock_post_object(url, body, **kwargs): assert t["device_name"] == ["HAL9000"] _was_called = True return MockResponse({"request_id": "497ABX"}) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -1350,7 +1352,7 @@ def mock_post_object(url, body, **kwargs): def test_alerts_bulk_dismiss_vmware(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/vmware/workflow/_criteria" @@ -1362,7 +1364,7 @@ def mock_post_object(url, body, **kwargs): assert t["device_name"] == ["HAL9000"] _was_called = True return MockResponse({"request_id": "497ABX"}) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -1377,7 +1379,7 @@ def mock_post_object(url, body, **kwargs): def test_alerts_bulk_dismiss_threat(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/threat/workflow/_criteria" @@ -1387,7 +1389,7 @@ def mock_post_object(url, body, **kwargs): assert body["comment"] == "Yessir" _was_called = True return MockResponse({"request_id": "497ABX"}) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -1401,7 +1403,7 @@ def mock_post_object(url, body, **kwargs): def test_alerts_bulk_undismiss_threat(monkeypatch): _was_called = False - + def mock_post_object(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/threat/workflow/_criteria" @@ -1411,7 +1413,7 @@ def mock_post_object(url, body, **kwargs): assert body["comment"] == "NoSir" _was_called = True return MockResponse({"request_id": "497ABX"}) - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) @@ -1421,11 +1423,11 @@ def mock_post_object(url, body, **kwargs): reqid = api.bulk_threat_update(["B0RG", "F3R3NG1"], "Fixed", "NoSir") assert _was_called assert reqid == "497ABX" - - + + def test_load_workflow(monkeypatch): _was_called = False - + def mock_get_object(url, parms=None, default=None): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" @@ -1434,7 +1436,7 @@ def mock_get_object(url, parms=None, default=None): resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} return resp - + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) monkeypatch.setattr(api, "get_object", mock_get_object) @@ -1444,4 +1446,3 @@ def mock_get_object(url, parms=None, default=None): workflow = api.select(WorkflowStatus, "497ABX") assert _was_called assert workflow.id_ == "497ABX" - \ No newline at end of file diff --git a/test/mocks.py b/test/mocks.py index 8d5f24df..22f19bd6 100755 --- a/test/mocks.py +++ b/test/mocks.py @@ -1,28 +1,35 @@ import pytest + class MockResponse(object): def __init__(self, contents, scode=200): self._contents = contents self.status_code = scode - + def json(self): return self._contents - + + def failing_mock_get_object(url, parms=None, default=None): pytest.fail("GET called for %s when it shouldn't be" % url) - + + def failing_mock_post_object(url, body, **kwargs): pytest.fail("POST called for %s when it shouldn't be" % url) - + + def failing_mock_put_object(url, body, **kwargs): pytest.fail("PUT called for %s when it shouldn't be" % url) - + + def failing_mock_delete_object(url): pytest.fail("DELETE called for %s when it shouldn't be" % url) -_methods = {"GET":failing_mock_get_object, "POST":failing_mock_post_object, - "PUT":failing_mock_put_object, "DELETE":failing_mock_delete_object} - + +_methods = {"GET": failing_mock_get_object, "POST": failing_mock_post_object, + "PUT": failing_mock_put_object, "DELETE": failing_mock_delete_object} + + class ConnectionMocks: @classmethod def get(cls, name): From 7cd7fdbd2b3207acb6e2546e5a9cb92e497c83bd Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Fri, 22 Nov 2019 14:06:50 -0700 Subject: [PATCH 056/197] extensive updates as requested by Becca, including splitting some source files, renaming methods, rebuilding tests, and changing examples to match --- examples/README.md | 16 + examples/psc/bulk_update_alerts.py | 2 +- .../psc/bulk_update_cbanalytics_alerts.py | 2 +- examples/psc/bulk_update_vmware_alerts.py | 2 +- examples/psc/bulk_update_watchlist_alerts.py | 2 +- examples/psc/download_device_list.py | 8 +- .../alertsv6.py} | 64 +- examples/psc/list_alert_facets.py | 2 +- examples/psc/list_alerts.py | 2 +- examples/psc/list_cbanalytics_alert_facets.py | 2 +- examples/psc/list_cbanalytics_alerts.py | 2 +- examples/psc/list_devices.py | 8 +- examples/psc/list_vmware_alert_facets.py | 2 +- examples/psc/list_vmware_alerts.py | 2 +- examples/psc/list_watchlist_alert_facets.py | 2 +- examples/psc/list_watchlist_alerts.py | 2 +- runtests.bat | 2 +- runtests.sh | 2 +- src/cbapi/psc/{query.py => alerts_query.py} | 742 ++------- src/cbapi/psc/base_query.py | 270 +++ src/cbapi/psc/devices_query.py | 361 ++++ src/cbapi/psc/livequery/query.py | 4 +- src/cbapi/psc/models.py | 21 +- src/cbapi/psc/rest_api.py | 46 +- test/cbapi/psc/livequery/test_models.py | 224 +-- test/cbapi/psc/livequery/test_rest_api.py | 119 +- test/cbapi/psc/test_alertsv6_api.py | 535 ++++++ test/cbapi/psc/test_devicev6_api.py | 383 +++++ test/cbapi/psc/test_models.py | 294 ++-- test/cbapi/psc/test_rest_api.py | 1448 ----------------- test/cbtest.py | 38 + test/mocks.py | 36 - 32 files changed, 2044 insertions(+), 2601 deletions(-) create mode 100755 examples/README.md rename examples/psc/{alertsv6common.py => helpers/alertsv6.py} (80%) rename src/cbapi/psc/{query.py => alerts_query.py} (51%) create mode 100755 src/cbapi/psc/base_query.py create mode 100755 src/cbapi/psc/devices_query.py create mode 100755 test/cbapi/psc/test_alertsv6_api.py create mode 100755 test/cbapi/psc/test_devicev6_api.py delete mode 100755 test/cbapi/psc/test_rest_api.py create mode 100755 test/cbtest.py delete mode 100755 test/mocks.py diff --git a/examples/README.md b/examples/README.md new file mode 100755 index 00000000..cd2e34e6 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,16 @@ +# Running the Example Scripts + +To run the example scripts, first set up the CBAPI with either the `pip install cbapi` or `python setup.py develop` +commands as detailed in the top-level `README.md` file. You may also set your `PYTHONPATH` environment variable to +point to the `{cbapi}/src` directory, where `{cbapi}` refers to the top-level directory where you have cloned +the CBAPI repository. + +You should also have an API key and have set up a `credentials` file as detailed in the "API Token" section of the +top-level `README.md` file. + +Once you have done so, you should be able to run any example script with the command: + + python scriptname.py [arguments] + +Executing any script with the `--help` argument should give you a detailed message about the arguments that can +be supplied to the script when it is executed. diff --git a/examples/psc/bulk_update_alerts.py b/examples/psc/bulk_update_alerts.py index 80ae7a7d..2c3ec81c 100755 --- a/examples/psc/bulk_update_alerts.py +++ b/examples/psc/bulk_update_alerts.py @@ -4,7 +4,7 @@ from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import BaseAlert, WorkflowStatus -from alertsv6common import setup_parser_with_basic_criteria, load_basic_criteria +from helpers.alertsv6 import setup_parser_with_basic_criteria, load_basic_criteria def main(): diff --git a/examples/psc/bulk_update_cbanalytics_alerts.py b/examples/psc/bulk_update_cbanalytics_alerts.py index 8ac4599f..147558c5 100755 --- a/examples/psc/bulk_update_cbanalytics_alerts.py +++ b/examples/psc/bulk_update_cbanalytics_alerts.py @@ -4,7 +4,7 @@ from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import CBAnalyticsAlert, WorkflowStatus -from alertsv6common import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria +from helpers.alertsv6 import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria def main(): diff --git a/examples/psc/bulk_update_vmware_alerts.py b/examples/psc/bulk_update_vmware_alerts.py index 0158acf4..a1cb0b58 100755 --- a/examples/psc/bulk_update_vmware_alerts.py +++ b/examples/psc/bulk_update_vmware_alerts.py @@ -4,7 +4,7 @@ from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import VMwareAlert, WorkflowStatus -from alertsv6common import setup_parser_with_vmware_criteria, load_vmware_criteria +from helpers.alertsv6 import setup_parser_with_vmware_criteria, load_vmware_criteria def main(): diff --git a/examples/psc/bulk_update_watchlist_alerts.py b/examples/psc/bulk_update_watchlist_alerts.py index 9589a815..bef036c5 100755 --- a/examples/psc/bulk_update_watchlist_alerts.py +++ b/examples/psc/bulk_update_watchlist_alerts.py @@ -4,7 +4,7 @@ from time import sleep from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import WatchlistAlert, WorkflowStatus -from alertsv6common import setup_parser_with_watchlist_criteria, load_watchlist_criteria +from helpers.alertsv6 import setup_parser_with_watchlist_criteria, load_watchlist_criteria def main(): diff --git a/examples/psc/download_device_list.py b/examples/psc/download_device_list.py index 23bf2552..25be3a43 100755 --- a/examples/psc/download_device_list.py +++ b/examples/psc/download_device_list.py @@ -26,13 +26,13 @@ def main(): if args.query: query = query.where(args.query) if args.ad_group_id: - query = query.ad_group_ids(args.ad_group_id) + query = query.set_ad_group_ids(args.ad_group_id) if args.policy_id: - query = query.policy_ids(args.policy_id) + query = query.set_policy_ids(args.policy_id) if args.status: - query = query.status(args.status) + query = query.set_status(args.status) if args.priority: - query = query.target_priorities(args.priority) + query = query.set_target_priorities(args.priority) if args.sort_by: direction = "DESC" if args.reverse else "ASC" query = query.sort_by(args.sort_by, direction) diff --git a/examples/psc/alertsv6common.py b/examples/psc/helpers/alertsv6.py similarity index 80% rename from examples/psc/alertsv6common.py rename to examples/psc/helpers/alertsv6.py index ba49228e..3d789669 100755 --- a/examples/psc/alertsv6common.py +++ b/examples/psc/helpers/alertsv6.py @@ -82,78 +82,78 @@ def load_basic_criteria(query, args): if args.query: query = query.where(args.query) if args.category: - query = query.categories(args.category) + query = query.set_categories(args.category) if args.deviceid: - query = query.device_ids(args.deviceid) + query = query.set_device_ids(args.deviceid) if args.devicename: - query = query.device_names(args.devicename) + query = query.set_device_names(args.devicename) if args.os: - query = query.device_os(args.os) + query = query.set_device_os(args.os) if args.osversion: - query = query.device_os_version(args.osversion) + query = query.set_device_os_versions(args.osversion) if args.username: - query = query.device_username(args.username) + query = query.set_device_username(args.username) if args.group: - query = query.group_results(True) + query = query.set_group_results(True) if args.alertid: - query = query.alert_ids(args.alertid) + query = query.set_alert_ids(args.alertid) if args.legacyalertid: - query = query.legacy_alert_ids(args.legacyalertid) + query = query.set_legacy_alert_ids(args.legacyalertid) if args.severity: - query = query.minimum_severity(args.severity) + query = query.set_minimum_severity(args.severity) if args.policyid: - query = query.policy_ids(args.policyid) + query = query.set_policy_ids(args.policyid) if args.policyname: - query = query.policy_names(args.policyname) + query = query.set_policy_names(args.policyname) if args.processname: - query = query.process_names(args.processname) + query = query.set_process_names(args.processname) if args.processhash: - query = query.process_sha256(args.processhash) + query = query.set_process_sha256(args.processhash) if args.reputation: - query = query.reputations(args.reputation) + query = query.set_reputations(args.reputation) if args.tag: - query = query.tags(args.tag) + query = query.set_tags(args.tag) if args.priority: - query = query.target_priorities(args.priority) + query = query.set_target_priorities(args.priority) if args.threatid: - query = query.threat_ids(args.threatid) + query = query.set_threat_ids(args.threatid) if args.type: - query = query.types(args.type) + query = query.set_types(args.type) if args.workflow: - query = query.workflows(args.workflow) + query = query.set_workflows(args.workflow) def load_cbanalytics_criteria(query, args): load_basic_criteria(query, args) if args.blockedthreat: - query = query.blocked_threat_categories(args.blockedthreat) + query = query.set_blocked_threat_categories(args.blockedthreat) if args.location: - query = query.device_locations(args.location) + query = query.set_device_locations(args.location) if args.killchain: - query = query.kill_chain_statuses(args.killchain) + query = query.set_kill_chain_statuses(args.killchain) if args.notblockedthreat: - query = query.not_blocked_threat_categories(args.notblockedthreat) + query = query.set_not_blocked_threat_categories(args.notblockedthreat) if args.policyapplied: - query = query.policy_applied(args.policyapplied) + query = query.set_policy_applied(args.policyapplied) if args.reason: - query = query.reason_code(args.reason) + query = query.set_reason_code(args.reason) if args.runstate: - query = query.run_states(args.runstate) + query = query.set_run_states(args.runstate) if args.sensoraction: - query = query.sensor_actions(args.sensoraction) + query = query.set_sensor_actions(args.sensoraction) if args.vector: - query = query.threat_cause_vectors(args.vector) + query = query.set_threat_cause_vectors(args.vector) def load_vmware_criteria(query, args): load_basic_criteria(query, args) if args.groupid: - query = query.group_ids(args.groupid) + query = query.set_group_ids(args.groupid) def load_watchlist_criteria(query, args): load_basic_criteria(query, args) if args.watchlistid: - query = query.watchlist_ids(args.watchlistid) + query = query.set_watchlist_ids(args.watchlistid) if args.watchlistname: - query = query.watchlist_names(args.watchlistname) + query = query.set_watchlist_names(args.watchlistname) diff --git a/examples/psc/list_alert_facets.py b/examples/psc/list_alert_facets.py index 62de7fcc..957e92da 100755 --- a/examples/psc/list_alert_facets.py +++ b/examples/psc/list_alert_facets.py @@ -3,7 +3,7 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import BaseAlert -from alertsv6common import setup_parser_with_basic_criteria, load_basic_criteria +from helpers.alertsv6 import setup_parser_with_basic_criteria, load_basic_criteria def main(): diff --git a/examples/psc/list_alerts.py b/examples/psc/list_alerts.py index a71ece6e..ff2fd079 100755 --- a/examples/psc/list_alerts.py +++ b/examples/psc/list_alerts.py @@ -3,7 +3,7 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import BaseAlert -from alertsv6common import setup_parser_with_basic_criteria, load_basic_criteria +from helpers.alertsv6 import setup_parser_with_basic_criteria, load_basic_criteria def main(): diff --git a/examples/psc/list_cbanalytics_alert_facets.py b/examples/psc/list_cbanalytics_alert_facets.py index 688d33a1..d654671b 100755 --- a/examples/psc/list_cbanalytics_alert_facets.py +++ b/examples/psc/list_cbanalytics_alert_facets.py @@ -3,7 +3,7 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import CBAnalyticsAlert -from alertsv6common import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria +from helpers.alertsv6 import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria def main(): diff --git a/examples/psc/list_cbanalytics_alerts.py b/examples/psc/list_cbanalytics_alerts.py index 29f50896..a45c62d8 100755 --- a/examples/psc/list_cbanalytics_alerts.py +++ b/examples/psc/list_cbanalytics_alerts.py @@ -3,7 +3,7 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import CBAnalyticsAlert -from alertsv6common import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria +from helpers.alertsv6 import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria def main(): diff --git a/examples/psc/list_devices.py b/examples/psc/list_devices.py index 49b3961c..24a5bb18 100755 --- a/examples/psc/list_devices.py +++ b/examples/psc/list_devices.py @@ -22,13 +22,13 @@ def main(): if args.query: query = query.where(args.query) if args.ad_group_id: - query = query.ad_group_ids(args.ad_group_id) + query = query.set_ad_group_ids(args.ad_group_id) if args.policy_id: - query = query.policy_ids(args.policy_id) + query = query.set_policy_ids(args.policy_id) if args.status: - query = query.status(args.status) + query = query.set_status(args.status) if args.priority: - query = query.target_priorities(args.priority) + query = query.set_target_priorities(args.priority) if args.sort_by: direction = "DESC" if args.reverse else "ASC" query = query.sort_by(args.sort_by, direction) diff --git a/examples/psc/list_vmware_alert_facets.py b/examples/psc/list_vmware_alert_facets.py index 67fa4ed5..c8420037 100755 --- a/examples/psc/list_vmware_alert_facets.py +++ b/examples/psc/list_vmware_alert_facets.py @@ -3,7 +3,7 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import VMwareAlert -from alertsv6common import setup_parser_with_vmware_criteria, load_vmware_criteria +from helpers.alertsv6 import setup_parser_with_vmware_criteria, load_vmware_criteria def main(): diff --git a/examples/psc/list_vmware_alerts.py b/examples/psc/list_vmware_alerts.py index 986f984c..ee0fb44e 100755 --- a/examples/psc/list_vmware_alerts.py +++ b/examples/psc/list_vmware_alerts.py @@ -3,7 +3,7 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import VMwareAlert -from alertsv6common import setup_parser_with_vmware_criteria, load_vmware_criteria +from helpers.alertsv6 import setup_parser_with_vmware_criteria, load_vmware_criteria def main(): diff --git a/examples/psc/list_watchlist_alert_facets.py b/examples/psc/list_watchlist_alert_facets.py index f5b8b803..35776ef1 100755 --- a/examples/psc/list_watchlist_alert_facets.py +++ b/examples/psc/list_watchlist_alert_facets.py @@ -3,7 +3,7 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import WatchlistAlert -from alertsv6common import setup_parser_with_watchlist_criteria, load_watchlist_criteria +from helpers.alertsv6 import setup_parser_with_watchlist_criteria, load_watchlist_criteria def main(): diff --git a/examples/psc/list_watchlist_alerts.py b/examples/psc/list_watchlist_alerts.py index 2d830645..708efa07 100755 --- a/examples/psc/list_watchlist_alerts.py +++ b/examples/psc/list_watchlist_alerts.py @@ -3,7 +3,7 @@ import sys from cbapi.example_helpers import build_cli_parser, get_cb_psc_object from cbapi.psc.models import WatchlistAlert -from alertsv6common import setup_parser_with_watchlist_criteria, load_watchlist_criteria +from helpers.alertsv6 import setup_parser_with_watchlist_criteria, load_watchlist_criteria def main(): diff --git a/runtests.bat b/runtests.bat index 7c9dceaf..6163babf 100755 --- a/runtests.bat +++ b/runtests.bat @@ -1,5 +1,5 @@ @echo off setlocal set PYTHONPATH=src -pytest test/ +pytest -v test/ endlocal diff --git a/runtests.sh b/runtests.sh index a46f46f2..1efaf470 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,2 +1,2 @@ #!/bin/bash -PYTHONPATH=src pytest test/ +PYTHONPATH=src pytest -v test/ diff --git a/src/cbapi/psc/query.py b/src/cbapi/psc/alerts_query.py similarity index 51% rename from src/cbapi/psc/query.py rename to src/cbapi/psc/alerts_query.py index 6d29386e..eade340b 100755 --- a/src/cbapi/psc/query.py +++ b/src/cbapi/psc/alerts_query.py @@ -1,588 +1,18 @@ -from cbapi.errors import ApiError, MoreThanOneResultError -import logging -import functools -from six import string_types -from solrq import Q - -log = logging.getLogger(__name__) - - -class QueryBuilder(object): - """ - Provides a flexible interface for building prepared queries for the CB - PSC backend. - - This object can be instantiated directly, or can be managed implicitly - through the :py:meth:`select` API. - """ - - def __init__(self, **kwargs): - if kwargs: - self._query = Q(**kwargs) - else: - self._query = None - self._raw_query = None - - def _guard_query_params(func): - """Decorates the query construction methods of *QueryBuilder*, preventing - them from being called with parameters that would result in an internally - inconsistent query. - """ - - @functools.wraps(func) - def wrap_guard_query_change(self, q, **kwargs): - if self._raw_query is not None and (kwargs or isinstance(q, Q)): - raise ApiError("Cannot modify a raw query with structured parameters") - if self._query is not None and isinstance(q, string_types): - raise ApiError("Cannot modify a structured query with a raw parameter") - return func(self, q, **kwargs) - - return wrap_guard_query_change - - @_guard_query_params - def where(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - if self._raw_query is None: - self._raw_query = [] - self._raw_query.append(q) - elif isinstance(q, Q) or kwargs: - if self._query is not None: - raise ApiError("Use .and_() or .or_() for an extant solrq.Q object") - if kwargs: - q = Q(**kwargs) - self._query = q - else: - raise ApiError(".where() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def and_(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - self.where(q) - elif isinstance(q, Q) or kwargs: - if kwargs: - q = Q(**kwargs) - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".and_() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def or_(self, q, **kwargs): - """Adds a disjunctive filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - q = Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query | q - else: - raise ApiError(".or_() only accepts solrq.Q objects") - - return self - - @_guard_query_params - def not_(self, q, **kwargs): - """Adds a negative filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - q = ~Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".not_() only accepts solrq.Q objects") - - def _collapse(self): - """The query can be represented by either an array of strings - (_raw_query) which is concatenated and passed directly to Solr, or - a solrq.Q object (_query) which is then converted into a string to - pass to Solr. This function will perform the appropriate conversions to - end up with the 'q' string sent into the POST request to the - PSC-R query endpoint.""" - if self._raw_query is not None: - return " ".join(self._raw_query) - elif self._query is not None: - return str(self._query) - else: - return None # return everything - - -class PSCQueryBase: - """ - Represents the base of all LiveQuery query classes. - """ - - def __init__(self, doc_class, cb): - self._doc_class = doc_class - self._cb = cb - - -class QueryBuilderSupportMixin: - """ - A mixin that supplies wrapper methods to access the _query_builder. - """ - def where(self, q=None, **kwargs): - """Add a filter to this query. - - :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q: - return self - if isinstance(q, QueryBuilder): - self._query_builder = q - else: - self._query_builder.where(q, **kwargs) - return self - - def and_(self, q=None, **kwargs): - """Add a conjunctive filter to this query. - - :param q: Query string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") - - self._query_builder.and_(q, **kwargs) - return self - - def or_(self, q=None, **kwargs): - """Add a disjunctive filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".or_() expects a solrq.Q or kwargs") - - self._query_builder.or_(q, **kwargs) - return self - - def not_(self, q=None, **kwargs): - """Adds a negated filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q and not kwargs: - raise ApiError(".not_() expects a solrq.Q, or kwargs") - - self._query_builder.not_(q, **kwargs) - return self - - -class IterableQueryMixin: - """ - A mix-in to provide iterability to a query. - """ - def all(self): - return self._perform_query() - - def first(self): - allres = list(self) - res = allres[:1] - if not len(res): - return None - return res[0] - - def one(self): - allres = list(self) - res = allres[:2] - if len(res) == 0: - raise MoreThanOneResultError( - message="0 results for query {0:s}".format(self._query) - ) - if len(res) > 1: - raise MoreThanOneResultError( - message="{0:d} results found for query {1:s}".format( - len(self), self._query - ) - ) - return res[0] - - def __len__(self): - return 0 - - def __getitem__(self, item): - return None - - def __iter__(self): - return self._perform_query() - - -class DeviceSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): - """ - Represents a query that is used to locate Device objects. - """ - valid_os = ["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"] - valid_statuses = ["PENDING", "REGISTERED", "UNINSTALLED", "DEREGISTERED", - "ACTIVE", "INACTIVE", "ERROR", "ALL", "BYPASS_ON", - "BYPASS", "QUARANTINE", "SENSOR_OUTOFDATE", - "DELETED", "LIVE"] - valid_priorities = ["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"] - valid_directions = ["ASC", "DESC"] - - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_builder = QueryBuilder() - self._criteria = {} - self._time_filter = {} - self._exclusions = {} - self._sortcriteria = {} - - def _update_criteria(self, key, newlist): - oldlist = self._criteria.get(key, []) - self._criteria[key] = oldlist + newlist - - def _update_exclusions(self, key, newlist): - oldlist = self._exclusions.get(key, []) - self._exclusions[key] = oldlist + newlist - - def ad_group_ids(self, ad_group_ids): - """ - Restricts the devices that this query is performed on to the specified - AD group IDs. - - :param ad_group_ids: list of ints - :return: This instance - """ - if not all(isinstance(ad_group_id, int) for ad_group_id in ad_group_ids): - raise ApiError("One or more invalid AD group IDs") - self._update_criteria("ad_group_id", ad_group_ids) - return self - - def device_ids(self, device_ids): - """ - Restricts the devices that this query is performed on to the specified - device IDs. - - :param ad_group_ids: list of ints - :return: This instance - """ - if not all(isinstance(device_id, int) for device_id in device_ids): - raise ApiError("One or more invalid device IDs") - self._update_criteria("id", device_ids) - return self - - def last_contact_time(self, *args, **kwargs): - """ - Restricts the devices that this query is performed on to the specified - last contact time (either specified as a start and end point or as a - range). - - :return: This instance - """ - if kwargs.get("start", None) and kwargs.get("end", None): - if kwargs.get("range", None): - raise ApiError("cannot specify range= in addition to start= and end=") - stime = kwargs["start"] - if not isinstance(stime, str): - stime = stime.isoformat() - etime = kwargs["end"] - if not isinstance(etime, str): - etime = etime.isoformat() - self._time_filter = {"start": stime, "end": etime} - elif kwargs.get("range", None): - if kwargs.get("start", None) or kwargs.get("end", None): - raise ApiError("cannot specify start= or end= in addition to range=") - self._time_filter = {"range": kwargs["range"]} - else: - raise ApiError("must specify either start= and end= or range=") - return self - - def os(self, operating_systems): - """ - Restricts the devices that this query is performed on to the specified - operating systems. - - :param operating_systems: list of operating systems - :return: This instance - """ - if not all((osval in DeviceSearchQuery.valid_os) for osval in operating_systems): - raise ApiError("One or more invalid operating systems") - self._update_criteria("os", operating_systems) - return self - - def policy_ids(self, policy_ids): - """ - Restricts the devices that this query is performed on to the specified - policy IDs. - - :param policy_ids: list of ints - :return: This instance - """ - if not all(isinstance(policy_id, int) for policy_id in policy_ids): - raise ApiError("One or more invalid policy IDs") - self._update_criteria("policy_id", policy_ids) - return self - - def status(self, statuses): - """ - Restricts the devices that this query is performed on to the specified - status values. - - :param statuses: list of strings - :return: This instance - """ - if not all((stat in DeviceSearchQuery.valid_statuses) for stat in statuses): - raise ApiError("One or more invalid status values") - self._update_criteria("status", statuses) - return self - - def target_priorities(self, target_priorities): - """ - Restricts the devices that this query is performed on to the specified - target priority values. - - :param target_priorities: list of strings - :return: This instance - """ - if not all((prio in DeviceSearchQuery.valid_priorities) for prio in target_priorities): - raise ApiError("One or more invalid target priority values") - self._update_criteria("target_priority", target_priorities) - return self - - def exclude_sensor_versions(self, sensor_versions): - """ - Restricts the devices that this query is performed on to exclude specified - sensor versions. - - :param sensor_versions: List of sensor versions to exclude - :return: This instance - """ - if not all(isinstance(v, str) for v in sensor_versions): - raise ApiError("One or more invalid sensor versions") - self._update_exclusions("sensor_version", sensor_versions) - return self - - def sort_by(self, key, direction="ASC"): - """Sets the sorting behavior on a query's results. - - Example:: - - >>> cb.select(Device).sort_by("name") - - :param key: the key in the schema to sort by - :param direction: the sort order, either "ASC" or "DESC" - :rtype: :py:class:`DeviceSearchQuery` - """ - if direction not in DeviceSearchQuery.valid_directions: - raise ApiError("invalid sort direction specified") - self._sortcriteria = {"field": key, "order": direction} - return self - - def _build_request(self, from_row, max_rows): - mycrit = self._criteria - if self._time_filter: - mycrit["last_contact_time"] = self._time_filter - request = {"criteria": mycrit, "exclusions": self._exclusions} - request["query"] = self._query_builder._collapse() - if from_row > 0: - request["start"] = from_row - if max_rows >= 0: - request["rows"] = max_rows - if self._sortcriteria != {}: - request["sort"] = [self._sortcriteria] - return request - - def _build_url(self, tail_end): - url = self._doc_class.urlobject.format( - self._cb.credentials.org_key) + tail_end - return url - - def _count(self): - if self._count_valid: - return self._total_results - - url = self._build_url("/_search") - request = self._build_request(0, -1) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - return self._total_results - - def _perform_query(self, from_row=0, max_rows=-1): - url = self._build_url("/_search") - current = from_row - numrows = 0 - still_querying = True - while still_querying: - request = self._build_request(current, max_rows) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - results = result.get("results", []) - for item in results: - yield self._doc_class(self._cb, item["id"], item) - current += 1 - numrows += 1 - - if max_rows > 0 and numrows == max_rows: - still_querying = False - break - - from_row = current - if current >= self._total_results: - still_querying = False - break - - def download(self): - """ - Uses the query parameters that have been set to download all - device listings in CSV format. - - Example:: - - >>> cb.select(Device).status(["ALL"]).download() - - :return: The CSV raw data as returned from the server. - """ - tmp = self._criteria.get("status", []) - if not tmp: - raise ApiError("at least one status must be specified to download") - query_params = {"status": ",".join(tmp)} - tmp = self._criteria.get("ad_group_id", []) - if tmp: - query_params["ad_group_id"] = ",".join([str(t) for t in tmp]) - tmp = self._criteria.get("policy_id", []) - if tmp: - query_params["policy_id"] = ",".join([str(t) for t in tmp]) - tmp = self._criteria.get("target_priority", []) - if tmp: - query_params["target_priority"] = ",".join(tmp) - tmp = self._query_builder._collapse() - if tmp: - query_params["query_string"] = tmp - if self._sortcriteria: - query_params["sort_field"] = self._sortcriteria["field"] - query_params["sort_order"] = self._sortcriteria["order"] - url = self._build_url("/_search/download") - # AGRB 10/3/2019 - Header is TEMPORARY until bug is fixed in API. Remove when fix deployed. - return self._cb.get_raw_data(url, query_params, headers={"Content-Type": "application/json"}) - - def _bulk_device_action(self, action_type, options=None): - request = {"action_type": action_type, "search": self._build_request(0, -1)} - if options: - request["options"] = options - return self._cb._raw_device_action(request) - - def background_scan(self, flag): - """ - Set the background scan option for the specified devices. - - :param boolean flag: True to turn background scan on, False to turn it off. - """ - return self._bulk_device_action("BACKGROUND_SCAN", self._cb._action_toggle(flag)) - - def bypass(self, flag): - """ - Set the bypass option for the specified devices. - - :param boolean flag: True to enable bypass, False to disable it. - """ - return self._bulk_device_action("BYPASS", self._cb._action_toggle(flag)) - - def delete_sensor(self): - """ - Delete the specified sensor devices. - """ - return self._bulk_device_action("DELETE_SENSOR") - - def uninstall_sensor(self): - """ - Uninstall the specified sensor devices. - """ - return self._bulk_device_action("UNINSTALL_SENSOR") - - def quarantine(self, flag): - """ - Set the quarantine option for the specified devices. - - :param boolean flag: True to enable quarantine, False to disable it. - """ - return self._bulk_device_action("QUARANTINE", self._cb._action_toggle(flag)) - - def update_policy(self, policy_id): - """ - Set the current policy for the specified devices. - - :param int policy_id: ID of the policy to set for the devices. - """ - return self._bulk_device_action("UPDATE_POLICY", {"policy_id": policy_id}) - - def update_sensor_version(self, sensor_version): - """ - Update the sensor version for the specified devices. - - :param dict sensor_version: New version properties for the sensor. - """ - return self._bulk_device_action("UPDATE_SENSOR_VERSION", - {"sensor_version": sensor_version}) +from cbapi.errors import ApiError +from .base_query import PSCQueryBase, QueryBuilder, QueryBuilderSupportMixin, IterableQueryMixin +from .devices_query import DeviceSearchQuery class BaseAlertSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): """ Represents a query that is used to locate BaseAlert objects. """ - valid_categories = ["THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", "CRITICAL"] - valid_reputations = ["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", "NOT_LISTED", "ADAPTIVE_WHITE_LIST", + VALID_CATEGORIES = ["THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", "CRITICAL"] + VALID_REPUTATIONS = ["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", "NOT_LISTED", "ADAPTIVE_WHITE_LIST", "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", "COMPANY_BLACK_LIST"] - valid_alerttypes = ["CB_ANALYTICS", "VMWARE", "WATCHLIST"] - valid_workflow_vals = ["OPEN", "DISMISSED"] - valid_facet_fields = ["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", + VALID_ALERTTYPES = ["CB_ANALYTICS", "VMWARE", "WATCHLIST"] + VALID_WORKFLOW_VALS = ["OPEN", "DISMISSED"] + VALID_FACET_FIELDS = ["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", "POLICY_APPLIED", "SENSOR_ACTION"] @@ -596,10 +26,18 @@ def __init__(self, doc_class, cb): self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/workflow/_criteria" def _update_criteria(self, key, newlist): + """ + Updates the criteria being collected for a query. Assumes the specified criteria item is + defined as a list; the list passed in will be set as the value for this criteria item, or + appended to the existing one if there is one. + + :param str key: The key for the criteria item to be set + :param list newlist: List of values to be set for the criteria item + """ oldlist = self._criteria.get(key, []) self._criteria[key] = oldlist + newlist - def categories(self, cats): + def set_categories(self, cats): """ Restricts the alerts that this query is performed on to the specified categories. @@ -607,12 +45,12 @@ def categories(self, cats): "THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", and "CRITICAL." :return: This instance """ - if not all((c in BaseAlertSearchQuery.valid_categories) for c in cats): + if not all((c in BaseAlertSearchQuery.VALID_CATEGORIES) for c in cats): raise ApiError("One or more invalid category values") self._update_criteria("category", cats) return self - def create_time(self, *args, **kwargs): + def set_create_time(self, *args, **kwargs): """ Restricts the alerts that this query is performed on to the specified creation time (either specified as a start and end point or as a @@ -638,7 +76,7 @@ def create_time(self, *args, **kwargs): raise ApiError("must specify either start= and end= or range=") return self - def device_ids(self, device_ids): + def set_device_ids(self, device_ids): """ Restricts the alerts that this query is performed on to the specified device IDs. @@ -651,7 +89,7 @@ def device_ids(self, device_ids): self._update_criteria("device_id", device_ids) return self - def device_names(self, device_names): + def set_device_names(self, device_names): """ Restricts the alerts that this query is performed on to the specified device names. @@ -664,7 +102,7 @@ def device_names(self, device_names): self._update_criteria("device_name", device_names) return self - def device_os(self, device_os): + def set_device_os(self, device_os): """ Restricts the alerts that this query is performed on to the specified device operating systems. @@ -673,12 +111,12 @@ def device_os(self, device_os): "WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", and "OTHER." :return: This instance """ - if not all((osval in DeviceSearchQuery.valid_os) for osval in device_os): + if not all((osval in DeviceSearchQuery.VALID_OS) for osval in device_os): raise ApiError("One or more invalid operating systems") self._update_criteria("device_os", device_os) return self - def device_os_versions(self, device_os_versions): + def set_device_os_versions(self, device_os_versions): """ Restricts the alerts that this query is performed on to the specified device operating system versions. @@ -691,7 +129,7 @@ def device_os_versions(self, device_os_versions): self._update_criteria("device_os_version", device_os_versions) return self - def device_username(self, users): + def set_device_username(self, users): """ Restricts the alerts that this query is performed on to the specified user names. @@ -704,17 +142,17 @@ def device_username(self, users): self._update_criteria("device_username", users) return self - def group_results(self, flag): + def set_group_results(self, do_group): """ Specifies whether or not to group the results of the query. - :param flag boolean: True to group the results, False to not do so. + :param do_group boolean: True to group the results, False to not do so. :return: This instance """ - self._criteria["group_results"] = True if flag else False + self._criteria["group_results"] = True if do_group else False return self - def alert_ids(self, alert_ids): + def set_alert_ids(self, alert_ids): """ Restricts the alerts that this query is performed on to the specified alert IDs. @@ -727,7 +165,7 @@ def alert_ids(self, alert_ids): self._update_criteria("id", alert_ids) return self - def legacy_alert_ids(self, alert_ids): + def set_legacy_alert_ids(self, alert_ids): """ Restricts the alerts that this query is performed on to the specified legacy alert IDs. @@ -740,7 +178,7 @@ def legacy_alert_ids(self, alert_ids): self._update_criteria("legacy_alert_id", alert_ids) return self - def minimum_severity(self, severity): + def set_minimum_severity(self, severity): """ Restricts the alerts that this query is performed on to the specified minimum severity level. @@ -751,7 +189,7 @@ def minimum_severity(self, severity): self._criteria["minimum_severity"] = severity return self - def policy_ids(self, policy_ids): + def set_policy_ids(self, policy_ids): """ Restricts the alerts that this query is performed on to the specified policy IDs. @@ -764,7 +202,7 @@ def policy_ids(self, policy_ids): self._update_criteria("policy_id", policy_ids) return self - def policy_names(self, policy_names): + def set_policy_names(self, policy_names): """ Restricts the alerts that this query is performed on to the specified policy names. @@ -777,7 +215,7 @@ def policy_names(self, policy_names): self._update_criteria("policy_name", policy_names) return self - def process_names(self, process_names): + def set_process_names(self, process_names): """ Restricts the alerts that this query is performed on to the specified process names. @@ -790,7 +228,7 @@ def process_names(self, process_names): self._update_criteria("process_name", process_names) return self - def process_sha256(self, shas): + def set_process_sha256(self, shas): """ Restricts the alerts that this query is performed on to the specified process SHA-256 hash values. @@ -803,7 +241,7 @@ def process_sha256(self, shas): self._update_criteria("process_sha256", shas) return self - def reputations(self, reps): + def set_reputations(self, reps): """ Restricts the alerts that this query is performed on to the specified reputation values. @@ -814,12 +252,12 @@ def reputations(self, reps): "TRUSTED_WHITE_LIST", and "COMPANY_BLACK_LIST". :return: This instance """ - if not all((r in BaseAlertSearchQuery.valid_reputations) for r in reps): + if not all((r in BaseAlertSearchQuery.VALID_REPUTATIONS) for r in reps): raise ApiError("One or more invalid reputation values") self._update_criteria("reputation", reps) return self - def tags(self, tags): + def set_tags(self, tags): """ Restricts the alerts that this query is performed on to the specified tag values. @@ -832,7 +270,7 @@ def tags(self, tags): self._update_criteria("tag", tags) return self - def target_priorities(self, priorities): + def set_target_priorities(self, priorities): """ Restricts the alerts that this query is performed on to the specified target priority values. @@ -841,12 +279,12 @@ def target_priorities(self, priorities): "LOW", "MEDIUM", "HIGH", and "MISSION_CRITICAL". :return: This instance """ - if not all((prio in DeviceSearchQuery.valid_priorities) for prio in priorities): + if not all((prio in DeviceSearchQuery.VALID_PRIORITIES) for prio in priorities): raise ApiError("One or more invalid priority values") self._update_criteria("target_value", priorities) return self - def threat_ids(self, threats): + def set_threat_ids(self, threats): """ Restricts the alerts that this query is performed on to the specified threat ID values. @@ -859,7 +297,7 @@ def threat_ids(self, threats): self._update_criteria("threat_id", threats) return self - def types(self, alerttypes): + def set_types(self, alerttypes): """ Restricts the alerts that this query is performed on to the specified alert type values. @@ -868,12 +306,12 @@ def types(self, alerttypes): "CB_ANALYTICS", "VMWARE", and "WATCHLIST". :return: This instance """ - if not all((t in BaseAlertSearchQuery.valid_alerttypes) for t in alerttypes): + if not all((t in BaseAlertSearchQuery.VALID_ALERTTYPES) for t in alerttypes): raise ApiError("One or more invalid alert type values") self._update_criteria("type", alerttypes) return self - def workflows(self, workflow_vals): + def set_workflows(self, workflow_vals): """ Restricts the alerts that this query is performed on to the specified workflow status values. @@ -882,7 +320,7 @@ def workflows(self, workflow_vals): "OPEN" and "DISMISSED". :return: This instance """ - if not all((t in BaseAlertSearchQuery.valid_workflow_vals) for t in workflow_vals): + if not all((t in BaseAlertSearchQuery.VALID_WORKFLOW_VALS) for t in workflow_vals): raise ApiError("One or more invalid workflow status values") self._update_criteria("workflow", workflow_vals) return self @@ -909,12 +347,20 @@ def sort_by(self, key, direction="ASC"): :param direction: the sort order, either "ASC" or "DESC" :rtype: :py:class:`BaseAlertSearchQuery` """ - if direction not in DeviceSearchQuery.valid_directions: + if direction not in DeviceSearchQuery.VALID_DIRECTIONS: raise ApiError("invalid sort direction specified") self._sortcriteria = {"field": key, "order": direction} return self def _build_request(self, from_row, max_rows, add_sort=True): + """ + Creates the request body for an API call. + + :param int from_row: The row to start the query at. + :param int max_rows: The maximum number of rows to be returned. + :param boolean add_sort: If True(default), the sort criteria will be added as part of the request. + :return: A dict containing the complete request body. + """ request = {"criteria": self._build_criteria()} request["query"] = self._query_builder._collapse() if from_row > 0: @@ -926,10 +372,20 @@ def _build_request(self, from_row, max_rows, add_sort=True): return request def _build_url(self, tail_end): + """ + Creates the URL to be used for an API call. + + :param str tail_end: String to be appended to the end of the generated URL. + """ url = self._doc_class.urlobject.format(self._cb.credentials.org_key) + tail_end return url def _count(self): + """ + Returns the number of results from the run of this query. + + :return: The number of results from the run of this query. + """ if self._count_valid: return self._total_results @@ -944,6 +400,12 @@ def _count(self): return self._total_results def _perform_query(self, from_row=0, max_rows=-1): + """ + Performs the query and returns the results of the query in an iterable fashion. + + :param int from_row: The row to start the query at (default 0). + :param int max_rows: The maximum number of rows to be returned (default -1, meaning "all"). + """ url = self._build_url("/_search") current = from_row numrows = 0 @@ -983,7 +445,7 @@ def facets(self, fieldlist, max_rows=0): :param max_rows int: The maximum number of rows to return. 0 means return all rows. :return: A list of facet information specified as dicts. """ - if not all((field in BaseAlertSearchQuery.valid_facet_fields) for field in fieldlist): + if not all((field in BaseAlertSearchQuery.VALID_FACET_FIELDS) for field in fieldlist): raise ApiError("One or more invalid term field names") request = self._build_request(0, -1, False) request["terms"] = {"fields": fieldlist, "rows": max_rows} @@ -993,6 +455,14 @@ def facets(self, fieldlist, max_rows=0): return result.get("results", []) def _update_status(self, status, remediation, comment): + """ + Updates the status of all alerts matching the given query. + + :param str state: The status to put the alerts into, either "OPEN" or "DISMISSED". + :param remediation str: The remediation state to set for all alerts. + :param comment str: The comment to set for all alerts. + :return: The request ID, which may be used to select a WorkflowStatus object. + """ request = {"state": status, "criteria": self._build_criteria(), "query": self._query_builder._collapse()} if remediation is not None: request["remediation_state"] = remediation @@ -1031,7 +501,7 @@ def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/watchlist/workflow/_criteria" - def watchlist_ids(self, ids): + def set_watchlist_ids(self, ids): """ Restricts the alerts that this query is performed on to the specified watchlist ID values. @@ -1044,7 +514,7 @@ def watchlist_ids(self, ids): self._update_criteria("watchlist_id", ids) return self - def watchlist_names(self, names): + def set_watchlist_names(self, names): """ Restricts the alerts that this query is performed on to the specified watchlist name values. @@ -1062,21 +532,21 @@ class CBAnalyticsAlertSearchQuery(BaseAlertSearchQuery): """ Represents a query that is used to locate CBAnalyticsAlert objects. """ - valid_threat_categories = ["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", "RISKY_PROGRAM"] - valid_locations = ["ONSITE", "OFFSITE", "UNKNOWN"] - valid_kill_chain_statuses = ["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", "INSTALL_RUN", + VALID_THREAT_CATEGORIES = ["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", "RISKY_PROGRAM"] + VALID_LOCATIONS = ["ONSITE", "OFFSITE", "UNKNOWN"] + VALID_KILL_CHAIN_STATUSES = ["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", "INSTALL_RUN", "COMMAND_AND_CONTROL", "EXECUTE_GOAL", "BREACH"] - valid_policy_applied = ["APPLIED", "NOT_APPLIED"] - valid_run_states = ["DID_NOT_RUN", "RAN", "UNKNOWN"] - valid_sensor_actions = ["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", "TERMINATE", "DENY"] - valid_threat_cause_vectors = ["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", + VALID_POLICY_APPLIED = ["APPLIED", "NOT_APPLIED"] + VALID_RUN_STATES = ["DID_NOT_RUN", "RAN", "UNKNOWN"] + VALID_SENSOR_ACTIONS = ["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", "TERMINATE", "DENY"] + VALID_THREAT_CAUSE_VECTORS = ["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", "APP_STORE", "THIRD_PARTY"] def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/cbanalytics/workflow/_criteria" - def blocked_threat_categories(self, categories): + def set_blocked_threat_categories(self, categories): """ Restricts the alerts that this query is performed on to the specified threat categories that were blocked. @@ -1085,13 +555,13 @@ def blocked_threat_categories(self, categories): "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". :return: This instance. """ - if not all((category in CBAnalyticsAlertSearchQuery.valid_threat_categories) + if not all((category in CBAnalyticsAlertSearchQuery.VALID_THREAT_CATEGORIES) for category in categories): raise ApiError("One or more invalid threat categories") self._update_criteria("blocked_threat_category", categories) return self - def device_locations(self, locations): + def set_device_locations(self, locations): """ Restricts the alerts that this query is performed on to the specified device locations. @@ -1100,13 +570,13 @@ def device_locations(self, locations): and "UNKNOWN". :return: This instance. """ - if not all((location in CBAnalyticsAlertSearchQuery.valid_locations) + if not all((location in CBAnalyticsAlertSearchQuery.VALID_LOCATIONS) for location in locations): raise ApiError("One or more invalid device locations") self._update_criteria("device_location", locations) return self - def kill_chain_statuses(self, statuses): + def set_kill_chain_statuses(self, statuses): """ Restricts the alerts that this query is performed on to the specified kill chain statuses. @@ -1116,13 +586,13 @@ def kill_chain_statuses(self, statuses): "EXECUTE_GOAL", and "BREACH". :return: This instance. """ - if not all((status in CBAnalyticsAlertSearchQuery.valid_kill_chain_statuses) + if not all((status in CBAnalyticsAlertSearchQuery.VALID_KILL_CHAIN_STATUSES) for status in statuses): raise ApiError("One or more invalid kill chain status values") self._update_criteria("kill_chain_status", statuses) return self - def not_blocked_threat_categories(self, categories): + def set_not_blocked_threat_categories(self, categories): """ Restricts the alerts that this query is performed on to the specified threat categories that were NOT blocked. @@ -1131,13 +601,13 @@ def not_blocked_threat_categories(self, categories): "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". :return: This instance. """ - if not all((category in CBAnalyticsAlertSearchQuery.valid_threat_categories) + if not all((category in CBAnalyticsAlertSearchQuery.VALID_THREAT_CATEGORIES) for category in categories): raise ApiError("One or more invalid threat categories") self._update_criteria("not_blocked_threat_category", categories) return self - def policy_applied(self, applied_statuses): + def set_policy_applied(self, applied_statuses): """ Restricts the alerts that this query is performed on to the specified status values showing whether policies were applied. @@ -1146,13 +616,13 @@ def policy_applied(self, applied_statuses): "APPLIED" and "NOT_APPLIED". :return: This instance. """ - if not all((s in CBAnalyticsAlertSearchQuery.valid_policy_applied) + if not all((s in CBAnalyticsAlertSearchQuery.VALID_POLICY_APPLIED) for s in applied_statuses): raise ApiError("One or more invalid policy-applied values") self._update_criteria("policy_applied", applied_statuses) return self - def reason_code(self, reason): + def set_reason_code(self, reason): """ Restricts the alerts that this query is performed on to the specified reason codes (enum values). @@ -1165,7 +635,7 @@ def reason_code(self, reason): self._update_criteria("reason_code", reason) return self - def run_states(self, states): + def set_run_states(self, states): """ Restricts the alerts that this query is performed on to the specified run states. @@ -1173,13 +643,13 @@ def run_states(self, states): and "UNKNOWN". :return: This instance. """ - if not all((s in CBAnalyticsAlertSearchQuery.valid_run_states) + if not all((s in CBAnalyticsAlertSearchQuery.VALID_RUN_STATES) for s in states): raise ApiError("One or more invalid run states") self._update_criteria("run_state", states) return self - def sensor_actions(self, actions): + def set_sensor_actions(self, actions): """ Restricts the alerts that this query is performed on to the specified sensor actions. @@ -1187,13 +657,13 @@ def sensor_actions(self, actions): "ALLOW", "ALLOW_AND_LOG", "TERMINATE", and "DENY". :return: This instance. """ - if not all((action in CBAnalyticsAlertSearchQuery.valid_sensor_actions) + if not all((action in CBAnalyticsAlertSearchQuery.VALID_SENSOR_ACTIONS) for action in actions): raise ApiError("One or more invalid sensor actions") self._update_criteria("sensor_action", actions) return self - def threat_cause_vectors(self, vectors): + def set_threat_cause_vectors(self, vectors): """ Restricts the alerts that this query is performed on to the specified threat cause vectors. @@ -1202,7 +672,7 @@ def threat_cause_vectors(self, vectors): "UNKNOWN", "APP_STORE", and "THIRD_PARTY". :return: This instance. """ - if not all((vector in CBAnalyticsAlertSearchQuery.valid_threat_cause_vectors) + if not all((vector in CBAnalyticsAlertSearchQuery.VALID_THREAT_CAUSE_VECTORS) for vector in vectors): raise ApiError("One or more invalid threat cause vectors") self._update_criteria("threat_cause_vector", vectors) @@ -1217,7 +687,7 @@ def __init__(self, doc_class, cb): super().__init__(doc_class, cb) self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/vmware/workflow/_criteria" - def group_ids(self, groupids): + def set_group_ids(self, groupids): """ Restricts the alerts that this query is performed on to the specified AppDefense-assigned alarm group IDs. diff --git a/src/cbapi/psc/base_query.py b/src/cbapi/psc/base_query.py new file mode 100755 index 00000000..f028fb30 --- /dev/null +++ b/src/cbapi/psc/base_query.py @@ -0,0 +1,270 @@ +from cbapi.errors import ApiError, MoreThanOneResultError +import functools +from six import string_types +from solrq import Q + + +class QueryBuilder(object): + """ + Provides a flexible interface for building prepared queries for the CB + PSC backend. + + This object can be instantiated directly, or can be managed implicitly + through the :py:meth:`select` API. + """ + + def __init__(self, **kwargs): + if kwargs: + self._query = Q(**kwargs) + else: + self._query = None + self._raw_query = None + + def _guard_query_params(func): + """Decorates the query construction methods of *QueryBuilder*, preventing + them from being called with parameters that would result in an internally + inconsistent query. + """ + + @functools.wraps(func) + def wrap_guard_query_change(self, q, **kwargs): + if self._raw_query is not None and (kwargs or isinstance(q, Q)): + raise ApiError("Cannot modify a raw query with structured parameters") + if self._query is not None and isinstance(q, string_types): + raise ApiError("Cannot modify a structured query with a raw parameter") + return func(self, q, **kwargs) + + return wrap_guard_query_change + + @_guard_query_params + def where(self, q, **kwargs): + """Adds a conjunctive filter to a query. + + :param q: string or `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: QueryBuilder object + :rtype: :py:class:`QueryBuilder` + """ + if isinstance(q, string_types): + if self._raw_query is None: + self._raw_query = [] + self._raw_query.append(q) + elif isinstance(q, Q) or kwargs: + if self._query is not None: + raise ApiError("Use .and_() or .or_() for an extant solrq.Q object") + if kwargs: + q = Q(**kwargs) + self._query = q + else: + raise ApiError(".where() only accepts strings or solrq.Q objects") + + return self + + @_guard_query_params + def and_(self, q, **kwargs): + """Adds a conjunctive filter to a query. + + :param q: string or `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: QueryBuilder object + :rtype: :py:class:`QueryBuilder` + """ + if isinstance(q, string_types): + self.where(q) + elif isinstance(q, Q) or kwargs: + if kwargs: + q = Q(**kwargs) + if self._query is None: + self._query = q + else: + self._query = self._query & q + else: + raise ApiError(".and_() only accepts strings or solrq.Q objects") + + return self + + @_guard_query_params + def or_(self, q, **kwargs): + """Adds a disjunctive filter to a query. + + :param q: `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: QueryBuilder object + :rtype: :py:class:`QueryBuilder` + """ + if kwargs: + q = Q(**kwargs) + + if isinstance(q, Q): + if self._query is None: + self._query = q + else: + self._query = self._query | q + else: + raise ApiError(".or_() only accepts solrq.Q objects") + + return self + + @_guard_query_params + def not_(self, q, **kwargs): + """Adds a negative filter to a query. + + :param q: `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: QueryBuilder object + :rtype: :py:class:`QueryBuilder` + """ + if kwargs: + q = ~Q(**kwargs) + + if isinstance(q, Q): + if self._query is None: + self._query = q + else: + self._query = self._query & q + else: + raise ApiError(".not_() only accepts solrq.Q objects") + + def _collapse(self): + """The query can be represented by either an array of strings + (_raw_query) which is concatenated and passed directly to Solr, or + a solrq.Q object (_query) which is then converted into a string to + pass to Solr. This function will perform the appropriate conversions to + end up with the 'q' string sent into the POST request to the + PSC-R query endpoint.""" + if self._raw_query is not None: + return " ".join(self._raw_query) + elif self._query is not None: + return str(self._query) + else: + return None # return everything + + +class PSCQueryBase: + """ + Represents the base of all LiveQuery query classes. + """ + + def __init__(self, doc_class, cb): + self._doc_class = doc_class + self._cb = cb + + +class QueryBuilderSupportMixin: + """ + A mixin that supplies wrapper methods to access the _query_builder. + """ + def where(self, q=None, **kwargs): + """Add a filter to this query. + + :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: Query object + :rtype: :py:class:`Query` + """ + + if not q: + return self + if isinstance(q, QueryBuilder): + self._query_builder = q + else: + self._query_builder.where(q, **kwargs) + return self + + def and_(self, q=None, **kwargs): + """Add a conjunctive filter to this query. + + :param q: Query string or `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: Query object + :rtype: :py:class:`Query` + """ + if not q and not kwargs: + raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") + + self._query_builder.and_(q, **kwargs) + return self + + def or_(self, q=None, **kwargs): + """Add a disjunctive filter to this query. + + :param q: `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: Query object + :rtype: :py:class:`Query` + """ + if not q and not kwargs: + raise ApiError(".or_() expects a solrq.Q or kwargs") + + self._query_builder.or_(q, **kwargs) + return self + + def not_(self, q=None, **kwargs): + """Adds a negated filter to this query. + + :param q: `solrq.Q` object + :param kwargs: Arguments to construct a `solrq.Q` with + :return: Query object + :rtype: :py:class:`Query` + """ + + if not q and not kwargs: + raise ApiError(".not_() expects a solrq.Q, or kwargs") + + self._query_builder.not_(q, **kwargs) + return self + + +class IterableQueryMixin: + """ + A mix-in to provide iterability to a query. + """ + def all(self): + """ + Returns all the items of a query as a list. + + :return: List of query items + """ + return self._perform_query() + + def first(self): + """ + Returns the first item that would be returned as the result of a query. + + :return: First query item + """ + allres = list(self) + res = allres[:1] + if not len(res): + return None + return res[0] + + def one(self): + """ + Returns the only item that would be returned by a query. + + :return: Sole query return item + :raises MoreThanOneResultError: If the query returns zero items, or more than one item + """ + allres = list(self) + res = allres[:2] + if len(res) == 0: + raise MoreThanOneResultError( + message="0 results for query {0:s}".format(self._query) + ) + if len(res) > 1: + raise MoreThanOneResultError( + message="{0:d} results found for query {1:s}".format( + len(self), self._query + ) + ) + return res[0] + + def __len__(self): + return 0 + + def __getitem__(self, item): + return None + + def __iter__(self): + return self._perform_query() diff --git a/src/cbapi/psc/devices_query.py b/src/cbapi/psc/devices_query.py new file mode 100755 index 00000000..bd5af53a --- /dev/null +++ b/src/cbapi/psc/devices_query.py @@ -0,0 +1,361 @@ +from cbapi.errors import ApiError +from .base_query import PSCQueryBase, QueryBuilder, QueryBuilderSupportMixin, IterableQueryMixin + + +class DeviceSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): + """ + Represents a query that is used to locate Device objects. + """ + VALID_OS = ["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"] + VALID_STATUSES = ["PENDING", "REGISTERED", "UNINSTALLED", "DEREGISTERED", + "ACTIVE", "INACTIVE", "ERROR", "ALL", "BYPASS_ON", + "BYPASS", "QUARANTINE", "SENSOR_OUTOFDATE", + "DELETED", "LIVE"] + VALID_PRIORITIES = ["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"] + VALID_DIRECTIONS = ["ASC", "DESC"] + + def __init__(self, doc_class, cb): + super().__init__(doc_class, cb) + self._query_builder = QueryBuilder() + self._criteria = {} + self._time_filter = {} + self._exclusions = {} + self._sortcriteria = {} + + def _update_criteria(self, key, newlist): + """ + Updates the criteria being collected for a query. Assumes the specified criteria item is + defined as a list; the list passed in will be set as the value for this criteria item, or + appended to the existing one if there is one. + + :param str key: The key for the criteria item to be set + :param list newlist: List of values to be set for the criteria item + """ + oldlist = self._criteria.get(key, []) + self._criteria[key] = oldlist + newlist + + def _update_exclusions(self, key, newlist): + """ + Updates the exclusion criteria being collected for a query. Assumes the specified criteria item is + defined as a list; the list passed in will be set as the value for this criteria item, or + appended to the existing one if there is one. + + :param str key: The key for the criteria item to be set + :param list newlist: List of values to be set for the criteria item + """ + oldlist = self._exclusions.get(key, []) + self._exclusions[key] = oldlist + newlist + + def set_ad_group_ids(self, ad_group_ids): + """ + Restricts the devices that this query is performed on to the specified + AD group IDs. + + :param ad_group_ids: list of ints + :return: This instance + """ + if not all(isinstance(ad_group_id, int) for ad_group_id in ad_group_ids): + raise ApiError("One or more invalid AD group IDs") + self._update_criteria("ad_group_id", ad_group_ids) + return self + + def set_device_ids(self, device_ids): + """ + Restricts the devices that this query is performed on to the specified + device IDs. + + :param ad_group_ids: list of ints + :return: This instance + """ + if not all(isinstance(device_id, int) for device_id in device_ids): + raise ApiError("One or more invalid device IDs") + self._update_criteria("id", device_ids) + return self + + def set_last_contact_time(self, *args, **kwargs): + """ + Restricts the devices that this query is performed on to the specified + last contact time (either specified as a start and end point or as a + range). + + :return: This instance + """ + if kwargs.get("start", None) and kwargs.get("end", None): + if kwargs.get("range", None): + raise ApiError("cannot specify range= in addition to start= and end=") + stime = kwargs["start"] + if not isinstance(stime, str): + stime = stime.isoformat() + etime = kwargs["end"] + if not isinstance(etime, str): + etime = etime.isoformat() + self._time_filter = {"start": stime, "end": etime} + elif kwargs.get("range", None): + if kwargs.get("start", None) or kwargs.get("end", None): + raise ApiError("cannot specify start= or end= in addition to range=") + self._time_filter = {"range": kwargs["range"]} + else: + raise ApiError("must specify either start= and end= or range=") + return self + + def set_os(self, operating_systems): + """ + Restricts the devices that this query is performed on to the specified + operating systems. + + :param operating_systems: list of operating systems + :return: This instance + """ + if not all((osval in DeviceSearchQuery.VALID_OS) for osval in operating_systems): + raise ApiError("One or more invalid operating systems") + self._update_criteria("os", operating_systems) + return self + + def set_policy_ids(self, policy_ids): + """ + Restricts the devices that this query is performed on to the specified + policy IDs. + + :param policy_ids: list of ints + :return: This instance + """ + if not all(isinstance(policy_id, int) for policy_id in policy_ids): + raise ApiError("One or more invalid policy IDs") + self._update_criteria("policy_id", policy_ids) + return self + + def set_status(self, statuses): + """ + Restricts the devices that this query is performed on to the specified + status values. + + :param statuses: list of strings + :return: This instance + """ + if not all((stat in DeviceSearchQuery.VALID_STATUSES) for stat in statuses): + raise ApiError("One or more invalid status values") + self._update_criteria("status", statuses) + return self + + def set_target_priorities(self, target_priorities): + """ + Restricts the devices that this query is performed on to the specified + target priority values. + + :param target_priorities: list of strings + :return: This instance + """ + if not all((prio in DeviceSearchQuery.VALID_PRIORITIES) for prio in target_priorities): + raise ApiError("One or more invalid target priority values") + self._update_criteria("target_priority", target_priorities) + return self + + def set_exclude_sensor_versions(self, sensor_versions): + """ + Restricts the devices that this query is performed on to exclude specified + sensor versions. + + :param sensor_versions: List of sensor versions to exclude + :return: This instance + """ + if not all(isinstance(v, str) for v in sensor_versions): + raise ApiError("One or more invalid sensor versions") + self._update_exclusions("sensor_version", sensor_versions) + return self + + def sort_by(self, key, direction="ASC"): + """Sets the sorting behavior on a query's results. + + Example:: + + >>> cb.select(Device).sort_by("name") + + :param key: the key in the schema to sort by + :param direction: the sort order, either "ASC" or "DESC" + :rtype: :py:class:`DeviceSearchQuery` + """ + if direction not in DeviceSearchQuery.VALID_DIRECTIONS: + raise ApiError("invalid sort direction specified") + self._sortcriteria = {"field": key, "order": direction} + return self + + def _build_request(self, from_row, max_rows): + """ + Creates the request body for an API call. + + :param int from_row: The row to start the query at. + :param int max_rows: The maximum number of rows to be returned. + :return: A dict containing the complete request body. + """ + mycrit = self._criteria + if self._time_filter: + mycrit["last_contact_time"] = self._time_filter + request = {"criteria": mycrit, "exclusions": self._exclusions} + request["query"] = self._query_builder._collapse() + if from_row > 0: + request["start"] = from_row + if max_rows >= 0: + request["rows"] = max_rows + if self._sortcriteria != {}: + request["sort"] = [self._sortcriteria] + return request + + def _build_url(self, tail_end): + """ + Creates the URL to be used for an API call. + + :param str tail_end: String to be appended to the end of the generated URL. + """ + url = self._doc_class.urlobject.format(self._cb.credentials.org_key) + tail_end + return url + + def _count(self): + """ + Returns the number of results from the run of this query. + + :return: The number of results from the run of this query. + """ + if self._count_valid: + return self._total_results + + url = self._build_url("/_search") + request = self._build_request(0, -1) + resp = self._cb.post_object(url, body=request) + result = resp.json() + + self._total_results = result["num_found"] + self._count_valid = True + + return self._total_results + + def _perform_query(self, from_row=0, max_rows=-1): + """ + Performs the query and returns the results of the query in an iterable fashion. + + :param int from_row: The row to start the query at (default 0). + :param int max_rows: The maximum number of rows to be returned (default -1, meaning "all"). + """ + url = self._build_url("/_search") + current = from_row + numrows = 0 + still_querying = True + while still_querying: + request = self._build_request(current, max_rows) + resp = self._cb.post_object(url, body=request) + result = resp.json() + + self._total_results = result["num_found"] + self._count_valid = True + + results = result.get("results", []) + for item in results: + yield self._doc_class(self._cb, item["id"], item) + current += 1 + numrows += 1 + + if max_rows > 0 and numrows == max_rows: + still_querying = False + break + + from_row = current + if current >= self._total_results: + still_querying = False + break + + def download(self): + """ + Uses the query parameters that have been set to download all + device listings in CSV format. + + Example:: + + >>> cb.select(Device).status(["ALL"]).download() + + :return: The CSV raw data as returned from the server. + """ + tmp = self._criteria.get("status", []) + if not tmp: + raise ApiError("at least one status must be specified to download") + query_params = {"status": ",".join(tmp)} + tmp = self._criteria.get("ad_group_id", []) + if tmp: + query_params["ad_group_id"] = ",".join([str(t) for t in tmp]) + tmp = self._criteria.get("policy_id", []) + if tmp: + query_params["policy_id"] = ",".join([str(t) for t in tmp]) + tmp = self._criteria.get("target_priority", []) + if tmp: + query_params["target_priority"] = ",".join(tmp) + tmp = self._query_builder._collapse() + if tmp: + query_params["query_string"] = tmp + if self._sortcriteria: + query_params["sort_field"] = self._sortcriteria["field"] + query_params["sort_order"] = self._sortcriteria["order"] + url = self._build_url("/_search/download") + # AGRB 10/3/2019 - Header is TEMPORARY until bug is fixed in API. Remove when fix deployed. + return self._cb.get_raw_data(url, query_params, headers={"Content-Type": "application/json"}) + + def _bulk_device_action(self, action_type, options=None): + """ + Perform a bulk action on all devices matching the current search criteria. + + :param str action_type: The action type to be performed. + :param dict options: Options for the bulk device action. Default None. + """ + request = {"action_type": action_type, "search": self._build_request(0, -1)} + if options: + request["options"] = options + return self._cb._raw_device_action(request) + + def background_scan(self, scan): + """ + Set the background scan option for the specified devices. + + :param boolean scan: True to turn background scan on, False to turn it off. + """ + return self._bulk_device_action("BACKGROUND_SCAN", self._cb._action_toggle(scan)) + + def bypass(self, enable): + """ + Set the bypass option for the specified devices. + + :param boolean enable: True to enable bypass, False to disable it. + """ + return self._bulk_device_action("BYPASS", self._cb._action_toggle(enable)) + + def delete_sensor(self): + """ + Delete the specified sensor devices. + """ + return self._bulk_device_action("DELETE_SENSOR") + + def uninstall_sensor(self): + """ + Uninstall the specified sensor devices. + """ + return self._bulk_device_action("UNINSTALL_SENSOR") + + def quarantine(self, enable): + """ + Set the quarantine option for the specified devices. + + :param boolean enable: True to enable quarantine, False to disable it. + """ + return self._bulk_device_action("QUARANTINE", self._cb._action_toggle(enable)) + + def update_policy(self, policy_id): + """ + Set the current policy for the specified devices. + + :param int policy_id: ID of the policy to set for the devices. + """ + return self._bulk_device_action("UPDATE_POLICY", {"policy_id": policy_id}) + + def update_sensor_version(self, sensor_version): + """ + Update the sensor version for the specified devices. + + :param dict sensor_version: New version properties for the sensor. + """ + return self._bulk_device_action("UPDATE_SENSOR_VERSION", {"sensor_version": sensor_version}) diff --git a/src/cbapi/psc/livequery/query.py b/src/cbapi/psc/livequery/query.py index 6fbf9f17..f4a5a05e 100644 --- a/src/cbapi/psc/livequery/query.py +++ b/src/cbapi/psc/livequery/query.py @@ -1,6 +1,6 @@ from cbapi.errors import ApiError -from cbapi.psc.query import QueryBuilder, PSCQueryBase -from cbapi.psc.query import QueryBuilderSupportMixin, IterableQueryMixin +from cbapi.psc.base_query import QueryBuilder, PSCQueryBase +from cbapi.psc.base_query import QueryBuilderSupportMixin, IterableQueryMixin import logging from six import string_types diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 958184ef..50805d4b 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -1,7 +1,8 @@ from cbapi.models import MutableBaseModel, UnrefreshableModel from cbapi.errors import ServerError -from cbapi.psc.query import DeviceSearchQuery, BaseAlertSearchQuery, WatchlistAlertSearchQuery, \ - CBAnalyticsAlertSearchQuery, VMwareAlertSearchQuery +from cbapi.psc.devices_query import DeviceSearchQuery +from cbapi.psc.alerts_query import BaseAlertSearchQuery, WatchlistAlertSearchQuery, \ + CBAnalyticsAlertSearchQuery, VMwareAlertSearchQuery from copy import deepcopy import logging @@ -208,7 +209,7 @@ class BaseAlert(PSCMutableModel): def __init__(self, cb, model_unique_id, initial_data=None): super(BaseAlert, self).__init__(cb, model_unique_id, initial_data) - self._workflow = Workflow(cb, initial_data.get("workflow", None)) + self._workflow = Workflow(cb, initial_data.get("workflow", None) if initial_data else None) if model_unique_id is not None and initial_data is None: self._refresh() @@ -229,6 +230,13 @@ def workflow_(self): return self._workflow def _update_workflow_status(self, state, remediation, comment): + """ + Update the workflow status of this alert. + + :param str state: The state to set for this alert, either "OPEN" or "DISMISSED". + :param remediation str: The remediation status to set for the alert. + :param comment str: The comment to set for the alert. + """ request = {"state": state} if remediation: request["remediation_state"] = remediation @@ -259,6 +267,13 @@ def update(self, remediation=None, comment=None): self._update_workflow_status("OPEN", remediation, comment) def _update_threat_workflow_status(self, state, remediation, comment): + """ + Update the workflow status of all alerts with the same threat ID, past or future. + + :param str state: The state to set for this alert, either "OPEN" or "DISMISSED". + :param remediation str: The remediation status to set for the alert. + :param comment str: The comment to set for the alert. + """ request = {"state": state} if remediation: request["remediation_state"] = remediation diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index ded3a9e2..aa12f794 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -41,6 +41,13 @@ def _request_lr_session(self, sensor_id): # ---- Device API def _raw_device_action(self, request): + """ + Invokes the API method for a device action. + + :param dict request: The request body to be passed as JSON to the API method. + :return: The parsed JSON output from the request. + :raises ServerError: If the API method returns an HTTP error code. + """ url = "/appservices/v6/orgs/{0}/device_actions".format(self.credentials.org_key) resp = self.post_object(url, body=request) if resp.status_code == 200: @@ -51,34 +58,47 @@ def _raw_device_action(self, request): raise ServerError(error_code=resp.status_code, message="Device action error: {0}".format(resp.content)) def _device_action(self, device_ids, action_type, options=None): + """ + Executes a device action on multiple device IDs. + + :param list device_ids: The list of device IDs to execute the action on. + :param str action_type: The action type to be performed. + :param dict options: Options for the bulk device action. Default None. + """ request = {"action_type": action_type, "device_id": device_ids} if options: request["options"] = options return self._raw_device_action(request) def _action_toggle(self, flag): + """ + Converts a boolean flag value into a "toggle" option. + + :param boolean flag: The value to be converted. + :return: A dict containing the appropriate "toggle" element. + """ if flag: return {"toggle": "ON"} else: return {"toggle": "OFF"} - def device_background_scan(self, device_ids, flag): + def device_background_scan(self, device_ids, scan): """ Set the background scan option for the specified devices. :param list device_ids: List of IDs of devices to be set. - :param boolean flag: True to turn background scan on, False to turn it off. + :param boolean scan: True to turn background scan on, False to turn it off. """ - return self._device_action(device_ids, "BACKGROUND_SCAN", self._action_toggle(flag)) + return self._device_action(device_ids, "BACKGROUND_SCAN", self._action_toggle(scan)) - def device_bypass(self, device_ids, flag): + def device_bypass(self, device_ids, enable): """ Set the bypass option for the specified devices. :param list device_ids: List of IDs of devices to be set. - :param boolean flag: True to enable bypass, False to disable it. + :param boolean enable: True to enable bypass, False to disable it. """ - return self._device_action(device_ids, "BYPASS", self._action_toggle(flag)) + return self._device_action(device_ids, "BYPASS", self._action_toggle(enable)) def device_delete_sensor(self, device_ids): """ @@ -96,14 +116,14 @@ def device_uninstall_sensor(self, device_ids): """ return self._device_action(device_ids, "UNINSTALL_SENSOR") - def device_quarantine(self, device_ids, flag): + def device_quarantine(self, device_ids, enable): """ Set the quarantine option for the specified devices. :param list device_ids: List of IDs of devices to be set. - :param boolean flag: True to enable quarantine, False to disable it. + :param boolean enable: True to enable quarantine, False to disable it. """ - return self._device_action(device_ids, "QUARANTINE", self._action_toggle(flag)) + return self._device_action(device_ids, "QUARANTINE", self._action_toggle(enable)) def device_update_policy(self, device_ids, policy_id): """ @@ -138,6 +158,14 @@ def alert_search_suggestions(self, query): return output["suggestions"] def _bulk_threat_update_status(self, threat_ids, status, remediation, comment): + """ + Update the status of alerts associated with multiple threat IDs, past and future. + + :param list threat_ids: List of string threat IDs. + :param str status: The status to set for all alerts, either "OPEN" or "DISMISSED". + :param str remediation: The remediation state to set for all alerts. + :param str comment: The comment to set for all alerts. + """ if not all(isinstance(t, str) for t in threat_ids): raise ApiError("One or more invalid threat ID values") request = {"state": status, "threat_id": threat_ids} diff --git a/test/cbapi/psc/livequery/test_models.py b/test/cbapi/psc/livequery/test_models.py index c172a87f..cc227c2e 100755 --- a/test/cbapi/psc/livequery/test_models.py +++ b/test/cbapi/psc/livequery/test_models.py @@ -3,29 +3,21 @@ from cbapi.psc.livequery.models import Run, Result from cbapi.psc.livequery.query import ResultQuery, FacetQuery from cbapi.errors import ApiError -from test.mocks import MockResponse, ConnectionMocks +from test.cbtest import StubResponse, patch_cbapi def test_run_refresh(monkeypatch): _was_called = False - def mock_get_object(url, parms=None, default=None): + def _get_run(url, parms=None, default=None): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" - assert parms is None - assert default is None _was_called = True - return {"org_key": "Z100", "name": "FoobieBletch", - "id": "abcdefg", "status": "COMPLETE"} - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", - "id": "abcdefg", "status": "ACTIVE"}) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + return {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "COMPLETE"} + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_run) + run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "ACTIVE"}) rc = run.refresh() assert _was_called assert rc @@ -38,22 +30,16 @@ def mock_get_object(url, parms=None, default=None): def test_run_stop(monkeypatch): _was_called = False - def mock_put_object(url, body, **kwargs): + def _execute_stop(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/status" - assert body["status"] == "CANCELLED" + assert body == {"status": "CANCELLED"} _was_called = True - return MockResponse({"org_key": "Z100", "name": "FoobieBletch", - "id": "abcdefg", "status": "CANCELLED"}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", - "id": "abcdefg", "status": "ACTIVE"}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", mock_put_object) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + return StubResponse({"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "CANCELLED"}) + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, PUT=_execute_stop) + run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "ACTIVE"}) rc = run.stop() assert _was_called assert rc @@ -66,21 +52,16 @@ def mock_put_object(url, body, **kwargs): def test_run_stop_failed(monkeypatch): _was_called = False - def mock_put_object(url, body, **kwargs): + def _execute_stop(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/status" - assert body["status"] == "CANCELLED" + assert body == {"status": "CANCELLED"} _was_called = True - return MockResponse({"error_message": "The query is not presently running."}, 409) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", - "id": "abcdefg", "status": "CANCELLED"}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", mock_put_object) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + return StubResponse({"error_message": "The query is not presently running."}, 409) + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, PUT=_execute_stop) + run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "CANCELLED"}) rc = run.stop() assert _was_called assert not rc @@ -89,22 +70,17 @@ def mock_put_object(url, body, **kwargs): def test_run_delete(monkeypatch): _was_called = False - def mock_delete_object(url): + def _execute_delete(url): nonlocal _was_called - if _was_called: - pytest.fail("delete should not be called twice!") assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" + if _was_called: + pytest.fail("_execute_delete should not be called twice!") _was_called = True - return MockResponse(None) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", - "id": "abcdefg", "status": "ACTIVE"}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", mock_delete_object) + return StubResponse(None) + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, DELETE=_execute_delete) + run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "ACTIVE"}) rc = run.delete() assert _was_called assert rc @@ -122,20 +98,15 @@ def mock_delete_object(url): def test_run_delete_failed(monkeypatch): _was_called = False - def mock_delete_object(url): + def _execute_delete(url): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" _was_called = True - return MockResponse(None, 403) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", - "id": "abcdefg", "status": "ACTIVE"}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", mock_delete_object) + return StubResponse(None, 403) + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, DELETE=_execute_delete) + run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "ACTIVE"}) rc = run.delete() assert _was_called assert not rc @@ -145,31 +116,22 @@ def mock_delete_object(url): def test_result_device_summaries(monkeypatch): _was_called = False - def mock_post_object(url, body, **kwargs): + def _run_summaries(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/results/device_summaries/_search" - assert body["query"] == "foo" - t = body["criteria"] - assert t["device_name"] == ["AxCx", "A7X"] - t = body["sort"][0] - assert t["field"] == "device_name" - assert t["order"] == "ASC" + assert body == {"query": "foo", "criteria": {"device_name": ["AxCx", "A7X"]}, + "sort": [{"field": "device_name", "order": "ASC"}], "start": 0} _was_called = True - metrics = [{"key": "aaa", "value": 0.0}, {"key": "bbb", "value": 0.0}] - res1 = {"id": "ghijklm", "total_results": 2, "device_id": 314159, "metrics": metrics} - res2 = {"id": "mnopqrs", "total_results": 3, "device_id": 271828, "metrics": metrics} - return MockResponse({"org_key": "Z100", "num_found": 2, "results": [res1, res2]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - tmp_id = {"id": "abcdefg"} - result = Result(api, {"id": "abcdefg", "device": tmp_id, "fields": {}, "metrics": {}}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = result.query_device_summaries().where("foo").criteria(device_name=["AxCx", "A7X"]) - query = query.sort_by("device_name") + return StubResponse({"org_key": "Z100", "num_found": 2, + "results": [{"id": "ghijklm", "total_results": 2, "device_id": 314159, + "metrics": [{"key": "aaa", "value": 0.0}, {"key": "bbb", "value": 0.0}]}, + {"id": "mnopqrs", "total_results": 3, "device_id": 271828, + "metrics": [{"key": "aaa", "value": 0.0}, {"key": "bbb", "value": 0.0}]}]}) + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_summaries) + result = Result(api, {"id": "abcdefg", "device": {"id": "abcdefg"}, "fields": {}, "metrics": {}}) + query = result.query_device_summaries().where("foo").criteria(device_name=["AxCx", "A7X"]).sort_by("device_name") assert isinstance(query, ResultQuery) count = 0 for item in query.all(): @@ -189,37 +151,26 @@ def mock_post_object(url, body, **kwargs): def test_result_query_result_facets(monkeypatch): _was_called = False - def mock_post_object(url, body, **kwargs): + def _run_facets(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/results/_facet" - assert body["query"] == "xyzzy" - t = body["criteria"] - assert t["device_name"] == ["AxCx", "A7X"] - t = body["terms"] - assert t["fields"] == ["alpha", "bravo", "charlie"] + assert body == {"query": "xyzzy", "criteria": {"device_name": ["AxCx", "A7X"]}, + "terms": {"fields": ["alpha", "bravo", "charlie"]}} _was_called = True - v1 = {"total": 1, "id": "alpha1", "name": "alpha1"} - v2 = {"total": 2, "id": "alpha2", "name": "alpha2"} - term1 = {"field": "alpha", "values": [v1, v2]} - v1 = {"total": 1, "id": "bravo1", "name": "bravo1"} - v2 = {"total": 2, "id": "bravo2", "name": "bravo2"} - term2 = {"field": "bravo", "values": [v1, v2]} - v1 = {"total": 1, "id": "charlie1", "name": "charlie1"} - v2 = {"total": 2, "id": "charlie2", "name": "charlie2"} - term3 = {"field": "charlie", "values": [v1, v2]} - return MockResponse({"terms": [term1, term2, term3]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - tmp_id = {"id": "abcdefg"} - result = Result(api, {"id": "abcdefg", "device": tmp_id, "fields": {}, "metrics": {}}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = result.query_result_facets().where("xyzzy") - query = query.facet_field("alpha").facet_field(["bravo", "charlie"]) - query = query.criteria(device_name=["AxCx", "A7X"]) + return StubResponse({"terms": [{"field": "alpha", "values": [{"total": 1, "id": "alpha1", "name": "alpha1"}, + {"total": 2, "id": "alpha2", "name": "alpha2"}]}, + {"field": "bravo", "values": [{"total": 1, "id": "bravo1", "name": "bravo1"}, + {"total": 2, "id": "bravo2", "name": "bravo2"}]}, + {"field": "charlie", "values": [{"total": 1, "id": "charlie1", + "name": "charlie1"}, + {"total": 2, "id": "charlie2", + "name": "charlie2"}]}]}) + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_facets) + result = Result(api, {"id": "abcdefg", "device": {"id": "abcdefg"}, "fields": {}, "metrics": {}}) + query = result.query_result_facets().where("xyzzy").facet_field("alpha").facet_field(["bravo", "charlie"]) \ + .criteria(device_name=["AxCx", "A7X"]) assert isinstance(query, FacetQuery) count = 0 for item in query.all(): @@ -243,37 +194,26 @@ def mock_post_object(url, body, **kwargs): def test_result_query_device_summary_facets(monkeypatch): _was_called = False - def mock_post_object(url, body, **kwargs): + def _run_facets(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/results/device_summaries/_facet" - assert body["query"] == "xyzzy" - t = body["criteria"] - assert t["device_name"] == ["AxCx", "A7X"] - t = body["terms"] - assert t["fields"] == ["alpha", "bravo", "charlie"] + assert body == {"query": "xyzzy", "criteria": {"device_name": ["AxCx", "A7X"]}, + "terms": {"fields": ["alpha", "bravo", "charlie"]}} _was_called = True - v1 = {"total": 1, "id": "alpha1", "name": "alpha1"} - v2 = {"total": 2, "id": "alpha2", "name": "alpha2"} - term1 = {"field": "alpha", "values": [v1, v2]} - v1 = {"total": 1, "id": "bravo1", "name": "bravo1"} - v2 = {"total": 2, "id": "bravo2", "name": "bravo2"} - term2 = {"field": "bravo", "values": [v1, v2]} - v1 = {"total": 1, "id": "charlie1", "name": "charlie1"} - v2 = {"total": 2, "id": "charlie2", "name": "charlie2"} - term3 = {"field": "charlie", "values": [v1, v2]} - return MockResponse({"terms": [term1, term2, term3]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - tmp_id = {"id": "abcdefg"} - result = Result(api, {"id": "abcdefg", "device": tmp_id, "fields": {}, "metrics": {}}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = result.query_device_summary_facets().where("xyzzy") - query = query.facet_field("alpha").facet_field(["bravo", "charlie"]) - query = query.criteria(device_name=["AxCx", "A7X"]) + return StubResponse({"terms": [{"field": "alpha", "values": [{"total": 1, "id": "alpha1", "name": "alpha1"}, + {"total": 2, "id": "alpha2", "name": "alpha2"}]}, + {"field": "bravo", "values": [{"total": 1, "id": "bravo1", "name": "bravo1"}, + {"total": 2, "id": "bravo2", "name": "bravo2"}]}, + {"field": "charlie", "values": [{"total": 1, "id": "charlie1", + "name": "charlie1"}, + {"total": 2, "id": "charlie2", + "name": "charlie2"}]}]}) + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_facets) + result = Result(api, {"id": "abcdefg", "device": {"id": "abcdefg"}, "fields": {}, "metrics": {}}) + query = result.query_device_summary_facets().where("xyzzy").facet_field("alpha") \ + .facet_field(["bravo", "charlie"]).criteria(device_name=["AxCx", "A7X"]) assert isinstance(query, FacetQuery) count = 0 for item in query.all(): diff --git a/test/cbapi/psc/livequery/test_rest_api.py b/test/cbapi/psc/livequery/test_rest_api.py index 05458e3b..b9d38a17 100755 --- a/test/cbapi/psc/livequery/test_rest_api.py +++ b/test/cbapi/psc/livequery/test_rest_api.py @@ -3,32 +3,25 @@ from cbapi.psc.livequery.models import Run from cbapi.psc.livequery.query import RunQuery, RunHistoryQuery from cbapi.errors import ApiError, CredentialError -from test.mocks import MockResponse, ConnectionMocks +from test.cbtest import StubResponse, patch_cbapi def test_no_org_key(): with pytest.raises(CredentialError): - CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - ssl_verify=True) # note: no org_key + CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", ssl_verify=True) # note: no org_key def test_simple_get(monkeypatch): _was_called = False - def mock_get_object(url, parms=None, default=None): + def _get_run(url, parms=None, default=None): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" - assert parms is None - assert default is None _was_called = True return {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"} - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_run) run = api.select(Run, "abcdefg") assert _was_called assert run.org_key == "Z100" @@ -39,19 +32,15 @@ def mock_get_object(url, parms=None, default=None): def test_query(monkeypatch): _was_called = False - def mock_post_object(url, body, **kwargs): + def _run_query(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs" - assert body["sql"] == "select * from whatever;" + assert body == {"sql": "select * from whatever;", "device_filter": {}} _was_called = True - return MockResponse({"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + return StubResponse({"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}) + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) query = api.query("select * from whatever;") assert isinstance(query, RunQuery) run = query.submit() @@ -64,29 +53,19 @@ def mock_post_object(url, body, **kwargs): def test_query_with_everything(monkeypatch): _was_called = False - def mock_post_object(url, body, **kwargs): + def _run_query(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs" - assert body["sql"] == "select * from whatever;" - assert body["name"] == "AmyWasHere" - assert body["notify_on_finish"] - df = body["device_filter"] - assert df["device_ids"] == [1, 2, 3] - assert df["device_types"] == ["Alpha", "Bravo", "Charlie"] - assert df["policy_ids"] == [16, 27, 38] + assert body == {"sql": "select * from whatever;", "name": "AmyWasHere", "notify_on_finish": True, + "device_filter": {"device_ids": [1, 2, 3], "device_types": ["Alpha", "Bravo", "Charlie"], + "policy_ids": [16, 27, 38]}} _was_called = True - return MockResponse({"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.query("select * from whatever;").device_ids([1, 2, 3]) - query = query.device_types(["Alpha", "Bravo", "Charlie"]) - query = query.policy_ids([16, 27, 38]) - query = query.name("AmyWasHere").notify_on_finish() + return StubResponse({"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}) + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) + query = api.query("select * from whatever;").device_ids([1, 2, 3]).device_types(["Alpha", "Bravo", "Charlie"]) \ + .policy_ids([16, 27, 38]).name("AmyWasHere").notify_on_finish() assert isinstance(query, RunQuery) run = query.submit() assert _was_called @@ -96,24 +75,21 @@ def mock_post_object(url, body, **kwargs): def test_query_device_ids_broken(): - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) query = api.query("select * from whatever;") with pytest.raises(ApiError): query = query.device_ids(["Bogus"]) def test_query_device_types_broken(): - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) query = api.query("select * from whatever;") with pytest.raises(ApiError): query = query.device_types([420]) def test_query_policy_ids_broken(): - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) query = api.query("select * from whatever;") with pytest.raises(ApiError): query = query.policy_ids(["Bogus"]) @@ -122,22 +98,18 @@ def test_query_policy_ids_broken(): def test_query_history(monkeypatch): _was_called = False - def mock_post_object(url, body, **kwargs): + def _run_query(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/_search" - assert body["query"] == "xyzzy" + assert body == {"query": "xyzzy", "start": 0} _was_called = True - run1 = {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"} - run2 = {"org_key": "Z100", "name": "Aoxomoxoa", "id": "cdefghi"} - run3 = {"org_key": "Z100", "name": "Read_Me", "id": "efghijk"} - return MockResponse({"org_key": "Z100", "num_found": 3, "results": [run1, run2, run3]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + return StubResponse({"org_key": "Z100", "num_found": 3, + "results": [{"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}, + {"org_key": "Z100", "name": "Aoxomoxoa", "id": "cdefghi"}, + {"org_key": "Z100", "name": "Read_Me", "id": "efghijk"}]}) + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) query = api.query_history("xyzzy") assert isinstance(query, RunHistoryQuery) count = 0 @@ -159,25 +131,18 @@ def mock_post_object(url, body, **kwargs): def test_query_history_with_everything(monkeypatch): _was_called = False - def mock_post_object(url, body, **kwargs): + def _run_query(url, body, **kwargs): nonlocal _was_called assert url == "/livequery/v1/orgs/Z100/runs/_search" - assert body["query"] == "xyzzy" - t = body["sort"][0] - assert t["field"] == "id" - assert t["order"] == "ASC" + assert body == {"query": "xyzzy", "sort": [{"field": "id", "order": "ASC"}], "start": 0} _was_called = True - run1 = {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"} - run2 = {"org_key": "Z100", "name": "Aoxomoxoa", "id": "cdefghi"} - run3 = {"org_key": "Z100", "name": "Read_Me", "id": "efghijk"} - return MockResponse({"org_key": "Z100", "num_found": 3, "results": [run1, run2, run3]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + return StubResponse({"org_key": "Z100", "num_found": 3, + "results": [{"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}, + {"org_key": "Z100", "name": "Aoxomoxoa", "id": "cdefghi"}, + {"org_key": "Z100", "name": "Read_Me", "id": "efghijk"}]}) + + api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) query = api.query_history("xyzzy").sort_by("id") assert isinstance(query, RunHistoryQuery) count = 0 diff --git a/test/cbapi/psc/test_alertsv6_api.py b/test/cbapi/psc/test_alertsv6_api.py new file mode 100755 index 00000000..c787bba5 --- /dev/null +++ b/test/cbapi/psc/test_alertsv6_api.py @@ -0,0 +1,535 @@ +import pytest +from cbapi.errors import ApiError +from cbapi.psc.models import BaseAlert, CBAnalyticsAlert, VMwareAlert, WatchlistAlert, WorkflowStatus +from cbapi.psc.rest_api import CbPSCBaseAPI +from test.cbtest import StubResponse, patch_cbapi + + +def test_query_basealert_with_all_bells_and_whistles(monkeypatch): + _was_called = False + + def _run_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/_search" + assert body == {"query": "Blort", + "criteria": {"category": ["SERIOUS", "CRITICAL"], "device_id": [6023], "device_name": ["HAL"], + "device_os": ["LINUX"], "device_os_version": ["0.1.2"], + "device_username": ["JRN"], "group_results": True, "id": ["S0L0"], + "legacy_alert_id": ["S0L0_1"], "minimum_severity": 6, "policy_id": [8675309], + "policy_name": ["Strict"], "process_name": ["IEXPLORE.EXE"], + "process_sha256": ["0123456789ABCDEF0123456789ABCDEF"], + "reputation": ["SUSPECT_MALWARE"], "tag": ["Frood"], "target_value": ["HIGH"], + "threat_id": ["B0RG"], "type": ["WATCHLIST"], "workflow": ["OPEN"]}, + "sort": [{"field": "name", "order": "DESC"}]} + _was_called = True + return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", + "workflow": {"state": "OPEN"}}], "num_found": 1}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) + query = api.select(BaseAlert).where("Blort").set_categories(["SERIOUS", "CRITICAL"]).set_device_ids([6023]) \ + .set_device_names(["HAL"]).set_device_os(["LINUX"]).set_device_os_versions(["0.1.2"]) \ + .set_device_username(["JRN"]).set_group_results(True).set_alert_ids(["S0L0"]) \ + .set_legacy_alert_ids(["S0L0_1"]).set_minimum_severity(6).set_policy_ids([8675309]) \ + .set_policy_names(["Strict"]).set_process_names(["IEXPLORE.EXE"]) \ + .set_process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).set_reputations(["SUSPECT_MALWARE"]) \ + .set_tags(["Frood"]).set_target_priorities(["HIGH"]).set_threat_ids(["B0RG"]).set_types(["WATCHLIST"]) \ + .set_workflows(["OPEN"]).sort_by("name", "DESC") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_basealert_with_create_time_as_start_end(monkeypatch): + _was_called = False + + def _run_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/_search" + assert body == {"query": "Blort", + "criteria": {"create_time": {"start": "2019-09-30T12:34:56", "end": "2019-10-01T12:00:12"}}} + _was_called = True + return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", + "workflow": {"state": "OPEN"}}], "num_found": 1}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) + query = api.select(BaseAlert).where("Blort").set_create_time(start="2019-09-30T12:34:56", + end="2019-10-01T12:00:12") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_basealert_with_create_time_as_range(monkeypatch): + _was_called = False + + def _run_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/_search" + assert body == {"query": "Blort", "criteria": {"create_time": {"range": "-3w"}}} + _was_called = True + return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", + "workflow": {"state": "OPEN"}}], "num_found": 1}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) + query = api.select(BaseAlert).where("Blort").set_create_time(range="-3w") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_basealert_facets(monkeypatch): + _was_called = False + + def _run_facet_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/_facet" + assert body["query"] == "Blort" + t = body["criteria"] + assert t["workflow"] == ["OPEN"] + t = body["terms"] + assert t["rows"] == 0 + assert t["fields"] == ["REPUTATION", "STATUS"] + _was_called = True + return StubResponse({"results": [{"field": {}, + "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, + {"field": {}, + "values": [{"id": "status", "name": "statusX", "total": 9}]}]}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_facet_query) + query = api.select(BaseAlert).where("Blort").set_workflows(["OPEN"]) + f = query.facets(["REPUTATION", "STATUS"]) + assert _was_called + assert f == [{"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, + {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]}] + + +def test_query_basealert_invalid_create_time_combinations(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(BaseAlert).set_create_time() + with pytest.raises(ApiError): + api.select(BaseAlert).set_create_time(start="2019-09-30T12:34:56", + end="2019-10-01T12:00:12", range="-3w") + with pytest.raises(ApiError): + api.select(BaseAlert).set_create_time(start="2019-09-30T12:34:56", range="-3w") + with pytest.raises(ApiError): + api.select(BaseAlert).set_create_time(end="2019-10-01T12:00:12", range="-3w") + + +def test_query_basealert_invalid_criteria_values(): + tests = [ + {"method": "set_categories", "arg": ["DOUBLE_DARE"]}, + {"method": "set_device_ids", "arg": ["Bogus"]}, + {"method": "set_device_names", "arg": [42]}, + {"method": "set_device_os", "arg": ["TI994A"]}, + {"method": "set_device_os_versions", "arg": [8808]}, + {"method": "set_device_username", "arg": [-1]}, + {"method": "set_alert_ids", "arg": [9001]}, + {"method": "set_legacy_alert_ids", "arg": [9001]}, + {"method": "set_policy_ids", "arg": ["Bogus"]}, + {"method": "set_policy_names", "arg": [323]}, + {"method": "set_process_names", "arg": [7071]}, + {"method": "set_process_sha256", "arg": [123456789]}, + {"method": "set_reputations", "arg": ["MICROSOFT_FUDWARE"]}, + {"method": "set_tags", "arg": [-1]}, + {"method": "set_target_priorities", "arg": ["DOGWASH"]}, + {"method": "set_threat_ids", "arg": [4096]}, + {"method": "set_types", "arg": ["ERBOSOFT"]}, + {"method": "set_workflows", "arg": ["IN_LIMBO"]}, + ] + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + query = api.select(BaseAlert) + for t in tests: + meth = getattr(query, t["method"], None) + with pytest.raises(ApiError): + meth(t["arg"]) + + +def test_query_cbanalyticsalert_with_all_bells_and_whistles(monkeypatch): + _was_called = False + + def _run_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/_search" + assert body == {"query": "Blort", + "criteria": {"category": ["SERIOUS", "CRITICAL"], "device_id": [6023], "device_name": ["HAL"], + "device_os": ["LINUX"], "device_os_version": ["0.1.2"], + "device_username": ["JRN"], "group_results": True, "id": ["S0L0"], + "legacy_alert_id": ["S0L0_1"], "minimum_severity": 6, "policy_id": [8675309], + "policy_name": ["Strict"], "process_name": ["IEXPLORE.EXE"], + "process_sha256": ["0123456789ABCDEF0123456789ABCDEF"], + "reputation": ["SUSPECT_MALWARE"], "tag": ["Frood"], "target_value": ["HIGH"], + "threat_id": ["B0RG"], "type": ["WATCHLIST"], "workflow": ["OPEN"], + "blocked_threat_category": ["RISKY_PROGRAM"], "device_location": ["ONSITE"], + "kill_chain_status": ["EXECUTE_GOAL"], + "not_blocked_threat_category": ["NEW_MALWARE"], "policy_applied": ["APPLIED"], + "reason_code": ["ATTACK_VECTOR"], "run_state": ["RAN"], "sensor_action": ["DENY"], + "threat_cause_vector": ["WEB"]}, "sort": [{"field": "name", "order": "DESC"}]} + _was_called = True + return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", + "workflow": {"state": "OPEN"}}], "num_found": 1}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) + query = api.select(CBAnalyticsAlert).where("Blort").set_categories(["SERIOUS", "CRITICAL"]) \ + .set_device_ids([6023]).set_device_names(["HAL"]).set_device_os(["LINUX"]).set_device_os_versions(["0.1.2"]) \ + .set_device_username(["JRN"]).set_group_results(True).set_alert_ids(["S0L0"]).set_legacy_alert_ids(["S0L0_1"]) \ + .set_minimum_severity(6).set_policy_ids([8675309]).set_policy_names(["Strict"]) \ + .set_process_names(["IEXPLORE.EXE"]).set_process_sha256(["0123456789ABCDEF0123456789ABCDEF"]) \ + .set_reputations(["SUSPECT_MALWARE"]).set_tags(["Frood"]).set_target_priorities(["HIGH"]) \ + .set_threat_ids(["B0RG"]).set_types(["WATCHLIST"]).set_workflows(["OPEN"]) \ + .set_blocked_threat_categories(["RISKY_PROGRAM"]).set_device_locations(["ONSITE"]) \ + .set_kill_chain_statuses(["EXECUTE_GOAL"]).set_not_blocked_threat_categories(["NEW_MALWARE"]) \ + .set_policy_applied(["APPLIED"]).set_reason_code(["ATTACK_VECTOR"]).set_run_states(["RAN"]) \ + .set_sensor_actions(["DENY"]).set_threat_cause_vectors(["WEB"]).sort_by("name", "DESC") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_cbanalyticsalert_facets(monkeypatch): + _was_called = False + + def _run_facet_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/_facet" + assert body == {"query": "Blort", "criteria": {"workflow": ["OPEN"]}, + "terms": {"rows": 0, "fields": ["REPUTATION", "STATUS"]}} + _was_called = True + return StubResponse({"results": [{"field": {}, + "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, + {"field": {}, + "values": [{"id": "status", "name": "statusX", "total": 9}]}]}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_facet_query) + query = api.select(CBAnalyticsAlert).where("Blort").set_workflows(["OPEN"]) + f = query.facets(["REPUTATION", "STATUS"]) + assert _was_called + assert f == [{"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, + {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]}] + + +def test_query_cbanalyticsalert_invalid_criteria_values(): + tests = [ + {"method": "set_blocked_threat_categories", "arg": ["MINOR"]}, + {"method": "set_device_locations", "arg": ["NARNIA"]}, + {"method": "set_kill_chain_statuses", "arg": ["SPAWN_COPIES"]}, + {"method": "set_not_blocked_threat_categories", "arg": ["MINOR"]}, + {"method": "set_policy_applied", "arg": ["MAYBE"]}, + {"method": "set_reason_code", "arg": [55]}, + {"method": "set_run_states", "arg": ["MIGHT_HAVE"]}, + {"method": "set_sensor_actions", "arg": ["FLIP_A_COIN"]}, + {"method": "set_threat_cause_vectors", "arg": ["NETWORK"]} + ] + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + query = api.select(CBAnalyticsAlert) + for t in tests: + meth = getattr(query, t["method"], None) + with pytest.raises(ApiError): + meth(t["arg"]) + + +def test_query_vmwarealert_with_all_bells_and_whistles(monkeypatch): + _was_called = False + + def _run_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/vmware/_search" + assert body == {"query": "Blort", + "criteria": {"category": ["SERIOUS", "CRITICAL"], "device_id": [6023], "device_name": ["HAL"], + "device_os": ["LINUX"], "device_os_version": ["0.1.2"], + "device_username": ["JRN"], "group_results": True, "id": ["S0L0"], + "legacy_alert_id": ["S0L0_1"], "minimum_severity": 6, "policy_id": [8675309], + "policy_name": ["Strict"], "process_name": ["IEXPLORE.EXE"], + "process_sha256": ["0123456789ABCDEF0123456789ABCDEF"], + "reputation": ["SUSPECT_MALWARE"], "tag": ["Frood"], "target_value": ["HIGH"], + "threat_id": ["B0RG"], "type": ["WATCHLIST"], "workflow": ["OPEN"], + "group_id": [14]}, "sort": [{"field": "name", "order": "DESC"}]} + _was_called = True + return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", + "workflow": {"state": "OPEN"}}], "num_found": 1}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) + query = api.select(VMwareAlert).where("Blort").set_categories(["SERIOUS", "CRITICAL"]).set_device_ids([6023]) \ + .set_device_names(["HAL"]).set_device_os(["LINUX"]).set_device_os_versions(["0.1.2"]) \ + .set_device_username(["JRN"]).set_group_results(True).set_alert_ids(["S0L0"]) \ + .set_legacy_alert_ids(["S0L0_1"]).set_minimum_severity(6).set_policy_ids([8675309]) \ + .set_policy_names(["Strict"]).set_process_names(["IEXPLORE.EXE"]) \ + .set_process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).set_reputations(["SUSPECT_MALWARE"]) \ + .set_tags(["Frood"]).set_target_priorities(["HIGH"]).set_threat_ids(["B0RG"]).set_types(["WATCHLIST"]) \ + .set_workflows(["OPEN"]).set_group_ids([14]).sort_by("name", "DESC") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_vmwarealert_facets(monkeypatch): + _was_called = False + + def _run_facet_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/vmware/_facet" + assert body == {"query": "Blort", "criteria": {"workflow": ["OPEN"]}, + "terms": {"rows": 0, "fields": ["REPUTATION", "STATUS"]}} + _was_called = True + return StubResponse({"results": [{"field": {}, + "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, + {"field": {}, + "values": [{"id": "status", "name": "statusX", "total": 9}]}]}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_facet_query) + query = api.select(VMwareAlert).where("Blort").set_workflows(["OPEN"]) + f = query.facets(["REPUTATION", "STATUS"]) + assert _was_called + assert f == [{"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, + {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]}] + + +def test_query_vmwarealert_invalid_group_ids(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(VMwareAlert).set_group_ids(["Bogus"]) + + +def test_query_watchlistalert_with_all_bells_and_whistles(monkeypatch): + _was_called = False + + def _run_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/_search" + assert body == {"query": "Blort", + "criteria": {"category": ["SERIOUS", "CRITICAL"], "device_id": [6023], "device_name": ["HAL"], + "device_os": ["LINUX"], "device_os_version": ["0.1.2"], + "device_username": ["JRN"], "group_results": True, "id": ["S0L0"], + "legacy_alert_id": ["S0L0_1"], "minimum_severity": 6, "policy_id": [8675309], + "policy_name": ["Strict"], "process_name": ["IEXPLORE.EXE"], + "process_sha256": ["0123456789ABCDEF0123456789ABCDEF"], + "reputation": ["SUSPECT_MALWARE"], "tag": ["Frood"], "target_value": ["HIGH"], + "threat_id": ["B0RG"], "type": ["WATCHLIST"], "workflow": ["OPEN"], + "watchlist_id": ["100"], "watchlist_name": ["Gandalf"]}, + "sort": [{"field": "name", "order": "DESC"}]} + _was_called = True + return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", + "workflow": {"state": "OPEN"}}], "num_found": 1}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) + query = api.select(WatchlistAlert).where("Blort").set_categories(["SERIOUS", "CRITICAL"]).set_device_ids([6023]) \ + .set_device_names(["HAL"]).set_device_os(["LINUX"]).set_device_os_versions(["0.1.2"]) \ + .set_device_username(["JRN"]).set_group_results(True).set_alert_ids(["S0L0"]) \ + .set_legacy_alert_ids(["S0L0_1"]).set_minimum_severity(6).set_policy_ids([8675309]) \ + .set_policy_names(["Strict"]).set_process_names(["IEXPLORE.EXE"]) \ + .set_process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).set_reputations(["SUSPECT_MALWARE"]) \ + .set_tags(["Frood"]).set_target_priorities(["HIGH"]).set_threat_ids(["B0RG"]).set_types(["WATCHLIST"]) \ + .set_workflows(["OPEN"]).set_watchlist_ids(["100"]).set_watchlist_names(["Gandalf"]).sort_by("name", "DESC") + a = query.one() + assert _was_called + assert a.id == "S0L0" + assert a.org_key == "Z100" + assert a.threat_id == "B0RG" + assert a.workflow_.state == "OPEN" + + +def test_query_watchlistalert_facets(monkeypatch): + _was_called = False + + def _run_facet_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/_facet" + assert body == {"query": "Blort", "criteria": {"workflow": ["OPEN"]}, + "terms": {"rows": 0, "fields": ["REPUTATION", "STATUS"]}} + _was_called = True + return StubResponse({"results": [{"field": {}, + "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, + {"field": {}, + "values": [{"id": "status", "name": "statusX", "total": 9}]}]}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_facet_query) + query = api.select(WatchlistAlert).where("Blort").set_workflows(["OPEN"]) + f = query.facets(["REPUTATION", "STATUS"]) + assert _was_called + assert f == [{"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, + {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]}] + + +def test_query_watchlistalert_invalid_criteria_values(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(WatchlistAlert).set_watchlist_ids([888]) + with pytest.raises(ApiError): + api.select(WatchlistAlert).set_watchlist_names([69]) + + +def test_alerts_bulk_dismiss(monkeypatch): + _was_called = False + + def _do_dismiss(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/workflow/_criteria" + assert body == {"query": "Blort", "state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir", + "criteria": {"device_name": ["HAL9000"]}} + _was_called = True + return StubResponse({"request_id": "497ABX"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_do_dismiss) + q = api.select(BaseAlert).where("Blort").set_device_names(["HAL9000"]) + reqid = q.dismiss("Fixed", "Yessir") + assert _was_called + assert reqid == "497ABX" + + +def test_alerts_bulk_undismiss(monkeypatch): + _was_called = False + + def _do_update(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/workflow/_criteria" + assert body == {"query": "Blort", "state": "OPEN", "remediation_state": "Fixed", "comment": "NoSir", + "criteria": {"device_name": ["HAL9000"]}} + _was_called = True + return StubResponse({"request_id": "497ABX"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_do_update) + q = api.select(BaseAlert).where("Blort").set_device_names(["HAL9000"]) + reqid = q.update("Fixed", "NoSir") + assert _was_called + assert reqid == "497ABX" + + +def test_alerts_bulk_dismiss_watchlist(monkeypatch): + _was_called = False + + def _do_dismiss(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/workflow/_criteria" + assert body == {"query": "Blort", "state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir", + "criteria": {"device_name": ["HAL9000"]}} + _was_called = True + return StubResponse({"request_id": "497ABX"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_do_dismiss) + q = api.select(WatchlistAlert).where("Blort").set_device_names(["HAL9000"]) + reqid = q.dismiss("Fixed", "Yessir") + assert _was_called + assert reqid == "497ABX" + + +def test_alerts_bulk_dismiss_cbanalytics(monkeypatch): + _was_called = False + + def _do_dismiss(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/workflow/_criteria" + assert body == {"query": "Blort", "state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir", + "criteria": {"device_name": ["HAL9000"]}} + _was_called = True + return StubResponse({"request_id": "497ABX"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_do_dismiss) + q = api.select(CBAnalyticsAlert).where("Blort").set_device_names(["HAL9000"]) + reqid = q.dismiss("Fixed", "Yessir") + assert _was_called + assert reqid == "497ABX" + + +def test_alerts_bulk_dismiss_vmware(monkeypatch): + _was_called = False + + def _do_dismiss(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/alerts/vmware/workflow/_criteria" + assert body == {"query": "Blort", "state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir", + "criteria": {"device_name": ["HAL9000"]}} + _was_called = True + return StubResponse({"request_id": "497ABX"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_do_dismiss) + q = api.select(VMwareAlert).where("Blort").set_device_names(["HAL9000"]) + reqid = q.dismiss("Fixed", "Yessir") + assert _was_called + assert reqid == "497ABX" + + +def test_alerts_bulk_dismiss_threat(monkeypatch): + _was_called = False + + def _do_dismiss(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/threat/workflow/_criteria" + assert body == {"threat_id": ["B0RG", "F3R3NG1"], "state": "DISMISSED", "remediation_state": "Fixed", + "comment": "Yessir"} + _was_called = True + return StubResponse({"request_id": "497ABX"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_do_dismiss) + reqid = api.bulk_threat_dismiss(["B0RG", "F3R3NG1"], "Fixed", "Yessir") + assert _was_called + assert reqid == "497ABX" + + +def test_alerts_bulk_undismiss_threat(monkeypatch): + _was_called = False + + def _do_update(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/threat/workflow/_criteria" + assert body == {"threat_id": ["B0RG", "F3R3NG1"], "state": "OPEN", "remediation_state": "Fixed", + "comment": "NoSir"} + _was_called = True + return StubResponse({"request_id": "497ABX"}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_do_update) + reqid = api.bulk_threat_update(["B0RG", "F3R3NG1"], "Fixed", "NoSir") + assert _was_called + assert reqid == "497ABX" + + +def test_load_workflow(monkeypatch): + _was_called = False + + def _get_workflow(url, parms=None, default=None): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" + _was_called = True + return {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED", + "workflow": {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}} + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_workflow) + workflow = api.select(WorkflowStatus, "497ABX") + assert _was_called + assert workflow.id_ == "497ABX" diff --git a/test/cbapi/psc/test_devicev6_api.py b/test/cbapi/psc/test_devicev6_api.py new file mode 100755 index 00000000..a73570e7 --- /dev/null +++ b/test/cbapi/psc/test_devicev6_api.py @@ -0,0 +1,383 @@ +import pytest +from cbapi.errors import ApiError +from cbapi.psc.models import Device +from cbapi.psc.rest_api import CbPSCBaseAPI +from test.cbtest import StubResponse, patch_cbapi + + +def test_get_device(monkeypatch): + _was_called = False + + def _get_device(url): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/devices/6023" + _was_called = True + return {"device_id": 6023, "organization_name": "thistestworks"} + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_device) + rc = api.select(Device, 6023) + assert _was_called + assert isinstance(rc, Device) + assert rc.device_id == 6023 + assert rc.organization_name == "thistestworks" + + +def test_device_background_scan(monkeypatch): + _was_called = False + + def _call_background_scan(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "BACKGROUND_SCAN", "device_id": [6023], "options": {"toggle": "ON"}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_call_background_scan) + api.device_background_scan([6023], True) + assert _was_called + + +def test_device_bypass(monkeypatch): + _was_called = False + + def _call_bypass(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "BYPASS", "device_id": [6023], "options": {"toggle": "OFF"}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_call_bypass) + api.device_bypass([6023], False) + assert _was_called + + +def test_device_delete_sensor(monkeypatch): + _was_called = False + + def _call_delete_sensor(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "DELETE_SENSOR", "device_id": [6023]} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_call_delete_sensor) + api.device_delete_sensor([6023]) + assert _was_called + + +def test_device_uninstall_sensor(monkeypatch): + _was_called = False + + def _call_uninstall_sensor(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "UNINSTALL_SENSOR", "device_id": [6023]} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_call_uninstall_sensor) + api.device_uninstall_sensor([6023]) + assert _was_called + + +def test_device_quarantine(monkeypatch): + _was_called = False + + def _call_quarantine(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "QUARANTINE", "device_id": [6023], "options": {"toggle": "ON"}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_call_quarantine) + api.device_quarantine([6023], True) + assert _was_called + + +def test_device_update_policy(monkeypatch): + _was_called = False + + def _call_update_policy(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "UPDATE_POLICY", "device_id": [6023], "options": {"policy_id": 8675309}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_call_update_policy) + api.device_update_policy([6023], 8675309) + assert _was_called + + +def test_device_update_sensor_version(monkeypatch): + _was_called = False + + def _call_update_sensor_version(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "UPDATE_SENSOR_VERSION", "device_id": [6023], + "options": {"sensor_version": {"RHEL": "2.3.4.5"}}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_call_update_sensor_version) + api.device_update_sensor_version([6023], {"RHEL": "2.3.4.5"}) + assert _was_called + + +def test_query_device_with_all_bells_and_whistles(monkeypatch): + _was_called = False + + def _run_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/devices/_search" + assert body == {"query": "foobar", + "criteria": {"ad_group_id": [14, 25], "os": ["LINUX"], "policy_id": [8675309], + "status": ["ALL"], "target_priority": ["HIGH"]}, + "exclusions": {"sensor_version": ["0.1"]}, + "sort": [{"field": "name", "order": "DESC"}]} + _was_called = True + return StubResponse({"results": [{"id": 6023, "organization_name": "thistestworks"}], + "num_found": 1}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) + query = api.select(Device).where("foobar").set_ad_group_ids([14, 25]).set_os(["LINUX"]) \ + .set_policy_ids([8675309]).set_status(["ALL"]).set_target_priorities(["HIGH"]) \ + .set_exclude_sensor_versions(["0.1"]).sort_by("name", "DESC") + d = query.one() + assert _was_called + assert d.id == 6023 + assert d.organization_name == "thistestworks" + + +def test_query_device_with_last_contact_time_as_start_end(monkeypatch): + _was_called = False + + def _run_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/devices/_search" + assert body == {"query": "foobar", + "criteria": {"last_contact_time": {"start": "2019-09-30T12:34:56", + "end": "2019-10-01T12:00:12"}}, "exclusions": {}} + _was_called = True + return StubResponse({"results": [{"id": 6023, "organization_name": "thistestworks"}], + "num_found": 1}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) + query = api.select(Device).where("foobar") \ + .set_last_contact_time(start="2019-09-30T12:34:56", end="2019-10-01T12:00:12") + d = query.one() + assert _was_called + assert d.id == 6023 + assert d.organization_name == "thistestworks" + + +def test_query_device_with_last_contact_time_as_range(monkeypatch): + _was_called = False + + def _run_query(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/devices/_search" + assert body == {"query": "foobar", "criteria": {"last_contact_time": {"range": "-3w"}}, "exclusions": {}} + _was_called = True + return StubResponse({"results": [{"id": 6023, "organization_name": "thistestworks"}], + "num_found": 1}) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_run_query) + query = api.select(Device).where("foobar").set_last_contact_time(range="-3w") + d = query.one() + assert _was_called + assert d.id == 6023 + assert d.organization_name == "thistestworks" + + +def test_query_device_invalid_last_contact_time_combinations(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).set_last_contact_time() + with pytest.raises(ApiError): + api.select(Device).set_last_contact_time(start="2019-09-30T12:34:56", end="2019-10-01T12:00:12", + range="-3w") + with pytest.raises(ApiError): + api.select(Device).set_last_contact_time(start="2019-09-30T12:34:56", range="-3w") + with pytest.raises(ApiError): + api.select(Device).set_last_contact_time(end="2019-10-01T12:00:12", range="-3w") + + +def test_query_device_invalid_criteria_values(): + tests = [ + {"method": "set_ad_group_ids", "arg": ["Bogus"]}, + {"method": "set_policy_ids", "arg": ["Bogus"]}, + {"method": "set_os", "arg": ["COMMODORE_64"]}, + {"method": "set_status", "arg": ["Bogus"]}, + {"method": "set_target_priorities", "arg": ["Bogus"]}, + {"method": "set_exclude_sensor_versions", "arg": [12703]} + ] + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + query = api.select(Device) + for t in tests: + meth = getattr(query, t["method"], None) + with pytest.raises(ApiError): + meth(t["arg"]) + + +def test_query_device_invalid_sort_direction(): + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + with pytest.raises(ApiError): + api.select(Device).sort_by("policy_name", "BOGUS") + + +def test_query_device_download(monkeypatch): + _was_called = False + + def _run_download(url, query_params, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/devices/_search/download" + assert query_params == {"status": "ALL", "ad_group_id": "14,25", "policy_id": "8675309", + "target_priority": "HIGH", "query_string": "foobar", "sort_field": "name", + "sort_order": "DESC"} + _was_called = True + return "123456789,123456789,123456789" + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, RAW_GET=_run_download) + rc = api.select(Device).where("foobar").set_ad_group_ids([14, 25]).set_policy_ids([8675309]) \ + .set_status(["ALL"]).set_target_priorities(["HIGH"]).sort_by("name", "DESC").download() + assert _was_called + assert rc == "123456789,123456789,123456789" + + +def test_query_device_do_background_scan(monkeypatch): + _was_called = False + + def _background_scan(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "BACKGROUND_SCAN", + "search": {"query": "foobar", "criteria": {}, "exclusions": {}}, "options": {"toggle": "ON"}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_background_scan) + api.select(Device).where("foobar").background_scan(True) + assert _was_called + + +def test_query_device_do_bypass(monkeypatch): + _was_called = False + + def _bypass(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "BYPASS", + "search": {"query": "foobar", "criteria": {}, "exclusions": {}}, "options": {"toggle": "OFF"}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_bypass) + api.select(Device).where("foobar").bypass(False) + assert _was_called + + +def test_query_device_do_delete_sensor(monkeypatch): + _was_called = False + + def _delete_sensor(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "DELETE_SENSOR", + "search": {"query": "foobar", "criteria": {}, "exclusions": {}}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_delete_sensor) + api.select(Device).where("foobar").delete_sensor() + assert _was_called + + +def test_query_device_do_uninstall_sensor(monkeypatch): + _was_called = False + + def _uninstall_sensor(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "UNINSTALL_SENSOR", + "search": {"query": "foobar", "criteria": {}, "exclusions": {}}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_uninstall_sensor) + api.select(Device).where("foobar").uninstall_sensor() + assert _was_called + + +def test_query_device_do_quarantine(monkeypatch): + _was_called = False + + def _quarantine(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "QUARANTINE", + "search": {"query": "foobar", "criteria": {}, "exclusions": {}}, "options": {"toggle": "ON"}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_quarantine) + api.select(Device).where("foobar").quarantine(True) + assert _was_called + + +def test_query_device_do_update_policy(monkeypatch): + _was_called = False + + def _update_policy(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "UPDATE_POLICY", + "search": {"query": "foobar", "criteria": {}, "exclusions": {}}, + "options": {"policy_id": 8675309}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_update_policy) + api.select(Device).where("foobar").update_policy(8675309) + assert _was_called + + +def test_query_device_do_update_sensor_version(monkeypatch): + _was_called = False + + def _update_sensor_version(url, body, **kwargs): + nonlocal _was_called + assert url == "/appservices/v6/orgs/Z100/device_actions" + assert body == {"action_type": "UPDATE_SENSOR_VERSION", + "search": {"query": "foobar", "criteria": {}, "exclusions": {}}, + "options": {"sensor_version": {"RHEL": "2.3.4.5"}}} + _was_called = True + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", + org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_update_sensor_version) + api.select(Device).where("foobar").update_sensor_version({"RHEL": "2.3.4.5"}) + assert _was_called diff --git a/test/cbapi/psc/test_models.py b/test/cbapi/psc/test_models.py index 99cdfb4e..4b86dede 100755 --- a/test/cbapi/psc/test_models.py +++ b/test/cbapi/psc/test_models.py @@ -1,10 +1,10 @@ import pytest from cbapi.psc.models import Device, BaseAlert, WorkflowStatus from cbapi.psc.rest_api import CbPSCBaseAPI -from test.mocks import MockResponse, ConnectionMocks +from test.cbtest import StubResponse, patch_cbapi -class MockScheduler: +class StubScheduler: def __init__(self, expected_id): self.expected_id = expected_id self.was_called = False @@ -16,230 +16,165 @@ def request_session(self, sensor_id): def test_Device_lr_session(monkeypatch): - _device_data = {"id": 6023} - def mock_get_object(url, parms=None, default=None): - nonlocal _device_data + def _get_session(url, parms=None, default=None): assert url == "/appservices/v6/orgs/Z100/devices/6023" - return _device_data + return {"id": 6023} - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - sked = MockScheduler(6023) + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + sked = StubScheduler(6023) api._lr_scheduler = sked - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - dev = Device(api, 6023, _device_data) + patch_cbapi(monkeypatch, api, GET=_get_session) + dev = Device(api, 6023, {"id": 6023}) sess = dev.lr_session() assert sess["itworks"] assert sked.was_called def test_Device_background_scan(monkeypatch): - _device_data = {"id": 6023} _was_called = False - def mock_get_object(url, parms=None, default=None): - nonlocal _device_data + def _get_device(url, parms=None, default=None): assert url == "/appservices/v6/orgs/Z100/devices/6023" - return _device_data + return {"id": 6023} - def mock_post_object(url, body, **kwargs): + def _background_scan(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "BACKGROUND_SCAN" - assert body["device_id"] == [6023] - t = body["options"] - assert t["toggle"] == "ON" + assert body == {"action_type": "BACKGROUND_SCAN", "device_id": [6023], "options": {"toggle": "ON"}} _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - dev = Device(api, 6023, _device_data) + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_device, POST=_background_scan) + dev = Device(api, 6023, {"id": 6023}) dev.background_scan(True) assert _was_called def test_Device_bypass(monkeypatch): - _device_data = {"id": 6023} _was_called = False - def mock_get_object(url, parms=None, default=None): - nonlocal _device_data + def _get_device(url, parms=None, default=None): assert url == "/appservices/v6/orgs/Z100/devices/6023" - return _device_data + return {"id": 6023} - def mock_post_object(url, body, **kwargs): + def _bypass(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "BYPASS" - assert body["device_id"] == [6023] - t = body["options"] - assert t["toggle"] == "OFF" + assert body == {"action_type": "BYPASS", "device_id": [6023], "options": {"toggle": "OFF"}} _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - dev = Device(api, 6023, _device_data) + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_device, POST=_bypass) + dev = Device(api, 6023, {"id": 6023}) dev.bypass(False) assert _was_called def test_Device_delete_sensor(monkeypatch): - _device_data = {"id": 6023} _was_called = False - def mock_get_object(url, parms=None, default=None): - nonlocal _device_data + def _get_device(url, parms=None, default=None): assert url == "/appservices/v6/orgs/Z100/devices/6023" - return _device_data + return {"id": 6023} - def mock_post_object(url, body, **kwargs): + def _delete_sensor(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "DELETE_SENSOR" - assert body["device_id"] == [6023] + assert body == {"action_type": "DELETE_SENSOR", "device_id": [6023]} _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - dev = Device(api, 6023, _device_data) + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_device, POST=_delete_sensor) + dev = Device(api, 6023, {"id": 6023}) dev.delete_sensor() assert _was_called def test_Device_uninstall_sensor(monkeypatch): - _device_data = {"id": 6023} _was_called = False - def mock_get_object(url, parms=None, default=None): - nonlocal _device_data + def _get_device(url, parms=None, default=None): assert url == "/appservices/v6/orgs/Z100/devices/6023" - return _device_data + return {"id": 6023} - def mock_post_object(url, body, **kwargs): + def _uninstall_sensor(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "UNINSTALL_SENSOR" - assert body["device_id"] == [6023] + assert body == {"action_type": "UNINSTALL_SENSOR", "device_id": [6023]} _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - dev = Device(api, 6023, _device_data) + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_device, POST=_uninstall_sensor) + dev = Device(api, 6023, {"id": 6023}) dev.uninstall_sensor() assert _was_called def test_Device_quarantine(monkeypatch): - _device_data = {"id": 6023} _was_called = False - def mock_get_object(url, parms=None, default=None): - nonlocal _device_data + def _get_device(url, parms=None, default=None): assert url == "/appservices/v6/orgs/Z100/devices/6023" - return _device_data + return {"id": 6023} - def mock_post_object(url, body, **kwargs): + def _quarantine(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "QUARANTINE" - assert body["device_id"] == [6023] - t = body["options"] - assert t["toggle"] == "ON" + assert body == {"action_type": "QUARANTINE", "device_id": [6023], "options": {"toggle": "ON"}} _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - dev = Device(api, 6023, _device_data) + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_device, POST=_quarantine) + dev = Device(api, 6023, {"id": 6023}) dev.quarantine(True) assert _was_called def test_Device_update_policy(monkeypatch): - _device_data = {"id": 6023} _was_called = False - def mock_get_object(url, parms=None, default=None): - nonlocal _device_data + def _get_device(url, parms=None, default=None): assert url == "/appservices/v6/orgs/Z100/devices/6023" - return _device_data + return {"id": 6023} - def mock_post_object(url, body, **kwargs): + def _update_policy(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "UPDATE_POLICY" - assert body["device_id"] == [6023] - t = body["options"] - assert t["policy_id"] == 8675309 + assert body == {"action_type": "UPDATE_POLICY", "device_id": [6023], "options": {"policy_id": 8675309}} _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - dev = Device(api, 6023, _device_data) + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_device, POST=_update_policy) + dev = Device(api, 6023, {"id": 6023}) dev.update_policy(8675309) assert _was_called def test_Device_update_sensor_version(monkeypatch): - _device_data = {"id": 6023} _was_called = False - def mock_get_object(url, parms=None, default=None): - nonlocal _device_data + def _get_device(url, parms=None, default=None): assert url == "/appservices/v6/orgs/Z100/devices/6023" - return _device_data + return {"id": 6023} - def mock_post_object(url, body, **kwargs): + def _update_sensor_version(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "UPDATE_SENSOR_VERSION" - assert body["device_id"] == [6023] - t = body["options"] - t2 = t["sensor_version"] - assert t2["RHEL"] == "2.3.4.5" + assert body == {"action_type": "UPDATE_SENSOR_VERSION", "device_id": [6023], + "options": {"sensor_version": {"RHEL": "2.3.4.5"}}} _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - dev = Device(api, 6023, _device_data) + return StubResponse(None, 204) + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_device, POST=_update_sensor_version) + dev = Device(api, 6023, {"id": 6023}) dev.update_sensor_version({"RHEL": "2.3.4.5"}) assert _was_called @@ -247,22 +182,16 @@ def mock_post_object(url, body, **kwargs): def test_BaseAlert_dismiss(monkeypatch): _was_called = False - def mock_post_object(url, body, **kwargs): + def _do_dismiss(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/ESD14U2C/workflow" - assert body["state"] == "DISMISSED" - assert body["remediation_state"] == "Fixed" - assert body["comment"] == "Yessir" + assert body == {"state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir"} _was_called = True - return MockResponse({"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + return StubResponse({"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_do_dismiss) alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "workflow": {"state": "OPEN"}}) alert.dismiss("Fixed", "Yessir") assert _was_called @@ -276,22 +205,16 @@ def mock_post_object(url, body, **kwargs): def test_BaseAlert_undismiss(monkeypatch): _was_called = False - def mock_post_object(url, body, **kwargs): + def _do_update(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/alerts/ESD14U2C/workflow" - assert body["state"] == "OPEN" - assert body["remediation_state"] == "Fixed" - assert body["comment"] == "NoSir" + assert body == {"state": "OPEN", "remediation_state": "Fixed", "comment": "NoSir"} _was_called = True - return MockResponse({"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", + return StubResponse({"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_do_update) alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "workflow": {"state": "DISMISS"}}) alert.update("Fixed", "NoSir") assert _was_called @@ -305,22 +228,16 @@ def mock_post_object(url, body, **kwargs): def test_BaseAlert_dismiss_threat(monkeypatch): _was_called = False - def mock_post_object(url, body, **kwargs): + def _do_dismiss(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/threat/B0RG/workflow" - assert body["state"] == "DISMISSED" - assert body["remediation_state"] == "Fixed" - assert body["comment"] == "Yessir" + assert body == {"state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir"} _was_called = True - return MockResponse({"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + return StubResponse({"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_do_dismiss) alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "threat_id": "B0RG", "workflow": {"state": "OPEN"}}) wf = alert.dismiss_threat("Fixed", "Yessir") assert _was_called @@ -334,22 +251,16 @@ def mock_post_object(url, body, **kwargs): def test_BaseAlert_undismiss_threat(monkeypatch): _was_called = False - def mock_post_object(url, body, **kwargs): + def _do_update(url, body, **kwargs): nonlocal _was_called assert url == "/appservices/v6/orgs/Z100/threat/B0RG/workflow" - assert body["state"] == "OPEN" - assert body["remediation_state"] == "Fixed" - assert body["comment"] == "NoSir" + assert body == {"state": "OPEN", "remediation_state": "Fixed", "comment": "NoSir"} _was_called = True - return MockResponse({"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", + return StubResponse({"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, POST=_do_update) alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "threat_id": "B0RG", "workflow": {"state": "OPEN"}}) wf = alert.update_threat("Fixed", "NoSir") assert _was_called @@ -363,7 +274,7 @@ def mock_post_object(url, body, **kwargs): def test_WorkflowStatus(monkeypatch): _times_called = 0 - def mock_get_object(url, parms=None, default=None): + def _get_workflow(url, parms=None, default=None): nonlocal _times_called assert url == "/appservices/v6/orgs/Z100/workflow/status/W00K13" if _times_called >= 0 and _times_called <= 3: @@ -373,19 +284,14 @@ def mock_get_object(url, parms=None, default=None): elif _times_called >= 7 and _times_called <= 9: _stat = "FINISHED" else: - pytest.fail("mock_get_object called too many times") - resp = {"errors": [], "failed_ids": [], "id": "W00K13", "num_hits": 0, "num_success": 0, "status": _stat} - resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} + pytest.fail("_get_workflow called too many times") _times_called = _times_called + 1 - return resp - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) + return {"errors": [], "failed_ids": [], "id": "W00K13", "num_hits": 0, "num_success": 0, "status": _stat, + "workflow": {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", + "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}} + + api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) + patch_cbapi(monkeypatch, api, GET=_get_workflow) wfstat = WorkflowStatus(api, "W00K13") assert wfstat.workflow_.changed_by == "Robocop" assert wfstat.workflow_.state == "DISMISSED" diff --git a/test/cbapi/psc/test_rest_api.py b/test/cbapi/psc/test_rest_api.py deleted file mode 100755 index 40b50bf6..00000000 --- a/test/cbapi/psc/test_rest_api.py +++ /dev/null @@ -1,1448 +0,0 @@ -import pytest -from cbapi.errors import ApiError -from cbapi.psc.models import Device, BaseAlert, CBAnalyticsAlert, VMwareAlert, WatchlistAlert, WorkflowStatus -from cbapi.psc.rest_api import CbPSCBaseAPI -from test.mocks import ConnectionMocks, MockResponse - -# -# --- Device v6 Tests -# - - -def test_get_device(monkeypatch): - _was_called = False - - def mock_get_object(url): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/devices/6023" - _was_called = True - return {"device_id": 6023, "organization_name": "thistestworks"} - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - rc = api.select(Device, 6023) - assert _was_called - assert isinstance(rc, Device) - assert rc.device_id == 6023 - assert rc.organization_name == "thistestworks" - - -def test_device_background_scan(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "BACKGROUND_SCAN" - assert body["device_id"] == [6023] - t = body["options"] - assert t["toggle"] == "ON" - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_background_scan([6023], True) - assert _was_called - - -def test_device_bypass(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "BYPASS" - assert body["device_id"] == [6023] - t = body["options"] - assert t["toggle"] == "OFF" - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_bypass([6023], False) - assert _was_called - - -def test_device_delete_sensor(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "DELETE_SENSOR" - assert body["device_id"] == [6023] - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_delete_sensor([6023]) - assert _was_called - - -def test_device_uninstall_sensor(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "UNINSTALL_SENSOR" - assert body["device_id"] == [6023] - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_uninstall_sensor([6023]) - assert _was_called - - -def test_device_quarantine(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "QUARANTINE" - assert body["device_id"] == [6023] - t = body["options"] - assert t["toggle"] == "ON" - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_quarantine([6023], True) - assert _was_called - - -def test_device_update_policy(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "UPDATE_POLICY" - assert body["device_id"] == [6023] - t = body["options"] - assert t["policy_id"] == 8675309 - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_update_policy([6023], 8675309) - assert _was_called - - -def test_device_update_sensor_version(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "UPDATE_SENSOR_VERSION" - assert body["device_id"] == [6023] - t = body["options"] - t2 = t["sensor_version"] - assert t2["RHEL"] == "2.3.4.5" - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.device_update_sensor_version([6023], {"RHEL": "2.3.4.5"}) - assert _was_called - - -def test_query_device_with_all_bells_and_whistles(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/devices/_search" - assert body["query"] == "foobar" - t = body.get("criteria", {}) - assert t["ad_group_id"] == [14, 25] - assert t["os"] == ["LINUX"] - assert t["policy_id"] == [8675309] - assert t["status"] == ["ALL"] - assert t["target_priority"] == ["HIGH"] - t = body.get("exclusions", {}) - assert t["sensor_version"] == ["0.1"] - t = body.get("sort", []) - t2 = t[0] - assert t2["field"] == "name" - assert t2["order"] == "DESC" - _was_called = True - body = {"id": 6023, "organization_name": "thistestworks"} - envelope = {"results": [body], "num_found": 1} - return MockResponse(envelope) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(Device).where("foobar").ad_group_ids([14, 25]) \ - .os(["LINUX"]).policy_ids([8675309]).status(["ALL"]) \ - .target_priorities(["HIGH"]).exclude_sensor_versions(["0.1"]) \ - .sort_by("name", "DESC") - d = query.one() - assert _was_called - assert d.id == 6023 - assert d.organization_name == "thistestworks" - - -def test_query_device_with_last_contact_time_as_start_end(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/devices/_search" - assert body["query"] == "foobar" - t = body.get("criteria", {}) - t2 = t.get("last_contact_time", {}) - assert t2["start"] == "2019-09-30T12:34:56" - assert t2["end"] == "2019-10-01T12:00:12" - _was_called = True - body = {"id": 6023, "organization_name": "thistestworks"} - envelope = {"results": [body], "num_found": 1} - return MockResponse(envelope) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(Device).where("foobar") \ - .last_contact_time(start="2019-09-30T12:34:56", end="2019-10-01T12:00:12") - d = query.one() - assert _was_called - assert d.id == 6023 - assert d.organization_name == "thistestworks" - - -def test_query_device_with_last_contact_time_as_range(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/devices/_search" - assert body["query"] == "foobar" - t = body.get("criteria", {}) - t2 = t.get("last_contact_time", {}) - assert t2["range"] == "-3w" - _was_called = True - body = {"id": 6023, "organization_name": "thistestworks"} - envelope = {"results": [body], "num_found": 1} - return MockResponse(envelope) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(Device).where("foobar").last_contact_time(range="-3w") - d = query.one() - assert _was_called - assert d.id == 6023 - assert d.organization_name == "thistestworks" - - -def test_query_device_invalid_ad_group_ids(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).ad_group_ids(["Bogus"]) - - -def test_query_device_invalid_policy_ids(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).policy_ids(["Bogus"]) - - -def test_query_device_last_contact_time_no_params_ok(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).last_contact_time() - - -def test_query_device_last_contact_time_range_specified_bad(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).last_contact_time(start="2019-09-30T12:34:56", - end="2019-10-01T12:00:12", range="-3w") - - -def test_query_device_last_contact_time_start_specified_bad(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).last_contact_time(start="2019-09-30T12:34:56", - range="-3w") - - -def test_query_device_last_contact_time_end_specified_bad(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).last_contact_time(end="2019-10-01T12:00:12", range="-3w") - - -def test_query_device_invalid_os(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).os(["COMMODORE_64"]) - - -def test_query_device_invalid_status(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).status(["Bogus"]) - - -def test_query_device_invalid_priority(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).target_priorities(["Bogus"]) - - -def test_query_device_invalid_exclude_sensor_version(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).exclude_sensor_versions([12703]) - - -def test_query_device_invalid_sort_direction(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).sort_by("policy_name", "BOGUS") - - -def test_query_device_download(monkeypatch): - _was_called = False - - def mock_get_raw_data(url, query_params, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/devices/_search/download" - assert query_params["status"] == "ALL" - assert query_params["ad_group_id"] == "14,25" - assert query_params["policy_id"] == "8675309" - assert query_params["target_priority"] == "HIGH" - assert query_params["query_string"] == "foobar" - assert query_params["sort_field"] == "name" - assert query_params["sort_order"] == "DESC" - _was_called = True - return "123456789,123456789,123456789" - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_raw_data", mock_get_raw_data) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - rc = api.select(Device).where("foobar").ad_group_ids([14, 25]) \ - .policy_ids([8675309]).status(["ALL"]).target_priorities(["HIGH"]) \ - .sort_by("name", "DESC").download() - assert _was_called - assert rc == "123456789,123456789,123456789" - - -def test_query_device_do_background_scan(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "BACKGROUND_SCAN" - t = body["search"] - assert t["query"] == "foobar" - t = body["options"] - assert t["toggle"] == "ON" - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.select(Device).where("foobar").background_scan(True) - assert _was_called - - -def test_query_device_do_bypass(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "BYPASS" - t = body["search"] - assert t["query"] == "foobar" - t = body["options"] - assert t["toggle"] == "OFF" - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.select(Device).where("foobar").bypass(False) - assert _was_called - - -def test_query_device_do_delete_sensor(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "DELETE_SENSOR" - t = body["search"] - assert t["query"] == "foobar" - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.select(Device).where("foobar").delete_sensor() - assert _was_called - - -def test_query_device_do_uninstall_sensor(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "UNINSTALL_SENSOR" - t = body["search"] - assert t["query"] == "foobar" - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.select(Device).where("foobar").uninstall_sensor() - assert _was_called - - -def test_query_device_do_quarantine(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "QUARANTINE" - t = body["search"] - assert t["query"] == "foobar" - t = body["options"] - assert t["toggle"] == "ON" - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.select(Device).where("foobar").quarantine(True) - assert _was_called - - -def test_query_device_do_update_policy(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "UPDATE_POLICY" - t = body["search"] - assert t["query"] == "foobar" - t = body["options"] - assert t["policy_id"] == 8675309 - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.select(Device).where("foobar").update_policy(8675309) - assert _was_called - - -def test_query_device_do_update_sensor_version(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body["action_type"] == "UPDATE_SENSOR_VERSION" - t = body["search"] - assert t["query"] == "foobar" - t = body["options"] - t2 = t["sensor_version"] - assert t2["RHEL"] == "2.3.4.5" - _was_called = True - return MockResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - api.select(Device).where("foobar").update_sensor_version({"RHEL": "2.3.4.5"}) - assert _was_called - -# -# --- Alerts v6 Tests -# - - -def test_query_basealert_with_all_bells_and_whistles(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/_search" - assert body["query"] == "Blort" - t = body["criteria"] - assert t["category"] == ["SERIOUS", "CRITICAL"] - assert t["device_id"] == [6023] - assert t["device_name"] == ["HAL"] - assert t["device_os"] == ["LINUX"] - assert t["device_os_version"] == ["0.1.2"] - assert t["device_username"] == ["JRN"] - assert t.get("group_results", False) - assert t["id"] == ["S0L0"] - assert t["legacy_alert_id"] == ["S0L0_1"] - assert t.get("minimum_severity", -1) == 6 - assert t["policy_id"] == [8675309] - assert t["policy_name"] == ["Strict"] - assert t["process_name"] == ["IEXPLORE.EXE"] - assert t["process_sha256"] == ["0123456789ABCDEF0123456789ABCDEF"] - assert t["reputation"] == ["SUSPECT_MALWARE"] - assert t["tag"] == ["Frood"] - assert t["target_value"] == ["HIGH"] - assert t["threat_id"] == ["B0RG"] - assert t["type"] == ["WATCHLIST"] - assert t["workflow"] == ["OPEN"] - t = body["sort"] - t2 = t[0] - assert t2["field"] == "name" - assert t2["order"] == "DESC" - _was_called = True - body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = {"results": [body], "num_found": 1} - return MockResponse(envelope) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(BaseAlert).where("Blort").categories(["SERIOUS", "CRITICAL"]).device_ids([6023]) \ - .device_names(["HAL"]).device_os(["LINUX"]).device_os_versions(["0.1.2"]).device_username(["JRN"]) \ - .group_results(True).alert_ids(["S0L0"]).legacy_alert_ids(["S0L0_1"]).minimum_severity(6) \ - .policy_ids([8675309]).policy_names(["Strict"]).process_names(["IEXPLORE.EXE"]) \ - .process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).reputations(["SUSPECT_MALWARE"]) \ - .tags(["Frood"]).target_priorities(["HIGH"]).threat_ids(["B0RG"]).types(["WATCHLIST"]) \ - .workflows(["OPEN"]).sort_by("name", "DESC") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_basealert_with_create_time_as_start_end(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/_search" - assert body["query"] == "Blort" - t = body["criteria"] - t2 = t.get("create_time", {}) - assert t2["start"] == "2019-09-30T12:34:56" - assert t2["end"] == "2019-10-01T12:00:12" - _was_called = True - body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = {"results": [body], "num_found": 1} - return MockResponse(envelope) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(BaseAlert).where("Blort") \ - .create_time(start="2019-09-30T12:34:56", end="2019-10-01T12:00:12") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_basealert_with_create_time_as_range(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/_search" - assert body["query"] == "Blort" - t = body["criteria"] - t2 = t.get("create_time", {}) - assert t2["range"] == "-3w" - _was_called = True - body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = {"results": [body], "num_found": 1} - return MockResponse(envelope) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(BaseAlert).where("Blort").create_time(range="-3w") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_basealert_facets(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/_facet" - assert body["query"] == "Blort" - t = body["criteria"] - assert t["workflow"] == ["OPEN"] - t = body["terms"] - assert t["rows"] == 0 - assert t["fields"] == ["REPUTATION", "STATUS"] - _was_called = True - dto1 = {"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]} - dto2 = {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]} - return MockResponse({"results": [dto1, dto2]}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(BaseAlert).where("Blort").workflows(["OPEN"]) - f = query.facets(["REPUTATION", "STATUS"]) - assert _was_called - t = f[0] - assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] - t = f[1] - assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] - - -def test_query_basealert_invalid_category(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).categories(["DOUBLE_DARE"]) - - -def test_query_basealert_create_time_no_params_ok(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).create_time() - - -def test_query_basealert_create_time_range_specified_bad(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).create_time(start="2019-09-30T12:34:56", - end="2019-10-01T12:00:12", range="-3w") - - -def test_query_basealert_create_time_start_specified_bad(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).create_time(start="2019-09-30T12:34:56", range="-3w") - - -def test_query_basealert_create_time_end_specified_bad(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).create_time(end="2019-10-01T12:00:12", range="-3w") - - -def test_query_basealert_invalid_device_ids(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).device_ids(["Bogus"]) - - -def test_query_basealert_invalid_device_names(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).device_names([42]) - - -def test_query_basealert_invalid_device_os(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).device_os(["TI994A"]) - - -def test_query_basealert_invalid_device_os_versions(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).device_os_versions([8808]) - - -def test_query_basealert_invalid_device_username(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).device_username([-1]) - - -def test_query_basealert_invalid_alert_ids(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).alert_ids([9001]) - - -def test_query_basealert_invalid_legacy_alert_ids(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).legacy_alert_ids([9001]) - - -def test_query_basealert_invalid_policy_ids(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).policy_ids(["Bogus"]) - - -def test_query_basealert_invalid_policy_names(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).policy_names([323]) - - -def test_query_basealert_invalid_process_names(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).process_names([7071]) - - -def test_query_basealert_invalid_process_sha256(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).process_sha256([123456789]) - - -def test_query_basealert_invalid_reputations(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).reputations(["MICROSOFT_FUDWARE"]) - - -def test_query_basealert_invalid_tags(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).tags([990909]) - - -def test_query_basealert_invalid_target_priorities(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).target_priorities(["DOGWASH"]) - - -def test_query_basealert_invalid_threat_ids(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).threat_ids([4096]) - - -def test_query_basealert_invalid_types(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).types(["ERBOSOFT"]) - - -def test_query_basealert_invalid_workflows(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).workflows(["IN_LIMBO"]) - - -def test_query_cbanalyticsalert_with_all_bells_and_whistles(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/_search" - assert body["query"] == "Blort" - t = body["criteria"] - assert t["category"] == ["SERIOUS", "CRITICAL"] - assert t["device_id"] == [6023] - assert t["device_name"] == ["HAL"] - assert t["device_os"] == ["LINUX"] - assert t["device_os_version"] == ["0.1.2"] - assert t["device_username"] == ["JRN"] - assert t.get("group_results", False) - assert t["id"] == ["S0L0"] - assert t["legacy_alert_id"] == ["S0L0_1"] - assert t.get("minimum_severity", -1) == 6 - assert t["policy_id"] == [8675309] - assert t["policy_name"] == ["Strict"] - assert t["process_name"] == ["IEXPLORE.EXE"] - assert t["process_sha256"] == ["0123456789ABCDEF0123456789ABCDEF"] - assert t["reputation"] == ["SUSPECT_MALWARE"] - assert t["tag"] == ["Frood"] - assert t["target_value"] == ["HIGH"] - assert t["threat_id"] == ["B0RG"] - assert t["type"] == ["WATCHLIST"] - assert t["workflow"] == ["OPEN"] - assert t["blocked_threat_category"] == ["RISKY_PROGRAM"] - assert t["device_location"] == ["ONSITE"] - assert t["kill_chain_status"] == ["EXECUTE_GOAL"] - assert t["not_blocked_threat_category"] == ["NEW_MALWARE"] - assert t["policy_applied"] == ["APPLIED"] - assert t["reason_code"] == ["ATTACK_VECTOR"] - assert t["run_state"] == ["RAN"] - assert t["sensor_action"] == ["DENY"] - assert t["threat_cause_vector"] == ["WEB"] - - t = body["sort"] - t2 = t[0] - assert t2["field"] == "name" - assert t2["order"] == "DESC" - _was_called = True - body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = {"results": [body], "num_found": 1} - return MockResponse(envelope) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(CBAnalyticsAlert).where("Blort").categories(["SERIOUS", "CRITICAL"]).device_ids([6023]) \ - .device_names(["HAL"]).device_os(["LINUX"]).device_os_versions(["0.1.2"]).device_username(["JRN"]) \ - .group_results(True).alert_ids(["S0L0"]).legacy_alert_ids(["S0L0_1"]).minimum_severity(6) \ - .policy_ids([8675309]).policy_names(["Strict"]).process_names(["IEXPLORE.EXE"]) \ - .process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).reputations(["SUSPECT_MALWARE"]) \ - .tags(["Frood"]).target_priorities(["HIGH"]).threat_ids(["B0RG"]).types(["WATCHLIST"]) \ - .workflows(["OPEN"]).blocked_threat_categories(["RISKY_PROGRAM"]).device_locations(["ONSITE"]) \ - .kill_chain_statuses(["EXECUTE_GOAL"]).not_blocked_threat_categories(["NEW_MALWARE"]) \ - .policy_applied(["APPLIED"]).reason_code(["ATTACK_VECTOR"]).run_states(["RAN"]) \ - .sensor_actions(["DENY"]).threat_cause_vectors(["WEB"]).sort_by("name", "DESC") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_cbanalyticsalert_facets(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/_facet" - assert body["query"] == "Blort" - t = body["criteria"] - assert t["workflow"] == ["OPEN"] - t = body["terms"] - assert t["rows"] == 0 - assert t["fields"] == ["REPUTATION", "STATUS"] - _was_called = True - dto1 = {"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]} - dto2 = {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]} - return MockResponse({"results": [dto1, dto2]}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(CBAnalyticsAlert).where("Blort").workflows(["OPEN"]) - f = query.facets(["REPUTATION", "STATUS"]) - assert _was_called - t = f[0] - assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] - t = f[1] - assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] - - -def test_query_cbanalyticsalert_invalid_blocked_threat_categories(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(CBAnalyticsAlert).blocked_threat_categories(["MINOR"]) - - -def test_query_cbanalyticsalert_invalid_device_locations(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(CBAnalyticsAlert).device_locations(["NARNIA"]) - - -def test_query_cbanalyticsalert_invalid_kill_chain_statuses(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(CBAnalyticsAlert).kill_chain_statuses(["SPAWN_COPIES"]) - - -def test_query_cbanalyticsalert_invalid_not_blocked_threat_categories(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(CBAnalyticsAlert).not_blocked_threat_categories(["MINOR"]) - - -def test_query_cbanalyticsalert_invalid_policy_applied(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(CBAnalyticsAlert).policy_applied(["MAYBE"]) - - -def test_query_cbanalyticsalert_invalid_reason_code(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(CBAnalyticsAlert).reason_code([55]) - - -def test_query_cbanalyticsalert_invalid_run_states(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(CBAnalyticsAlert).run_states(["MIGHT_HAVE"]) - - -def test_query_cbanalyticsalert_invalid_sensor_actions(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(CBAnalyticsAlert).sensor_actions(["FLIP_A_COIN"]) - - -def test_query_cbanalyticsalert_invalid_threat_cause_vectors(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(CBAnalyticsAlert).threat_cause_vectors(["NETWORK"]) - - -def test_query_vmwarealert_with_all_bells_and_whistles(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/vmware/_search" - assert body["query"] == "Blort" - t = body["criteria"] - assert t["category"] == ["SERIOUS", "CRITICAL"] - assert t["device_id"] == [6023] - assert t["device_name"] == ["HAL"] - assert t["device_os"] == ["LINUX"] - assert t["device_os_version"] == ["0.1.2"] - assert t["device_username"] == ["JRN"] - assert t.get("group_results", False) - assert t["id"] == ["S0L0"] - assert t["legacy_alert_id"] == ["S0L0_1"] - assert t.get("minimum_severity", -1) == 6 - assert t["policy_id"] == [8675309] - assert t["policy_name"] == ["Strict"] - assert t["process_name"] == ["IEXPLORE.EXE"] - assert t["process_sha256"] == ["0123456789ABCDEF0123456789ABCDEF"] - assert t["reputation"] == ["SUSPECT_MALWARE"] - assert t["tag"] == ["Frood"] - assert t["target_value"] == ["HIGH"] - assert t["threat_id"] == ["B0RG"] - assert t["type"] == ["WATCHLIST"] - assert t["workflow"] == ["OPEN"] - assert t["group_id"] == [14] - t = body["sort"] - t2 = t[0] - assert t2["field"] == "name" - assert t2["order"] == "DESC" - _was_called = True - body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = {"results": [body], "num_found": 1} - return MockResponse(envelope) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(VMwareAlert).where("Blort").categories(["SERIOUS", "CRITICAL"]).device_ids([6023]) \ - .device_names(["HAL"]).device_os(["LINUX"]).device_os_versions(["0.1.2"]).device_username(["JRN"]) \ - .group_results(True).alert_ids(["S0L0"]).legacy_alert_ids(["S0L0_1"]).minimum_severity(6) \ - .policy_ids([8675309]).policy_names(["Strict"]).process_names(["IEXPLORE.EXE"]) \ - .process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).reputations(["SUSPECT_MALWARE"]) \ - .tags(["Frood"]).target_priorities(["HIGH"]).threat_ids(["B0RG"]).types(["WATCHLIST"]) \ - .workflows(["OPEN"]).group_ids([14]).sort_by("name", "DESC") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_vmwarealert_facets(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/vmware/_facet" - assert body["query"] == "Blort" - t = body["criteria"] - assert t["workflow"] == ["OPEN"] - t = body["terms"] - assert t["rows"] == 0 - assert t["fields"] == ["REPUTATION", "STATUS"] - _was_called = True - dto1 = {"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]} - dto2 = {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]} - return MockResponse({"results": [dto1, dto2]}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(VMwareAlert).where("Blort").workflows(["OPEN"]) - f = query.facets(["REPUTATION", "STATUS"]) - assert _was_called - t = f[0] - assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] - t = f[1] - assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] - - -def test_query_vmwarealert_invalid_group_ids(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(VMwareAlert).group_ids(["Bogus"]) - - -def test_query_watchlistalert_with_all_bells_and_whistles(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/_search" - assert body["query"] == "Blort" - t = body["criteria"] - assert t["category"] == ["SERIOUS", "CRITICAL"] - assert t["device_id"] == [6023] - assert t["device_name"] == ["HAL"] - assert t["device_os"] == ["LINUX"] - assert t["device_os_version"] == ["0.1.2"] - assert t["device_username"] == ["JRN"] - assert t.get("group_results", False) - assert t["id"] == ["S0L0"] - assert t["legacy_alert_id"] == ["S0L0_1"] - assert t.get("minimum_severity", -1) == 6 - assert t["policy_id"] == [8675309] - assert t["policy_name"] == ["Strict"] - assert t["process_name"] == ["IEXPLORE.EXE"] - assert t["process_sha256"] == ["0123456789ABCDEF0123456789ABCDEF"] - assert t["reputation"] == ["SUSPECT_MALWARE"] - assert t["tag"] == ["Frood"] - assert t["target_value"] == ["HIGH"] - assert t["threat_id"] == ["B0RG"] - assert t["type"] == ["WATCHLIST"] - assert t["workflow"] == ["OPEN"] - assert t["watchlist_id"] == ["100"] - assert t["watchlist_name"] == ["Gandalf"] - t = body["sort"] - t2 = t[0] - assert t2["field"] == "name" - assert t2["order"] == "DESC" - _was_called = True - body = {"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", "workflow": {"state": "OPEN"}} - envelope = {"results": [body], "num_found": 1} - return MockResponse(envelope) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(WatchlistAlert).where("Blort").categories(["SERIOUS", "CRITICAL"]).device_ids([6023]) \ - .device_names(["HAL"]).device_os(["LINUX"]).device_os_versions(["0.1.2"]).device_username(["JRN"]) \ - .group_results(True).alert_ids(["S0L0"]).legacy_alert_ids(["S0L0_1"]).minimum_severity(6) \ - .policy_ids([8675309]).policy_names(["Strict"]).process_names(["IEXPLORE.EXE"]) \ - .process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).reputations(["SUSPECT_MALWARE"]) \ - .tags(["Frood"]).target_priorities(["HIGH"]).threat_ids(["B0RG"]).types(["WATCHLIST"]) \ - .workflows(["OPEN"]).watchlist_ids(["100"]).watchlist_names(["Gandalf"]).sort_by("name", "DESC") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_watchlistalert_facets(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/_facet" - assert body["query"] == "Blort" - t = body["criteria"] - assert t["workflow"] == ["OPEN"] - t = body["terms"] - assert t["rows"] == 0 - assert t["fields"] == ["REPUTATION", "STATUS"] - _was_called = True - dto1 = {"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]} - dto2 = {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]} - return MockResponse({"results": [dto1, dto2]}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.select(WatchlistAlert).where("Blort").workflows(["OPEN"]) - f = query.facets(["REPUTATION", "STATUS"]) - assert _was_called - t = f[0] - assert t["values"] == [{"id": "reputation", "name": "reputationX", "total": 4}] - t = f[1] - assert t["values"] == [{"id": "status", "name": "statusX", "total": 9}] - - -def test_query_watchlistalert_invalid_watchlist_ids(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(WatchlistAlert).watchlist_ids([888]) - - -def test_query_watchlistalert_invalid_watchlist_names(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(WatchlistAlert).watchlist_names([69]) - - -def test_alerts_bulk_dismiss(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/workflow/_criteria" - assert body["query"] == "Blort" - assert body["state"] == "DISMISSED" - assert body["remediation_state"] == "Fixed" - assert body["comment"] == "Yessir" - t = body["criteria"] - assert t["device_name"] == ["HAL9000"] - _was_called = True - return MockResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - q = api.select(BaseAlert).where("Blort").device_names(["HAL9000"]) - reqid = q.dismiss("Fixed", "Yessir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_undismiss(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/workflow/_criteria" - assert body["query"] == "Blort" - assert body["state"] == "OPEN" - assert body["remediation_state"] == "Fixed" - assert body["comment"] == "NoSir" - t = body["criteria"] - assert t["device_name"] == ["HAL9000"] - _was_called = True - return MockResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - q = api.select(BaseAlert).where("Blort").device_names(["HAL9000"]) - reqid = q.update("Fixed", "NoSir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_dismiss_watchlist(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/workflow/_criteria" - assert body["query"] == "Blort" - assert body["state"] == "DISMISSED" - assert body["remediation_state"] == "Fixed" - assert body["comment"] == "Yessir" - t = body["criteria"] - assert t["device_name"] == ["HAL9000"] - _was_called = True - return MockResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - q = api.select(WatchlistAlert).where("Blort").device_names(["HAL9000"]) - reqid = q.dismiss("Fixed", "Yessir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_dismiss_cbanalytics(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/workflow/_criteria" - assert body["query"] == "Blort" - assert body["state"] == "DISMISSED" - assert body["remediation_state"] == "Fixed" - assert body["comment"] == "Yessir" - t = body["criteria"] - assert t["device_name"] == ["HAL9000"] - _was_called = True - return MockResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - q = api.select(CBAnalyticsAlert).where("Blort").device_names(["HAL9000"]) - reqid = q.dismiss("Fixed", "Yessir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_dismiss_vmware(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/vmware/workflow/_criteria" - assert body["query"] == "Blort" - assert body["state"] == "DISMISSED" - assert body["remediation_state"] == "Fixed" - assert body["comment"] == "Yessir" - t = body["criteria"] - assert t["device_name"] == ["HAL9000"] - _was_called = True - return MockResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - q = api.select(VMwareAlert).where("Blort").device_names(["HAL9000"]) - reqid = q.dismiss("Fixed", "Yessir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_dismiss_threat(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/threat/workflow/_criteria" - assert body["threat_id"] == ["B0RG", "F3R3NG1"] - assert body["state"] == "DISMISSED" - assert body["remediation_state"] == "Fixed" - assert body["comment"] == "Yessir" - _was_called = True - return MockResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - reqid = api.bulk_threat_dismiss(["B0RG", "F3R3NG1"], "Fixed", "Yessir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_undismiss_threat(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/threat/workflow/_criteria" - assert body["threat_id"] == ["B0RG", "F3R3NG1"] - assert body["state"] == "OPEN" - assert body["remediation_state"] == "Fixed" - assert body["comment"] == "NoSir" - _was_called = True - return MockResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - reqid = api.bulk_threat_update(["B0RG", "F3R3NG1"], "Fixed", "NoSir") - assert _was_called - assert reqid == "497ABX" - - -def test_load_workflow(monkeypatch): - _was_called = False - - def mock_get_object(url, parms=None, default=None): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" - _was_called = True - resp = {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED"} - resp["workflow"] = {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"} - return resp - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - workflow = api.select(WorkflowStatus, "497ABX") - assert _was_called - assert workflow.id_ == "497ABX" diff --git a/test/cbtest.py b/test/cbtest.py new file mode 100755 index 00000000..cdfe9467 --- /dev/null +++ b/test/cbtest.py @@ -0,0 +1,38 @@ +import pytest + + +class StubResponse(object): + def __init__(self, contents, scode=200): + self._contents = contents + self.status_code = scode + + def json(self): + return self._contents + + +def _failing_get_object(url, parms=None, default=None): + pytest.fail("GET called for %s when it shouldn't be" % url) + + +def _failing_get_raw_data(url, query_params, **kwargs): + pytest.fail("Raw GET called for %s when it shouldn't be" % url) + + +def _failing_post_object(url, body, **kwargs): + pytest.fail("POST called for %s when it shouldn't be" % url) + + +def _failing_put_object(url, body, **kwargs): + pytest.fail("PUT called for %s when it shouldn't be" % url) + + +def _failing_delete_object(url): + pytest.fail("DELETE called for %s when it shouldn't be" % url) + + +def patch_cbapi(monkeypatch, api, **kwargs): + monkeypatch.setattr(api, "get_object", kwargs.get('GET', _failing_get_object)) + monkeypatch.setattr(api, "get_raw_data", kwargs.get('RAW_GET', _failing_get_raw_data)) + monkeypatch.setattr(api, "post_object", kwargs.get('POST', _failing_post_object)) + monkeypatch.setattr(api, "put_object", kwargs.get('PUT', _failing_put_object)) + monkeypatch.setattr(api, "delete_object", kwargs.get('DELETE', _failing_delete_object)) diff --git a/test/mocks.py b/test/mocks.py deleted file mode 100755 index 22f19bd6..00000000 --- a/test/mocks.py +++ /dev/null @@ -1,36 +0,0 @@ -import pytest - - -class MockResponse(object): - def __init__(self, contents, scode=200): - self._contents = contents - self.status_code = scode - - def json(self): - return self._contents - - -def failing_mock_get_object(url, parms=None, default=None): - pytest.fail("GET called for %s when it shouldn't be" % url) - - -def failing_mock_post_object(url, body, **kwargs): - pytest.fail("POST called for %s when it shouldn't be" % url) - - -def failing_mock_put_object(url, body, **kwargs): - pytest.fail("PUT called for %s when it shouldn't be" % url) - - -def failing_mock_delete_object(url): - pytest.fail("DELETE called for %s when it shouldn't be" % url) - - -_methods = {"GET": failing_mock_get_object, "POST": failing_mock_post_object, - "PUT": failing_mock_put_object, "DELETE": failing_mock_delete_object} - - -class ConnectionMocks: - @classmethod - def get(cls, name): - return _methods[name] From f05e5e8faa820483482125bc9774df95f098b6ee Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Fri, 22 Nov 2019 14:08:54 -0700 Subject: [PATCH 057/197] changed the docs to match the new names for some API methods --- docs/psc-api.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/psc-api.rst b/docs/psc-api.rst index b122a3c1..0034acd3 100755 --- a/docs/psc-api.rst +++ b/docs/psc-api.rst @@ -27,7 +27,7 @@ Device objects, which can be used to locate a list of Devices. *Example:* >>> cbapi = CbPSCBaseAPI(...) - >>> devices = cbapi.select(Device).os("LINUX").status("ALL") + >>> devices = cbapi.select(Device).set_os("LINUX").status("ALL") Selects all devices running Linux from the current organization. @@ -59,7 +59,7 @@ search for more specialized alert types: *Example:* >>> cbapi = CbPSCBaseAPI(...) - >>> alerts = cbapi.select(BaseAlert).device_os(["WINDOWS"]).process_name(["IEXPLORE.EXE"]) + >>> alerts = cbapi.select(BaseAlert).set_device_os(["WINDOWS"]).set_process_name(["IEXPLORE.EXE"]) Selects all alerts on a Windows device running the Internet Explorer process. @@ -73,7 +73,7 @@ finished. *Example:* >>> cbapi = CbPSCBaseAPI(...) - >>> query = cbapi.select(BaseAlert).process_name(["IEXPLORE.EXE"]) + >>> query = cbapi.select(BaseAlert).set_process_name(["IEXPLORE.EXE"]) >>> reqid = query.dismiss("Using Chrome") >>> stat = cbapi.select(WorkflowStatus, reqid) >>> while not stat.finished: From 9cb64a28aa494171d4db31553bc5e515ace93f01 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Mon, 25 Nov 2019 13:44:49 -0700 Subject: [PATCH 058/197] cleaned up flake8 errors in docs/conf.py --- docs/conf.py | 91 ++++++++++++++++++++++++++-------------------------- 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7acc801d..8be55876 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,18 +12,18 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys -import os +# import sys (imports not needed) +# import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -44,7 +44,7 @@ source_suffix = '.rst' # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' @@ -72,9 +72,9 @@ # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -83,27 +83,27 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'tango' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True @@ -124,14 +124,14 @@ } # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. # " v documentation" by default. -#html_title = u'cbapi v0.9.1' +# html_title = u'cbapi v0.9.1' # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. @@ -140,7 +140,7 @@ # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -150,64 +150,64 @@ # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not None, a 'Last updated on:' timestamp is inserted at every page # bottom, using the given strftime format. # The empty string is equivalent to '%b %d, %Y'. -#html_last_updated_fmt = None +# html_last_updated_fmt = None # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # 'ja' uses this config value. # 'zh' user can custom change `jieba` dictionary path. -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. htmlhelp_basename = 'CarbonBlackAPI-PythonBindingsdoc' @@ -222,23 +222,23 @@ # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- @@ -251,7 +251,7 @@ ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -266,16 +266,16 @@ ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False latex_elements = { # Additional stuff for the LaTeX preamble. @@ -291,11 +291,12 @@ # Latex figure (float) alignment # 'figure_align': 'htbp', 'preamble': "".join(( - '\DeclareUnicodeCharacter{25A0}{=}', # Solid box + '\\DeclareUnicodeCharacter{25A0}{=}', # Solid box )), } autoclass_content = 'both' + def setup(app): app.add_stylesheet('css/custom.css') From 8880156a12c733d9f1f7725dd1d720ea9b6cf09f Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 26 Nov 2019 14:43:01 -0700 Subject: [PATCH 059/197] two minor updates for readability --- src/cbapi/psc/alerts_query.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/cbapi/psc/alerts_query.py b/src/cbapi/psc/alerts_query.py index eade340b..194f83c5 100755 --- a/src/cbapi/psc/alerts_query.py +++ b/src/cbapi/psc/alerts_query.py @@ -10,7 +10,7 @@ class BaseAlertSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQuery VALID_CATEGORIES = ["THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", "CRITICAL"] VALID_REPUTATIONS = ["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", "NOT_LISTED", "ADAPTIVE_WHITE_LIST", "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", "COMPANY_BLACK_LIST"] - VALID_ALERTTYPES = ["CB_ANALYTICS", "VMWARE", "WATCHLIST"] + VALID_ALERT_TYPES = ["CB_ANALYTICS", "VMWARE", "WATCHLIST"] VALID_WORKFLOW_VALS = ["OPEN", "DISMISSED"] VALID_FACET_FIELDS = ["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", @@ -37,17 +37,17 @@ def _update_criteria(self, key, newlist): oldlist = self._criteria.get(key, []) self._criteria[key] = oldlist + newlist - def set_categories(self, cats): + def set_categories(self, categories): """ Restricts the alerts that this query is performed on to the specified categories. - :param cats list: List of categories to be restricted to. Valid categories are - "THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", and "CRITICAL." + :param categories list: List of categories to be restricted to. Valid categories are + "THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", and "CRITICAL." :return: This instance """ - if not all((c in BaseAlertSearchQuery.VALID_CATEGORIES) for c in cats): + if not all((c in BaseAlertSearchQuery.VALID_CATEGORIES) for c in categories): raise ApiError("One or more invalid category values") - self._update_criteria("category", cats) + self._update_criteria("category", categories) return self def set_create_time(self, *args, **kwargs): @@ -306,7 +306,7 @@ def set_types(self, alerttypes): "CB_ANALYTICS", "VMWARE", and "WATCHLIST". :return: This instance """ - if not all((t in BaseAlertSearchQuery.VALID_ALERTTYPES) for t in alerttypes): + if not all((t in BaseAlertSearchQuery.VALID_ALERT_TYPES) for t in alerttypes): raise ApiError("One or more invalid alert type values") self._update_criteria("type", alerttypes) return self From 1f05a5506822e2afc0f71f2cc35d0233104b4cf7 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 26 Nov 2019 15:05:35 -0700 Subject: [PATCH 060/197] flake8'ed setversion.py --- setversion.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/setversion.py b/setversion.py index 6201103d..9c85a49d 100755 --- a/setversion.py +++ b/setversion.py @@ -9,6 +9,7 @@ import argparse from datetime import date + def readme_rewriter(line, ctxt): expr = ctxt.get("readme_expr", None) if not expr: @@ -72,7 +73,7 @@ def init_rewriter(line, ctxt): if expr.match(line): return "__version__ = '{0}'\n".format(ctxt["version"]) return None - + def rewrite_file(infilename, rewritefunc, ctxt): outfilename = infilename + ".new" @@ -93,15 +94,18 @@ def rewrite_file(infilename, rewritefunc, ctxt): if not ctxt["nodelete"]: os.remove(infilename) os.rename(outfilename, infilename) - - + + def main(): parser = argparse.ArgumentParser(description="Set the version number in CbAPI source and documentation.\n" - "Execute this on a release or hotfix branch to update the version numbers in the source.", - epilog="After running, edit docs/changelog.rst and add the new changelog information under the new heading.") + "Execute this on a release or hotfix branch to update " + "the version numbers in the source.", + epilog="After running, edit docs/changelog.rst and add the new changelog " + "information under the new heading.") parser.add_argument("version", help="New version number to add") - parser.add_argument("-n", "--nodelete", action="store_true", help="Do not delete existing files, leave new files with .new extension") - + parser.add_argument("-n", "--nodelete", action="store_true", + help="Do not delete existing files, leave new files with .new extension") + args = parser.parse_args() ctxt = {"version": args.version, "nodelete": args.nodelete} rewrite_file("README.md", readme_rewriter, ctxt) @@ -110,6 +114,7 @@ def main(): rewrite_file("setup.py", setup_rewriter, ctxt) rewrite_file("src/cbapi/__init__.py", init_rewriter, ctxt) return 0 - + + if __name__ == "__main__": sys.exit(main()) From f31e25a144598870ad525f156e81e580bf0fc9a1 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 3 Dec 2019 10:40:41 -0700 Subject: [PATCH 061/197] added version number validation and fixed the marker line in changelog --- docs/changelog.rst | 4 ++-- setversion.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 743a96ca..7bcbb639 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,5 +1,7 @@ CbAPI Changelog =============== +.. top-of-changelog (DO NOT REMOVE THIS COMMENT) + CbAPI 1.5.6 - Released November 19, 2019 ---------------------------------------- @@ -16,8 +18,6 @@ Updates * CB ThreatHunter * Fix List object that was not callable. -.. top-of-changelog (DO NOT REMOVE THIS COMMENT) - CbAPI 1.5.4 - Released October 24, 2019 ---------------------------------------- diff --git a/setversion.py b/setversion.py index 9c85a49d..1ae1e664 100755 --- a/setversion.py +++ b/setversion.py @@ -107,6 +107,12 @@ def main(): help="Do not delete existing files, leave new files with .new extension") args = parser.parse_args() + + vnexpr = re.compile(r"^[1-9]\d*\.\d+\.\d+$") + if not vnexpr.match(args.version): + print("Invalid version number {0}: must be three numeric values separated by dots\n".format(args.version)) + return 1 + ctxt = {"version": args.version, "nodelete": args.nodelete} rewrite_file("README.md", readme_rewriter, ctxt) rewrite_file("docs/changelog.rst", changelog_rewriter, ctxt) From de7cb29d1845d90fc9bbce9bd8988444e202b67e Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 3 Dec 2019 11:04:30 -0700 Subject: [PATCH 062/197] added -b/--backup option to script --- setversion.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/setversion.py b/setversion.py index 1ae1e664..3239f7da 100755 --- a/setversion.py +++ b/setversion.py @@ -92,7 +92,10 @@ def rewrite_file(infilename, rewritefunc, ctxt): infile.close() outfile.close() if not ctxt["nodelete"]: - os.remove(infilename) + if ctxt["backup"]: + os.rename(infilename, infilename + ".bak") + else: + os.remove(infilename) os.rename(outfilename, infilename) @@ -105,6 +108,8 @@ def main(): parser.add_argument("version", help="New version number to add") parser.add_argument("-n", "--nodelete", action="store_true", help="Do not delete existing files, leave new files with .new extension") + parser.add_argument("-b", "--backup", action="store_true", + help="Keep old versions of files around with a .bak extension") args = parser.parse_args() @@ -113,7 +118,7 @@ def main(): print("Invalid version number {0}: must be three numeric values separated by dots\n".format(args.version)) return 1 - ctxt = {"version": args.version, "nodelete": args.nodelete} + ctxt = {"version": args.version, "nodelete": args.nodelete, "backup": args.backup} rewrite_file("README.md", readme_rewriter, ctxt) rewrite_file("docs/changelog.rst", changelog_rewriter, ctxt) rewrite_file("docs/conf.py", doc_conf_rewriter, ctxt) From eebacba1480c783abb943869692b7e65c0adc994 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 3 Dec 2019 11:41:55 -0700 Subject: [PATCH 063/197] added -r/--renameonly option to support Alex's suggested workflow --- setversion.py | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/setversion.py b/setversion.py index 3239f7da..5f489499 100755 --- a/setversion.py +++ b/setversion.py @@ -77,20 +77,25 @@ def init_rewriter(line, ctxt): def rewrite_file(infilename, rewritefunc, ctxt): outfilename = infilename + ".new" - infile = open(infilename, "r") - outfile = open(outfilename, "w") - try: - s = infile.readline() - while s: - s2 = rewritefunc(s, ctxt) - if s2: - outfile.write(s2) - else: - outfile.write(s) + if not ctxt["renameonly"]: + infile = open(infilename, "r") + outfile = open(outfilename, "w") + try: s = infile.readline() - finally: - infile.close() - outfile.close() + while s: + s2 = rewritefunc(s, ctxt) + if s2: + outfile.write(s2) + else: + outfile.write(s) + s = infile.readline() + finally: + infile.close() + outfile.close() + else: + if not os.access(outfilename, os.F_OK): + print("warning: new file {0} does not exist to be renamed".format(outfilename)) + return if not ctxt["nodelete"]: if ctxt["backup"]: os.rename(infilename, infilename + ".bak") @@ -110,15 +115,22 @@ def main(): help="Do not delete existing files, leave new files with .new extension") parser.add_argument("-b", "--backup", action="store_true", help="Keep old versions of files around with a .bak extension") + parser.add_argument("-r", "--renameonly", action="store_true", + help="Do rename of .new files only; don't rewrite") args = parser.parse_args() - vnexpr = re.compile(r"^[1-9]\d*\.\d+\.\d+$") - if not vnexpr.match(args.version): - print("Invalid version number {0}: must be three numeric values separated by dots\n".format(args.version)) + if args.renameonly and args.nodelete: + print("cannot specify --renameonly and --nodelete together") return 1 - ctxt = {"version": args.version, "nodelete": args.nodelete, "backup": args.backup} + if not args.renameonly: + vnexpr = re.compile(r"^[1-9]\d*\.\d+\.\d+$") + if not vnexpr.match(args.version): + print("Invalid version number {0}: must be three numeric values separated by dots\n".format(args.version)) + return 1 + + ctxt = {"version": args.version, "nodelete": args.nodelete, "backup": args.backup, "renameonly": args.renameonly} rewrite_file("README.md", readme_rewriter, ctxt) rewrite_file("docs/changelog.rst", changelog_rewriter, ctxt) rewrite_file("docs/conf.py", doc_conf_rewriter, ctxt) From 6a89ea01f8e30b38f8b522363ea8aedcd3c3ed63 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 3 Dec 2019 11:46:16 -0700 Subject: [PATCH 064/197] re-flake8'd the setversion.py script --- setversion.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setversion.py b/setversion.py index 5f489499..af09e302 100755 --- a/setversion.py +++ b/setversion.py @@ -119,17 +119,17 @@ def main(): help="Do rename of .new files only; don't rewrite") args = parser.parse_args() - + if args.renameonly and args.nodelete: print("cannot specify --renameonly and --nodelete together") return 1 - + if not args.renameonly: vnexpr = re.compile(r"^[1-9]\d*\.\d+\.\d+$") if not vnexpr.match(args.version): print("Invalid version number {0}: must be three numeric values separated by dots\n".format(args.version)) return 1 - + ctxt = {"version": args.version, "nodelete": args.nodelete, "backup": args.backup, "renameonly": args.renameonly} rewrite_file("README.md", readme_rewriter, ctxt) rewrite_file("docs/changelog.rst", changelog_rewriter, ctxt) From a63030c1d1dad2b428ea32ffead69fc0e7939e4f Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 3 Dec 2019 11:56:57 -0700 Subject: [PATCH 065/197] make the script use an "accept" command, which is more like what Alex wants --- setversion.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setversion.py b/setversion.py index af09e302..53d35bb6 100755 --- a/setversion.py +++ b/setversion.py @@ -77,7 +77,7 @@ def init_rewriter(line, ctxt): def rewrite_file(infilename, rewritefunc, ctxt): outfilename = infilename + ".new" - if not ctxt["renameonly"]: + if not ctxt["accept"]: infile = open(infilename, "r") outfile = open(outfilename, "w") try: @@ -110,27 +110,27 @@ def main(): "the version numbers in the source.", epilog="After running, edit docs/changelog.rst and add the new changelog " "information under the new heading.") - parser.add_argument("version", help="New version number to add") + parser.add_argument("version", help="New version number to add, or \"accept\" to accept changes to .new files") parser.add_argument("-n", "--nodelete", action="store_true", help="Do not delete existing files, leave new files with .new extension") parser.add_argument("-b", "--backup", action="store_true", help="Keep old versions of files around with a .bak extension") - parser.add_argument("-r", "--renameonly", action="store_true", - help="Do rename of .new files only; don't rewrite") args = parser.parse_args() - if args.renameonly and args.nodelete: - print("cannot specify --renameonly and --nodelete together") - return 1 - - if not args.renameonly: + is_accept = False + if args.version == "accept": + if args.nodelete: + print("cannot specify --nodelete with accept command") + return 1 + is_accept = True + else: vnexpr = re.compile(r"^[1-9]\d*\.\d+\.\d+$") if not vnexpr.match(args.version): print("Invalid version number {0}: must be three numeric values separated by dots\n".format(args.version)) return 1 - ctxt = {"version": args.version, "nodelete": args.nodelete, "backup": args.backup, "renameonly": args.renameonly} + ctxt = {"version": args.version, "nodelete": args.nodelete, "backup": args.backup, "accept": is_accept} rewrite_file("README.md", readme_rewriter, ctxt) rewrite_file("docs/changelog.rst", changelog_rewriter, ctxt) rewrite_file("docs/conf.py", doc_conf_rewriter, ctxt) From 0d0d49cdb1863211b1225b730c636ef4b6e74660 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 3 Dec 2019 15:38:34 -0700 Subject: [PATCH 066/197] version update: 1.6.0 --- README.md | 2 +- docs/changelog.rst | 36 ++++++++++++++++++++++++++++++++++++ docs/conf.py | 4 ++-- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 41 insertions(+), 5 deletions(-) mode change 100644 => 100755 docs/changelog.rst mode change 100644 => 100755 docs/conf.py diff --git a/README.md b/README.md index 7fb36671..caa9b362 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.5.6** +**Latest Version: 1.6.0** These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. diff --git a/docs/changelog.rst b/docs/changelog.rst old mode 100644 new mode 100755 index 7bcbb639..24761a94 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,42 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.6.0 - Released December 3, 2019 +--------------------------------------- + +Updates + +* New Carbon Black Cloud API Support + * Support for Devices v6: + * List and search for devices + * Export device information to CSV + * Device control actions: quarantine, bypass, background scan, deregister/delete, update + * Support for Alerts v6: + * Search for and retrieve alerts + * Update alert status (dismiss alerts) + +Examples + +* Devices v6: + * psc/device_control.py + * psc/download_device_list.py + * psc/list_devices.py +* Alerts v6: + * psc/alert_search_suggestions.py + * psc/bulk_update_alerts.py + * psc/bulk_update_cbanalytics_alerts.py + * psc/bulk_update_threat_alerts.py + * psc/bulk_update_vmware_alerts.py + * psc/bulk_update_watchlist_alerts.py + * psc/list_alert_facets.py + * psc/list_alerts.py + * psc/list_cbanalytics_alert_facets.py + * psc/list_cbanalytics_alerts.py + * psc/list_vmware_alert_facets.py + * psc/list_vmware_alerts.py + * psc/list_watchlist_alert_facets.py + * psc/list_watchlist_alerts.py + CbAPI 1.5.6 - Released November 19, 2019 ---------------------------------------- diff --git a/docs/conf.py b/docs/conf.py old mode 100644 new mode 100755 index 8be55876..61014b09 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,9 +59,9 @@ # built documents. # # The short X.Y version. -version = u'1.5' +version = u'1.6' # The full version, including alpha/beta/rc tags. -release = u'1.5.6' +release = u'1.6.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index e491e338..20457295 100755 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ setup( name='cbapi', - version='1.5.6', + version='1.6.0' url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index 2c792e43..dfb702f7 100755 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -5,7 +5,7 @@ __title__ = 'cbapi' __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' -__copyright__ = 'Copyright 2018-2019 Carbon Black' +__copyright__ = 'Copyright 2018-2019 VMware Carbon Black' __version__ = '1.6.0' # New API as of cbapi 0.9.0 From 7ddcd64f22d53959bdbc23e428b9f58864ceb43a Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 3 Dec 2019 15:49:49 -0700 Subject: [PATCH 067/197] setversion broke setup.py - fixed both scripts --- setup.py | 2 +- setversion.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 20457295..e6ce8af7 100755 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ setup( name='cbapi', - version='1.6.0' + version='1.6.0', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/setversion.py b/setversion.py index 53d35bb6..73d65a5b 100755 --- a/setversion.py +++ b/setversion.py @@ -61,7 +61,7 @@ def setup_rewriter(line, ctxt): ctxt["setup_expr"] = expr m = expr.match(line) if m: - return "{0}version='{1}'\n".format(m.group(1), ctxt["version"]) + return "{0}version='{1}',\n".format(m.group(1), ctxt["version"]) return None From 5f54583de50d34c0f476e852290b8799e907af87 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 3 Dec 2019 16:15:23 -0700 Subject: [PATCH 068/197] removed an unnecessary import --- src/cbapi/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cbapi/models.py b/src/cbapi/models.py index e0161339..59ecaad3 100644 --- a/src/cbapi/models.py +++ b/src/cbapi/models.py @@ -6,7 +6,6 @@ from cbapi.six import python_2_unicode_compatible -import sys import base64 import os.path from cbapi.six import iteritems, add_metaclass, integer_types From 6ffc6343393b1e11c8d4e6fdfb18efac36e63894 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Mon, 6 Jan 2020 15:06:07 -0700 Subject: [PATCH 069/197] LIVE_RESPONSE_KEY only supports integrationServices/v3/device --- src/cbapi/psc/cblr.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cbapi/psc/cblr.py b/src/cbapi/psc/cblr.py index e945ba68..380450f0 100644 --- a/src/cbapi/psc/cblr.py +++ b/src/cbapi/psc/cblr.py @@ -6,7 +6,6 @@ from cbapi.errors import TimeoutError from cbapi.live_response_api import CbLRManagerBase, CbLRSessionBase, poll_status -from cbapi.psc.models import Device OS_LIVE_RESPONSE_ENUM = { @@ -21,12 +20,14 @@ class LiveResponseSession(CbLRSessionBase): def __init__(self, cblr_manager, session_id, sensor_id, session_data=None): super(LiveResponseSession, self).__init__(cblr_manager, session_id, sensor_id, session_data=session_data) + from cbapi.psc.defense.models import Device device_info = self._cb.select(Device, self.sensor_id) self.os_type = OS_LIVE_RESPONSE_ENUM.get(device_info.deviceType, None) class WorkItem(object): def __init__(self, fn, sensor_id): + from cbapi.psc.defense.models import Device self.fn = fn if isinstance(sensor_id, Device): self.sensor_id = sensor_id.deviceId @@ -228,6 +229,7 @@ def submit_job(self, work_item): self.schedule_queue.put(work_item) def _spawn_new_workers(self): + from cbapi.psc.defense.models import Device if len(self._job_workers) >= self._max_workers: return From 5d2b500f4500b3ac54c84d44306d437828f9aec4 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Mon, 6 Jan 2020 15:33:17 -0700 Subject: [PATCH 070/197] Fix doc references --- docs/psc-api.rst | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/psc-api.rst b/docs/psc-api.rst index 0034acd3..891f3563 100755 --- a/docs/psc-api.rst +++ b/docs/psc-api.rst @@ -33,15 +33,15 @@ Selects all devices running Linux from the current organization. **Query Object:** -.. autoclass:: cbapi.psc.query.DeviceSearchQuery +.. autoclass:: cbapi.psc.devices_query.DeviceSearchQuery :members: - + **Model Object:** .. autoclass:: cbapi.psc.models.Device :members: :undoc-members: - + Alerts API ---------- @@ -60,7 +60,7 @@ search for more specialized alert types: >>> cbapi = CbPSCBaseAPI(...) >>> alerts = cbapi.select(BaseAlert).set_device_os(["WINDOWS"]).set_process_name(["IEXPLORE.EXE"]) - + Selects all alerts on a Windows device running the Internet Explorer process. Individual alerts may have their status changed using the dismiss() or update() @@ -78,21 +78,21 @@ finished. >>> stat = cbapi.select(WorkflowStatus, reqid) >>> while not stat.finished: >>> # wait for it to finish - + This dismisses all alerts which reference the Internet Explorer process. **Query Objects:** -.. autoclass:: cbapi.psc.query.BaseAlertSearchQuery +.. autoclass:: cbapi.psc.alerts_query.BaseAlertSearchQuery :members: - -.. autoclass:: cbapi.psc.query.CBAnalyticsAlertSearchQuery + +.. autoclass:: cbapi.psc.alerts_query.CBAnalyticsAlertSearchQuery :members: -.. autoclass:: cbapi.psc.query.VMwareAlertSearchQuery +.. autoclass:: cbapi.psc.alerts_query.VMwareAlertSearchQuery :members: -.. autoclass:: cbapi.psc.query.WatchlistAlertSearchQuery +.. autoclass:: cbapi.psc.alerts_query.WatchlistAlertSearchQuery :members: **Model Objects:** @@ -100,24 +100,23 @@ This dismisses all alerts which reference the Internet Explorer process. .. autoclass:: cbapi.psc.models.Workflow :members: :undoc-members: - + .. autoclass:: cbapi.psc.models.BaseAlert :members: :undoc-members: - + .. autoclass:: cbapi.psc.models.CBAnalyticsAlert :members: :undoc-members: - + .. autoclass:: cbapi.psc.models.VMwareAlert :members: :undoc-members: - + .. autoclass:: cbapi.psc.models.WatchlistAlert :members: :undoc-members: - + .. autoclass:: cbapi.psc.models.WorkflowStatus :members: :undoc-members: - From f1826cb0995b4bfb3288031a618a2703344da948 Mon Sep 17 00:00:00 2001 From: becca <55801254+bvasil-cb@users.noreply.github.com> Date: Tue, 7 Jan 2020 14:03:03 -0700 Subject: [PATCH 071/197] Create a pull request template --- .github/pull_request_template.md | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..541292e9 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,51 @@ +## Pull request checklist + +Please check if your PR fulfills the following requirements: +- [ ] Docs have been reviewed and added / updated if needed (for bug fixes / features) +- [ ] Tests have been added that prove the fix is effective or that the feature works. +- [ ] New and existing tests pass locally with the changes. +- [ ] Code follows the style guidelines of this project (PEP8, clean code). +- [ ] Linter has passed locally and any fixes were made for failures. +- [ ] A self-review of the code has been done. + +## Pull request type + + + +Please check the type of change your PR introduces: +- [ ] Bugfix +- [ ] Feature +- [ ] Code style update (formatting, renaming) +- [ ] Refactoring (no functional changes, no api changes) +- [ ] Build related changes +- [ ] Documentation content changes (not tied to bugs/features) +- [ ] Other (please describe): + + +## What is the ticket or issue number? + + +- Ticket Number: N/A + +- Issue Number: N/A + +## What is the current behavior? + + +## What is the new behavior? + + +## Does this introduce a breaking change? + +- [ ] Yes +- [ ] No + + + +## How Has This Been Tested? + + + +## Other information: + + From bc31973c762b6ac33e44f9ab0d6d4629d6e9a445 Mon Sep 17 00:00:00 2001 From: becca <55801254+bvasil-cb@users.noreply.github.com> Date: Tue, 7 Jan 2020 14:50:14 -0700 Subject: [PATCH 072/197] Update pull_request_template.md --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 541292e9..141ba550 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,7 @@ ## Pull request checklist Please check if your PR fulfills the following requirements: + - [ ] Docs have been reviewed and added / updated if needed (for bug fixes / features) - [ ] Tests have been added that prove the fix is effective or that the feature works. - [ ] New and existing tests pass locally with the changes. From 9ec7ec55627ae2ff34e684e0d5d3fc90beadd97e Mon Sep 17 00:00:00 2001 From: becca <55801254+bvasil-cb@users.noreply.github.com> Date: Wed, 8 Jan 2020 10:45:05 -0700 Subject: [PATCH 073/197] Update pull_request_template.md --- .github/pull_request_template.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 141ba550..c82af637 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -30,11 +30,8 @@ Please check the type of change your PR introduces: - Issue Number: N/A -## What is the current behavior? - - -## What is the new behavior? - +## Pull Request Description + ## Does this introduce a breaking change? From 86fd9e7779bfc0e977ee727fce6a3c0e1709bc39 Mon Sep 17 00:00:00 2001 From: becca <55801254+bvasil-cb@users.noreply.github.com> Date: Wed, 8 Jan 2020 14:58:07 -0700 Subject: [PATCH 074/197] Update issue templates --- .github/ISSUE_TEMPLATE/bug-report.md | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 00000000..e3bb8782 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: bug +assignees: kebringer-cb, talfano-cb + +--- + +**I am seeing this behaviour on: (please complete the following information):** + - OS: [e.g. iOS] + - Carbon Black Product: [e.g. CB Protection, CB Defense] + - Version: [e.g. 22] + +**Describe the bug** +A clear and concise description of what the bug is. + +**Steps to Reproduce** +Steps to reproduce the behavior (Provide a log message if relevant): +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots or code to help explain your problem. + +**Additional context** +Add any other context about the problem here. From cc82e93da990fdb95dbeb7db213a6f226c2ae964 Mon Sep 17 00:00:00 2001 From: becca <55801254+bvasil-cb@users.noreply.github.com> Date: Wed, 8 Jan 2020 16:49:12 -0700 Subject: [PATCH 075/197] Update issue templates --- .github/ISSUE_TEMPLATE/bug-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index e3bb8782..9a53bbab 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -10,7 +10,7 @@ assignees: kebringer-cb, talfano-cb **I am seeing this behaviour on: (please complete the following information):** - OS: [e.g. iOS] - Carbon Black Product: [e.g. CB Protection, CB Defense] - - Version: [e.g. 22] + - Python Version: [e.g. 2.7] **Describe the bug** A clear and concise description of what the bug is. From 907a76e703d10b844e0fd2fca3f8cf1ed8c3dd04 Mon Sep 17 00:00:00 2001 From: Jason Garman Date: Thu, 9 Jan 2020 06:24:30 -0800 Subject: [PATCH 076/197] Fix #202 by overriding URL when performing non-GET requests --- src/cbapi/response/models.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 5e68920a..ef6a7b14 100644 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -308,6 +308,17 @@ def feed(self): def trigger_ioc(self): return self.ioc_attr + # override the Alert API URL to /api/v1/alert when performing POST/PUTs + def _build_api_request_uri(self, http_method="GET"): + if http_method == "GET": + return super(Alert, self)._build_api_request_uri(http_method) + else: + baseuri = "/api/v1/alert" + if self._model_unique_id is not None: + return baseuri + "/%s" % self._model_unique_id + else: + return baseuri + class Feed(MutableBaseModel, CreatableModelMixin): swagger_meta_file = "response/models/feed.yaml" From 6fd132db88fe218880daa68139d2605c6b555563 Mon Sep 17 00:00:00 2001 From: Casey Parman Date: Thu, 9 Jan 2020 11:16:21 -0500 Subject: [PATCH 077/197] udpated example documentation --- src/cbapi/psc/threathunter/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index a2fd62e1..0f732323 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -169,7 +169,7 @@ class Query(PaginatedQuery): Examples:: - >>> from cbapi.psc.threathunter import CbThreatHunterAPI + >>> from cbapi.psc.threathunter import CbThreatHunterAPI,Process >>> cb = CbThreatHunterAPI() >>> query = cb.select(Process) >>> query = query.where(process_name="notepad.exe") From fedf5ab682e2d303dae226b7f702726af2e82c53 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Mon, 13 Jan 2020 14:43:00 -0700 Subject: [PATCH 078/197] Release version 1.6.1 --- README.md | 2 +- docs/changelog.rst | 14 +++++++++++++- docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 17 insertions(+), 5 deletions(-) mode change 100755 => 100644 README.md mode change 100755 => 100644 docs/changelog.rst mode change 100755 => 100644 docs/conf.py mode change 100755 => 100644 setup.py mode change 100755 => 100644 src/cbapi/__init__.py diff --git a/README.md b/README.md old mode 100755 new mode 100644 index caa9b362..de8e003d --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.6.0** +**Latest Version: 1.6.1** These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. diff --git a/docs/changelog.rst b/docs/changelog.rst old mode 100755 new mode 100644 index 24761a94..2f5f82d4 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,18 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.6.1 - Released January 13, 2020 +--------------------------------------- + +Updates + +* CB Response + * Fix Alert.save() to use alert v1 API +* Carbon Black Cloud + * Fix Live Response flow to use integrationServices/v3/device to prevent need for multiple API keys +* CB ThreatHunter + * Update example for ThreatHunter Query + CbAPI 1.6.0 - Released December 3, 2019 --------------------------------------- @@ -15,7 +27,7 @@ Updates * Support for Alerts v6: * Search for and retrieve alerts * Update alert status (dismiss alerts) - + Examples * Devices v6: diff --git a/docs/conf.py b/docs/conf.py old mode 100755 new mode 100644 index 61014b09..f150cbf7 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.6' # The full version, including alpha/beta/rc tags. -release = u'1.6.0' +release = u'1.6.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index e6ce8af7..158b4cb2 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ setup( name='cbapi', - version='1.6.0', + version='1.6.1', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py old mode 100755 new mode 100644 index dfb702f7..a3e0d1a2 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2019 VMware Carbon Black' -__version__ = '1.6.0' +__version__ = '1.6.1' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From 3adae423a3842b1101da5cc5a036641da8ab2bb4 Mon Sep 17 00:00:00 2001 From: Luke Lyon <52218532+llyon-cb@users.noreply.github.com> Date: Tue, 14 Jan 2020 13:59:49 -0700 Subject: [PATCH 079/197] Threat Intelligence Polling (#199) Addition of ThreatIntel Module to TH Examples, and STIX/TAXII Example script --- .../threat_intelligence/README.md | 132 ++++++ .../threat_intelligence/Taxii_README.md | 42 ++ .../threat_intelligence/config.yml | 70 +++ .../threat_intelligence/feed_helper.py | 37 ++ .../threat_intelligence/get_feed_ids.py | 21 + .../threat_intelligence/requirements.txt | 9 + .../threat_intelligence/results.py | 77 ++++ .../threat_intelligence/stix_parse.py | 416 ++++++++++++++++++ .../threat_intelligence/stix_taxii.py | 366 +++++++++++++++ .../threat_intelligence/threatintel.py | 247 +++++++++++ 10 files changed, 1417 insertions(+) create mode 100644 examples/threathunter/threat_intelligence/README.md create mode 100644 examples/threathunter/threat_intelligence/Taxii_README.md create mode 100644 examples/threathunter/threat_intelligence/config.yml create mode 100644 examples/threathunter/threat_intelligence/feed_helper.py create mode 100644 examples/threathunter/threat_intelligence/get_feed_ids.py create mode 100644 examples/threathunter/threat_intelligence/requirements.txt create mode 100644 examples/threathunter/threat_intelligence/results.py create mode 100644 examples/threathunter/threat_intelligence/stix_parse.py create mode 100644 examples/threathunter/threat_intelligence/stix_taxii.py create mode 100644 examples/threathunter/threat_intelligence/threatintel.py diff --git a/examples/threathunter/threat_intelligence/README.md b/examples/threathunter/threat_intelligence/README.md new file mode 100644 index 00000000..5f30afe5 --- /dev/null +++ b/examples/threathunter/threat_intelligence/README.md @@ -0,0 +1,132 @@ +# ThreatIntel Module +Python3 module that can be used in the development of Threat Intelligence Connectors for the Carbon Black Cloud. + +## Requirements + +The file `requirements.txt` contains a list of dependencies for this project. After cloning this repository, run the following command from the `examples/threathunter/threat_intelligence` directory: + +```python +pip3 install -r ./requirements.txt +``` + + +## Introduction +This document describes how to use the ThreatIntel Python3 module for development of connectors that retrieve Threat Intelligence and import it into a Carbon Black Cloud instance. + +Throughout this document, there are references to Carbon Black ThreatHunter Feed and Report formats. Documentation on Feed and Report definitions is [available here.](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/#definitions) + +## Example + +An example of implementing this ThreatIntel module is [available here](Taxii_README.md). The example uses cabby to connect to a TAXII server, collect threat intelligence, and send it to a ThreatHunter Feed. + + +## Usage + +`threatintel.py` has two main uses: + +1. Report Validation with `threatintel.input_validation()` +2. Pushing Reports to a Carbon Black ThreatHunter Feed with `threatintel.push_to_cb()` + +### Report validation + +Each Report to be sent to the Carbon Black Cloud should be validated +before sending. The [ThreatHunter Report format](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/#definitions) is a JSON object with +five required and five optional values. + +|Required|Type|Optional|Type| +|---|---|---|---| +|`id`|string|`link`|string| +|`timestamp`|integer|`[tags]`|[str]| +|`title`|string|`iocs`|[IOC Format](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/#definitions)| +|`description`|string|`[iocs_v2]`|[[IOCv2 Format](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/#definitions)]| +|`severity`|integer|`visibility`|string| + +The `input_validation` function checks for the existence and type of the five +required values, and (if applicable) checks the optional values. The +function takes a list of dictionaries as input, and outputs a Boolean +indicating if validation was successful. + +### Pushing Reports to a Carbon Black ThreatHunter Feed + +The `push_to_cb` function takes a list of `AnalysisResult` objects (or objects of your own custom class) and a Carbon +Black ThreatHunter Feed ID as input, and writes output to the console. +The `AnalysisResult` class is defined in `results.py`, and requirements for a custom class are outlined in the Customization section below. + +`AnalysisResult` objects are expected to have the same properties as +ThreatHunter Reports (listed in the table above in Report Validation), with the addition of `iocs_v2`. The +`push_to_cb` function will convert `AnalysisResult` objects into +Report dictionaries, and then those dictionaries into ThreatHunter +Report objects. + +Report dictionaries are passed through the Report validation function +`input_validation` described above. Any improperly formatted report +dictionaries are saved to a file called `malformed_reports.json`. + +Upon successful sending of reports to a ThreatHunter Feed, you should +see something similar to the following INFO message in the console: + +`INFO:threatintel:Appended 1000 reports to ThreatHunter Feed AbCdEfGhIjKlMnOp` + + +### Using Validation and Pushing to ThreatHunter in your own code + +Import the module and supporting classes like any other python package, and instantiate a ThreatIntel object: + + ```python + from threatintel import ThreatIntel + from results import IOC_v2, AnalysisResult + ti = ThreatIntel() +``` + +Take the threat intelligence data from your source, and convert it into ``AnalysisResult`` objects. Then, attach the indicators of compromise, and store your data in a list. + +```python + myResults = [] + for intel in myThreatIntelligenceData: + result = AnalysisResult(analysis_name=intel.name, scan_time=intel.scan_time, score=intel.score, title=intel.title, description=intel.description) + #ioc_dict could be a collection of md5 hashes, dns values, file hashes, etc. + for ioc_key, ioc_val in intel.ioc_dict.items(): + result.attach_ioc_v2(values=ioc_val, field=ioc_key, link=link) + myResults.append(result) +``` + +Finally, push your threat intelligence data to a ThreatHunter Feed. +```python + ti.push_to_cb(feed_id='AbCdEfGhIjKlMnOp', results=myResults) +``` + +`ti.push_to_cb` automatically validates your input to ensure it has the values required for ThreatHunter. Validated reports will be sent to your specified ThreatHunter Feed, and any malformed reports will be available for review locally at `malformed_reports.json`. + + + +## Customization + +Although the `AnalysisResult` class is provided in `results.py` as an example, you may create your own custom class to use with `push_to_cb`. The class must have the following attributes to work with the provided `push_to_cb` and `input_validation` functions, as well as the ThreatHunter backend: + + +|Attribute|Type| +|---|---| +|`id`|string| +|`timestamp`|integer| +|`title`|string| +|`description`|string| +|`severity`|integer| +|`iocs_v2`|[[IOCv2 Format](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/#definitions)]| + +It is strongly recommended to use the provided `IOC_v2()` class from `results.py`. If you decide to use a custom `iocs_v2` class, that class must have a method called `as_dict` that returns `id`, `match_type`, `values`, `field`, and `link` as a dictionary. + + +## Writing a Custom Threat Intelligence Polling Connector + +An example of a custom Threat Intel connector that uses the `ThreatIntel` Python3 module is included in this repository as `stix_taxii.py`. Most use cases will warrant the use of the ThreatHunter `Report` attribute `iocs_v2`, so it is included in `ThreatIntel.push_to_cb()`. + +`ThreatIntel.push_to_cb()` and `AnalysisResult` can be adapted to include other ThreatHunter `Report` attributes like `link, tags, iocs, and visibility`. + + +## Troubleshooting + +### Credential Error +In order to use this code, you must have CBAPI installed and configured. If you receive the following message, visit the CBAPI GitHub repository for [instructions on setting up authentication](https://github.com/carbonblack/cbapi-python#api-token). + +### 504 Gateway Timeout Error +The [Carbon Black ThreatHunter Feed Manager API](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/) is used in this code. When posting to a Feed, there is a 60 second limit before the gateway terminates your connection. The amount of reports you can POST to a Feed is limited by your connection speed. In this case, you will have to split your threat intelligence into smaller collections until the request takes less than 60 seconds, and send each smaller collection to an individual ThreatHunter Feed. diff --git a/examples/threathunter/threat_intelligence/Taxii_README.md b/examples/threathunter/threat_intelligence/Taxii_README.md new file mode 100644 index 00000000..687d6c17 --- /dev/null +++ b/examples/threathunter/threat_intelligence/Taxii_README.md @@ -0,0 +1,42 @@ +# TAXII Connector +Connector for pulling and converting STIX information from TAXII Service Providers into CB Feeds. + +## Requirements/Installation + +The file `requirements.txt` contains a list of dependencies for this project. After cloning this repository, run the following command from the `examples/threathunter/threat_intelligence` directory: + +```python +pip3 install -r ./requirements.txt +``` + +## Introduction +This document describes how to configure the CB ThreatHunter TAXII connector. +This connector allows for the importing of STIX data by querying one or more TAXII services, retrieving that data and then converting it into CB feeds using the CB JSON format for IOCs. + +## Setup - TAXII Configuration File +The TAXII connector uses the configuration file `config.yml`. An example configuration file is available [here.](config.yml) An explanation of each entry in the configuration file is provided in the example. + + +## Running the Connector +The connector can be activated by running the Python3 file `stix_taxii.py`. The connector will attempt to connect to your TAXII service(s), poll the collection(s), retrieve the STIX data, and send it to the ThreatHunter Feed specified in your `config.yml` file. + +```python +python3 stix_taxii.py +``` + +This script supports updating each TAXII configuration's `start_date`, the date for which to start requesting data, via the command line with the argument `site_start_date`. To change the `stat_date` value for each site in your config file, you must supply the site name and desired `start_date` in `%Y-%m-%d %H:%M:%S` format. + +```python +python3 stix_taxii.py --site_start_date my_site_name_1 '2019-11-05 00:00:00' my_site_name_2 '2019-11-05 00:00:00' +``` + +This may be useful if the intention is to keep an up-to-date collection of STIX data in a ThreatHunter Feed. + +## Troubleshooting + +### Credential Error +In order to use this code, you must have CBAPI installed and configured. If you receive the following message, visit the CBAPI GitHub repository for [instructions on setting up authentication](https://github.com/carbonblack/cbapi-python#api-token). + +### 504 Gateway Timeout Error +The [Carbon Black ThreatHunter Feed Manager API](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/) is used in this code. When posting to a Feed, there is a 60 second limit before the gateway terminates your connection. The amount of reports you can POST to a Feed is limited by your connection speed. In this case, you will have to split your threat intelligence into smaller collections until the request takes less than 60 seconds, and send each smaller collection to an individual ThreatHunter Feed. + diff --git a/examples/threathunter/threat_intelligence/config.yml b/examples/threathunter/threat_intelligence/config.yml new file mode 100644 index 00000000..498dca51 --- /dev/null +++ b/examples/threathunter/threat_intelligence/config.yml @@ -0,0 +1,70 @@ +sites: + my_site_name_1: + # the feed_id of the ThreatHunter Feed you want to send ThreatIntel to + feed_id: + + # the address of the site (only server ip or dns; don't put https:// or a trailing slash) + site: + + # the path of the site for discovering what services are available + # this is supplied by your taxii provider + discovery_path: + + # the path of the site for listing what collections are available to you + # this is supplied by your taxii provider + collection_management_path: + + # the path of the site for polling a collection + # this is supplied by your taxii provider + poll_path: + + # change to true if you require https for your TAXII service connection + use_https: + + # by default, we validate SSL certificates. Turn this off by setting ssl_verify: False + ssl_verify: + + # (optional) if you need SSL certificates for authentication, set the path of the + # certificate and key here. Please leave blank to ignore. + cert_file: + key_file: + + # how to score each result. Accepts values [1,10], and defaults to 5 + default_score: + + # (optional) username for authorization with your taxii provider + username: + + # (optional) password for authorization with your taxii provider + password: + + # (optional) specify which collections to convert to feeds (comma-delimited) + collections: + + # (optional) the start date for which to start requesting data. + # Defaults to 2019-01-01 00:00:00 if you supply nothing + start_date: + + # (optional) the minutes to advance for each request. + # If you don't have a lot of data, you could advance your requests + # to every 60 minutes, or 1440 minutes for daily chunks + # defaults to 60 + size_of_request_in_minutes: + + # (optional) path to a CA SSL certificate + ca_cert: + + # (optional) if you need requests to go through a proxy, specify an http URL here + http_proxy_url: + + # (optional) if you need requests to go through a proxy, specify an https URL here + https_proxy_url: + + # (optional) number of reports to collect from each site. + # Leave blank for no limit + reports_limit: + + # control the number of attempts per-collection before giving up + # trying to get (empty/malformed) STIX data out of a TAXII server. + # defaults to 10 + fail_limit: diff --git a/examples/threathunter/threat_intelligence/feed_helper.py b/examples/threathunter/threat_intelligence/feed_helper.py new file mode 100644 index 00000000..cc732e40 --- /dev/null +++ b/examples/threathunter/threat_intelligence/feed_helper.py @@ -0,0 +1,37 @@ +"""Advances the `begin_date` and `end_date` fields while polling the TAXII server to iteratively get per-collection STIX content. + +This is tied to the `start_date` and `size_of_request_in_minutes` configuration options in your `config.yml`. +""" + +from datetime import datetime, timedelta, timezone + + +class FeedHelper(): + def __init__(self, start_date, size_of_request_in_minutes): + self.size_of_request_in_minutes = size_of_request_in_minutes + self.start_date = start_date.replace(tzinfo=timezone.utc) + self.end_date = self.start_date + \ + timedelta(minutes=self.size_of_request_in_minutes) + self.now = datetime.utcnow().replace(tzinfo=timezone.utc) + if self.end_date > self.now: + self.end_date = self.now + self.start = False + self.done = False + + def advance(self): + """Returns True if keep going, False if we already hit the end time and cannot advance.""" + if not self.start: + self.start = True + return True + + if self.done: + return False + + # continues shifting the time window by size_of_request_in_minutes until we hit current time, then stops + self.start_date = self.end_date + self.end_date += timedelta(minutes=self.size_of_request_in_minutes) + if self.end_date > self.now: + self.end_date = self.now + self.done = True + + return True diff --git a/examples/threathunter/threat_intelligence/get_feed_ids.py b/examples/threathunter/threat_intelligence/get_feed_ids.py new file mode 100644 index 00000000..488a29c9 --- /dev/null +++ b/examples/threathunter/threat_intelligence/get_feed_ids.py @@ -0,0 +1,21 @@ +"""Lists ThreatHunter Feed IDs available for results dispatch.""" + +from cbapi.psc.threathunter import CbThreatHunterAPI +from cbapi.psc.threathunter.models import Feed +import logging + +log = logging.getLogger(__name__) + + +def get_feed_ids(): + cb = CbThreatHunterAPI() + feeds = cb.select(Feed) + if not feeds: + log.info("No feeds are available for the org key {}".format(cb.credentials.org_key)) + else: + for feed in feeds: + log.info("Feed name: {:<20} \t Feed ID: {:>20}".format(feed.name, feed.id)) + + +if __name__ == '__main__': + get_feed_ids() diff --git a/examples/threathunter/threat_intelligence/requirements.txt b/examples/threathunter/threat_intelligence/requirements.txt new file mode 100644 index 00000000..d23ecc51 --- /dev/null +++ b/examples/threathunter/threat_intelligence/requirements.txt @@ -0,0 +1,9 @@ +cybox==2.1.0.18 +dataclasses>=0.6 +cabby==0.1.20 +stix==1.2.0.7 +lxml==4.4.1 +urllib3==1.22 +cbapi>=1.5.6 +python_dateutil==2.8.1 +PyYAML==5.1.2 diff --git a/examples/threathunter/threat_intelligence/results.py b/examples/threathunter/threat_intelligence/results.py new file mode 100644 index 00000000..f750191c --- /dev/null +++ b/examples/threathunter/threat_intelligence/results.py @@ -0,0 +1,77 @@ +import enum +import logging + + +class IOC_v2(): + """Models an indicator of compromise detected during an analysis. + + Every IOC belongs to an AnalysisResult. + """ + + def __init__(self, analysis, match_type, values, field, link): + self.id = analysis + self.match_type = match_type + self.values = values + self.field = field + self.link = link + + class MatchType(str, enum.Enum): + """ + Represents the valid matching strategies for an IOC. + """ + + Equality: str = "equality" + Regex: str = "regex" + Query: str = "query" + + def as_dict(self): + return { + "id": str(self.id), + "match_type": self.match_type, + "values": list(self.values), + "field": self.field, + "link": self.link, + } + + +class AnalysisResult(): + """Models the result of an analysis performed by a connector.""" + + def __init__(self, analysis_name, scan_time, score, title, description): + self.id = str(analysis_name) + self.timestamp = scan_time + self.title = title + self.description = description + self.severity = score + self.iocs = [] + self.iocs_v2 = [] + self.link = None + self.tags = None + self.visibility = None + self.connector_name = "STIX_TAXII" + + def attach_ioc_v2(self, *, match_type=IOC_v2.MatchType.Equality, values, field, link): + self.iocs_v2.append(IOC_v2(analysis=self.id, match_type=match_type, values=values, field=field, link=link)) + + def normalize(self): + """Normalizes this result to make it palatable for the CbTH backend.""" + + if self.severity <= 0 or self.severity > 10: + logging.warning("normalizing OOB score: {}".format(self.severity)) + self.severity = max(1, min(self.severity, 10)) + # NOTE: min 1 and not 0 + # else err 400 from cbapi: Report severity must be between 1 & 10 + return self + + def as_dict(self): + return {"IOCs_v2": [ioc_v2.as_dict() for ioc_v2 in self.iocs_v2], **super().as_dict()} + + def as_dict_full(self): + return { + "id": self.id, + "timestamp": self.timestamp, + "title": self.title, + "description": self.description, + "severity": self.severity, + "iocs_v2": [iocv2.as_dict() for iocv2 in self.iocs_v2] + } diff --git a/examples/threathunter/threat_intelligence/stix_parse.py b/examples/threathunter/threat_intelligence/stix_parse.py new file mode 100644 index 00000000..b872318d --- /dev/null +++ b/examples/threathunter/threat_intelligence/stix_parse.py @@ -0,0 +1,416 @@ +"""Parses STIX observables from the XML data returned by the TAXII server. + +The following IOC types are extracted from STIX data: + +* MD5 Hashes +* Domain Names +* IP-Addresses +* IP-Address Ranges +""" + +from cybox.objects.domain_name_object import DomainName +from cybox.objects.address_object import Address +from cybox.objects.file_object import File +from lxml import etree +from io import BytesIO +from stix.core import STIXPackage + +import logging +import string +import socket +import uuid +import time +import datetime +import dateutil +import dateutil.tz + +from cabby.constants import ( + CB_STIX_XML_111, CB_CAP_11, CB_SMIME, + CB_STIX_XML_10, CB_STIX_XML_101, CB_STIX_XML_11, CB_XENC_122002) + +CB_STIX_XML_12 = 'urn:stix.mitre.org:xml:1.2' + +BINDING_CHOICES = [CB_STIX_XML_111, CB_CAP_11, CB_SMIME, CB_STIX_XML_12, + CB_STIX_XML_10, CB_STIX_XML_101, CB_STIX_XML_11, + CB_XENC_122002] + + +logger = logging.getLogger(__name__) + + +domain_allowed_chars = string.printable[:-6] # Used by validate_domain_name function + + +def validate_domain_name(domain_name): + """Validates a domain name to ensure validity and saneness. + + Args: + domain_name: Domain name string to check. + + Returns: + True if checks pass, False otherwise. + """ + + if len(domain_name) > 255: + logger.warn( + "Excessively long domain name {} in IOC list".format(domain_name)) + return False + + if not all([c in domain_allowed_chars for c in domain_name]): + logger.warn("Malformed domain name {} in IOC list".format(domain_name)) + return False + + parts = domain_name.split('.') + if not parts: + logger.warn("Empty domain name found in IOC list") + return False + + for part in parts: + if len(part) < 1 or len(part) > 63: + logger.warn("Invalid label length {} in domain name {} for report %s".format( + part, domain_name)) + return False + + return True + + +def validate_md5sum(md5): + """Validates md5sum. + + Args: + md5sum: md5sum to check. + + Returns: + True if checks pass, False otherwise. + """ + + if 32 != len(md5): + logger.warn("Invalid md5 length for md5 {}".format(md5)) + return False + if not md5.isalnum(): + logger.warn("Malformed md5 {} in IOC list".format(md5)) + return False + for c in "ghijklmnopqrstuvwxyz": + if c in md5 or c.upper() in md5: + logger.warn("Malformed md5 {} in IOC list".format(md5)) + return False + + return True + + +def sanitize_id(id): + """Removes unallowed chars from an ID. + + Ids may only contain a-z, A-Z, 0-9, - and must have one character. + + Args: + id: the ID to be sanitized. + + Returns: + A sanitized ID. + """ + + return id.replace(':', '-') + + +def validate_ip_address(ip_address): + """Validates an IPv4 address.""" + + try: + socket.inet_aton(ip_address) + return True + except socket.error: + return False + + +def cybox_parse_observable(observable, indicator, timestamp, score): + """Parses a cybox observable and returns a list containing a report dictionary. + + cybox is a open standard language encoding info about cyber observables. + + Args: + observable: the cybox obserable to parse. + + Returns: + A report dictionary if the cybox observable has props of type: + + cybox.objects.address_object.Address, + cybox.objects.file_object.File, or + cybox.objects.domain_name_object.DomainName. + + Otherwise it will return an empty list. + + """ + reports = [] + + if observable.object_ and observable.object_.properties: + props = observable.object_.properties + logger.debug("{0} has props type: {1}".format(indicator, type(props))) + else: + logger.debug("{} has no props; skipping".format(indicator)) + return reports + + # + # sometimes the description is None + # + description = '' + if observable.description and observable.description.value: + description = str(observable.description.value) + + # + # if description is an empty string, then use the indicator's description + # NOTE: This was added for RecordedFuture + # + + if not description and indicator and indicator.description and indicator.description.value: + description = str(indicator.description.value) + + # + # if description is still empty, use the indicator's title + # + # if not description and indicator and indicator.title: + # description = str(indicator.title) + + # + # use the first reference as a link + # This was added for RecordedFuture + # + link = '' + if indicator and indicator.producer and indicator.producer.references: + for reference in indicator.producer.references: + link = reference + break + else: + split_title = indicator.title.split() + if split_title[2][0:8] == 'https://' or split_title[2][0:7] == 'http://': + link = split_title[2] + + # + # Sometimes the title is None, so generate a random UUID + # + + if observable.title: + title = observable.title + else: + title = str(uuid.uuid4()) + + # ID must be unique. Collisions cause 500 error on Carbon Black backend + id = str(uuid.uuid4()) + + if type(props) == DomainName: + # go into domainname function + reports = parse_domain_name_observable(observable, props, id, description, title, timestamp, link, score) + + elif type(props) == Address: + reports = parse_address_observable(observable, props, id, description, title, timestamp, link, score) + + elif type(props) == File: + reports = parse_file_observable(observable, props, id, description, title, timestamp, link, score) + + else: + return reports + + return reports + + +def parse_domain_name_observable(observable, props, id, description, title, timestamp, link, score): + + reports = [] + if props.value and props.value.value: + iocs = {'dns': []} + # + # Sometimes props.value.value is a list + # + + if type(props.value.value) is list: + for domain_name in props.value.value: + if validate_domain_name(domain_name.strip()): + iocs['dns'].append(domain_name.strip()) + else: + domain_name = props.value.value.strip() + if validate_domain_name(domain_name): + iocs['dns'].append(domain_name) + + if len(iocs['dns']) > 0: + reports.append({'iocs_v2': iocs, + 'id': sanitize_id(id), + 'description': description, + 'title': title, + 'timestamp': timestamp, + 'link': link, + 'score': score}) + return reports + + +def parse_address_observable(observable, props, id, description, title, timestamp, link, score): + + reports = [] + if props.category == 'ipv4-addr' and props.address_value: + iocs = {'ipv4': []} + + # + # Sometimes props.address_value.value is a list vs a string + # + if type(props.address_value.value) is list: + for ip in props.address_value.value: + if validate_ip_address(ip.strip()): + iocs['ipv4'].append(ip.strip()) + else: + ipv4 = props.address_value.value.strip() + if validate_ip_address(ipv4): + iocs['ipv4'].append(ipv4) + + if len(iocs['ipv4']) > 0: + reports.append({'iocs_v2': iocs, + 'id': sanitize_id(observable.id_), + 'description': description, + 'title': title, + 'timestamp': timestamp, + 'link': link, + 'score': score}) + + return reports + + +def parse_file_observable(observable, props, id, description, title, timestamp, link, score): + + reports = [] + iocs = {'md5': []} + if props.md5: + if type(props.md5) is list: + for md5 in props.md5: + if validate_md5sum(md5.strip()): + iocs['md5'].append(md5.strip()) + else: + if hasattr(props.md5, 'value'): + md5 = props.md5.value.strip() + else: + md5 = props.md5.strip() + if validate_md5sum(md5): + iocs['md5'].append(md5) + + if len(iocs['md5']) > 0: + reports.append({'iocs_v2': iocs, + 'id': sanitize_id(id), + 'description': description, + 'title': title, + 'timestamp': timestamp, + 'link': link, + 'score': score}) + + return reports + + +def get_stix_indicator_score(indicator, default_score): + """Returns a digit representing the indicator score. + + Converts from "high", "medium", or "low" into a digit, if necessary. + """ + + if not indicator.confidence: + return default_score + + confidence_val_str = str(indicator.confidence.value) + if confidence_val_str.isdigit(): + score = int(indicator.confidence.to_dict().get("value", default_score)) + return score + elif confidence_val_str.lower() == "high": + return 7 # 75 + elif confidence_val_str.lower() == "medium": + return 5 # 50 + elif confidence_val_str.lower() == "low": + return 2 # 25 + else: + return default_score + + +def get_stix_indicator_timestamp(indicator): + timestamp = 0 + if indicator.timestamp: + if indicator.timestamp.tzinfo: + timestamp = int((indicator.timestamp - + datetime.datetime(1970, 1, 1).replace( + tzinfo=dateutil.tz.tzutc())).total_seconds()) + else: + timestamp = int((indicator.timestamp - + datetime.datetime(1970, 1, 1)).total_seconds()) + return timestamp + + +def get_stix_package_timestamp(stix_package): + timestamp = 0 + if not stix_package or not stix_package.timestamp: + return timestamp + try: + timestamp = stix_package.timestamp + timestamp = int(time.mktime(timestamp.timetuple())) + except (TypeError, OverflowError, ValueError) as e: + logger.warning("Problem parsing stix timestamp: {}".format(e)) + return timestamp + + +def parse_stix_indicators(stix_package, default_score): + reports = [] + if not stix_package.indicators: + return reports + + for indicator in stix_package.indicators: + if not indicator or not indicator.observable: + continue + score = get_stix_indicator_score(indicator, default_score) + timestamp = get_stix_indicator_timestamp(indicator) + yield from cybox_parse_observable( + indicator.observable, indicator, timestamp, score) + + +def parse_stix_observables(stix_package, default_score): + reports = [] + if not stix_package.observables: + return reports + + timestamp = get_stix_package_timestamp(stix_package) + for observable in stix_package.observables: + if not observable: + continue + yield from cybox_parse_observable( # single element list + observable, None, timestamp, default_score) + + +def sanitize_stix(stix_xml): + ret_xml = b'' + try: + xml_root = etree.fromstring(stix_xml) + content = xml_root.find( + './/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content') + if content is not None and len(content) == 0 and len(list(content)) == 0: + # Content has no children. + # So lets make sure we parse the xml text for content and + # re-add it as valid XML so we can parse + _content = xml_root.find( + "{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content_Block/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content") + if _content: + new_stix_package = etree.fromstring(_content.text) + content.append(new_stix_package) + ret_xml = etree.tostring(xml_root) + except etree.ParseError as e: + logger.warning("Problem parsing stix: {}".format(e)) + return ret_xml + + +def parse_stix(stix_xml, default_score): + reports = [] + try: + stix_xml = sanitize_stix(stix_xml) + bio = BytesIO(stix_xml) + stix_package = STIXPackage.from_xml(bio) + if not stix_package: + logger.warning("Could not parse STIX xml") + return reports + if not stix_package.indicators and not stix_package.observables: + logger.info("No indicators or observables found in stix_xml") + return reports + yield from parse_stix_indicators(stix_package, default_score) + yield from parse_stix_observables(stix_package, default_score) + except etree.XMLSyntaxError as e: + logger.warning("Problem parsing stix: {}".format(e)) + return reports diff --git a/examples/threathunter/threat_intelligence/stix_taxii.py b/examples/threathunter/threat_intelligence/stix_taxii.py new file mode 100644 index 00000000..0cb820da --- /dev/null +++ b/examples/threathunter/threat_intelligence/stix_taxii.py @@ -0,0 +1,366 @@ +"""Connects to TAXII servers via cabby and formats the data received for dispatching to a Carbon Black feed.""" + +import argparse +import logging +from threatintel import ThreatIntel +from cabby.exceptions import NoURIProvidedError, ClientException +from cabby import create_client +from dataclasses import dataclass +import yaml +import os +from stix_parse import parse_stix, BINDING_CHOICES +from feed_helper import FeedHelper +from datetime import datetime +from results import AnalysisResult +import urllib3 + +logging.basicConfig(level=logging.DEBUG) + +handled_exceptions = (NoURIProvidedError, ClientException) + + +def load_config_from_file(): + """Loads YAML formatted configuration from config.yml in working directory.""" + + logging.debug("loading config from file") + config_filename = os.path.join(os.path.dirname((os.path.abspath(__file__))), "config.yml") + with open(config_filename, "r") as config_file: + config_data = yaml.load(config_file, Loader=yaml.SafeLoader) + logging.info(f"loaded config data: {config_data}") + return config_data + + +@dataclass(eq=True, frozen=True) +class TaxiiSiteConfig: + """Contains information needed to interface with a TAXII server. + + These values are loaded in from config.yml for each entry in the configuration file. + Each TaxiiSiteConnector has its own TaxiiSiteConfig. + """ + + feed_id: str = '' + site: str = '' + discovery_path: str = '' + collection_management_path: str = '' + poll_path: str = '' + use_https: bool = False + ssl_verify: bool = False + cert_file: str = None + key_file: str = None + default_score: int = 5 # [1,10] + username: str = None + password: str = None + collections: str = '*' + start_date: str = '2019-01-01 00:00:00' + size_of_request_in_minutes: int = 60 + ca_cert: str = None + http_proxy_url: str = None + https_proxy_url: str = None + reports_limit: int = None + fail_limit: int = 10 # num attempts per collection for polling & parsing + + +class TaxiiSiteConnector(): + """Connects to and pulls data from a TAXII server.""" + + def __init__(self, site_conf): + self.config = TaxiiSiteConfig(**site_conf) + self.client = None + + def create_taxii_client(self): + """Connects to a TAXII server using cabby and configuration entries.""" + + conf = self.config + if not conf.ssl_verify: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + try: + client = create_client(conf.site, + use_https=conf.use_https, + discovery_path=conf.discovery_path) + client.set_auth(username=conf.username, + password=conf.password, + verify_ssl=conf.ssl_verify, + ca_cert=conf.ca_cert, + cert_file=conf.cert_file, + key_file=conf.key_file) + + proxy_dict = dict() + if conf.http_proxy_url: + proxy_dict['http'] = conf.http_proxy_url + if conf.https_proxy_url: + proxy_dict['https'] = conf.https_proxy_url + if proxy_dict: + client.set_proxies(proxy_dict) + + self.client = client + + except handled_exceptions as e: + logging.error(f"Error creating client: {e}") + + def create_uri(self, config_path): + """Formats a URI for discovery, collection, or polling of a TAXII server. + + Args: + config_path: A URI path to a TAXII server's discovery, collection, or polling service. Defined in config.yml configuration file. + + Returns: + A full URI to one of a TAXII server's service paths. + """ + + uri = None + if self.config.site and config_path: + if self.config.use_https: + uri = 'https://' + else: + uri = 'http://' + uri = uri + self.config.site + config_path + return uri + + def query_collections(self): + """Returns a list of STIX collections available to the user to poll.""" + + collections = [] + try: + uri = self.create_uri(self.config.collection_management_path) + collections = self.client.get_collections( + uri=uri) # autodetect if uri=None + for collection in collections: + logging.info(f"Collection: {collection.name}, {collection.type}") + except handled_exceptions as e: + logging.warning(f"Problem fetching collections from taxii server: {e}") + return collections + + def poll_server(self, collection, feed_helper): + """Returns a STIX content block for a specific TAXII collection. + + Args: + collection: Name of a TAXII collection to poll. + feed_helper: FeedHelper object. + """ + + content_blocks = [] + uri = self.create_uri(self.config.poll_path) + try: + logging.info(f"Polling Collection: {collection.name}") + content_blocks = self.client.poll( + uri=uri, + collection_name=collection.name, + begin_date=feed_helper.start_date, + end_date=feed_helper.end_date, + content_bindings=BINDING_CHOICES) + except handled_exceptions as e: + logging.warning(f"problem polling taxii server: {e}") + return content_blocks + + def parse_collection_content(self, content_blocks): + """Yields a formatted report dictionary for each STIX content_block. + + Args: + content_block: A chunk of STIX data from the TAXII collection being polled. + """ + + for block in content_blocks: + yield from parse_stix(block.content, self.config.default_score) + + def import_collection(self, collection): + """Polls a single TAXII server collection. + + Starting at the start_date set in config.yml, a FeedHelper object will continue to grab chunks + of data from a collection until the report limit is reached or we reach the current datetime. + + Args: + collection: Name of a TAXII collection to poll. + + Yields: + Formatted report dictionaries from parse_collection_content(content_blocks) + for each content_block pulled from a single TAXII collection. + """ + + num_times_empty_content_blocks = 0 + advance = True + reports_limit = self.config.reports_limit + logging.info(f"reports limit: {reports_limit}") + feed_helper = FeedHelper(self.config.start_date, + self.config.size_of_request_in_minutes) + # config parameters `start_date` and `size_of_request_in_minutes` tell this Feed Helper + # where to start polling in the collection, and then will advance polling in chunks of + # `size_of_request_in_minutes` until we hit the most current `content_block`, + # or reports_limit is reached. + while feed_helper.advance(): + num_reports = 0 + num_times_empty_content_blocks = 0 + content_blocks = self.poll_server(collection, feed_helper) + reports = self.parse_collection_content(content_blocks) + for report in reports: + yield report + num_reports += 1 + if reports_limit is not None and num_reports >= reports_limit: + logging.info(f"Reports limit of {self.config.reports_limit} reached") + advance = False + break + + if not advance: + break + if collection.type == 'DATA_SET': # data is unordered, not a feed + logging.info(f"collection:{collection}; type data_set; breaking") + break + if num_reports == 0: + num_times_empty_content_blocks += 1 + if num_times_empty_content_blocks > self.config.fail_limit: + logging.error('Max fail limit reached; Exiting.') + break + if reports_limit is not None: + reports_limit -= num_reports + + def import_collections(self, available_collections): + """Polls each desired collection specified in config.yml. + + Args: + available_collections: list of collections available to a TAXII server user. + + Yields: + From import_collection(self, collection) for each desired collection. + """ + + desired_collections = self.config.collections + desired_collections = [x.strip() + for x in desired_collections.lower().split(',')] + + want_all = True if '*' in desired_collections else False + + for collection in available_collections: + if collection.type != 'DATA_FEED' and collection.type != 'DATA_SET': + logging.debug(f"collection:{collection}; type not feed or data") + continue + if not collection.available: + logging.debug(f"collection:{collection} not available") + continue + if want_all or collection.name.lower() in desired_collections: + yield from self.import_collection(collection) + + def generate_reports(self): + """Returns a list of report dictionaries for each desired collection specified in config.yml.""" + + reports = [] + + self.create_taxii_client() + if not self.client: + logging.error('Unable to create taxii client.') + return reports + + available_collections = self.query_collections() + if not available_collections: + logging.warning('Unable to find any collections.') + return reports + + reports = self.import_collections(available_collections) + if not reports: + logging.warning('Unable to import collections.') + return reports + + return reports + + +class StixTaxii(): + """Allows for interfacing with multiple TAXII servers. + + Instantiates separate TaxiiSiteConnector objects for each site specified in config.yml. + Formats report dictionaries into AnalysisResult objects with formatted IOC_v2 attirbutes. + Sends AnalysisResult objects to ThreatIntel.push_to_cb for dispatching to a feed. + """ + + def __init__(self, site_confs): + self.config = site_confs + self.client = None + + def result(self, **kwargs): + """Returns a new AnalysisResult with the given fields populated.""" + + result = AnalysisResult(**kwargs).normalize() + return result + + def configure_sites(self): + """Creates a TaxiiSiteConnector for each site in config.yml""" + + self.sites = {} + try: + for site_name, site_conf in self.config['sites'].items(): + self.sites[site_name] = TaxiiSiteConnector(site_conf) + logging.info(f"loaded site {site_name}") + except handled_exceptions as e: + + logging.error(f"Error in parsing config file: {e}") + + def format_report(self, reports): + """Converts a dictionary into an AnalysisResult. + + Args: + reports: list of report dictionaries containing an id, title, description, timestamp, score, link, and iocs_v2. + + Yields: + An AnalysisResult for each report dictionary. + """ + + for report in reports: + try: + analysis_name = report['id'] + title = report['title'] + description = report['description'] + scan_time = datetime.fromtimestamp(report['timestamp']) + score = report['score'] + link = report['link'] + ioc_dict = report['iocs_v2'] + result = self.result( + analysis_name=analysis_name, + scan_time=scan_time, + score=score, + title=title, + description=description) + for ioc_key, ioc_val in ioc_dict.items(): + result.attach_ioc_v2(values=ioc_val, field=ioc_key, link=link) + except handled_exceptions as e: + logging.warning(f"Problem in report formatting: {e}") + result = self.result( + analysis_name="exception_format_report", error=True) + yield result + + def collect_and_send_reports(self): + """Collects and sends formatted reports to ThreatIntel.push_to_cb for validation and dispatching to a feed.""" + + self.configure_sites() + ti = ThreatIntel() + for site_name, site_conn in self.sites.items(): + logging.info(f"Talking to {site_name} server") + reports = site_conn.generate_reports() + if not reports: + logging.error(f"No reports generated for {site_name}") + continue + else: + try: + ti.push_to_cb(feed_id=site_conn.config.feed_id, results=self.format_report(reports)) + except Exception as e: + logging.error(f"Failed to push reports to feed {site_conn.config.feed_id}: {e}") + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Modify configuration values via command line.') + parser.add_argument('--site_start_date', metavar='s', nargs='+', + help='the site name and desired start date to begin polling from') + args = parser.parse_args() + + config = load_config_from_file() + + if args.site_start_date: + for index in range(len(args.site_start_date)): + arg = args.site_start_date[index] + if arg in config['sites']: # if we see a name that matches a site Name + try: + new_time = datetime.strptime(args.site_start_date[index+1], "%Y-%m-%d %H:%M:%S") + config['sites'][arg]['start_date'] = new_time + logging.info(f"Updated the start_date for {arg} to {new_time}") + except ValueError as e: + logging.error(f"Failed to update start_date for {arg}: {e}") + + stix_taxii = StixTaxii(config) + stix_taxii.collect_and_send_reports() diff --git a/examples/threathunter/threat_intelligence/threatintel.py b/examples/threathunter/threat_intelligence/threatintel.py new file mode 100644 index 00000000..2b7b148d --- /dev/null +++ b/examples/threathunter/threat_intelligence/threatintel.py @@ -0,0 +1,247 @@ +"""Validates result dictionaries, creates ThreatHunter Reports, validates ThreatHunter Reports, and sends them to a ThreatHunter Feed. + +Also allows for conversion from result dictionaries into ThreatHunter `Report` objects. +""" + +import logging +import json +from cbapi.psc.threathunter import CbThreatHunterAPI, Report +from cbapi.errors import ApiError +from cbapi.psc.threathunter.models import Feed + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + + +class ThreatIntel: + def __init__(self): + self.cb = CbThreatHunterAPI(timeout=200) + + def push_to_cb(self, feed_id, results): + try: + feed = self.cb.select(Feed, feed_id) + except ApiError as e: + log.error(f"couldn't find CbTH feed {feed_id}: {e}") + return + + report_list_to_send = [] + reports = [] + malformed_reports = [] + + for result in results: + try: + report_dict = { + "id": str(result.id), + "timestamp": int(result.timestamp.timestamp()), + "title": str(result.title), + "description": str(result.description), + "severity": int(result.severity), + "iocs_v2": [ioc_v2.as_dict() for ioc_v2 in result.iocs_v2] + } + + if self.input_validation([report_dict]): + # create CB Report object + report = Report(self.cb, initial_data=report_dict, feed_id=feed_id) + report_list_to_send.append(report) + reports.append(report_dict) + else: + log.warning("Report Validation failed. Saving report to file for reference.") + malformed_reports.append(report_dict) + except Exception as e: + log.error(f"Failed to create a report dictionary from result object. {e}") + + log.debug(f"Num Reports: {len(report_list_to_send)}") + try: + with open('reports.json', 'w') as f: + json.dump(reports, f, indent=4) + except Exception as e: + log.error(f"Failed to write reports to file: {e}") + + log.debug("Sending results to Carbon Black Cloud.") + + try: + feed.append_reports(report_list_to_send) + log.info(f"Appended {len(report_list_to_send)} reports to ThreatHunter Feed {feed_id}") + except Exception as e: + log.debug(f"Failed sending {len(report_list_to_send)} reports: {e}") + + if malformed_reports: + log.warning("Some report(s) failed validation. See malformed_reports.json for reports that failed.") + try: + with open('malformed_reports.json', 'w') as f: + json.dump(malformed_reports, f, indent=4) + except Exception as e: + log.error(f"Failed to write malformed_reports to file: {e}") + + ########################################################## + # Input validation of reports + ########################################################## + + # Input validation for query IOC + def query_ioc_validation(self, query_ioc): + if isinstance(query_ioc, list): + fields = ["search_query", "index_type"] + field_data_types = [str, str] + + for query in query_ioc: + status = [[fields[0], False], [fields[1], False]] + + status[0][1] = isinstance(query[fields[0]], field_data_types[0]) + status[1][1] = isinstance(query[fields[1]], field_data_types[1]) + + if not status[0][1]: + log.warning(f"Missing Required Report Field: {str(status[0][0])}") + + if not status[1][1]: + log.warning(f"Field {str(status[1][0])} does not match correct data type") + + log.debug("Query IOC validation complete") + return True + else: + log.warning("Query_ioc does not match correct data type") + return False + + # Input validation for IOCs + def ioc_validation(self, ioc): + fields_opt = ["md5", "ipv4", "ipv6", "dns", "query"] + fields_data_type = [str, str, str, str] + + ioc_fields = list(ioc.keys()) + status_opt = [[fields_opt[i], False] for i in range(len(fields_opt))] + no_error = True + + # Iterate through ioc fields and check for validation + for ioc_field in ioc_fields: + if ioc_field in fields_opt and ioc[ioc_field]: + if ioc_field == 'query': + query_ioc_status = self.query_ioc_validation(ioc[ioc_field]) + no_error = no_error and query_ioc_status + else: + index = fields_opt.index(ioc_field) + status_opt[index][1] = all(isinstance(elem, fields_data_type[index]) for elem in ioc[ioc_field]) + no_error = no_error and status_opt[index][1] + + if not status_opt[index][1]: + log.warning(f"Field {str(ioc_field)} does not match correct data type") + else: + log.warning(f"Invalid field: {str(ioc_field)}") + no_error = no_error and False + + return no_error + + # Input validation for IOCsv2 + def iocv2_validation(self, iocv2): + if not isinstance(iocv2, list): + log.warning("IOCv2 must be a list of IOCv2 dictionaries") + return False + else: + fields_req = ["id", "match_type", "values"] + fields_opt = ["field", "link"] + no_error = True + + for ioc_dictionary in iocv2: + ioc_v2_fields = list(ioc_dictionary.keys()) + status_req = [[fields_req[i], False] for i in range(len(fields_req))] + status_opt = [[fields_opt[i], False] for i in range(len(fields_opt))] + + for ioc_v2_field in ioc_v2_fields: + if ioc_v2_field in fields_req: + if not ioc_dictionary[ioc_v2_field]: + log.warning(f"Required field {str(ioc_v2_field)} is empty") + no_error = False + else: + if ioc_v2_field == 'values' and isinstance(ioc_dictionary[ioc_v2_field], list): + status_req[2][1] = all(isinstance(elem, str) for elem in ioc_dictionary[ioc_v2_field]) + no_error = no_error and status_req[2][1] + + if not status_req[2][1]: + log.warning(f"Missing REQUIRED field or invalid data type: {str(ioc_v2_field)}") + else: + index = fields_req.index(ioc_v2_field) + status_req[index][1] = isinstance(ioc_dictionary[ioc_v2_field], str) + no_error = no_error and status_req[index][1] + + if not status_req[index][1]: + log.warning(f"Missing REQUIRED field or invalid data type: {str(ioc_v2_field)}") + elif ioc_v2_field in fields_opt: + if not ioc_dictionary[ioc_v2_field]: + log.warning(f"Optional field {str(ioc_v2_field)} is empty") + no_error = False + else: + iocv2_status = isinstance(ioc_dictionary[ioc_v2_field], str) + index = fields_opt.index(ioc_v2_field) + status_opt[index][1] = iocv2_status + no_error = no_error and iocv2_status + + if not iocv2_status: + log.warning(f"Missing field or invalid data type: {str(ioc_v2_field)}") + else: + log.warning(f"Invalid field in IOCv2: {str(ioc_v2_field)}") + no_error = False + + return no_error + + # Input a list of dictionaries representing + # reports to be checked and validated + def input_validation(self, reports): + if len(reports) < 1: + log.debug("There are no reports to be validated") + else: + + fields_req = ["id", "timestamp", "title", "description", "severity"] + fields_opt = ["link", "tags", "iocs", "iocs_v2", "visibility"] + + # Iterate though the array and check for invalidation of each field + for index, report in enumerate(reports): + no_error = True + status_req = [[fields_req[i], False] for i in range(len(fields_req))] + status_opt = [[fields_opt[i], False] for i in range(len(fields_opt))] + + for key, val in report.items(): + if key in fields_req: + if key == "timestamp" or key == "severity": + if isinstance(val, int): + idx = fields_req.index(key) + status_req[idx][1] = True + no_error = no_error and status_req[idx][1] + else: + log.warning(f"Missing REQUIRED field or invalid data type: {str(key)}") + no_error = no_error and False + else: + if isinstance(val, str): + idx = fields_req.index(key) + status_req[idx][1] = True + no_error = no_error and status_req[idx][1] + else: + log.warning(f"Missing REQUIRED field or invalid data type: {str(key)}") + no_error = no_error and False + elif key in fields_opt: + if key == "iocs": + ioc_status = self.ioc_validation(val) + status_opt[2][1] = ioc_status + no_error = no_error and ioc_status + elif key == "iocs_v2": + iocv2_status = self.iocv2_validation(val) + status_opt[3][1] = iocv2_status + no_error = no_error and iocv2_status + else: + if isinstance(val, str): + idx = fields_opt.index(key) + status_opt[idx][1] = True # this is always true? What's the point + no_error = no_error and status_opt[idx][1] + else: + log.warning(f"Missing field or invalid data type: {str(key)}") + else: + log.warning(f"Invalid Report field: {str(key)}. Please remove from report before dispatching.") + no_error = no_error and False + + log.debug(f"Report {str(index)} validation complete") + + if no_error: + log.debug("Report Validation found no errors in report schema") + else: + log.warning("Report Validation found an error in report schema") + return False + + log.debug("Report Validation check complete") + return True From 46b4a0b2455d677d7ad59db2f2906983aa5c31d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jan 2020 14:03:13 -0700 Subject: [PATCH 080/197] Bump urllib3 in /examples/threathunter/threat_intelligence (#210) Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.22 to 1.24.2. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/master/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.22...1.24.2) Signed-off-by: dependabot[bot] --- examples/threathunter/threat_intelligence/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/threathunter/threat_intelligence/requirements.txt b/examples/threathunter/threat_intelligence/requirements.txt index d23ecc51..bb0c132b 100644 --- a/examples/threathunter/threat_intelligence/requirements.txt +++ b/examples/threathunter/threat_intelligence/requirements.txt @@ -3,7 +3,7 @@ dataclasses>=0.6 cabby==0.1.20 stix==1.2.0.7 lxml==4.4.1 -urllib3==1.22 +urllib3==1.24.2 cbapi>=1.5.6 python_dateutil==2.8.1 PyYAML==5.1.2 From b0e97185768a24823b32e2968dd6c224ef4423ab Mon Sep 17 00:00:00 2001 From: Luke Lyon <52218532+llyon-cb@users.noreply.github.com> Date: Wed, 15 Jan 2020 09:53:44 -0700 Subject: [PATCH 081/197] Update requirements.txt --- examples/threathunter/threat_intelligence/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/threathunter/threat_intelligence/requirements.txt b/examples/threathunter/threat_intelligence/requirements.txt index bb0c132b..38bd9dbd 100644 --- a/examples/threathunter/threat_intelligence/requirements.txt +++ b/examples/threathunter/threat_intelligence/requirements.txt @@ -3,7 +3,7 @@ dataclasses>=0.6 cabby==0.1.20 stix==1.2.0.7 lxml==4.4.1 -urllib3==1.24.2 +urllib3>=1.24.2 cbapi>=1.5.6 python_dateutil==2.8.1 PyYAML==5.1.2 From 6e81507ff30a57eb1f13ae829c28e6ee339d2ad1 Mon Sep 17 00:00:00 2001 From: Luke Lyon <52218532+llyon-cb@users.noreply.github.com> Date: Wed, 15 Jan 2020 15:18:58 -0700 Subject: [PATCH 082/197] add date formatting info in config.yml --- examples/threathunter/threat_intelligence/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/threathunter/threat_intelligence/config.yml b/examples/threathunter/threat_intelligence/config.yml index 498dca51..d028e7d6 100644 --- a/examples/threathunter/threat_intelligence/config.yml +++ b/examples/threathunter/threat_intelligence/config.yml @@ -42,7 +42,8 @@ sites: collections: # (optional) the start date for which to start requesting data. - # Defaults to 2019-01-01 00:00:00 if you supply nothing + # Use %y-%m-%d %H:%M:%S format. + # Defaults to 2019-01-01 00:00:00 if you supply nothing. start_date: # (optional) the minutes to advance for each request. From e056f96390a91bdcb1c8adccbf2f5d56420bcbed Mon Sep 17 00:00:00 2001 From: Michael McGrew Date: Tue, 21 Jan 2020 15:22:20 -0800 Subject: [PATCH 083/197] Fixing bug where max_children is not passed to walk_children --- src/cbapi/response/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index ef6a7b14..766e02c2 100644 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -2579,7 +2579,8 @@ def children(self): "terminated": False }, is_suppressed=child.get("is_suppressed", False), - proc_data=child) + proc_data=child, + max_children=self.max_children) else: for cp in self.childprocs: yield cp @@ -3154,8 +3155,9 @@ def __init__(self, parent_process, timestamp, sequence, event_data, version=1): class CbChildProcEvent(CbEvent): - def __init__(self, parent_process, timestamp, sequence, event_data, is_suppressed=False, proc_data=None): + def __init__(self, parent_process, timestamp, sequence, event_data, is_suppressed=False, proc_data=None, max_children=15): super(CbChildProcEvent, self).__init__(parent_process, timestamp, sequence, event_data) + self.max_children = max_children self.event_type = u'Cb Child Process event' self.stat_titles.extend(['procguid', 'pid', 'path', 'md5']) self.is_suppressed = is_suppressed @@ -3197,7 +3199,7 @@ def process(self): child_unique_id = self.procguid return self.parent._cb.select(Process, child_unique_id, initial_data=proc_data, - suppressed_process=self.is_suppressed) + suppressed_process=self.is_suppressed, max_children=self.max_children) class CbCrossProcEvent(CbEvent): From 7e77a85642ad3aac0add0d1966b1548d372d0f31 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Tue, 28 Jan 2020 11:02:57 -0700 Subject: [PATCH 084/197] Move pytest to a test dependency --- setup.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 158b4cb2..e8c7e40d 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'cbapi.psc.livequery' ] -install_requires=[ +install_requires = [ 'requests', 'attrdict', 'cachetools', @@ -24,13 +24,16 @@ 'pika', 'prompt_toolkit', 'pygments', - 'pytest<=5.0', 'python-dateutil', 'protobuf', 'solrq', 'validators' ] +tests_requires = [ + 'pytest<=5.0' +] + if sys.version_info < (2, 7): install_requires.extend(['simplejson', 'total-ordering', 'ordereddict']) if sys.version_info < (3, 0): @@ -50,6 +53,7 @@ zip_safe=False, platforms='any', install_requires=install_requires, + tests_requires=tests_requires, classifiers=[ 'Environment :: Web Environment', 'Intended Audience :: Developers', From e61f81b4887392da6b5344408e288a62c9c6f86e Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Tue, 28 Jan 2020 11:18:52 -0700 Subject: [PATCH 085/197] Fix flake8 error in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e8c7e40d..d2706cd1 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ description='Carbon Black REST API Python Bindings', packages=packages, include_package_data=True, - package_dir = {'': 'src'}, + package_dir={'': 'src'}, zip_safe=False, platforms='any', install_requires=install_requires, From 90b95935f2760eecc720270c6839bde04520b575 Mon Sep 17 00:00:00 2001 From: Luke Lyon <52218532+llyon-cb@users.noreply.github.com> Date: Wed, 5 Feb 2020 09:30:12 -0700 Subject: [PATCH 086/197] CBAPI 1308 STIX/TAXII Fixes (#216) * error handling for empty config entries for start_date, collections * add support for URI objects received from TAXII server * change default size_of_request to 1440 minutes for speed * start_date now a req. parameter in the config file. Errors out if none supplied --- .../threat_intelligence/config.yml | 7 ++- .../threat_intelligence/feed_helper.py | 2 +- .../threat_intelligence/stix_parse.py | 54 ++++++++++++++++--- .../threat_intelligence/stix_taxii.py | 22 +++++--- 4 files changed, 68 insertions(+), 17 deletions(-) diff --git a/examples/threathunter/threat_intelligence/config.yml b/examples/threathunter/threat_intelligence/config.yml index d028e7d6..61f79250 100644 --- a/examples/threathunter/threat_intelligence/config.yml +++ b/examples/threathunter/threat_intelligence/config.yml @@ -41,15 +41,14 @@ sites: # (optional) specify which collections to convert to feeds (comma-delimited) collections: - # (optional) the start date for which to start requesting data. - # Use %y-%m-%d %H:%M:%S format. - # Defaults to 2019-01-01 00:00:00 if you supply nothing. + # the start date for which to start requesting data. + # Use %y-%m-%d %H:%M:%S format, for example: '2019-01-01 00:00:00' start_date: # (optional) the minutes to advance for each request. # If you don't have a lot of data, you could advance your requests # to every 60 minutes, or 1440 minutes for daily chunks - # defaults to 60 + # defaults to 1440 size_of_request_in_minutes: # (optional) path to a CA SSL certificate diff --git a/examples/threathunter/threat_intelligence/feed_helper.py b/examples/threathunter/threat_intelligence/feed_helper.py index cc732e40..c1ae1dbe 100644 --- a/examples/threathunter/threat_intelligence/feed_helper.py +++ b/examples/threathunter/threat_intelligence/feed_helper.py @@ -9,7 +9,7 @@ class FeedHelper(): def __init__(self, start_date, size_of_request_in_minutes): self.size_of_request_in_minutes = size_of_request_in_minutes - self.start_date = start_date.replace(tzinfo=timezone.utc) + self.start_date = datetime.strptime(start_date, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) self.end_date = self.start_date + \ timedelta(minutes=self.size_of_request_in_minutes) self.now = datetime.utcnow().replace(tzinfo=timezone.utc) diff --git a/examples/threathunter/threat_intelligence/stix_parse.py b/examples/threathunter/threat_intelligence/stix_parse.py index b872318d..b34006b9 100644 --- a/examples/threathunter/threat_intelligence/stix_parse.py +++ b/examples/threathunter/threat_intelligence/stix_parse.py @@ -11,6 +11,7 @@ from cybox.objects.domain_name_object import DomainName from cybox.objects.address_object import Address from cybox.objects.file_object import File +from cybox.objects.uri_object import URI from lxml import etree from io import BytesIO from stix.core import STIXPackage @@ -168,8 +169,8 @@ def cybox_parse_observable(observable, indicator, timestamp, score): # # if description is still empty, use the indicator's title # - # if not description and indicator and indicator.title: - # description = str(indicator.title) + if not description and indicator and indicator.title: + description = str(indicator.title) # # use the first reference as a link @@ -181,15 +182,23 @@ def cybox_parse_observable(observable, indicator, timestamp, score): link = reference break else: - split_title = indicator.title.split() - if split_title[2][0:8] == 'https://' or split_title[2][0:7] == 'http://': - link = split_title[2] + if indicator and indicator.title: + split_title = indicator.title.split() + if split_title[2][0:8] == 'https://' or split_title[2][0:7] == 'http://': + link = split_title[2] + elif observable and observable.title: + split_title = observable.title.split() + if split_title[1][0:8] == 'https://' or split_title[1][0:7] == 'http://': + link = split_title[1] + else: + link = '' + # # Sometimes the title is None, so generate a random UUID # - if observable.title: + if observable and observable.title: title = observable.title else: title = str(uuid.uuid4()) @@ -207,11 +216,44 @@ def cybox_parse_observable(observable, indicator, timestamp, score): elif type(props) == File: reports = parse_file_observable(observable, props, id, description, title, timestamp, link, score) + elif type(props) == URI: + reports = parse_uri_observable(observable, props, id, description, title, timestamp, link, score) + else: return reports return reports +def parse_uri_observable(observable, props, id, description, title, timestamp, link, score): + + reports = [] + + if props.value and props.value.value: + + iocs = {'dns': []} + # + # Sometimes props.value.value is a list + # + + if type(props.value.value) is list: + for domain_name in props.value.value: + if validate_domain_name(domain_name.strip()): + iocs['dns'].append(domain_name.strip()) + else: + domain_name = props.value.value.strip() + if validate_domain_name(domain_name): + iocs['dns'].append(domain_name) + + if len(iocs['dns']) > 0: + reports.append({'iocs_v2': iocs, + 'id': sanitize_id(id), + 'description': description, + 'title': title, + 'timestamp': timestamp, + 'link': link, + 'score': score}) + return reports + def parse_domain_name_observable(observable, props, id, description, title, timestamp, link, score): diff --git a/examples/threathunter/threat_intelligence/stix_taxii.py b/examples/threathunter/threat_intelligence/stix_taxii.py index 0cb820da..1eaf7f3e 100644 --- a/examples/threathunter/threat_intelligence/stix_taxii.py +++ b/examples/threathunter/threat_intelligence/stix_taxii.py @@ -14,7 +14,7 @@ from results import AnalysisResult import urllib3 -logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(filename='stix.log', filemode='w', level=logging.DEBUG) handled_exceptions = (NoURIProvidedError, ClientException) @@ -51,8 +51,8 @@ class TaxiiSiteConfig: username: str = None password: str = None collections: str = '*' - start_date: str = '2019-01-01 00:00:00' - size_of_request_in_minutes: int = 60 + start_date: str = None + size_of_request_in_minutes: int = 1440 ca_cert: str = None http_proxy_url: str = None https_proxy_url: str = None @@ -71,6 +71,9 @@ def create_taxii_client(self): """Connects to a TAXII server using cabby and configuration entries.""" conf = self.config + if not conf.start_date: + logging.error(f"A start_date is required for site {conf.site}. Exiting.") + return if not conf.ssl_verify: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) try: @@ -180,8 +183,12 @@ def import_collection(self, collection): advance = True reports_limit = self.config.reports_limit logging.info(f"reports limit: {reports_limit}") + if not self.config.size_of_request_in_minutes: + size_of_request_in_minutes = 1440 + else: + size_of_request_in_minutes = self.config.size_of_request_in_minutes feed_helper = FeedHelper(self.config.start_date, - self.config.size_of_request_in_minutes) + size_of_request_in_minutes) # config parameters `start_date` and `size_of_request_in_minutes` tell this Feed Helper # where to start polling in the collection, and then will advance polling in chunks of # `size_of_request_in_minutes` until we hit the most current `content_block`, @@ -222,7 +229,11 @@ def import_collections(self, available_collections): From import_collection(self, collection) for each desired collection. """ - desired_collections = self.config.collections + if not self.config.collections: + desired_collections = '*' + else: + desired_collections = self.config.collections + desired_collections = [x.strip() for x in desired_collections.lower().split(',')] @@ -361,6 +372,5 @@ def collect_and_send_reports(self): logging.info(f"Updated the start_date for {arg} to {new_time}") except ValueError as e: logging.error(f"Failed to update start_date for {arg}: {e}") - stix_taxii = StixTaxii(config) stix_taxii.collect_and_send_reports() From 58a07666e4451feec4c88c5152ddda0d13e2023a Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Tue, 18 Feb 2020 09:37:36 -0700 Subject: [PATCH 087/197] Fix example documentation bug --- src/cbapi/psc/devices_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/psc/devices_query.py b/src/cbapi/psc/devices_query.py index bd5af53a..6ddc431e 100755 --- a/src/cbapi/psc/devices_query.py +++ b/src/cbapi/psc/devices_query.py @@ -269,7 +269,7 @@ def download(self): Example:: - >>> cb.select(Device).status(["ALL"]).download() + >>> cb.select(Device).set_status(["ALL"]).download() :return: The CSV raw data as returned from the server. """ From b8c6ac9422b8b84d2846f648bf78b1e803ef8d40 Mon Sep 17 00:00:00 2001 From: dly-cb Date: Wed, 8 Apr 2020 10:41:21 -0600 Subject: [PATCH 088/197] Changes for Cb-response 7.1 --- src/cbapi/response/models.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 766e02c2..0b1d2b9e 100644 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -818,13 +818,18 @@ def unisolate(self, timeout=None): return True def _update_object(self): - # Workarounds for issuing a sensor queue flush in Cb Response 6.0 + # 1st Workarounds for issuing a sensor queue flush in Cb Response 6.0 # - We only want to reflect back the event_log_flush_time if the user explicitly set it to a new value # (therefore, set the event_log_flush_time to None if it isn't marked dirty) # - The event_log_flush_time must be sent in RFC822 format (not ISO 8601) for Cb Response 6.x servers. # # Note that even though we delete the event_log_flush_time here, it'll get re-initialized when we GET # the sensor after sending the PUT request. + # + # 2nd Workarounds for updating a sensor's ability to update attributes in Cb Response 7.1 + # - Only allowed fields are: ['network_isolation_enabled', 'restart_queued', 'uninstall', 'liveresponse_init', 'group_id', 'notes', 'event_log_flush_time'] + # - Instead of sending in entire sensorObject fields, we will only send in the above 7 + if "event_log_flush_time" in self._dirty_attributes and self._info.get("event_log_flush_time", None) is not None: if self._cb.cb_server_version > LooseVersion("6.0.0"): @@ -837,7 +842,34 @@ def _update_object(self): else: self._info["event_log_flush_time"] = None - return super(Sensor, self)._update_object() + if self.__class__.primary_key in self._dirty_attributes.keys() or self._model_unique_id is None: + new_object_info = deepcopy(self._info) + try: + if not self._new_object_needs_primary_key: + del(new_object_info[self.__class__.primary_key]) + except Exception: + pass + log.debug("Creating a new {0:s} object".format(self.__class__.__name__)) + ret = self._cb.api_json_request(self.__class__._new_object_http_method, self.urlobject, + data=new_object_info) + else: + log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) + http_method = self.__class__._change_object_http_method + + allowed_fields = { + 'network_isolation_enabled': self._info['network_isolation_enabled'], + 'restart_queued': self._info['restart_queued'], + 'uninstall': self._info['uninstall'], + 'liveresponse_init': self._info['liveresponse_init'], + 'group_id': self._info['group_id'], + 'notes': self._info['notes'], + 'event_log_flush_time': self._info['event_log_flush_time'] + } + + ret = self._cb.api_json_request(http_method, self._build_api_request_uri(http_method=http_method), + data=allowed_fields) + + return self._refresh_if_needed(ret) class SensorGroup(MutableBaseModel, CreatableModelMixin): From 6579e227463032b4405674a31292b4cc242896c6 Mon Sep 17 00:00:00 2001 From: dly-cb Date: Wed, 8 Apr 2020 11:51:41 -0600 Subject: [PATCH 089/197] Removed liveresponse_init --- src/cbapi/response/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 0b1d2b9e..2c82e377 100644 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -860,7 +860,6 @@ def _update_object(self): 'network_isolation_enabled': self._info['network_isolation_enabled'], 'restart_queued': self._info['restart_queued'], 'uninstall': self._info['uninstall'], - 'liveresponse_init': self._info['liveresponse_init'], 'group_id': self._info['group_id'], 'notes': self._info['notes'], 'event_log_flush_time': self._info['event_log_flush_time'] From ab3a9c2f6db2610e0e5c02b2b75e0a62785aadf2 Mon Sep 17 00:00:00 2001 From: dly-cb Date: Wed, 8 Apr 2020 15:17:58 -0600 Subject: [PATCH 090/197] Added release note --- README.md | 2 +- docs/changelog.rst | 7 +++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index de8e003d..05894073 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.6.1** +**Latest Version: 1.6.2** These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. diff --git a/docs/changelog.rst b/docs/changelog.rst index 2f5f82d4..417499f2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.6.2 - Released April 08, 2020 +------------------------------------- + +Updates + +.. (add your updates here) + CbAPI 1.6.1 - Released January 13, 2020 --------------------------------------- diff --git a/docs/conf.py b/docs/conf.py index f150cbf7..2ccc21c0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.6' # The full version, including alpha/beta/rc tags. -release = u'1.6.1' +release = u'1.6.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index d2706cd1..1cb038a7 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( name='cbapi', - version='1.6.1', + version='1.6.2', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index a3e0d1a2..99cda4a8 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2019 VMware Carbon Black' -__version__ = '1.6.1' +__version__ = '1.6.2' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From ba3c867b9fa9fafdd6d70fc7bec0dcfb11a12c1f Mon Sep 17 00:00:00 2001 From: dly-cb Date: Wed, 8 Apr 2020 15:55:09 -0600 Subject: [PATCH 091/197] Finalized release notes 1.6.2 --- docs/changelog.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 417499f2..109b8baa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,8 @@ CbAPI 1.6.2 - Released April 08, 2020 Updates -.. (add your updates here) +* CB Response + * Changes to align with limits placed on the sensor update function in CB Response 7.1.0. Release notes are available on User Exchange, the ID is `CB 28683 `_. CbAPI 1.6.1 - Released January 13, 2020 --------------------------------------- From 2cadf7208a31ff22b35aeca3e5011da5d4897093 Mon Sep 17 00:00:00 2001 From: Luke Lyon <52218532+llyon-cb@users.noreply.github.com> Date: Mon, 27 Apr 2020 11:11:24 -0700 Subject: [PATCH 092/197] CBAPI 1490 various bug fixes (#222) * update wording for params * check for datetime before modifying * update url check in title to regex * improve logging information, keep default vals of config * remove failure condition if optional field isn't supplied * revert to generic site name * updates to error handling/displaying to user for invalid TAXII or CBAPI details * fix for feed verification * update authentication link to point to DevNetwork * remove report_limit val from config, add more defensive type check for start_date, update READMEs' wording and links, make logging default to INFO, remove unreachable return and add comment about raising ApiError * raise ValueError on wrong config start_date input type --- .../threat_intelligence/README.md | 2 +- .../threat_intelligence/Taxii_README.md | 3 +- .../threat_intelligence/config.yml | 20 +++++++--- .../threat_intelligence/feed_helper.py | 10 ++++- .../threat_intelligence/stix_parse.py | 23 ++++++++---- .../threat_intelligence/stix_taxii.py | 37 ++++++++++++++----- .../threat_intelligence/threatintel.py | 16 ++++---- 7 files changed, 76 insertions(+), 35 deletions(-) diff --git a/examples/threathunter/threat_intelligence/README.md b/examples/threathunter/threat_intelligence/README.md index 5f30afe5..cf74a1dc 100644 --- a/examples/threathunter/threat_intelligence/README.md +++ b/examples/threathunter/threat_intelligence/README.md @@ -126,7 +126,7 @@ An example of a custom Threat Intel connector that uses the `ThreatIntel` Python ## Troubleshooting ### Credential Error -In order to use this code, you must have CBAPI installed and configured. If you receive the following message, visit the CBAPI GitHub repository for [instructions on setting up authentication](https://github.com/carbonblack/cbapi-python#api-token). +In order to use this code, you must have CBAPI installed and configured. If you receive an authentication error, visit the Developer Network Authentication Page for [instructions on setting up authentication](https://developer.carbonblack.com/reference/carbon-black-cloud/authentication/). See [ReadTheDocs](https://cbapi.readthedocs.io/en/latest/index.html#api-credentials) for instructions on configuring your credentials file. ### 504 Gateway Timeout Error The [Carbon Black ThreatHunter Feed Manager API](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/) is used in this code. When posting to a Feed, there is a 60 second limit before the gateway terminates your connection. The amount of reports you can POST to a Feed is limited by your connection speed. In this case, you will have to split your threat intelligence into smaller collections until the request takes less than 60 seconds, and send each smaller collection to an individual ThreatHunter Feed. diff --git a/examples/threathunter/threat_intelligence/Taxii_README.md b/examples/threathunter/threat_intelligence/Taxii_README.md index 687d6c17..fff2e434 100644 --- a/examples/threathunter/threat_intelligence/Taxii_README.md +++ b/examples/threathunter/threat_intelligence/Taxii_README.md @@ -35,8 +35,7 @@ This may be useful if the intention is to keep an up-to-date collection of STIX ## Troubleshooting ### Credential Error -In order to use this code, you must have CBAPI installed and configured. If you receive the following message, visit the CBAPI GitHub repository for [instructions on setting up authentication](https://github.com/carbonblack/cbapi-python#api-token). +In order to use this code, you must have CBAPI installed and configured. If you receive an authentication error, visit the Developer Network Authentication Page for [instructions on setting up authentication](https://developer.carbonblack.com/reference/carbon-black-cloud/authentication/). See [ReadTheDocs](https://cbapi.readthedocs.io/en/latest/index.html?highlight=credentials.psc#api-credentials) for instructions on configuring your credentials file. ### 504 Gateway Timeout Error The [Carbon Black ThreatHunter Feed Manager API](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/) is used in this code. When posting to a Feed, there is a 60 second limit before the gateway terminates your connection. The amount of reports you can POST to a Feed is limited by your connection speed. In this case, you will have to split your threat intelligence into smaller collections until the request takes less than 60 seconds, and send each smaller collection to an individual ThreatHunter Feed. - diff --git a/examples/threathunter/threat_intelligence/config.yml b/examples/threathunter/threat_intelligence/config.yml index 61f79250..121b204e 100644 --- a/examples/threathunter/threat_intelligence/config.yml +++ b/examples/threathunter/threat_intelligence/config.yml @@ -1,35 +1,41 @@ sites: my_site_name_1: # the feed_id of the ThreatHunter Feed you want to send ThreatIntel to + # example: 7wP8BEc2QsS8ciEqaRv7Ad feed_id: # the address of the site (only server ip or dns; don't put https:// or a trailing slash) + # example: limo.anomali.com site: # the path of the site for discovering what services are available # this is supplied by your taxii provider + # example: /api/v1/taxii/taxii-discovery-service/ discovery_path: # the path of the site for listing what collections are available to you # this is supplied by your taxii provider + # example: /api/v1/taxii/collection_management/ collection_management_path: # the path of the site for polling a collection # this is supplied by your taxii provider + # example: /api/v1/taxii/poll/ poll_path: - # change to true if you require https for your TAXII service connection + # if you require https for your TAXII service connection, set to true + # defaults to true use_https: - # by default, we validate SSL certificates. Turn this off by setting ssl_verify: False + # by default, we validate SSL certificates. Change to false to turn off SSL verification ssl_verify: # (optional) if you need SSL certificates for authentication, set the path of the - # certificate and key here. Please leave blank to ignore. + # certificate and key here. cert_file: key_file: - # how to score each result. Accepts values [1,10], and defaults to 5 + # (optional) how to score each result. Accepts values [1,10], and defaults to 5 default_score: # (optional) username for authorization with your taxii provider @@ -39,10 +45,12 @@ sites: password: # (optional) specify which collections to convert to feeds (comma-delimited) + # example: Abuse_ch_Ransomware_IPs_F135, DShield_Scanning_IPs_F150 collections: # the start date for which to start requesting data. - # Use %y-%m-%d %H:%M:%S format, for example: '2019-01-01 00:00:00' + # Use %y-%m-%d %H:%M:%S format + # example: 2019-01-01 00:00:00 start_date: # (optional) the minutes to advance for each request. @@ -64,7 +72,7 @@ sites: # Leave blank for no limit reports_limit: - # control the number of attempts per-collection before giving up + # (optional) control the number of failed attempts per-collection before giving up # trying to get (empty/malformed) STIX data out of a TAXII server. # defaults to 10 fail_limit: diff --git a/examples/threathunter/threat_intelligence/feed_helper.py b/examples/threathunter/threat_intelligence/feed_helper.py index c1ae1dbe..1484bce3 100644 --- a/examples/threathunter/threat_intelligence/feed_helper.py +++ b/examples/threathunter/threat_intelligence/feed_helper.py @@ -4,12 +4,20 @@ """ from datetime import datetime, timedelta, timezone +import logging +log = logging.getLogger(__name__) class FeedHelper(): def __init__(self, start_date, size_of_request_in_minutes): self.size_of_request_in_minutes = size_of_request_in_minutes - self.start_date = datetime.strptime(start_date, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) + if isinstance(start_date, datetime): + self.start_date = start_date.replace(tzinfo=timezone.utc) + elif isinstance(start_date, str): + self.start_date = datetime.strptime(start_date, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) + else: + log.error(f"Start_date must be a string or datetime object. Received a start_time config value with unsupported type: {type(start_date)}") + raise ValueError self.end_date = self.start_date + \ timedelta(minutes=self.size_of_request_in_minutes) self.now = datetime.utcnow().replace(tzinfo=timezone.utc) diff --git a/examples/threathunter/threat_intelligence/stix_parse.py b/examples/threathunter/threat_intelligence/stix_parse.py index b34006b9..56e391cd 100644 --- a/examples/threathunter/threat_intelligence/stix_parse.py +++ b/examples/threathunter/threat_intelligence/stix_parse.py @@ -24,6 +24,7 @@ import datetime import dateutil import dateutil.tz +import re from cabby.constants import ( CB_STIX_XML_111, CB_CAP_11, CB_SMIME, @@ -184,14 +185,19 @@ def cybox_parse_observable(observable, indicator, timestamp, score): else: if indicator and indicator.title: split_title = indicator.title.split() - if split_title[2][0:8] == 'https://' or split_title[2][0:7] == 'http://': - link = split_title[2] + title_found = True elif observable and observable.title: split_title = observable.title.split() - if split_title[1][0:8] == 'https://' or split_title[1][0:7] == 'http://': - link = split_title[1] + title_found = True else: - link = '' + title_found = False + + if title_found: + url_pattern = re.compile("^(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$") + for token in split_title: + if url_pattern.match(token): + link = token + break # @@ -243,7 +249,7 @@ def parse_uri_observable(observable, props, id, description, title, timestamp, l domain_name = props.value.value.strip() if validate_domain_name(domain_name): iocs['dns'].append(domain_name) - + if len(iocs['dns']) > 0: reports.append({'iocs_v2': iocs, 'id': sanitize_id(id), @@ -352,9 +358,10 @@ def get_stix_indicator_score(indicator, default_score): if not indicator.confidence: return default_score - confidence_val_str = str(indicator.confidence.value) + + confidence_val_str = indicator.confidence.value.__str__() if confidence_val_str.isdigit(): - score = int(indicator.confidence.to_dict().get("value", default_score)) + score = int(confidence_val_str) return score elif confidence_val_str.lower() == "high": return 7 # 75 diff --git a/examples/threathunter/threat_intelligence/stix_taxii.py b/examples/threathunter/threat_intelligence/stix_taxii.py index 1eaf7f3e..9e3012a1 100644 --- a/examples/threathunter/threat_intelligence/stix_taxii.py +++ b/examples/threathunter/threat_intelligence/stix_taxii.py @@ -2,8 +2,11 @@ import argparse import logging +import traceback from threatintel import ThreatIntel from cabby.exceptions import NoURIProvidedError, ClientException +from requests.exceptions import ConnectionError +from cbapi.errors import ApiError from cabby import create_client from dataclasses import dataclass import yaml @@ -12,11 +15,15 @@ from feed_helper import FeedHelper from datetime import datetime from results import AnalysisResult +from cbapi.psc.threathunter.models import Feed import urllib3 +import copy -logging.basicConfig(filename='stix.log', filemode='w', level=logging.DEBUG) - -handled_exceptions = (NoURIProvidedError, ClientException) +# logging.basicConfig(filename='stix.log', filemode='w', level=logging.DEBUG) +logging.basicConfig(filename='stix.log', filemode='w', format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', + datefmt='%Y-%m-%d:%H:%M:%S', + level=logging.INFO) +handled_exceptions = (NoURIProvidedError, ClientException, ConnectionError) def load_config_from_file(): @@ -26,8 +33,13 @@ def load_config_from_file(): config_filename = os.path.join(os.path.dirname((os.path.abspath(__file__))), "config.yml") with open(config_filename, "r") as config_file: config_data = yaml.load(config_file, Loader=yaml.SafeLoader) - logging.info(f"loaded config data: {config_data}") - return config_data + config_data_without_none_vals = copy.deepcopy(config_data) + for site_name, site_config_dict in config_data['sites'].items(): + for conf_key, conf_value in site_config_dict.items(): + if conf_value is None: + del config_data_without_none_vals['sites'][site_name][conf_key] + logging.info(f"loaded config data: {config_data_without_none_vals}") + return config_data_without_none_vals @dataclass(eq=True, frozen=True) @@ -43,8 +55,8 @@ class TaxiiSiteConfig: discovery_path: str = '' collection_management_path: str = '' poll_path: str = '' - use_https: bool = False - ssl_verify: bool = False + use_https: bool = True + ssl_verify: bool = True cert_file: str = None key_file: str = None default_score: int = 5 # [1,10] @@ -130,7 +142,7 @@ def query_collections(self): for collection in collections: logging.info(f"Collection: {collection.name}, {collection.type}") except handled_exceptions as e: - logging.warning(f"Problem fetching collections from taxii server: {e}") + logging.warning(f"Problem fetching collections from TAXII server. Check your TAXII Provider URL and username/password (if required to access TAXII server): {e}") return collections def poll_server(self, collection, feed_helper): @@ -341,6 +353,12 @@ def collect_and_send_reports(self): self.configure_sites() ti = ThreatIntel() for site_name, site_conn in self.sites.items(): + logging.debug(f"Verifying Feed {site_conn.config.feed_id} exists") + try: + ti.verify_feed_exists(site_conn.config.feed_id) + except ApiError as e: + logging.error(f"Couldn't find CbTH Feed {site_conn.config.feed_id}. Skipping {site_name}: {e}") + continue logging.info(f"Talking to {site_name} server") reports = site_conn.generate_reports() if not reports: @@ -350,9 +368,8 @@ def collect_and_send_reports(self): try: ti.push_to_cb(feed_id=site_conn.config.feed_id, results=self.format_report(reports)) except Exception as e: + logging.error(traceback.format_exc()) logging.error(f"Failed to push reports to feed {site_conn.config.feed_id}: {e}") - - if __name__ == '__main__': parser = argparse.ArgumentParser(description='Modify configuration values via command line.') diff --git a/examples/threathunter/threat_intelligence/threatintel.py b/examples/threathunter/threat_intelligence/threatintel.py index 2b7b148d..44ee67fb 100644 --- a/examples/threathunter/threat_intelligence/threatintel.py +++ b/examples/threathunter/threat_intelligence/threatintel.py @@ -10,20 +10,23 @@ from cbapi.psc.threathunter.models import Feed log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) - class ThreatIntel: def __init__(self): self.cb = CbThreatHunterAPI(timeout=200) - def push_to_cb(self, feed_id, results): + def verify_feed_exists(self, feed_id): + """Verify that a Feed exists""" try: feed = self.cb.select(Feed, feed_id) - except ApiError as e: - log.error(f"couldn't find CbTH feed {feed_id}: {e}") - return + return feed + except ApiError: + raise ApiError + def push_to_cb(self, feed_id, results): + feed = self.verify_feed_exists(feed_id) # will raise an ApiError if the feed cannot be found + if not feed: + return report_list_to_send = [] reports = [] malformed_reports = [] @@ -166,7 +169,6 @@ def iocv2_validation(self, iocv2): elif ioc_v2_field in fields_opt: if not ioc_dictionary[ioc_v2_field]: log.warning(f"Optional field {str(ioc_v2_field)} is empty") - no_error = False else: iocv2_status = isinstance(ioc_dictionary[ioc_v2_field], str) index = fields_opt.index(ioc_v2_field) From d34ac98d9f07970d68720e9db758bb6c19d74300 Mon Sep 17 00:00:00 2001 From: Kylie Ebringer <55465092+kebringer-cb@users.noreply.github.com> Date: Tue, 28 Apr 2020 12:56:00 -0600 Subject: [PATCH 093/197] Update README.md Updated support options --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 05894073..bed2aac6 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ an overview of the concepts that underly this API binding. ## Support -If you have questions on the Carbon Black API or these API Bindings, please contact us at dev-support@carbonblack.com. -Also review the documentation and guides available on the -[Carbon Black Developer Network website](https://developer.carbonblack.com) +1. View all API and integration offerings on the [Developer Network](https://developer.carbonblack.com) along with reference documentation, video tutorials, and how-to guides. +2. Use the [Developer Community Forum](https://community.carbonblack.com/community/resources/developer-relations) to discuss issues and get answers from other API developers in the Carbon Black Community. +3. Report bugs and change requests to [Carbon Black Support](http://carbonblack.com/resources/support). ## Requirements From 2729f465f1237d72ae20a8c2aec61ba851c62aee Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 5 May 2020 20:18:50 -0400 Subject: [PATCH 094/197] examples/threathunter: Make report optional during feed creation (#177) --- examples/threathunter/create_feed.py | 46 +++++++++++++++------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/examples/threathunter/create_feed.py b/examples/threathunter/create_feed.py index 2c143e80..619c7166 100644 --- a/examples/threathunter/create_feed.py +++ b/examples/threathunter/create_feed.py @@ -8,7 +8,7 @@ def main(): - parser = build_cli_parser("Create a CbTH feed and report from a stream of IOCs") + parser = build_cli_parser("Create a CbTH feed and, optionally, a report from a stream of IOCs") # Feed metadata arguments. parser.add_argument("--name", type=str, help="Feed name", required=True) @@ -16,13 +16,14 @@ def main(): parser.add_argument("--url", type=str, help="Feed provider url", required=True) parser.add_argument("--summary", type=str, help="Feed summary", required=True) parser.add_argument("--category", type=str, help="Feed category", required=True) - parser.add_argument("--source_label", type=str, help="Feed source label", required=True) + parser.add_argument("--source_label", type=str, help="Feed source label", default=None) parser.add_argument("--access", type=str, help="Feed access scope", default="private") # Report metadata arguments. + parser.add_argument("--read_report", action="store_true", help="Read a report from stdin") parser.add_argument("--rep_timestamp", type=int, help="Report timestamp", default=int(time.time())) - parser.add_argument("--rep_title", type=str, help="Report title", required=True) - parser.add_argument("--rep_desc", type=str, help="Report description", required=True) + parser.add_argument("--rep_title", type=str, help="Report title") + parser.add_argument("--rep_desc", type=str, help="Report description") parser.add_argument("--rep_severity", type=int, help="Report severity", default=1) parser.add_argument("--rep_link", type=str, help="Report link") parser.add_argument("--rep_tags", type=str, help="Report tags, comma separated") @@ -40,28 +41,31 @@ def main(): "access": args.access, } - rep_tags = [] - if args.rep_tags: - rep_tags = args.rep_tags.split(",") - - report = { - "timestamp": args.rep_timestamp, - "title": args.rep_title, - "description": args.rep_desc, - "severity": args.rep_severity, - "link": args.rep_link, - "tags": rep_tags, - "iocs_v2": [], # NOTE(ww): The feed server will convert IOCs to v2s for us. - } + reports = [] + if args.read_report: + rep_tags = [] + if args.rep_tags: + rep_tags = args.rep_tags.split(",") + + report = { + "timestamp": args.rep_timestamp, + "title": args.rep_title, + "description": args.rep_desc, + "severity": args.rep_severity, + "link": args.rep_link, + "tags": rep_tags, + "iocs_v2": [], # NOTE(ww): The feed server will convert IOCs to v2s for us. + } - report_id, iocs = read_iocs(cb) + report_id, iocs = read_iocs(cb) - report["id"] = report_id - report["iocs"] = iocs + report["id"] = report_id + report["iocs"] = iocs + reports.append(report) feed = { "feedinfo": feed_info, - "reports": [report] + "reports": reports } feed = cb.create(Feed, feed) From ecb69d7f993d1590d859a63a69149e825b31da9d Mon Sep 17 00:00:00 2001 From: Luke Lyon <52218532+llyon-cb@users.noreply.github.com> Date: Thu, 7 May 2020 09:21:35 -0700 Subject: [PATCH 095/197] Simplify report validation in ThreatIntel module, add conversion of severity percentage to int (#224) * update wording for params * check for datetime before modifying * update url check in title to regex * improve logging information, keep default vals of config * remove failure condition if optional field isn't supplied * revert to generic site name * updates to error handling/displaying to user for invalid TAXII or CBAPI details * fix for feed verification * update authentication link to point to DevNetwork * remove report_limit val from config, add more defensive type check for start_date, update READMEs' wording and links, make logging default to INFO, remove unreachable return and add comment about raising ApiError * raise ValueError on wrong config start_date input type * add Schema for report validation in ThreatIntel.push_to_cb * remove unneccesary logging statement: * add rounding of severity for sevs between 10->100 * remove references to removed validation function * fix rounding, fix linter * comment conversion of severity --- .../threat_intelligence/README.md | 15 +- .../threat_intelligence/requirements.txt | 1 + .../threat_intelligence/results.py | 10 +- .../threat_intelligence/schemas.py | 44 ++++ .../threat_intelligence/stix_taxii.py | 1 - .../threat_intelligence/threatintel.py | 193 ++---------------- 6 files changed, 71 insertions(+), 193 deletions(-) create mode 100644 examples/threathunter/threat_intelligence/schemas.py diff --git a/examples/threathunter/threat_intelligence/README.md b/examples/threathunter/threat_intelligence/README.md index cf74a1dc..fa862284 100644 --- a/examples/threathunter/threat_intelligence/README.md +++ b/examples/threathunter/threat_intelligence/README.md @@ -24,7 +24,7 @@ An example of implementing this ThreatIntel module is [available here](Taxii_REA `threatintel.py` has two main uses: -1. Report Validation with `threatintel.input_validation()` +1. Report Validation with `schemas.ReportSchema` 2. Pushing Reports to a Carbon Black ThreatHunter Feed with `threatintel.push_to_cb()` ### Report validation @@ -41,10 +41,9 @@ five required and five optional values. |`description`|string|`[iocs_v2]`|[[IOCv2 Format](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/#definitions)]| |`severity`|integer|`visibility`|string| -The `input_validation` function checks for the existence and type of the five -required values, and (if applicable) checks the optional values. The -function takes a list of dictionaries as input, and outputs a Boolean -indicating if validation was successful. +The `push_to_cb` function checks for the existence and type of the five +required values, and (if applicable) checks the optional values, through a Schema. +See `schemas.py` for the definitions. ### Pushing Reports to a Carbon Black ThreatHunter Feed @@ -58,9 +57,7 @@ ThreatHunter Reports (listed in the table above in Report Validation), with the Report dictionaries, and then those dictionaries into ThreatHunter Report objects. -Report dictionaries are passed through the Report validation function -`input_validation` described above. Any improperly formatted report -dictionaries are saved to a file called `malformed_reports.json`. +Any improperly formatted report dictionaries are saved to a file called `malformed_reports.json`. Upon successful sending of reports to a ThreatHunter Feed, you should see something similar to the following INFO message in the console: @@ -101,7 +98,7 @@ Finally, push your threat intelligence data to a ThreatHunter Feed. ## Customization -Although the `AnalysisResult` class is provided in `results.py` as an example, you may create your own custom class to use with `push_to_cb`. The class must have the following attributes to work with the provided `push_to_cb` and `input_validation` functions, as well as the ThreatHunter backend: +Although the `AnalysisResult` class is provided in `results.py` as an example, you may create your own custom class to use with `push_to_cb`. The class must have the following attributes to work with the provided `push_to_cb` function, as well as the ThreatHunter backend: |Attribute|Type| diff --git a/examples/threathunter/threat_intelligence/requirements.txt b/examples/threathunter/threat_intelligence/requirements.txt index 38bd9dbd..94b0c32b 100644 --- a/examples/threathunter/threat_intelligence/requirements.txt +++ b/examples/threathunter/threat_intelligence/requirements.txt @@ -7,3 +7,4 @@ urllib3>=1.24.2 cbapi>=1.5.6 python_dateutil==2.8.1 PyYAML==5.1.2 +schema diff --git a/examples/threathunter/threat_intelligence/results.py b/examples/threathunter/threat_intelligence/results.py index f750191c..e3d8d732 100644 --- a/examples/threathunter/threat_intelligence/results.py +++ b/examples/threathunter/threat_intelligence/results.py @@ -58,9 +58,13 @@ def normalize(self): if self.severity <= 0 or self.severity > 10: logging.warning("normalizing OOB score: {}".format(self.severity)) - self.severity = max(1, min(self.severity, 10)) - # NOTE: min 1 and not 0 - # else err 400 from cbapi: Report severity must be between 1 & 10 + if self.severity > 10 and self.severity < 100: + #assume it's a percentage + self.severity = round(self.severity/10) + else: + # any severity above 10 becomes 10, or below 1 becomes 1 + # Report severity must be between 1 & 10, else CBAPI throws 400 error + self.severity = max(1, min(self.severity, 10)) return self def as_dict(self): diff --git a/examples/threathunter/threat_intelligence/schemas.py b/examples/threathunter/threat_intelligence/schemas.py new file mode 100644 index 00000000..da361604 --- /dev/null +++ b/examples/threathunter/threat_intelligence/schemas.py @@ -0,0 +1,44 @@ +from schema import And, Or, Optional, Schema + + +IOCv2Schema = Schema( + { + "id": And(str, len), + "match_type": And(str, lambda type: type in ["query", "equality", "regex"]), + "values": And([str], len), + Optional("field"): str, + Optional("link"): str + } +) + +QueryIOCSchema = Schema( + { + "search_query": And(str, len), + Optional("index_type"): And(str, len) + } +) + +IOCSchema = Schema( + { + Optional("md5"): And([str], len), + Optional("ipv4"): And([str], len), + Optional("ipv6"): And([str], len), + Optional("dns"): And([str], len), + Optional("query"): [QueryIOCSchema] + } +) + +ReportSchema = Schema( + { + "id": And(str, len), + "timestamp": And(int, lambda n: n > 0), + "title": And(str, len), + "description": And(str, len), + "severity": And(int, lambda n: n > 0 and n < 11), + Optional("link"): str, + Optional("tags"): [str], + Optional("iocs_v2"): [IOCv2Schema], + Optional("iocs"): IOCSchema, + Optional("visibility"): str + } +) diff --git a/examples/threathunter/threat_intelligence/stix_taxii.py b/examples/threathunter/threat_intelligence/stix_taxii.py index 9e3012a1..95358071 100644 --- a/examples/threathunter/threat_intelligence/stix_taxii.py +++ b/examples/threathunter/threat_intelligence/stix_taxii.py @@ -194,7 +194,6 @@ def import_collection(self, collection): num_times_empty_content_blocks = 0 advance = True reports_limit = self.config.reports_limit - logging.info(f"reports limit: {reports_limit}") if not self.config.size_of_request_in_minutes: size_of_request_in_minutes = 1440 else: diff --git a/examples/threathunter/threat_intelligence/threatintel.py b/examples/threathunter/threat_intelligence/threatintel.py index 44ee67fb..c00672f9 100644 --- a/examples/threathunter/threat_intelligence/threatintel.py +++ b/examples/threathunter/threat_intelligence/threatintel.py @@ -8,15 +8,18 @@ from cbapi.psc.threathunter import CbThreatHunterAPI, Report from cbapi.errors import ApiError from cbapi.psc.threathunter.models import Feed +from schemas import ReportSchema +from schema import SchemaError log = logging.getLogger(__name__) + class ThreatIntel: def __init__(self): self.cb = CbThreatHunterAPI(timeout=200) def verify_feed_exists(self, feed_id): - """Verify that a Feed exists""" + """Verify that a Feed exists.""" try: feed = self.cb.select(Feed, feed_id) return feed @@ -41,13 +44,13 @@ def push_to_cb(self, feed_id, results): "severity": int(result.severity), "iocs_v2": [ioc_v2.as_dict() for ioc_v2 in result.iocs_v2] } - - if self.input_validation([report_dict]): + try: + ReportSchema.validate(report_dict) # create CB Report object report = Report(self.cb, initial_data=report_dict, feed_id=feed_id) report_list_to_send.append(report) reports.append(report_dict) - else: + except SchemaError as e: log.warning("Report Validation failed. Saving report to file for reference.") malformed_reports.append(report_dict) except Exception as e: @@ -62,11 +65,12 @@ def push_to_cb(self, feed_id, results): log.debug("Sending results to Carbon Black Cloud.") - try: - feed.append_reports(report_list_to_send) - log.info(f"Appended {len(report_list_to_send)} reports to ThreatHunter Feed {feed_id}") - except Exception as e: - log.debug(f"Failed sending {len(report_list_to_send)} reports: {e}") + if report_list_to_send: + try: + feed.append_reports(report_list_to_send) + log.info(f"Appended {len(report_list_to_send)} reports to ThreatHunter Feed {feed_id}") + except Exception as e: + log.debug(f"Failed sending {len(report_list_to_send)} reports: {e}") if malformed_reports: log.warning("Some report(s) failed validation. See malformed_reports.json for reports that failed.") @@ -76,174 +80,3 @@ def push_to_cb(self, feed_id, results): except Exception as e: log.error(f"Failed to write malformed_reports to file: {e}") - ########################################################## - # Input validation of reports - ########################################################## - - # Input validation for query IOC - def query_ioc_validation(self, query_ioc): - if isinstance(query_ioc, list): - fields = ["search_query", "index_type"] - field_data_types = [str, str] - - for query in query_ioc: - status = [[fields[0], False], [fields[1], False]] - - status[0][1] = isinstance(query[fields[0]], field_data_types[0]) - status[1][1] = isinstance(query[fields[1]], field_data_types[1]) - - if not status[0][1]: - log.warning(f"Missing Required Report Field: {str(status[0][0])}") - - if not status[1][1]: - log.warning(f"Field {str(status[1][0])} does not match correct data type") - - log.debug("Query IOC validation complete") - return True - else: - log.warning("Query_ioc does not match correct data type") - return False - - # Input validation for IOCs - def ioc_validation(self, ioc): - fields_opt = ["md5", "ipv4", "ipv6", "dns", "query"] - fields_data_type = [str, str, str, str] - - ioc_fields = list(ioc.keys()) - status_opt = [[fields_opt[i], False] for i in range(len(fields_opt))] - no_error = True - - # Iterate through ioc fields and check for validation - for ioc_field in ioc_fields: - if ioc_field in fields_opt and ioc[ioc_field]: - if ioc_field == 'query': - query_ioc_status = self.query_ioc_validation(ioc[ioc_field]) - no_error = no_error and query_ioc_status - else: - index = fields_opt.index(ioc_field) - status_opt[index][1] = all(isinstance(elem, fields_data_type[index]) for elem in ioc[ioc_field]) - no_error = no_error and status_opt[index][1] - - if not status_opt[index][1]: - log.warning(f"Field {str(ioc_field)} does not match correct data type") - else: - log.warning(f"Invalid field: {str(ioc_field)}") - no_error = no_error and False - - return no_error - - # Input validation for IOCsv2 - def iocv2_validation(self, iocv2): - if not isinstance(iocv2, list): - log.warning("IOCv2 must be a list of IOCv2 dictionaries") - return False - else: - fields_req = ["id", "match_type", "values"] - fields_opt = ["field", "link"] - no_error = True - - for ioc_dictionary in iocv2: - ioc_v2_fields = list(ioc_dictionary.keys()) - status_req = [[fields_req[i], False] for i in range(len(fields_req))] - status_opt = [[fields_opt[i], False] for i in range(len(fields_opt))] - - for ioc_v2_field in ioc_v2_fields: - if ioc_v2_field in fields_req: - if not ioc_dictionary[ioc_v2_field]: - log.warning(f"Required field {str(ioc_v2_field)} is empty") - no_error = False - else: - if ioc_v2_field == 'values' and isinstance(ioc_dictionary[ioc_v2_field], list): - status_req[2][1] = all(isinstance(elem, str) for elem in ioc_dictionary[ioc_v2_field]) - no_error = no_error and status_req[2][1] - - if not status_req[2][1]: - log.warning(f"Missing REQUIRED field or invalid data type: {str(ioc_v2_field)}") - else: - index = fields_req.index(ioc_v2_field) - status_req[index][1] = isinstance(ioc_dictionary[ioc_v2_field], str) - no_error = no_error and status_req[index][1] - - if not status_req[index][1]: - log.warning(f"Missing REQUIRED field or invalid data type: {str(ioc_v2_field)}") - elif ioc_v2_field in fields_opt: - if not ioc_dictionary[ioc_v2_field]: - log.warning(f"Optional field {str(ioc_v2_field)} is empty") - else: - iocv2_status = isinstance(ioc_dictionary[ioc_v2_field], str) - index = fields_opt.index(ioc_v2_field) - status_opt[index][1] = iocv2_status - no_error = no_error and iocv2_status - - if not iocv2_status: - log.warning(f"Missing field or invalid data type: {str(ioc_v2_field)}") - else: - log.warning(f"Invalid field in IOCv2: {str(ioc_v2_field)}") - no_error = False - - return no_error - - # Input a list of dictionaries representing - # reports to be checked and validated - def input_validation(self, reports): - if len(reports) < 1: - log.debug("There are no reports to be validated") - else: - - fields_req = ["id", "timestamp", "title", "description", "severity"] - fields_opt = ["link", "tags", "iocs", "iocs_v2", "visibility"] - - # Iterate though the array and check for invalidation of each field - for index, report in enumerate(reports): - no_error = True - status_req = [[fields_req[i], False] for i in range(len(fields_req))] - status_opt = [[fields_opt[i], False] for i in range(len(fields_opt))] - - for key, val in report.items(): - if key in fields_req: - if key == "timestamp" or key == "severity": - if isinstance(val, int): - idx = fields_req.index(key) - status_req[idx][1] = True - no_error = no_error and status_req[idx][1] - else: - log.warning(f"Missing REQUIRED field or invalid data type: {str(key)}") - no_error = no_error and False - else: - if isinstance(val, str): - idx = fields_req.index(key) - status_req[idx][1] = True - no_error = no_error and status_req[idx][1] - else: - log.warning(f"Missing REQUIRED field or invalid data type: {str(key)}") - no_error = no_error and False - elif key in fields_opt: - if key == "iocs": - ioc_status = self.ioc_validation(val) - status_opt[2][1] = ioc_status - no_error = no_error and ioc_status - elif key == "iocs_v2": - iocv2_status = self.iocv2_validation(val) - status_opt[3][1] = iocv2_status - no_error = no_error and iocv2_status - else: - if isinstance(val, str): - idx = fields_opt.index(key) - status_opt[idx][1] = True # this is always true? What's the point - no_error = no_error and status_opt[idx][1] - else: - log.warning(f"Missing field or invalid data type: {str(key)}") - else: - log.warning(f"Invalid Report field: {str(key)}. Please remove from report before dispatching.") - no_error = no_error and False - - log.debug(f"Report {str(index)} validation complete") - - if no_error: - log.debug("Report Validation found no errors in report schema") - else: - log.warning("Report Validation found an error in report schema") - return False - - log.debug("Report Validation check complete") - return True From 233fe172f70d9b59f59972d9a5a0b71fe8f4d210 Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Fri, 8 May 2020 13:30:39 +0000 Subject: [PATCH 096/197] Import ABC from collections.abc for Python 3 compatibility. --- tests/requests_cache/backends/storage/dbdict.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/requests_cache/backends/storage/dbdict.py b/tests/requests_cache/backends/storage/dbdict.py index 2c75264b..513db8bc 100755 --- a/tests/requests_cache/backends/storage/dbdict.py +++ b/tests/requests_cache/backends/storage/dbdict.py @@ -6,7 +6,6 @@ Dictionary-like objects for saving large data sets to `sqlite` database """ -from collections import MutableMapping import sqlite3 as sqlite from contextlib import contextmanager try: @@ -17,6 +16,10 @@ import cPickle as pickle except ImportError: import pickle +try: + from collections.abc import MutableMapping +except ImportError: + from collections import MutableMapping from requests_cache.compat import bytes From 3ae56130455611b88418ad45f42ffe1966727060 Mon Sep 17 00:00:00 2001 From: Paul Drapeau Date: Wed, 10 Jun 2020 11:39:34 -0400 Subject: [PATCH 097/197] Dell BIOS Verification examples Uses LR to pull BIOS image files from Dell Devices failing BIOS image verification --- .../DellBiosVerification/BiosVerification.py | 102 ++++++++++++++++++ .../cblr/DellBiosVerification/dellbios.bat | 4 + 2 files changed, 106 insertions(+) create mode 100755 examples/defense/cblr/DellBiosVerification/BiosVerification.py create mode 100644 examples/defense/cblr/DellBiosVerification/dellbios.bat diff --git a/examples/defense/cblr/DellBiosVerification/BiosVerification.py b/examples/defense/cblr/DellBiosVerification/BiosVerification.py new file mode 100755 index 00000000..b564ce98 --- /dev/null +++ b/examples/defense/cblr/DellBiosVerification/BiosVerification.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +# Carbon Black Cloud -Dell Bios Verification LiveResponse +# Copyright VMware 2020 +# May 2020 +# Version 0.1 +# pdrapeau [at] vmware . com +# +# usage: BiosVerification.py [-h] [-m MACHINENAME] [-g] [-o ORGPROFILE] +# +# optional arguments: +# -h, --help show this help message and exit +# -m MACHINENAME, --machinename MACHINENAME +# machinename to run host bios forensics on +# -g, --get Get BIOS images +# +# -o ORGPROFILE, --orgprofile ORGPROFILE +# Select your cbapi credential profile + +import os, sys, time, argparse +from cbapi.defense import * + +def live_response(cb, host=None, response=None): + + print ("") + + #Select the device you want to gather forensic data from + query_hostname = "hostNameExact:%s" % host + print ("[ * ] Establishing LiveResponse Session with Remote Host:") + + #Create a new device object to launch LR on + device = cb.select(Device).where(query_hostname).first() + print(" - Hostname: {}".format(device.name)) + print(" - OS Version: {}".format(device.osVersion)) + print(" - Sensor Version: {}".format(device.sensorVersion)) + print(" - AntiVirus Status: {}".format(device.avStatus)) + print(" - Internal IP Address: {}".format(device.lastInternalIpAddress)) + print(" - External IP Address: {}".format(device.lastExternalIpAddress)) + print ("") + + #Execute our LR session + with device.lr_session() as lr_session: + print ("[ * ] Uploading scripts to the remote host") + lr_session.put_file(open("dellbios.bat", "rb"), "C:\\Program Files\\Confer\\temp\\dellbios.bat") + + if response == "get": + print ("[ * ] Getting the images") + result = lr_session.create_process("cmd.exe /c .\\dellbios.bat", wait_for_output=True, remote_output_file_name=None, working_directory="C:\\Program Files\\Confer\\temp\\", wait_timeout=120, wait_for_completion=True).decode("utf-8") + print ("") + print("{}".format(result)) + + print ("[ * ] Removing scripts") + lr_session.create_process("powershell.exe del .\\dellbios.bat", wait_for_output=False, remote_output_file_name=None, working_directory="C:\\Program Files\\Confer\\temp\\", wait_timeout=30, wait_for_completion=False) + + + print ("[ * ] Downloading images") + zipdata = lr_session.get_file("C:\\Program Files\\Confer\\temp\\BiosImages.zip") + + print ("[ * ] Writing out " + host + "-BiosImages.zip") + zipfile = open(host + "-BiosImages.zip","wb") + zipfile.write(zipdata) + + print ("") + + + + else: + print ("[ * ] Nothing to do") + + + print ("[ * ] Cleaning up") + lr_session.create_process("powershell.exe del .\\BiosImages.zip", wait_for_output=False, remote_output_file_name=None, working_directory="C:\\Program Files\\Confer\\temp\\", wait_timeout=30, wait_for_completion=False) + lr_session.create_process("powershell.exe del C:\\tmpbios\\*.*", wait_for_output=False, remote_output_file_name=None, working_directory="C:\\Program Files\\Confer\\temp\\", wait_timeout=30, wait_for_completion=False) + + + print ("") + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-m", "--machinename", help = "machinename to run host forensics recon on") + parser.add_argument("-g", "--get", help = "Get the Dell BIOS Verification images", action = "store_true") + parser.add_argument('-o', '--orgprofile', help = "Select your cbapi credential profile", dest = "orgprofile", default = "default") + args = parser.parse_args() + + #Create the CbD LR API object + profile = CbDefenseAPI(profile="{}".format(args.orgprofile)) + cb_url = profile.credentials.url + cb_token = profile.credentials.token + cb_org_key = profile.credentials.org_key + cb_ssl = "True" + cb = CbDefenseAPI(url=cb_url, token=cb_token, orgId=cb_org_key, ssl_verify=cb_ssl) + + if args.machinename: + if args.get: + live_response(cb, host=args.machinename, response="get") + else: + print ("Nothing to do...") + else: + print ("[ ! ] You must specify a machinename with a --machinename parameter. IE ./BiosVerification.py --machinename cheese") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/defense/cblr/DellBiosVerification/dellbios.bat b/examples/defense/cblr/DellBiosVerification/dellbios.bat new file mode 100644 index 00000000..b4d8e57a --- /dev/null +++ b/examples/defense/cblr/DellBiosVerification/dellbios.bat @@ -0,0 +1,4 @@ +mkdir c:\tmpbios +del BiosImages.zip +"C:\Program Files\Dell\BiosVerification\Dell.TrustedDevice.Service.Console.exe" -exportall -export c:\tmpbios +powershell.exe -ExecutionPolicy Bypass Compress-Archive -Path c:\tmpbios\*.* -DestinationPath BiosImages.zip -Force \ No newline at end of file From dd300d7b8be1078b3b049853e8adcc88118487ab Mon Sep 17 00:00:00 2001 From: Paul Drapeau Date: Wed, 10 Jun 2020 11:56:48 -0400 Subject: [PATCH 098/197] Create README.md --- .../cblr/DellBiosVerification/README.md | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 examples/defense/cblr/DellBiosVerification/README.md diff --git a/examples/defense/cblr/DellBiosVerification/README.md b/examples/defense/cblr/DellBiosVerification/README.md new file mode 100644 index 00000000..1c8da5a8 --- /dev/null +++ b/examples/defense/cblr/DellBiosVerification/README.md @@ -0,0 +1,70 @@ +# BiosVerification.py Live Response API Script + +## References +https://www.dell.com/support/manuals/us/en/04/trusted-device/trusted_device/results-troubleshooting-and-remediation?guid=guid-240f1964-167a-41b0-9fb3-687dddbdb71f&lang=en-us + +## Summary + +This set of tools uses the VMware Carbon Black Security Cloud Live Response APIs to retrieve artifacts generated by the Dell Trusted Device SafeBIOS verification service. The SafeBIOS console application can generate BIOS image files to the filesystem when a verification failure event is detected. + +Incident responders can use this set of scripts to retrieve the image files for forensic analysis. + + +## Instructions + +Usage: + +To retrieve the BIOS image files from a device in a failed verification state via the Live Response API: + + +Copy the BiosVerification.py and dellbios.bat files to the same directory on the administrator system. +Install the cbapi Python bindings: https://github.com/carbonblack/cbapi-python +Create necessary API keys and configure credentials on the administrator system: https://cbapi.readthedocs.io/en/latest/getting-started.html +Run the provided BiosVerification.py utility with the following command line to target the failed system: +``` +BiosVerification.py --get --machinename +``` + +If failed BIOS image files are found the script will retrieve the image files to the local administrator system in a compressed archive named +``` +-BiosImages.zip +``` + +## Example + +``` +$ ./BiosVerification.py --get --machinename "DPENNY\LT-7400" + +[ * ] Establishing LiveResponse Session with Remote Host: + - Hostname: DPENNY\LT-7400 + - OS Version: Windows 10 x64 + - Sensor Version: 3.6.0.1201 + - AntiVirus Status: ['AV_ACTIVE', 'ONDEMAND_SCAN_DISABLED'] + - Internal IP Address: 172.16.0.196 + - External IP Address: 70.114.97.235 + +[ * ] Uploading scripts to the remote host +[ * ] Getting the images + + +c:\program files\confer\temp>mkdir c:\tmpbios +A subdirectory or file c:\tmpbios already exists. + +c:\program files\confer\temp>del BiosImages.zip +Could Not Find c:\program files\confer\temp\BiosImages.zip + +c:\program files\confer\temp>"C:\Program Files\Dell\BiosVerification\Dell.TrustedDevice.Service.Console.exe" -exportall -export c:\tmpbios +Wrote image to c:\tmpbios\BIOSImageCaptureBVS06092020_120255.bv + +c:\program files\confer\temp>powershell.exe -ExecutionPolicy Bypass Compress-Archive -Path c:\tmpbios\*.* -DestinationPath BiosImages.zip -Force + +[ * ] Removing scripts +[ * ] Downloading images +[ * ] Writing out DPENNY\LT-7400-BiosImages.zip + +[ * ] Cleaning up + +``` + + +This script is compatible with the full VMware Carbon Black Cloud API and requires the python cbapi \ No newline at end of file From 130a9068680828498715896365590b2013113fec Mon Sep 17 00:00:00 2001 From: Paul Drapeau Date: Wed, 10 Jun 2020 11:59:49 -0400 Subject: [PATCH 099/197] Update README.md --- examples/defense/cblr/DellBiosVerification/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/defense/cblr/DellBiosVerification/README.md b/examples/defense/cblr/DellBiosVerification/README.md index 1c8da5a8..c034c6fd 100644 --- a/examples/defense/cblr/DellBiosVerification/README.md +++ b/examples/defense/cblr/DellBiosVerification/README.md @@ -1,4 +1,4 @@ -# BiosVerification.py Live Response API Script +# Dell BiosVerification.py Live Response API Script ## References https://www.dell.com/support/manuals/us/en/04/trusted-device/trusted_device/results-troubleshooting-and-remediation?guid=guid-240f1964-167a-41b0-9fb3-687dddbdb71f&lang=en-us From 1ea0607698c55b89f0bf284a25544a43f1c46d69 Mon Sep 17 00:00:00 2001 From: Paul Drapeau Date: Wed, 10 Jun 2020 12:03:01 -0400 Subject: [PATCH 100/197] Update README.md --- .../defense/cblr/DellBiosVerification/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/defense/cblr/DellBiosVerification/README.md b/examples/defense/cblr/DellBiosVerification/README.md index c034c6fd..aff674cf 100644 --- a/examples/defense/cblr/DellBiosVerification/README.md +++ b/examples/defense/cblr/DellBiosVerification/README.md @@ -1,7 +1,10 @@ # Dell BiosVerification.py Live Response API Script ## References -https://www.dell.com/support/manuals/us/en/04/trusted-device/trusted_device/results-troubleshooting-and-remediation?guid=guid-240f1964-167a-41b0-9fb3-687dddbdb71f&lang=en-us + +Troubleshooting: https://www.dell.com/support/manuals/us/en/04/trusted-device/trusted_device/results-troubleshooting-and-remediation?guid=guid-240f1964-167a-41b0-9fb3-687dddbdb71f&lang=en-us + +Dell Trusted Device Installation Instructions: https://www.dell.com/support/manuals/us/en/04/trusted-device/trusted_device/installation?guid=guid-b9217d4f-6932-47d2-8db5-50633eb47691&lang=en-us ## Summary @@ -17,10 +20,10 @@ Usage: To retrieve the BIOS image files from a device in a failed verification state via the Live Response API: -Copy the BiosVerification.py and dellbios.bat files to the same directory on the administrator system. -Install the cbapi Python bindings: https://github.com/carbonblack/cbapi-python -Create necessary API keys and configure credentials on the administrator system: https://cbapi.readthedocs.io/en/latest/getting-started.html -Run the provided BiosVerification.py utility with the following command line to target the failed system: +1. Copy the BiosVerification.py and dellbios.bat files to the same directory on the administrator system. +2. Install the cbapi Python bindings: https://github.com/carbonblack/cbapi-python +3. Create necessary API keys and configure credentials on the administrator system: https://cbapi.readthedocs.io/en/latest/getting-started.html +4. Run the provided BiosVerification.py utility with the following command line to target the failed system: ``` BiosVerification.py --get --machinename ``` From ce7a9be27e6d4887884ea2b1d54f4aa56d49afc8 Mon Sep 17 00:00:00 2001 From: Paul Drapeau Date: Wed, 10 Jun 2020 12:07:47 -0400 Subject: [PATCH 101/197] Update README.md --- .../cblr/DellBiosVerification/README.md | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/examples/defense/cblr/DellBiosVerification/README.md b/examples/defense/cblr/DellBiosVerification/README.md index aff674cf..daf79bc8 100644 --- a/examples/defense/cblr/DellBiosVerification/README.md +++ b/examples/defense/cblr/DellBiosVerification/README.md @@ -2,15 +2,23 @@ ## References -Troubleshooting: https://www.dell.com/support/manuals/us/en/04/trusted-device/trusted_device/results-troubleshooting-and-remediation?guid=guid-240f1964-167a-41b0-9fb3-687dddbdb71f&lang=en-us +Dell Trusted Device Product Information: https://www.delltechnologies.com/endpointsecurity Dell Trusted Device Installation Instructions: https://www.dell.com/support/manuals/us/en/04/trusted-device/trusted_device/installation?guid=guid-b9217d4f-6932-47d2-8db5-50633eb47691&lang=en-us +Troubleshooting: https://www.dell.com/support/manuals/us/en/04/trusted-device/trusted_device/results-troubleshooting-and-remediation?guid=guid-240f1964-167a-41b0-9fb3-687dddbdb71f&lang=en-us + + + ## Summary -This set of tools uses the VMware Carbon Black Security Cloud Live Response APIs to retrieve artifacts generated by the Dell Trusted Device SafeBIOS verification service. The SafeBIOS console application can generate BIOS image files to the filesystem when a verification failure event is detected. +This set of tools uses the VMware Carbon Black Security Cloud Live Response APIs to retrieve +artifacts generated by the Dell Trusted Device SafeBIOS verification service. The Dell Trusted +Device agent saves BIOS image files to the filesystem when a verification failure event is +detected. -Incident responders can use this set of scripts to retrieve the image files for forensic analysis. +Incident responders can use this set of scripts to retrieve the BIOS image files for forensic +analysis. ## Instructions @@ -36,15 +44,15 @@ If failed BIOS image files are found the script will retrieve the image files to ## Example ``` -$ ./BiosVerification.py --get --machinename "DPENNY\LT-7400" +$ ./BiosVerification.py --get --machinename "x\LT-7400" [ * ] Establishing LiveResponse Session with Remote Host: - - Hostname: DPENNY\LT-7400 + - Hostname: x\LT-7400 - OS Version: Windows 10 x64 - Sensor Version: 3.6.0.1201 - AntiVirus Status: ['AV_ACTIVE', 'ONDEMAND_SCAN_DISABLED'] - Internal IP Address: 172.16.0.196 - - External IP Address: 70.114.97.235 + - External IP Address: x.x.x.x [ * ] Uploading scripts to the remote host [ * ] Getting the images @@ -63,7 +71,7 @@ c:\program files\confer\temp>powershell.exe -ExecutionPolicy Bypass Compress-Arc [ * ] Removing scripts [ * ] Downloading images -[ * ] Writing out DPENNY\LT-7400-BiosImages.zip +[ * ] Writing out x\LT-7400-BiosImages.zip [ * ] Cleaning up From 4d2f90fdce7e384785e12fa295e6070317a434b7 Mon Sep 17 00:00:00 2001 From: Paul Drapeau Date: Wed, 10 Jun 2020 12:47:50 -0400 Subject: [PATCH 102/197] Update README.md --- examples/defense/cblr/DellBiosVerification/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/defense/cblr/DellBiosVerification/README.md b/examples/defense/cblr/DellBiosVerification/README.md index daf79bc8..4a968c1c 100644 --- a/examples/defense/cblr/DellBiosVerification/README.md +++ b/examples/defense/cblr/DellBiosVerification/README.md @@ -78,4 +78,4 @@ c:\program files\confer\temp>powershell.exe -ExecutionPolicy Bypass Compress-Arc ``` -This script is compatible with the full VMware Carbon Black Cloud API and requires the python cbapi \ No newline at end of file +This script is compatible with the full VMware Carbon Black Cloud API and requires the python cbapi. \ No newline at end of file From 4aa40b03316728ebe204872d940c291a8b3f9192 Mon Sep 17 00:00:00 2001 From: Paul Drapeau Date: Wed, 10 Jun 2020 13:53:46 -0400 Subject: [PATCH 103/197] Updated README and simplified Object creation --- .../defense/cblr/DellBiosVerification/BiosVerification.py | 7 +------ examples/defense/cblr/DellBiosVerification/README.md | 5 +++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/examples/defense/cblr/DellBiosVerification/BiosVerification.py b/examples/defense/cblr/DellBiosVerification/BiosVerification.py index b564ce98..5d55ca73 100755 --- a/examples/defense/cblr/DellBiosVerification/BiosVerification.py +++ b/examples/defense/cblr/DellBiosVerification/BiosVerification.py @@ -83,12 +83,7 @@ def main(): args = parser.parse_args() #Create the CbD LR API object - profile = CbDefenseAPI(profile="{}".format(args.orgprofile)) - cb_url = profile.credentials.url - cb_token = profile.credentials.token - cb_org_key = profile.credentials.org_key - cb_ssl = "True" - cb = CbDefenseAPI(url=cb_url, token=cb_token, orgId=cb_org_key, ssl_verify=cb_ssl) + cb = CbDefenseAPI(profile="{}".format(args.orgprofile)) if args.machinename: if args.get: diff --git a/examples/defense/cblr/DellBiosVerification/README.md b/examples/defense/cblr/DellBiosVerification/README.md index 4a968c1c..1ef0fa35 100644 --- a/examples/defense/cblr/DellBiosVerification/README.md +++ b/examples/defense/cblr/DellBiosVerification/README.md @@ -30,8 +30,9 @@ To retrieve the BIOS image files from a device in a failed verification state vi 1. Copy the BiosVerification.py and dellbios.bat files to the same directory on the administrator system. 2. Install the cbapi Python bindings: https://github.com/carbonblack/cbapi-python -3. Create necessary API keys and configure credentials on the administrator system: https://cbapi.readthedocs.io/en/latest/getting-started.html -4. Run the provided BiosVerification.py utility with the following command line to target the failed system: +3. Create a Live Response API key https://developer.carbonblack.com/reference/carbon-black-cloud/authentication/ +4. Configure credentials on the administrator system: https://cbapi.readthedocs.io/en/latest/getting-started.html +5. Run the provided BiosVerification.py utility with the following command line to target the failed system: ``` BiosVerification.py --get --machinename ``` From d5555f9b84d1da0b05fda1ec15adf6c43e544eaa Mon Sep 17 00:00:00 2001 From: Thomas Bouve Date: Thu, 2 Jul 2020 17:50:57 +0200 Subject: [PATCH 104/197] Adding org_key value to urlobject property. --- src/cbapi/psc/threathunter/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index 0f732323..b84c1393 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -267,7 +267,7 @@ def _count(self): log.debug("args: {}".format(str(args))) - self._total_results = int(self._cb.post_object(self._doc_class.urlobject, body=args) + self._total_results = int(self._cb.post_object(self._doc_class.urlobject.format(self._cb.credentials.org_key), body=args) .json().get("response_header", {}).get("num_available", 0)) self._count_valid = True return self._total_results From 18d6d389029dde6145a22dc14047a738ffc5090d Mon Sep 17 00:00:00 2001 From: Filipe Spencer Lopes Date: Thu, 2 Jul 2020 17:59:36 +0200 Subject: [PATCH 105/197] Refactor to url (for org key population) --- src/cbapi/psc/threathunter/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index b84c1393..023e1a97 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -531,7 +531,7 @@ def _perform_query(self): results = self._cb.get_object(url, query_parameters=self._args) while results["incomplete_results"]: - result = self._cb.get_object(self._doc_class.urlobject, query_parameters=self._args) + result = self._cb.get_object(url, query_parameters=self._args) results["nodes"]["children"].extend(result["nodes"]["children"]) results["incomplete_results"] = result["incomplete_results"] From fa396dfe709b1b4f127b23842f8161e5729c0bb2 Mon Sep 17 00:00:00 2001 From: Thomas Bouve Date: Fri, 3 Jul 2020 14:37:44 +0200 Subject: [PATCH 106/197] Update to v2 based process and event searches --- src/cbapi/psc/threathunter/models.py | 16 ++++--- src/cbapi/psc/threathunter/query.py | 68 ++++++++++++++++++---------- 2 files changed, 54 insertions(+), 30 deletions(-) diff --git a/src/cbapi/psc/threathunter/models.py b/src/cbapi/psc/threathunter/models.py index 461380b9..dbd71815 100644 --- a/src/cbapi/psc/threathunter/models.py +++ b/src/cbapi/psc/threathunter/models.py @@ -20,7 +20,7 @@ class Process(UnrefreshableModel): """ default_sort = 'last_update desc' primary_key = "process_guid" - validation_url = "/threathunter/search/v1/orgs/{}/processes/search_validation" + validation_url = "/api/investigate/v1/orgs/{}/processes/search_validation" class Summary(UnrefreshableModel): """Represents a summary of organization-specific information for @@ -28,7 +28,7 @@ class Summary(UnrefreshableModel): """ default_sort = "last_update desc" primary_key = "process_guid" - urlobject_single = "/threathunter/search/v1/orgs/{}/processes/summary" + urlobject_single = "/api/investigate/v1/orgs/{}/processes/summary" def __init__(self, cb, model_unique_id): url = self.urlobject_single.format(cb.credentials.org_key) @@ -171,8 +171,8 @@ class Event(UnrefreshableModel): """Events can be queried for via ``CbThreatHunterAPI.select`` or though an already selected process with ``Process.events()``. """ - urlobject = '/threathunter/search/v1/orgs/{}/events/_search' - validation_url = '/threathunter/search/v1/orgs/{}/events/search_validation' + urlobject = '/api/investigate/v2/orgs/{}/events/{}/_search' + validation_url = '/api/investigate/v1/orgs/{}/events/search_validation' default_sort = 'last_update desc' primary_key = "process_guid" @@ -180,7 +180,7 @@ class Event(UnrefreshableModel): def _query_implementation(cls, cb): return Query(cls, cb) - def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=True): + def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=True): super(Event, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, force_init=force_init, full_doc=full_doc) @@ -197,8 +197,10 @@ def _query_implementation(cls, cb): return TreeQuery(cls, cb) def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=True): - super(Tree, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=force_init, full_doc=full_doc) + super(Tree, self).__init__( + cb, model_unique_id=model_unique_id, initial_data=initial_data, + force_init=force_init, full_doc=full_doc + ) @property def children(self): diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index 0f732323..3792bd56 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -254,22 +254,40 @@ def not_(self, q=None, **kwargs): def _get_query_parameters(self): args = self._default_args.copy() - args['q'] = self._query_builder._collapse() + args['query'] = self._query_builder._collapse() if self._query_builder._process_guid is not None: - args["cb.process_guid"] = self._query_builder._process_guid - args["fl"] = "*,parent_hash,parent_name,process_cmdline,backend_timestamp,device_external_ip,device_group," \ - + "device_internal_ip,device_os,process_effective_reputation,process_reputation,ttp" + args["process_guid"] = self._query_builder._process_guid + args["fields"] = [ + "*", + "parent_hash", + "parent_name", + "process_cmdline", + "backend_timestamp", + "device_external_ip", + "device_group", + "device_internal_ip", + "device_os", + "process_effective_reputation", + "process_reputation,ttp" + ] return args def _count(self): - args = {"search_params": self._get_query_parameters()} + args = self._get_query_parameters() log.debug("args: {}".format(str(args))) - self._total_results = int(self._cb.post_object(self._doc_class.urlobject, body=args) - .json().get("response_header", {}).get("num_available", 0)) + result = self._cb.post_object( + self._doc_class.urlobject.format( + self._cb.credentials.org_key, + args["process_guid"] + ), body=args + ).json() + + self._total_results = int(result.get('num_available', 0)) self._count_valid = True + return self._total_results def _validate(self, args): @@ -285,13 +303,13 @@ def _validate(self, args): def _search(self, start=0, rows=0): # iterate over total result set, 100 at a time args = self._get_query_parameters() - self._validate(args) + #self._validate(args) if start != 0: args['start'] = start args['rows'] = self._batch_size - args = {"search_params": args} + #args = {"search_params": args} current = start numrows = 0 @@ -299,14 +317,17 @@ def _search(self, start=0, rows=0): still_querying = True while still_querying: - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) + url = self._doc_class.urlobject.format( + self._cb.credentials.org_key, + args["process_guid"] + ) resp = self._cb.post_object(url, body=args) result = resp.json() - self._total_results = result.get("response_header", {}).get("num_available", 0) + self._total_results = result.get('num_available', 0) self._count_valid = True - results = result.get('docs', []) + results = result.get('results', []) for item in results: yield item @@ -379,12 +400,13 @@ def _submit(self): raise ApiError("Query already submitted: token {0}".format(self._query_token)) args = self._get_query_parameters() - self._validate(args) + #self._validate(args) + + url = "/api/investigate/v2/orgs/{}/processes/search_jobs".format(self._cb.credentials.org_key) + query_start = self._cb.post_object(url, body=args) - url = "/threathunter/search/v1/orgs/{}/processes/search_jobs".format(self._cb.credentials.org_key) - query_start = self._cb.post_object(url, body={"search_params": args}) + self._query_token = query_start.json().get("job_id") - self._query_token = query_start.json().get("query_id") self._timed_out = False self._submit_time = time.time() * 1000 @@ -392,7 +414,7 @@ def _still_querying(self): if not self._query_token: self._submit() - status_url = "/threathunter/search/v1/orgs/{}/processes/search_jobs/{}".format( + status_url = "/api/investigate/v1/orgs/{}/processes/search_jobs/{}".format( self._cb.credentials.org_key, self._query_token, ) @@ -421,13 +443,13 @@ def _count(self): if self._timed_out: raise TimeoutError(message="user-specified timeout exceeded while waiting for results") - result_url = "/threathunter/search/v1/orgs/{}/processes/search_jobs/{}/results".format( + result_url = "/api/investigate/v2/orgs/{}/processes/search_jobs/{}/results".format( self._cb.credentials.org_key, self._query_token, ) result = self._cb.get_object(result_url) - self._total_results = result.get('response_header', {}).get('num_available', 0) + self._total_results = result.get('num_available', 0) self._count_valid = True return self._total_results @@ -447,7 +469,7 @@ def _search(self, start=0, rows=0): current = start rows_fetched = 0 still_fetching = True - result_url_template = "/threathunter/search/v1/orgs/{}/processes/search_jobs/{}/results".format( + result_url_template = "/api/investigate/v2/orgs/{}/processes/search_jobs/{}/results".format( self._cb.credentials.org_key, self._query_token ) @@ -461,10 +483,10 @@ def _search(self, start=0, rows=0): result = self._cb.get_object(result_url, query_parameters=query_parameters) - self._total_results = result.get('response_header', {}).get('num_available', 0) + self._total_results = result.get('num_available', 0) self._count_valid = True - results = result.get('data', []) + results = result.get('results', []) for item in results: yield item @@ -531,7 +553,7 @@ def _perform_query(self): results = self._cb.get_object(url, query_parameters=self._args) while results["incomplete_results"]: - result = self._cb.get_object(self._doc_class.urlobject, query_parameters=self._args) + result = self._cb.get_object(url, query_parameters=self._args) results["nodes"]["children"].extend(result["nodes"]["children"]) results["incomplete_results"] = result["incomplete_results"] From 6f31adce660c5fcbbc2e03bf9e6bec52bcd9f137 Mon Sep 17 00:00:00 2001 From: Thomas Bouve Date: Mon, 6 Jul 2020 09:31:08 +0200 Subject: [PATCH 107/197] Enabling _validate based again. --- src/cbapi/psc/threathunter/query.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index 3792bd56..d02ecfa5 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -295,7 +295,7 @@ def _validate(self, args): return url = self._doc_class.validation_url.format(self._cb.credentials.org_key) - validated = self._cb.get_object(url, query_parameters=args) + validated = self._cb.get_object(url, query_parameters={'q': args['query']}) if not validated.get("valid"): raise ApiError("Invalid query: {}: {}".format(args, validated["invalid_message"])) @@ -303,13 +303,13 @@ def _validate(self, args): def _search(self, start=0, rows=0): # iterate over total result set, 100 at a time args = self._get_query_parameters() - #self._validate(args) + self._validate(args) if start != 0: args['start'] = start args['rows'] = self._batch_size - #args = {"search_params": args} + # args = {"search_params": args} current = start numrows = 0 @@ -400,7 +400,7 @@ def _submit(self): raise ApiError("Query already submitted: token {0}".format(self._query_token)) args = self._get_query_parameters() - #self._validate(args) + self._validate(args) url = "/api/investigate/v2/orgs/{}/processes/search_jobs".format(self._cb.credentials.org_key) query_start = self._cb.post_object(url, body=args) From c63d6a9bff325dddf2555e9ca26b209bb6ffd2f5 Mon Sep 17 00:00:00 2001 From: Thomas Bouve Date: Mon, 6 Jul 2020 09:55:43 +0200 Subject: [PATCH 108/197] Update _validate logic --- src/cbapi/psc/threathunter/query.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index d02ecfa5..d5b5440f 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -295,7 +295,11 @@ def _validate(self, args): return url = self._doc_class.validation_url.format(self._cb.credentials.org_key) - validated = self._cb.get_object(url, query_parameters={'q': args['query']}) + + if args.get('query', False): + args['q'] = args['query'] + + validated = self._cb.get_object(url, query_parameters=args) if not validated.get("valid"): raise ApiError("Invalid query: {}: {}".format(args, validated["invalid_message"])) From 420656ff2f8f68eb2756845bf2754fa2eb0318fa Mon Sep 17 00:00:00 2001 From: Thomas Bouve Date: Mon, 6 Jul 2020 17:05:18 +0200 Subject: [PATCH 109/197] Add missing __len__ implementation --- src/cbapi/psc/alerts_query.py | 2 ++ src/cbapi/psc/base_query.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cbapi/psc/alerts_query.py b/src/cbapi/psc/alerts_query.py index 194f83c5..9a350f1f 100755 --- a/src/cbapi/psc/alerts_query.py +++ b/src/cbapi/psc/alerts_query.py @@ -24,6 +24,8 @@ def __init__(self, doc_class, cb): self._time_filter = {} self._sortcriteria = {} self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/workflow/_criteria" + self._count_valid = False + self._total_results = 0 def _update_criteria(self, key, newlist): """ diff --git a/src/cbapi/psc/base_query.py b/src/cbapi/psc/base_query.py index f028fb30..452d9fd9 100755 --- a/src/cbapi/psc/base_query.py +++ b/src/cbapi/psc/base_query.py @@ -261,7 +261,7 @@ def one(self): return res[0] def __len__(self): - return 0 + return self._count() def __getitem__(self, item): return None From 43b88c141a84ed9dc02f1bd4b4f1cb36cdc99305 Mon Sep 17 00:00:00 2001 From: Luke Lyon <52218532+llyon-cb@users.noreply.github.com> Date: Wed, 8 Jul 2020 10:09:31 -0600 Subject: [PATCH 110/197] [EA-16489] Update IOC 'field' to use IOCv2 field names (#238) * modify IOCv2 field names to enable TH Event Searches --- .../threat_intelligence/stix_parse.py | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/examples/threathunter/threat_intelligence/stix_parse.py b/examples/threathunter/threat_intelligence/stix_parse.py index 56e391cd..8663c549 100644 --- a/examples/threathunter/threat_intelligence/stix_parse.py +++ b/examples/threathunter/threat_intelligence/stix_parse.py @@ -137,8 +137,9 @@ def cybox_parse_observable(observable, indicator, timestamp, score): A report dictionary if the cybox observable has props of type: cybox.objects.address_object.Address, - cybox.objects.file_object.File, or - cybox.objects.domain_name_object.DomainName. + cybox.objects.file_object.File, + cybox.objects.domain_name_object.DomainName, or + cybox.objects.uri_object.URI Otherwise it will return an empty list. @@ -191,7 +192,7 @@ def cybox_parse_observable(observable, indicator, timestamp, score): title_found = True else: title_found = False - + if title_found: url_pattern = re.compile("^(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$") for token in split_title: @@ -236,7 +237,7 @@ def parse_uri_observable(observable, props, id, description, title, timestamp, l if props.value and props.value.value: - iocs = {'dns': []} + iocs = {'netconn_domain': []} # # Sometimes props.value.value is a list # @@ -244,13 +245,13 @@ def parse_uri_observable(observable, props, id, description, title, timestamp, l if type(props.value.value) is list: for domain_name in props.value.value: if validate_domain_name(domain_name.strip()): - iocs['dns'].append(domain_name.strip()) + iocs['netconn_domain'].append(domain_name.strip()) else: domain_name = props.value.value.strip() if validate_domain_name(domain_name): - iocs['dns'].append(domain_name) + iocs['netconn_domain'].append(domain_name) - if len(iocs['dns']) > 0: + if len(iocs['netconn_domain']) > 0: reports.append({'iocs_v2': iocs, 'id': sanitize_id(id), 'description': description, @@ -265,7 +266,7 @@ def parse_domain_name_observable(observable, props, id, description, title, time reports = [] if props.value and props.value.value: - iocs = {'dns': []} + iocs = {'netconn_domain': []} # # Sometimes props.value.value is a list # @@ -273,13 +274,13 @@ def parse_domain_name_observable(observable, props, id, description, title, time if type(props.value.value) is list: for domain_name in props.value.value: if validate_domain_name(domain_name.strip()): - iocs['dns'].append(domain_name.strip()) + iocs['netconn_domain'].append(domain_name.strip()) else: domain_name = props.value.value.strip() if validate_domain_name(domain_name): - iocs['dns'].append(domain_name) + iocs['netconn_domain'].append(domain_name) - if len(iocs['dns']) > 0: + if len(iocs['netconn_domain']) > 0: reports.append({'iocs_v2': iocs, 'id': sanitize_id(id), 'description': description, @@ -294,7 +295,7 @@ def parse_address_observable(observable, props, id, description, title, timestam reports = [] if props.category == 'ipv4-addr' and props.address_value: - iocs = {'ipv4': []} + iocs = {'netconn_ipv4': []} # # Sometimes props.address_value.value is a list vs a string @@ -302,13 +303,13 @@ def parse_address_observable(observable, props, id, description, title, timestam if type(props.address_value.value) is list: for ip in props.address_value.value: if validate_ip_address(ip.strip()): - iocs['ipv4'].append(ip.strip()) + iocs['netconn_ipv4'].append(ip.strip()) else: ipv4 = props.address_value.value.strip() if validate_ip_address(ipv4): - iocs['ipv4'].append(ipv4) + iocs['netconn_ipv4'].append(ipv4) - if len(iocs['ipv4']) > 0: + if len(iocs['netconn_ipv4']) > 0: reports.append({'iocs_v2': iocs, 'id': sanitize_id(observable.id_), 'description': description, @@ -323,21 +324,21 @@ def parse_address_observable(observable, props, id, description, title, timestam def parse_file_observable(observable, props, id, description, title, timestamp, link, score): reports = [] - iocs = {'md5': []} + iocs = {'hash': []} if props.md5: if type(props.md5) is list: - for md5 in props.md5: - if validate_md5sum(md5.strip()): - iocs['md5'].append(md5.strip()) + for hash in props.md5: + if validate_md5sum(hash.strip()): + iocs['hash'].append(hash.strip()) else: if hasattr(props.md5, 'value'): - md5 = props.md5.value.strip() + hash = props.md5.value.strip() else: - md5 = props.md5.strip() - if validate_md5sum(md5): - iocs['md5'].append(md5) + hash = props.md5.strip() + if validate_md5sum(hash): + iocs['hash'].append(hash) - if len(iocs['md5']) > 0: + if len(iocs['hash']) > 0: reports.append({'iocs_v2': iocs, 'id': sanitize_id(id), 'description': description, From 07ed4d86baaad042cf571390452c7b87282e1898 Mon Sep 17 00:00:00 2001 From: klazaga <62656947+klazaga@users.noreply.github.com> Date: Wed, 8 Jul 2020 13:09:53 -0400 Subject: [PATCH 111/197] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bed2aac6..b358c580 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ an overview of the concepts that underly this API binding. ## Support -1. View all API and integration offerings on the [Developer Network](https://developer.carbonblack.com) along with reference documentation, video tutorials, and how-to guides. -2. Use the [Developer Community Forum](https://community.carbonblack.com/community/resources/developer-relations) to discuss issues and get answers from other API developers in the Carbon Black Community. -3. Report bugs and change requests to [Carbon Black Support](http://carbonblack.com/resources/support). +1. View all API and integration offerings on the [Developer Network](https://developer.carbonblack.com/) along with reference documentation, video tutorials, and how-to guides. +2. Use the [Developer Community Forum](https://community.carbonblack.com/t5/Developer-Relations/bd-p/developer-relations) to discuss issues and get answers from other API developers in the Carbon Black Community. +3. Report bugs and change requests to [Carbon Black Support](http://carbonblack.com/resources/support/). ## Requirements From dd988a7d15e61838ec03c46c519f45bbad94988c Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Wed, 8 Jul 2020 13:29:36 -0600 Subject: [PATCH 112/197] Expose segment properties --- src/cbapi/psc/threathunter/query.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index 0f732323..5e9d79d7 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -304,6 +304,8 @@ def _search(self, start=0, rows=0): result = resp.json() self._total_results = result.get("response_header", {}).get("num_available", 0) + self._total_segments = result.get("response_header", {}).get("total_segments", 0) + self._processed_segments = result.get("response_header", {}).get("processed_segments", 0) self._count_valid = True results = result.get('docs', []) From 8cd1fe54ce565e416d392e7286b1576e38cb2fdf Mon Sep 17 00:00:00 2001 From: Luke Lyon <52218532+llyon-cb@users.noreply.github.com> Date: Thu, 9 Jul 2020 14:58:21 -0600 Subject: [PATCH 113/197] Add headers to csv writer in Enterprise EDR example script (#227) * add headers to CSV writer * update to how we handle a process with no children --- examples/threathunter/process_exporter.py | 7 ++++++- src/cbapi/psc/threathunter/models.py | 11 +++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/examples/threathunter/process_exporter.py b/examples/threathunter/process_exporter.py index e86b113f..b0d017e7 100644 --- a/examples/threathunter/process_exporter.py +++ b/examples/threathunter/process_exporter.py @@ -44,12 +44,17 @@ def main(): with open(args.f, 'w') as outfile: for p in processes: json.dump(p.original_document, outfile) + print(p.original_document) else: + headers = set() + headers.update(*(d.original_document.keys() for d in processes)) with open(args.f, 'w') as outfile: - csvwriter = csv.writer(outfile) + csvwriter = csv.DictWriter(outfile, fieldnames=headers) + csvwriter.writeheader() for p in processes: csvwriter.writerow(p.original_document) + if __name__ == "__main__": sys.exit(main()) diff --git a/src/cbapi/psc/threathunter/models.py b/src/cbapi/psc/threathunter/models.py index 461380b9..1ddd3c06 100644 --- a/src/cbapi/psc/threathunter/models.py +++ b/src/cbapi/psc/threathunter/models.py @@ -112,10 +112,13 @@ def children(self): :return: Returns a list of process objects :rtype: list of :py:class:`Process` """ - return [ - Process(self._cb, initial_data=child) - for child in self.summary.children - ] + if isinstance(self.summary.children, list): + return [ + Process(self._cb, initial_data=child) + for child in self.summary.children + ] + else: + return [] @property def siblings(self): From 9ab1930ff0898b36514ad0712b7e337bed0358bb Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Fri, 10 Jul 2020 15:18:14 -0600 Subject: [PATCH 114/197] Fix python2/3 integer_types --- src/cbapi/models.py | 2 +- src/cbapi/six.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cbapi/models.py b/src/cbapi/models.py index 59ecaad3..e24ca39c 100644 --- a/src/cbapi/models.py +++ b/src/cbapi/models.py @@ -144,7 +144,7 @@ def __init__(self, field_name, multiplier=1.0): def __get__(self, instance, instance_type=None): d = super(EpochDateTimeFieldDescriptor, self).__get__(instance, instance_type) - if type(d) is float or isinstance(type(d), integer_types): + if type(d) is float or type(d) in integer_types: epoch_seconds = d / self.multiplier return datetime.utcfromtimestamp(epoch_seconds) else: diff --git a/src/cbapi/six.py b/src/cbapi/six.py index 77f4957e..7769534c 100644 --- a/src/cbapi/six.py +++ b/src/cbapi/six.py @@ -39,7 +39,7 @@ if PY3: string_types = str, - integer_types = int, + integer_types = (int,), class_types = type, text_type = str binary_type = bytes From 73325f2879cfcd8c6a4d7d6a4881bfb0cee157d5 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Fri, 10 Jul 2020 15:48:30 -0600 Subject: [PATCH 115/197] Remove duplicate type --- src/cbapi/response/models/process.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cbapi/response/models/process.yaml b/src/cbapi/response/models/process.yaml index 71f1befe..44a605e9 100644 --- a/src/cbapi/response/models/process.yaml +++ b/src/cbapi/response/models/process.yaml @@ -58,7 +58,6 @@ properties: format: int32 group: type: string - type: string netconn_count: type: integer format: int32 From 7d6eac0c6da4a2bedb10299529dd8d7707e7a022 Mon Sep 17 00:00:00 2001 From: Thomas Bouve Date: Mon, 13 Jul 2020 15:54:14 +0200 Subject: [PATCH 116/197] Update sort_by() to support search v2 --- src/cbapi/psc/threathunter/query.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index ee8b3af1..c37d51ba 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -299,6 +299,9 @@ def _validate(self, args): if args.get('query', False): args['q'] = args['query'] + # v2 search sort key does not work with v1 validation + args.pop('sort', None) + validated = self._cb.get_object(url, query_parameters=args) if not validated.get("valid"): @@ -366,8 +369,7 @@ def __init__(self, doc_class, cb): self._query_token = None self._timeout = 0 self._timed_out = False - self._sort_by = "backend_timestamp" # Requires default to prevent unstable fetching of results - self._sort_direction = "ASC" + self._sort = [] def sort_by(self, key, direction="ASC"): """Sets the sorting behavior on a query's results. @@ -380,11 +382,18 @@ def sort_by(self, key, direction="ASC"): :param direction: the sort order, either "ASC" or "DESC" :rtype: :py:class:`AsyncProcessQuery` """ - self._sort_by = key - self._sort_direction = direction + found = False + + for sort_item in self._sort: + if sort_item['field'] == key: + sort_item['order'] = direction + found = True + + if not found: + self._sort.append({'field': key, 'order': direction}) + + self._default_args['sort'] = self._sort - # Append to search_job query - self._default_args['sort'] = '{} {}'.format(key, direction) return self def timeout(self, msecs): From e9a5b32be6c8a023daa301727db21e24094f4637 Mon Sep 17 00:00:00 2001 From: Rafael Lukas Maers Date: Tue, 20 Nov 2018 15:59:00 +0100 Subject: [PATCH 117/197] Set reasonable defaults in connection.BaseAPI Use library defaults unless specified otherwise, except for the number of pools since a subclass instance always corresponds to one server / host / domain. --- src/cbapi/connection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cbapi/connection.py b/src/cbapi/connection.py index 356f1d30..90370fec 100644 --- a/src/cbapi/connection.py +++ b/src/cbapi/connection.py @@ -4,7 +4,7 @@ import requests import sys -from requests.adapters import HTTPAdapter, DEFAULT_POOLBLOCK, DEFAULT_RETRIES, DEFAULT_POOLSIZE +from requests.adapters import HTTPAdapter, DEFAULT_POOLBLOCK, DEFAULT_RETRIES, DEFAULT_POOLSIZE, DEFAULT_POOL_TIMEOUT try: from requests.packages.urllib3.util.ssl_ import create_urllib3_context @@ -229,9 +229,9 @@ def __init__(self, *args, **kwargs): self.credential_profile_name = kwargs.pop("profile", None) self.credentials = self.credential_store.get_credentials(self.credential_profile_name) - timeout = kwargs.pop("timeout", None) - max_retries = kwargs.pop("max_retries", None) - pool_connections = kwargs.pop("pool_connections", DEFAULT_POOLSIZE) + timeout = kwargs.pop("timeout", DEFAULT_POOL_TIMEOUT) + max_retries = kwargs.pop("max_retries", DEFAULT_RETRIES) + pool_connections = kwargs.pop("pool_connections", 1) pool_maxsize = kwargs.pop("pool_maxsize", DEFAULT_POOLSIZE) pool_block = kwargs.pop("pool_block", DEFAULT_POOLBLOCK) From e809fe57297d038c84aecfdb2bad1ccd3e552363 Mon Sep 17 00:00:00 2001 From: Rafael Lukas Maers Date: Thu, 4 Apr 2019 11:25:17 +0200 Subject: [PATCH 118/197] Reraise requests.ConnectionError as errors.ConnectionError This makes it easier to handle down the line, but keep the error message handling by subclassing errors.ConnectionError from errors.ApiError. --- src/cbapi/connection.py | 2 +- src/cbapi/errors.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cbapi/connection.py b/src/cbapi/connection.py index 90370fec..8c8b9855 100644 --- a/src/cbapi/connection.py +++ b/src/cbapi/connection.py @@ -177,7 +177,7 @@ def http_request(self, method, url, **kwargs): except requests.Timeout as timeout_error: raise TimeoutError(uri=uri, original_exception=timeout_error) except requests.ConnectionError as connection_error: - raise ApiError("Received a network connection error from {0:s}: {1:s}".format(self.server, + raise ConnectionError("Received a network connection error from {0:s}: {1:s}".format(self.server, str(connection_error)), original_exception=connection_error) except Exception as e: diff --git a/src/cbapi/errors.py b/src/cbapi/errors.py index 8273b30a..840df656 100644 --- a/src/cbapi/errors.py +++ b/src/cbapi/errors.py @@ -2,10 +2,6 @@ from cbapi.six import python_2_unicode_compatible -class ConnectionError(Exception): - pass - - class ApiError(Exception): def __init__(self, message=None, original_exception=None): self.original_exception = original_exception @@ -87,6 +83,10 @@ def __str__(self): return "Unauthorized (Check API creds): attempted to {0:s} {1:s}".format(self.action, self.uri) +class ConnectionError(ApiError): + pass + + class CredentialError(ApiError): pass From 361ce56e9f8c2af4d1fa61fa61e0d6117f6026dc Mon Sep 17 00:00:00 2001 From: Rafael Lukas Maers Date: Wed, 2 Oct 2019 08:24:26 +0200 Subject: [PATCH 119/197] Fix oldmodels.BaseModel.get method Pass along the default value to the _attribute method. --- src/cbapi/oldmodels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/oldmodels.py b/src/cbapi/oldmodels.py index 82cab04a..ee76bff2 100644 --- a/src/cbapi/oldmodels.py +++ b/src/cbapi/oldmodels.py @@ -145,7 +145,7 @@ def _attribute(self, attrname, default=None): def get(self, attrname, default_val=None): try: - return self._attribute(attrname) + return self._attribute(attrname, default_val) except AttributeError: return default_val From 01f109794719438b00b2f0d993dee41774c53df3 Mon Sep 17 00:00:00 2001 From: Rafael Lukas Maers Date: Fri, 4 Oct 2019 22:17:23 +0200 Subject: [PATCH 120/197] Improve detection of queries with malformed syntax Add and use ClientError and QuerySyntaxError exceptions to improve the detection of client errors (HTTP 4xx, as opposed to server errors HTTP 5xx) and queries with malformed syntax; respectively. --- src/cbapi/connection.py | 18 +++++++++++++++--- src/cbapi/errors.py | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/cbapi/connection.py b/src/cbapi/connection.py index 8c8b9855..036d0e87 100644 --- a/src/cbapi/connection.py +++ b/src/cbapi/connection.py @@ -30,7 +30,8 @@ from cbapi.six.moves import urllib from .auth import CredentialStoreFactory, Credentials -from .errors import ServerError, TimeoutError, ApiError, ObjectNotFoundError, UnauthorizedError, ConnectionError +from .errors import ClientError, QuerySyntaxError, ServerError, TimeoutError, ApiError, ObjectNotFoundError, \ + UnauthorizedError, CredentialError, ConnectionError from . import __version__ from .cache.lru import lru_cache_function @@ -40,6 +41,13 @@ log = logging.getLogger(__name__) +def try_json(resp): + try: + return resp.json() + except: + return dict() + + def check_python_tls_compatibility(): try: CbAPISessionAdapter(force_tls_1_2=True) @@ -184,12 +192,16 @@ def http_request(self, method, url, **kwargs): raise ApiError("Unknown exception when connecting to server: {0:s}".format(str(e)), original_exception=e) else: - if r.status_code == 404: + if r.status_code >= 500: + raise ServerError(error_code=r.status_code, message=r.text) + elif r.status_code == 404: raise ObjectNotFoundError(uri=uri, message=r.text) elif r.status_code == 401: raise UnauthorizedError(uri=uri, action=method, message=r.text) + elif r.status_code == 400 and try_json(r).get('reason') == 'query_malformed_syntax': + raise QuerySyntaxError(uri=uri, message=r.text) elif r.status_code >= 400: - raise ServerError(error_code=r.status_code, message=r.text) + raise ClientError(error_code=r.status_code, message=r.text) return r def get(self, url, **kwargs): diff --git a/src/cbapi/errors.py b/src/cbapi/errors.py index 840df656..d9ca54cd 100644 --- a/src/cbapi/errors.py +++ b/src/cbapi/errors.py @@ -11,9 +11,47 @@ def __str__(self): return self.message +@python_2_unicode_compatible +class ClientError(ApiError): + """A ClientError is raised when an HTTP 4xx error code is returned from the Carbon Black server.""" + + def __init__(self, error_code, message, result=None, original_exception=None): + super(ClientError, self).__init__(message=message, original_exception=original_exception) + + self.error_code = error_code + self.result = result + + def __str__(self): + msg = "Received error code {0:d} from API".format(self.error_code) + if self.message: + msg += ": {0:s}".format(self.message) + else: + msg += " (No further information provided)" + + if self.result: + msg += ". {}".format(self.result) + return msg + + +@python_2_unicode_compatible +class QuerySyntaxError(ApiError): + """The request contains a query with malformed syntax.""" + + def __init__(self, uri, message=None, original_exception=None): + super(QuerySyntaxError, self).__init__(message=message, original_exception=original_exception) + self.uri = uri + + def __str__(self): + msg = "Received query syntax error for {0:s}".format(self.uri) + if self.message: + msg += ": {0:s}".format(self.message) + + return msg + + @python_2_unicode_compatible class ServerError(ApiError): - """A ServerError is raised when an HTTP error code is returned from the Carbon Black server.""" + """A ServerError is raised when an HTTP 5xx error code is returned from the Carbon Black server.""" def __init__(self, error_code, message, result=None, original_exception=None): super(ServerError, self).__init__(message=message, original_exception=original_exception) From ebfacd24ad085090f4a6a3194c4d55bb2fa4f45c Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 7 Jul 2020 13:12:01 -0600 Subject: [PATCH 121/197] set .gitignore for IntelliJ IDEA --- .gitignore | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f79adb10..7397522a 100644 --- a/.gitignore +++ b/.gitignore @@ -60,11 +60,13 @@ target/ #Ipython Notebook .ipynb_checkpoints - -.idea/ *.ipynb .DS_Store +# IntelliJ IDEA +/.idea +*.iml + # Eclipse/PyDev /.project /.pydevproject From c03662ceebaaad618af3ab571de60ae2fbb32c99 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Thu, 9 Jul 2020 13:56:06 -0600 Subject: [PATCH 122/197] changed files now pass flake8 --- src/cbapi/connection.py | 268 +++++++++++++++++++++++++++++++++++++--- src/cbapi/errors.py | 118 ++++++++++++++++++ src/cbapi/oldmodels.py | 142 +++++++++++++++++++-- 3 files changed, 503 insertions(+), 25 deletions(-) diff --git a/src/cbapi/connection.py b/src/cbapi/connection.py index 036d0e87..6c6253d7 100644 --- a/src/cbapi/connection.py +++ b/src/cbapi/connection.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +"""Manages the CBAPI connection to the server.""" + from __future__ import absolute_import import requests @@ -31,7 +33,7 @@ from .auth import CredentialStoreFactory, Credentials from .errors import ClientError, QuerySyntaxError, ServerError, TimeoutError, ApiError, ObjectNotFoundError, \ - UnauthorizedError, CredentialError, ConnectionError + UnauthorizedError, ConnectionError from . import __version__ from .cache.lru import lru_cache_function @@ -42,13 +44,28 @@ def try_json(resp): + """ + Return a parsed JSON representation of the input. + + Args: + resp (str): Input to be parsed. + + Returns: + object: The parsed JSON result, or an empty dict if the value is not valid JSON. + """ try: return resp.json() - except: + except ValueError: return dict() def check_python_tls_compatibility(): + """ + Verify which level of TLS/SSL that this version of the code is compatible with. + + Returns: + str: The maximum level of TLS/SSL that this version is compatible with. + """ try: CbAPISessionAdapter(force_tls_1_2=True) except Exception: @@ -69,7 +86,21 @@ def check_python_tls_compatibility(): class CbAPISessionAdapter(HTTPAdapter): + """Adapter object used to handle TLS connections to the CB server.""" + def __init__(self, verify_hostname=True, force_tls_1_2=False, max_retries=DEFAULT_RETRIES, **pool_kwargs): + """ + Initialize the CbAPISessionManager. + + Args: + verify_hostname (boolean): True if we want to verify the hostname. + force_tls_1_2 (boolean): True to force the use of TLS 1.2. + max_retries (int): Maximum number of retries. + **pool_kwargs: Additional arguments. + + Raises: + ApiError: If the library versions are too old to force the use of TLS 1.2. + """ self._cbapi_verify_hostname = verify_hostname self._cbapi_force_tls_1_2 = force_tls_1_2 @@ -79,6 +110,18 @@ def __init__(self, verify_hostname=True, force_tls_1_2=False, max_retries=DEFAUL super(CbAPISessionAdapter, self).__init__(max_retries=max_retries, **pool_kwargs) def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs): + """ + Initialize the connection pool manager. + + Args: + connections (int): Initial number of connections to be used. + maxsize (int): Maximum size of the connection pool. + block (object): Blocking policy. + **pool_kwargs: Additional arguments for the connection pool. + + Returns: + object: TBD + """ if self._cbapi_force_tls_1_2 and REQUESTS_HAS_URLLIB_SSL_CONTEXT: # Force the use of TLS v1.2 when talking to this Cb Response server. context = create_urllib3_context(ciphers=('TLSv1.2:!aNULL:!eNULL:!MD5')) @@ -98,7 +141,23 @@ def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool class Connection(object): + """Object that encapsulates the HTTP connection to the CB server.""" + def __init__(self, credentials, integration_name=None, timeout=None, max_retries=None, **pool_kwargs): + """ + Initialize the Connection object. + + Args: + credentials (object): The credentials to use for the connection. + integration_name (str): The integration name being used. + timeout (int): The timeout value to use for HTTP requests on this connection. + max_retries (int): The maximum number of times to retry a request. + **pool_kwargs: Additional arguments to be used to initialize connection pooling. + + Raises: + ApiError: If there's an internal error initializing the connection. + ConnectionError: If there's a problem with the credentials. + """ if not credentials.url or not credentials.url.startswith("https://"): raise ConnectionError("Server URL must be a URL: eg. https://localhost") @@ -159,6 +218,27 @@ def __init__(self, credentials, integration_name=None, timeout=None, max_retries self.proxies['https'] = credentials.proxy def http_request(self, method, url, **kwargs): + """ + Submit a HTTP request to the server. + + Args: + method (str): The method name to use for the HTTP request. + url (str): The URL to submit the request to. + **kwargs: Additional arguments for the request. + + Returns: + object: Result of the HTTP request. + + Raises: + ApiError: An unknown problem was detected. + ClientError: The server returned an error code in the 4xx range, indicating a problem with the request. + ConnectionError: A problem was seen with the HTTP connection. + ObjectNotFoundError: The specified object was not found on the server. + QuerySyntaxError: The query passed in had invalid syntax. + ServerError: The server returned an error code in the 5xx range, indicating a problem on the server side. + TimeoutError: The HTTP request timed out. + UnauthorizedError: The stored credentials do not permit access to the specified request. + """ method = method.upper() verify_ssl = kwargs.pop('verify', None) or self.ssl_verify @@ -185,9 +265,9 @@ def http_request(self, method, url, **kwargs): except requests.Timeout as timeout_error: raise TimeoutError(uri=uri, original_exception=timeout_error) except requests.ConnectionError as connection_error: - raise ConnectionError("Received a network connection error from {0:s}: {1:s}".format(self.server, - str(connection_error)), - original_exception=connection_error) + raise ConnectionError("Received a network connection error from {0:s}: {1:s}" + .format(self.server, str(connection_error)), + original_exception=connection_error) except Exception as e: raise ApiError("Unknown exception when connecting to server: {0:s}".format(str(e)), original_exception=e) @@ -205,21 +285,69 @@ def http_request(self, method, url, **kwargs): return r def get(self, url, **kwargs): + """ + Submit a GET request on this connection. + + Args: + url (str): The URL to submit the request to. + **kwargs: Additional arguments for the request. + + Returns: + object: Result of the HTTP request. + """ return self.http_request("GET", url, **kwargs) def post(self, url, **kwargs): + """ + Submit a POST request on this connection. + + Args: + url (str): The URL to submit the request to. + **kwargs: Additional arguments for the request. + + Returns: + object: Result of the HTTP request. + """ return self.http_request("POST", url, **kwargs) def put(self, url, **kwargs): + """ + Submit a PUT request on this connection. + + Args: + url (str): The URL to submit the request to. + **kwargs: Additional arguments for the request. + + Returns: + object: Result of the HTTP request. + """ return self.http_request("PUT", url, **kwargs) def delete(self, url, **kwargs): + """ + Submit a DELETE request on this connection. + + Args: + url (str): The URL to submit the request to. + **kwargs: Additional arguments for the request. + + Returns: + object: Result of the HTTP request. + """ return self.http_request("DELETE", url, **kwargs) class BaseAPI(object): - """baseapi""" + """The base API object used by all CBAPI objects to communicate with the server.""" + def __init__(self, *args, **kwargs): + """ + Initialize the base API information. + + Args: + *args: TBD + **kwargs: Additional arguments. + """ product_name = kwargs.pop("product_name", None) credential_file = kwargs.pop("credential_file", None) integration_name = kwargs.pop("integration_name", None) @@ -252,15 +380,36 @@ def __init__(self, *args, **kwargs): pool_maxsize=pool_maxsize, pool_block=pool_block) def raise_unless_json(self, ret, expected): + """ + Raise a ServerError unless we got back an HTTP 200 response with JSON containing all the expected values. + + Args: + ret (object): Return value to be checked. + expected (dict): Expected keys and values that need to be found in the JSON response. + + Raises: + ServerError: If the HTTP response is anything but 200, or if the expected values are not found. + """ if ret.status_code == 200: message = ret.json() for k, v in iteritems(expected): if k not in message or message[k] != v: raise ServerError(ret.status_code, message) else: - raise ServerError(ret.status_code, "".format(ret.content), ) + raise ServerError(ret.status_code, "{0}".format(ret.content), ) def get_object(self, uri, query_parameters=None, default=None): + """ + Submit a GET request to the server and parse the result as JSON before returning. + + Args: + uri (str): The URI to send the GET request to. + query_parameters (object): Parameters for the query. + default (object): What gets returned in the event of an empty response. + + Returns: + object: Result of the GET request. + """ if query_parameters: if isinstance(query_parameters, dict): query_parameters = convert_query_params(query_parameters) @@ -279,6 +428,18 @@ def get_object(self, uri, query_parameters=None, default=None): raise ServerError(error_code=result.status_code, message="Unknown error: {0}".format(result.content)) def get_raw_data(self, uri, query_parameters=None, default=None, **kwargs): + """ + Submit a GET request to the server and return the result without parsing it. + + Args: + uri (str): The URI to send the GET request to. + query_parameters (object): Parameters for the query. + default (object): What gets returned in the event of an empty response. + **kwargs: + + Returns: + object: Result of the GET request. + """ if query_parameters: if isinstance(query_parameters, dict): query_parameters = convert_query_params(query_parameters) @@ -295,6 +456,20 @@ def get_raw_data(self, uri, query_parameters=None, default=None, **kwargs): raise ServerError(error_code=result.status_code, message="Unknown error: {0}".format(result.content)) def api_json_request(self, method, uri, **kwargs): + """ + Submit a request to the server. + + Args: + method (str): HTTP method to use. + uri (str): URI to submit the request to. + **kwargs (dict): Additional arguments. + + Returns: + object: Result of the operation. + + Raises: + ServerError: If there's an error output from the server. + """ headers = kwargs.pop("headers", {}) raw_data = None @@ -308,7 +483,7 @@ def api_json_request(self, method, uri, **kwargs): try: resp = result.json() - except Exception: + except ValueError: return result if "errorMessage" in resp: @@ -317,21 +492,57 @@ def api_json_request(self, method, uri, **kwargs): return result def post_object(self, uri, body, **kwargs): + """ + Send a POST request to the specified URI. + + Args: + uri (str): The URI to send the POST request to. + body (object): The data to be sent in the body of the POST request. + **kwargs: + + Returns: + object: The return data from the POST request. + """ return self.api_json_request("POST", uri, data=body, **kwargs) def put_object(self, uri, body, **kwargs): + """ + Send a PUT request to the specified URI. + + Args: + uri (str): The URI to send the PUT request to. + body (object): The data to be sent in the body of the PUT request. + **kwargs: + + Returns: + object: The return data from the PUT request. + """ return self.api_json_request("PUT", uri, data=body, **kwargs) def delete_object(self, uri): + """ + Send a DELETE request to the specified URI. + + Args: + uri (str): The URI to send the DELETE request to. + + Returns: + object: The return data from the DELETE request. + """ return self.api_json_request("DELETE", uri) def select(self, cls, unique_id=None, *args, **kwargs): - """Prepares a query against the Carbon Black data store. + """ + Prepare a query against the Carbon Black data store. - :param class cls: The Model class (for example, Computer, Process, Binary, FileInstance) to query - :param unique_id: (optional) The unique id of the object to retrieve, to retrieve a single object by ID + Args: + cls (class): The Model class (for example, Computer, Process, Binary, FileInstance) to query + unique_id (optional): The unique id of the object to retrieve, to retrieve a single object by ID + *args: + **kwargs: - :returns: An instance of the Model class if a unique_id is provided, otherwise a Query object + Returns: + object: An instance of the Model class if a unique_id is provided, otherwise a Query object """ if unique_id is not None: return select_instance(self, cls, unique_id, *args, **kwargs) @@ -339,12 +550,18 @@ def select(self, cls, unique_id=None, *args, **kwargs): return self._perform_query(cls, **kwargs) def create(self, cls, data=None): - """Creates a new object. + """ + Create a new object. - :param class cls: The Model class (only some models can be created, for example, Feed, Notification, ...) + Args: + cls (class): The Model class (only some models can be created, for example, Feed, Notification, ...) + data (object): The data used to initialize the new object - :returns: An empty instance of the Model class - :raises ApiError: if the Model cannot be created + Returns: + Model: An empty instance of the model class. + + Raises: + ApiError: If the Model cannot be created. """ if issubclass(cls, CreatableModelMixin): n = cls(self) @@ -360,6 +577,12 @@ def _perform_query(self, cls, **kwargs): @property def url(self): + """ + Return the connection URL. + + Returns: + str: The connection URL. + """ return self.session.server @@ -367,4 +590,17 @@ def url(self): # TODO: how does this interfere with mutable objects? @lru_cache_function(max_size=1024, expiration=1*60) def select_instance(api, cls, unique_id, *args, **kwargs): + """ + Select a cached instance of an object. + + Args: + api: TBD + cls: TBD + unique_id: TBD + *args: + **kwargs: + + Returns: + TBD + """ return cls(api, unique_id, *args, **kwargs) diff --git a/src/cbapi/errors.py b/src/cbapi/errors.py index d9ca54cd..921bcb54 100644 --- a/src/cbapi/errors.py +++ b/src/cbapi/errors.py @@ -1,13 +1,30 @@ #!/usr/bin/env python +"""Exceptions that are thrown by CBAPI operations.""" + from cbapi.six import python_2_unicode_compatible class ApiError(Exception): + """Base class for all CBAPI errors; also raised for generic internal errors.""" + def __init__(self, message=None, original_exception=None): + """ + Initialize the ApiError. + + Args: + message (str): The actual error message. + original_exception (Exception): The exception that caused this one to be raised. + """ self.original_exception = original_exception self.message = str(message) def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ return self.message @@ -16,12 +33,27 @@ class ClientError(ApiError): """A ClientError is raised when an HTTP 4xx error code is returned from the Carbon Black server.""" def __init__(self, error_code, message, result=None, original_exception=None): + """ + Initialize the ClientError. + + Args: + error_code (int): The error code that was received from the server. + message (str): The actual error message. + result (object): The result of the operation from the server. + original_exception (Exception): The exception that caused this one to be raised. + """ super(ClientError, self).__init__(message=message, original_exception=original_exception) self.error_code = error_code self.result = result def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ msg = "Received error code {0:d} from API".format(self.error_code) if self.message: msg += ": {0:s}".format(self.message) @@ -38,10 +70,24 @@ class QuerySyntaxError(ApiError): """The request contains a query with malformed syntax.""" def __init__(self, uri, message=None, original_exception=None): + """ + Initialize the QuerySyntaxError. + + Args: + uri (str): The URI of the action that failed. + message (str): The error message. + original_exception (Exception): The exception that caused this one to be raised. + """ super(QuerySyntaxError, self).__init__(message=message, original_exception=original_exception) self.uri = uri def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ msg = "Received query syntax error for {0:s}".format(self.uri) if self.message: msg += ": {0:s}".format(self.message) @@ -54,12 +100,27 @@ class ServerError(ApiError): """A ServerError is raised when an HTTP 5xx error code is returned from the Carbon Black server.""" def __init__(self, error_code, message, result=None, original_exception=None): + """ + Initialize the ServerError. + + Args: + error_code (int): The error code that was received from the server. + message (str): The actual error message. + result (object): The result of the operation from the server. + original_exception (Exception): The exception that caused this one to be raised. + """ super(ServerError, self).__init__(message=message, original_exception=original_exception) self.error_code = error_code self.result = result def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ msg = "Received error code {0:d} from API".format(self.error_code) if self.message: msg += ": {0:s}".format(self.message) @@ -76,10 +137,24 @@ class ObjectNotFoundError(ApiError): """The requested object could not be found in the Carbon Black datastore.""" def __init__(self, uri, message=None, original_exception=None): + """ + Initialize the ObjectNotFoundError. + + Args: + uri (str): The URI of the action that failed. + message (str): The error message. + original_exception (Exception): The exception that caused this one to be raised. + """ super(ObjectNotFoundError, self).__init__(message=message, original_exception=original_exception) self.uri = uri def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ msg = "Received 404 (Object Not Found) for {0:s}".format(self.uri) if self.message: msg += ": {0:s}".format(self.message) @@ -89,12 +164,29 @@ def __str__(self): @python_2_unicode_compatible class TimeoutError(ApiError): + """A requested operation timed out.""" + def __init__(self, uri=None, error_code=None, message=None, original_exception=None): + """ + Initialize the TimeoutError. + + Args: + uri (str): The URI of the action that timed out. + error_code (int): The error code that was received from the server. + message (str): The error message. + original_exception (Exception): The exception that caused this one to be raised. + """ super(TimeoutError, self).__init__(message=message, original_exception=original_exception) self.uri = uri self.error_code = error_code def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ if self.uri: msg = "Timed out when requesting {0:s} from API".format(self.uri) if self.error_code: @@ -109,12 +201,29 @@ def __str__(self): @python_2_unicode_compatible class UnauthorizedError(ApiError): + """The action that was attempted was not authorized.""" + def __init__(self, uri, message=None, action="read", original_exception=None): + """ + Initialize the UnauthorizedError. + + Args: + uri (str): The URI of the action that was not authorized. + message (str): The error message. + action (str): The action that was being performed that was not authorized. + original_exception (Exception): The exception that caused this one to be raised. + """ super(UnauthorizedError, self).__init__(message=message, original_exception=original_exception) self.uri = uri self.action = action def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ if self.message: return "Check your API Credentials: " + str(self.message) @@ -122,21 +231,30 @@ def __str__(self): class ConnectionError(ApiError): + """There was an error in the connection to the server.""" + pass class CredentialError(ApiError): + """The credentials had an unspecified error.""" + pass class InvalidObjectError(ApiError): + """An invalid object was received by the server.""" + pass class InvalidHashError(Exception): + """An invalid hash value was used.""" + pass class MoreThanOneResultError(ApiError): """Only one object was requested, but multiple matches were found in the Carbon Black datastore.""" + pass diff --git a/src/cbapi/oldmodels.py b/src/cbapi/oldmodels.py index ee76bff2..defb4d41 100644 --- a/src/cbapi/oldmodels.py +++ b/src/cbapi/oldmodels.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +"""Base model objects for the CBAPI.""" + from __future__ import absolute_import from functools import wraps @@ -14,14 +16,28 @@ class CreatableModelMixin(object): + """TBD.""" + pass # TODO: this doesn't exactly do what I want... this needs to be cleaned up before release def immutable(cls): + """Decorate a model object in such a way that allows it to be specified as "immutable".""" cls.__frozen = False def frozensetattr(self, key, value): + """ + Set an attribute on this object, but disallow it if the object is frozen. + + Args: + self (object): The object being changed by this attribute setting operation. + key (str): The attribute being changed. + value (object): The new value being set for the attribute. + + Returns: + TBD + """ if self.__frozen and not hasattr(self, key): print("Class {} is frozen. Cannot set {} = {}" .format(cls.__name__, key, value)) @@ -29,6 +45,15 @@ def frozensetattr(self, key, value): object.__setattr__(self, key, value) def init_decorator(func): + """ + Initialize the decorator on a function. + + Args: + func (func): Function to be wrapped by this decorator. + + Returns: + func: The wrapped function. + """ @wraps(func) def wrapper(self, *args, **kwargs): func(self, *args, **kwargs) @@ -43,7 +68,18 @@ def wrapper(self, *args, **kwargs): @python_2_unicode_compatible class BaseModel(object): + """The base "model" class for all objects that the CBAPI works with.""" + def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False): + """ + Initialize the BaseModel object. + + Args: + cb (BaseAPI): Reference to the CBAPI object. + model_unique_id (object): Unique ID for the model object. + initial_data (dict): Initial data for the model object. + force_init (bool): True to force the object to be re-initialized from the server. + """ self._cb = cb if model_unique_id is not None: self._model_unique_id = str(model_unique_id) @@ -78,13 +114,22 @@ def _stat_titles(self): @classmethod def new_object(cls, cb, item): + """ + Create a new instance of the object from item data. + + Args: + cb (BaseAPI): Reference to the CBAPI object. + item (dict): Item data, as retrieved from the server. + + Returns: + object: New instance of the object. + """ # TODO: do we ever need to evaluate item['unique_id'] which is the id + segment id? # TODO: is there a better way to handle this? (see how this is called from Query._search()) return cb.select(cls, item['id'], initial_data=item) def refresh(self): - """Refresh the object from the Carbon Black server. - """ + """Refresh the object from the Carbon Black server.""" self._retrieve_cb_info() def _retrieve_cb_info(self, query_parameters=None): @@ -106,20 +151,37 @@ def _build_api_request_uri(self): @property def webui_link(self): - """Returns a link associated with this object in the Carbon Black user interface. + """ + Return a link associated with this object in the Carbon Black user interface. - :returns: URL that can be used to view the object in the Carbon Black web user interface or None if the Model - does not support generating a Web user interface URL + Returns: + str: URL that can be used to view the object in the Carbon Black web user interface, or None if the Model + does not support generating a Web user interface URL """ return None def __dir__(self): + """ + Fetch a list of all attribute names defined for this object. + + Returns: + list: A list of strings, each of which represents an attribute name on the object. + """ if not self._full_init: self._retrieve_cb_info() return list(set().union(self.__dict__.keys(), self._info.keys())) def __getattr__(self, attrname): + """ + Return the value of an attribute on this object. + + Args: + attrname (str): Name of the attribute to return. + + Returns: + object: The attribute value. + """ try: object.__getattribute__(self, "_base_initialized") except AttributeError: @@ -144,12 +206,28 @@ def _attribute(self, attrname, default=None): return default def get(self, attrname, default_val=None): + """ + Return the value of an attribute on this object. + + Args: + attrname (str): Name of the attribute to be returned. + default_val (object): The default value to be returned if the attribute is undefined. + + Returns: + object: The attribute value, which may be default_val. + """ try: return self._attribute(attrname, default_val) except AttributeError: return default_val def __str__(self): + """ + Return a representation of this object as a string. + + Returns: + str: A representation of this object as a string. + """ ret = '{0:s}.{1:s}:\n'.format(self.__class__.__module__, self.__class__.__name__) if self.webui_link: ret += "-> available via web UI at %s\n" % self.webui_link @@ -160,17 +238,35 @@ def __str__(self): return ret def __repr__(self): + """ + Return a representation of this object as a string. + + Returns: + str: A representation of this object as a string. + """ return "<%s.%s: id %s> @ %s" % (self.__class__.__module__, self.__class__.__name__, self._model_unique_id, self._cb.session.server) @property def original_document(self): + """ + Return the original document associated with this object. + + Returns: + object: Original document associated with this object. + """ if not self._full_init: self._retrieve_cb_info() return self._info def to_html(self): + """ + Return a representation of this object as HTML. + + Returns: + str: HTML representation of the object. + """ ret = u"

%s

" % self.__class__.__name__ ret += u"\n" for a in self._stat_titles: @@ -186,7 +282,17 @@ def _repr_html_(self): class MutableModel(BaseModel): + """A model object that implements mutability, the ability to change its values.""" + def __init__(self, cb, model_unique_id, initial_data=None): + """ + Initialize the MutableModel. + + Args: + cb (BaseAPI): Reference to the CBAPI object. + model_unique_id (object): Unique ID for the model object. + initial_data (dict): Initial data for the model object. + """ super(MutableModel, self).__init__(cb, model_unique_id, initial_data) self._dirty_attributes = {} self._mutable_initialized = True @@ -199,6 +305,16 @@ def _target_val(self, attrname, val): return val def __setattr__(self, attrname, val): + """ + Set an attribute on the model object. + + Args: + attrname (str): Name of the attribute to be set. + val (object): Value of the object to be set. + + Returns: + TBD + """ try: object.__getattribute__(self, "_mutable_initialized") except AttributeError: @@ -238,8 +354,11 @@ def __setattr__(self, attrname, val): super(MutableModel, self).__setattr__(attrname, val) def is_dirty(self): - """Returns True if this object has unsaved changes. Use :py:meth:`MutableModel.save` to upload the changes to - the Carbon Black server.""" + """ + Return True if this object has unsaved changes. + + Use MutableModel.save to upload the changes to the Carbon Black server. + """ return len(self._dirty_attributes) > 0 def _update_object(self): @@ -283,6 +402,7 @@ def _update_object(self): return self._model_unique_id def refresh(self): + """Refresh the contents of this obejct from the server.""" super(MutableModel, self).refresh() self._dirty_attributes = {} @@ -300,9 +420,11 @@ def _delete_object(self): raise ServerError(ret.status_code, message, result="Did not delete {0:s}.".format(str(self))) def save(self): - """Save changes to this object to the Carbon Black server. + """ + Save changes to this object to the Carbon Black server. - :raises ServerError: if an error was returned by the Carbon Black server + Raises: + ServerError: If an error was returned by the Carbon Black server. """ if not self.is_dirty(): return @@ -310,6 +432,7 @@ def save(self): return self._update_object() def reset(self): + """Reset all changes made to this object.""" for k, v in iteritems(self._dirty_attributes): self._info[k] = v @@ -317,6 +440,7 @@ def reset(self): # TODO: How do we delete this object from our LRU cache? def delete(self): + """Delete this object.""" return self._delete_object() def _join(self, join_cls, field_name): From 9f00b41f9d98ef30880d1a53928c54312d6c7ba4 Mon Sep 17 00:00:00 2001 From: davewagner2 Date: Wed, 11 Mar 2020 17:31:50 +0000 Subject: [PATCH 123/197] Account for sensor queue depth when scheduling jobs --- src/cbapi/live_response_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cbapi/live_response_api.py b/src/cbapi/live_response_api.py index 8d06f947..fd1d2d8c 100644 --- a/src/cbapi/live_response_api.py +++ b/src/cbapi/live_response_api.py @@ -797,7 +797,9 @@ def _spawn_new_workers(self): sensors = [s for s in self._cb.select(Sensor) if s.id in self._unscheduled_jobs and s.id not in self._job_workers and s.status == "Online"] - sensors_to_schedule = sorted(sensors, key=lambda x: x.next_checkin_time)[:schedule_max] + sensors_to_schedule = sorted(sensors, key=lambda x: ( + int(x.num_storefiles_bytes) + int(x.num_eventlog_bytes), x.next_checkin_time + ))[:schedule_max] log.debug("Spawning new workers to handle these sensors: {0}".format(sensors_to_schedule)) for sensor in sensors_to_schedule: From ed98dea6e4cbeb269663a1681f4f5447cae86756 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Fri, 10 Jul 2020 13:34:16 -0600 Subject: [PATCH 124/197] cleaned up docstrings and Flake8-reported issues --- src/cbapi/live_response_api.py | 446 ++++++++++++++++++++++++++------- 1 file changed, 353 insertions(+), 93 deletions(-) diff --git a/src/cbapi/live_response_api.py b/src/cbapi/live_response_api.py index fd1d2d8c..04632411 100644 --- a/src/cbapi/live_response_api.py +++ b/src/cbapi/live_response_api.py @@ -1,3 +1,6 @@ + +"""The Live Response API and associated objects.""" + from __future__ import absolute_import import json @@ -24,7 +27,15 @@ class LiveResponseError(Exception): + """Exception raised for errors with Live Response.""" + def __init__(self, details): + """ + Initialize the LiveResponseError. + + Args: + details (object): Details of the specific error. + """ message_list = [] self.details = details @@ -53,13 +64,30 @@ def __init__(self, details): self.message = ": ".join(message_list) def __str__(self): + """ + Return the string equivalent of this exception (the exception's message). + + Returns: + str: The exception's message. + """ return self.message class CbLRSessionBase(object): + """A Live Response session that interacts with a remote machine.""" + MAX_RETRY_COUNT = 5 def __init__(self, cblr_manager, session_id, sensor_id, session_data=None): + """ + Initialize the CbLRSessionBase. + + Args: + cblr_manager (CbLRManagerBase): The Live Response manager governing this session. + session_id (str): The ID of this session. + sensor_id (int): The ID of the sensor (remote machine) we're connected to. + session_data (dict): Additional session data. + """ self.session_id = session_id self.sensor_id = sensor_id self._cblr_manager = cblr_manager @@ -73,16 +101,32 @@ def __init__(self, cblr_manager, session_id, sensor_id, session_data=None): self.cblr_base = self._cblr_manager.cblr_base def __enter__(self): + """Enter the Live Response session context.""" return self def __exit__(self, exc_type, exc_val, exc_tb): + """ + Exit the Live Response session context. + + Args: + exc_type (str): Exception type, if any. + exc_val (Exception): Exception value, if any. + exc_tb (str): Exception traceback, if any. + """ self.close() def close(self): + """Close the Live Response session.""" self._cblr_manager.close_session(self.sensor_id, self.session_id) self._closed = True def get_session_archive(self): + """ + Get the archive data of the current session. + + Returns: + object: Contains the archive data of the current session. + """ response = self._cb.session.get("{cblr_base}/session/{0}/archive".format(self.session_id, cblr_base=self.cblr_base), stream=True) response.raw.decode_content = True @@ -92,6 +136,17 @@ def get_session_archive(self): # File operations # def get_raw_file(self, file_name, timeout=None, delay=None): + """ + Retrieve contents of the specified file on the remote machine. + + Args: + file_name (str): Name of the file to be retrieved. + timeout (int): Timeout for the operation. + delay (float): TBD + + Returns: + object: Contains the data of the file. + """ data = {"name": "get file", "object": file_name} resp = self._lr_post_command(data).json() @@ -108,11 +163,15 @@ def get_raw_file(self, file_name, timeout=None, delay=None): def get_file(self, file_name, timeout=None, delay=None): """ - Retrieve contents of the specified file name + Retrieve contents of the specified file on the remote machine. - :param str file_name: Name of the file - :return: Content of the specified file name - :rtype: str + Args: + file_name (str): Name of the file to be retrieved. + timeout (int): Timeout for the operation. + delay (float): TBD + + Returns: + str: Contents of the specified file. """ fp = self.get_raw_file(file_name, timeout=timeout, delay=delay) content = fp.read() @@ -122,10 +181,10 @@ def get_file(self, file_name, timeout=None, delay=None): def delete_file(self, filename): """ - Delete the specified file name + Delete the specified file name on the remote machine. - :param str filename: Name of the file - :return: None + Args: + filename (str): Name of the file to be deleted. """ data = {"name": "delete file", "object": filename} resp = self._lr_post_command(data).json() @@ -133,17 +192,16 @@ def delete_file(self, filename): self._poll_command(command_id) def put_file(self, infp, remote_filename): - """ - Create a new file on the remote endpoint with the specified data - - :Example: + r""" + Create a new file on the remote machine with the specified data. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... lr_session.put_file(open("test.txt", "rb"), r"c:\test.txt") - :param str infp: Python file-like containing data to upload to the remote endpoint - :param str remote_filename: File name to create on the remote endpoint - :return: None + Args: + infp (object): Python file-like containing data to upload to the remote endpoint. + remote_filename (str): File name to create on the remote endpoint. """ data = {"name": "put file", "object": remote_filename} file_id = self._upload_file(infp) @@ -154,11 +212,10 @@ def put_file(self, infp, remote_filename): self._poll_command(command_id) def list_directory(self, dir_name): - """ - List the contents of a directory - - :Example: + r""" + List the contents of a directory on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... pprint.pprint(lr_session.list_directory('C:\\\\temp\\\\')) [{u'attributes': [u'DIRECTORY'], @@ -180,9 +237,11 @@ def list_directory(self, dir_name): u'last_write_time': 1476390668, u'size': 0}] - :param str dir_name: Directory to list. This parameter should end with '\' - :return: Returns a directory listing - :rtype: list + Args: + dir_name (str): Directory to list. This parameter should end with the path separator. + + Returns: + list: A list of dicts, each one describing a directory entry. """ data = {"name": "directory list", "object": dir_name} resp = self._lr_post_command(data).json() @@ -191,10 +250,10 @@ def list_directory(self, dir_name): def create_directory(self, dir_name): """ - Create a directory on the remote endpoint + Create a directory on the remote machine. - :param str dir_name: New directory name - :return: None + Args: + dir_name (str): The new directory name. """ data = {"name": "create directory", "object": dir_name} resp = self._lr_post_command(data).json() @@ -202,6 +261,15 @@ def create_directory(self, dir_name): self._poll_command(command_id) def path_join(self, *dirnames): + """ + Join multiple directory names into a single one. + + Args: + *dirnames (list): List of directory names to join together. + + Returns: + str: The joined directory name. + """ if self.os_type == 1: # Windows return "\\".join(dirnames) @@ -210,27 +278,37 @@ def path_join(self, *dirnames): return "/".join(dirnames) def path_islink(self, fi): + """ + Determine if the path is a link. Not implemented. + + Args: + fi (str): File to check. + + Returns: + True if the file is a link, False if not. + """ # TODO: implement return False def walk(self, top, topdown=True, onerror=None, followlinks=False): - """ - Perform a full directory walk with recursion into subdirectories - - :Example: + r""" + Perform a full directory walk with recursion into subdirectories on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... for entry in lr_session.walk(directory_name): ... print(entry) ('C:\\temp\\', [u'dir1', u'dir2'], [u'file1.txt']) - :param str top: Directory to recurse - :param bool topdown: if True, start output from top level directory - :param bool onerror: Callback if an error occurs. - This function is called with one argument (the exception that occurred) - :param bool followlinks: Follow symbolic links - :return: Returns output in the follow tuple format: (Directory Name, [dirnames], [filenames]) - :rtype: tuple + Args: + top (str): Directory to recurse on. + topdown (bool): If True, start output from top level directory. + onerror (func): Callback if an error occurs. This function is called with one argument (the exception + that occurred). + followlinks (bool): True to follow symbolic links. + + Returns: + list: List of tuples containing directory name, subdirectory names, file names. """ try: allfiles = self.list_directory(self.path_join(top, "*")) @@ -266,11 +344,13 @@ def walk(self, top, topdown=True, onerror=None, followlinks=False): def kill_process(self, pid): """ - Terminate a process on the remote endpoint + Terminate a process on the remote machine. - :param pid: Process ID to terminate - :return: True if success, False if failure - :rtype: bool + Args: + pid (int): Process ID to be terminated. + + Returns: + bool: True if success, False if failure. """ data = {"name": "kill", "object": pid} resp = self._lr_post_command(data).json() @@ -286,24 +366,25 @@ def kill_process(self, pid): def create_process(self, command_string, wait_for_output=True, remote_output_file_name=None, working_directory=None, wait_timeout=30, wait_for_completion=True): """ - Create a new process with the specified command string. - - :Example: + Create a new process on the remote machine with the specified command string. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... print(lr_session.create_process(r'cmd.exe /c "ping.exe 192.168.1.1"')) Pinging 192.168.1.1 with 32 bytes of data: Reply from 192.168.1.1: bytes=32 time<1ms TTL=64 - :param str command_string: command string used for the create process operation - :param bool wait_for_output: Block on output from the new process (execute in foreground). This will also set - wait_for_completion (below). - :param str remote_output_file_name: The remote output file name used for process output - :param str working_directory: The working directory of the create process operation - :param int wait_timeout: Time out used for this live response command - :param bool wait_for_completion: Wait until the process is completed before returning - :return: returns the output of the command string - :rtype: str + Args: + command_string (str): Command string used for the create process operation. + wait_for_output (bool): True to block on output from the new process (execute in foreground). + This will also set wait_for_completion (below). + remote_output_file_name (str): The remote output file name used for process output. + working_directory (str): The working directory of the create process operation. + wait_timeout (int): Timeout used for this command. + wait_for_completion (bool): True to wait until the process is completed before returning. + + Returns: + str: The output of the process. """ # process is: # - create a temporary file name @@ -345,11 +426,10 @@ def create_process(self, command_string, wait_for_output=True, remote_output_fil return None def list_processes(self): - """ - List currently running processes - - :Example: + r""" + List currently running processes on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... print(lr_session.list_processes()[0]) {u'command_line': u'', @@ -362,8 +442,8 @@ def list_processes(self): u'sid': u's-1-5-18', u'username': u'NT AUTHORITY\\SYSTEM'} - :return: returns a list of running processes - :rtype: list + Returns: + list: A list of dicts describing the processes. """ data = {"name": "process list"} resp = self._lr_post_command(data).json() @@ -378,11 +458,10 @@ def list_processes(self): # "values" is a list containing a dictionary for each registry value in the key # "sub_keys" is a list containing one entry for each sub_key def list_registry_keys_and_values(self, regkey): - """ - Enumerate subkeys and values of the specified registry key. - - :Example: + r""" + Enumerate subkeys and values of the specified registry key on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: >>> pprint.pprint(lr_session.list_registry_keys_and_values('HKLM\\SYSTEM\\CurrentControlSet\\services\\ACPI')) {'sub_keys': [u'Parameters', u'Enum'], @@ -411,9 +490,12 @@ def list_registry_keys_and_values(self, regkey): u'value_name': u'Tag', u'value_type': u'REG_DWORD'}]} - :param str regkey: The registry key to enumerate - :return: returns a dictionary with 2 keys (sub_keys and values) - :rtype: dict + Args: + regkey (str): The registry key to enumerate. + + Returns: + dict: A dictionary with two keys, 'sub_keys' (a list of subkey names) and 'values' (a list of dicts + containing value data, name, and type). """ data = {"name": "reg enum key", "object": regkey} resp = self._lr_post_command(data).json() @@ -426,11 +508,13 @@ def list_registry_keys_and_values(self, regkey): # returns a list containing a dictionary for each registry value in the key def list_registry_keys(self, regkey): """ - Enumerate all registry values from the specified registry key. + Enumerate all registry values from the specified registry key on the remote machine. - :param regkey: The registry key to enumearte - :return: returns a list of values - :rtype: list + Args: + regkey (str): The registry key to enumerate. + + Returns: + list: List of values for the registry key. """ data = {"name": "reg enum key", "object": regkey} resp = self._lr_post_command(data).json() @@ -440,18 +524,19 @@ def list_registry_keys(self, regkey): # returns a dictionary with the registry value def get_registry_value(self, regkey): - """ - Returns the associated value of the specified registry key - - :Example: + r""" + Return the associated value of the specified registry key on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: >>> pprint.pprint(lr_session.get_registry_value('HKLM\\SYSTEM\\CurrentControlSet\\services\\ACPI\\Start')) {u'value_data': 0, u'value_name': u'Start', u'value_type': u'REG_DWORD'} - :param str regkey: The registry key to retrieve - :return: Returns a dictionary with keys of: value_data, value_name, value_type - :rtype: dict + Args: + regkey (str): The registry key to retrieve. + + Returns: + dict: A dictionary with keys of: value_data, value_name, value_type. """ data = {"name": "reg query value", "object": regkey} resp = self._lr_post_command(data).json() @@ -460,19 +545,18 @@ def get_registry_value(self, regkey): return self._poll_command(command_id).get("value", {}) def set_registry_value(self, regkey, value, overwrite=True, value_type=None): - """ - Set a registry value of the specified registry key - - :Example: + r""" + Set a registry value on the specified registry key on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... lr_session.set_registry_value('HKLM\\\\SYSTEM\\\\CurrentControlSet\\\\services\\\\ACPI\\\\testvalue', 1) - :param str regkey: They registry key to set - :param obj value: The value data - :param bool overwrite: Overwrite value if True - :param str value_type: The type of value. Examples: REG_DWORD, REG_MULTI_SZ, REG_SZ - :return: None + Args: + regkey (str): The registry key to set. + value (object): The value data. + overwrite (bool): If True, any existing value will be overwritten. + value_type (str): The type of value. Examples: REG_DWORD, REG_MULTI_SZ, REG_SZ """ if value_type is None: if type(value) == int: @@ -493,10 +577,10 @@ def set_registry_value(self, regkey, value, overwrite=True, value_type=None): def create_registry_key(self, regkey): """ - Create a new registry + Create a new registry key on the remote machine. - :param str regkey: The registry key to create - :return: None + Args: + regkey (str): The registry key to create. """ data = {"name": "reg create key", "object": regkey} resp = self._lr_post_command(data).json() @@ -505,10 +589,10 @@ def create_registry_key(self, regkey): def delete_registry_key(self, regkey): """ - Delete a registry key + Delete a registry key on the remote machine. - :param str regkey: The registry key to delete - :return: None + Args: + regkey (str): The registry key to delete. """ data = {"name": "reg delete key", "object": regkey} resp = self._lr_post_command(data).json() @@ -517,10 +601,10 @@ def delete_registry_key(self, regkey): def delete_registry_value(self, regkey): """ - Delete a registry value + Delete a registry value on the remote machine. - :param str regkey: the registry value to delete - :return: None + Args: + regkey (str): The registry value to delete. """ data = {"name": "reg delete value", "object": regkey} resp = self._lr_post_command(data).json() @@ -531,12 +615,30 @@ def delete_registry_value(self, regkey): # Physical memory capture # def memdump(self, local_filename, remote_filename=None, compress=False): + """ + Perform a memory dump operation on the remote machine. + + Args: + local_filename (str): Name of the file the memory dump will be transferred to on the local machine. + remote_filename (str): Name of the file the memory dump will be stored in on the remote machine. + compress (bool): True to compress the file on the remote system. + """ dump_object = self.start_memdump(remote_filename=remote_filename, compress=compress) dump_object.wait() dump_object.get(local_filename) dump_object.delete() def start_memdump(self, remote_filename=None, compress=True): + """ + Start a memory dump operation on the remote machine. + + Args: + remote_filename (str): Name of the file the memory dump will be stored in on the remote machine. + compress (bool): True to compress the file on the remote system. + + Returns: + LiveResponseMemdump: Controlling object for the memory dump operation. + """ if not remote_filename: remote_filename = self._random_file_name() @@ -602,7 +704,17 @@ def _lr_post_command(self, data): class LiveResponseMemdump(object): + """Object managing a memory dump on a remote machine.""" + def __init__(self, lr_session, memdump_id, remote_filename): + """ + Initialize the LiveResponseMemdump. + + Args: + lr_session (Session): The Live Response session to the machine doing the memory dump. + memdump_id (str): The ID of the memory dump being performed. + remote_filename (str): The file name the memory dump will be stored in on the remote machine. + """ self.lr_session = lr_session self.memdump_id = memdump_id self.remote_filename = remote_filename @@ -610,6 +722,12 @@ def __init__(self, lr_session, memdump_id, remote_filename): self._error = None def get(self, local_filename): + """ + Retrieve the remote memory dump to a local file. + + Args: + local_filename (str): Filename locally that will receive the memory dump. + """ if not self._done: self.wait() if self._error: @@ -619,20 +737,42 @@ def get(self, local_filename): shutil.copyfileobj(src, dst) def wait(self): + """Wait for the remote memory dump to complete.""" self.lr_session._poll_command(self.memdump_id, timeout=3600, delay=5) self._done = True def delete(self): + """Delete the memory dump file.""" self.lr_session.delete_file(self.remote_filename) def jobrunner(callable, cb, sensor_id): + """ + Wrap a callable object with a live response session. + + Args: + callable (object): The object to be wrapped. + cb (BaseAPI): The CBAPI object reference. + sensor_id (int): The sensor ID to use to get the session. + + Returns: + object: The wrapped object. + """ with cb.select(Sensor, sensor_id).lr_session() as sess: return callable(sess) class WorkItem(object): + """Work item for scheduling.""" + def __init__(self, fn, sensor_id): + """ + Initialize the WorkItem. + + Args: + fn (func): The function to be called to do the actual work. + sensor_id (object): The sensor ID or Sensor object the work item is directed for. + """ self.fn = fn if isinstance(sensor_id, Sensor): self.sensor_id = sensor_id.id @@ -643,19 +783,47 @@ def __init__(self, fn, sensor_id): class CompletionNotification(object): + """The notification that an operation is complete.""" + def __init__(self, sensor_id): + """ + Initialize the CompletionNotification. + + Args: + sensor_id (int): The sensor ID this notification is for. + """ self.sensor_id = sensor_id class WorkerStatus(object): + """Holds the status of an individual worker.""" + def __init__(self, sensor_id, status="ready", exception=None): + """ + Initialize the WorkerStatus. + + Args: + sensor_id (int): The sensor ID this status is for. + status (str): The current status value. + exception (Exception): Any exception that happened. + """ self.sensor_id = sensor_id self.status = status self.exception = exception class JobWorker(threading.Thread): + """Thread object that executes individual Live Response jobs.""" + def __init__(self, cb, sensor_id, result_queue): + """ + Initialize the JobWorker. + + Args: + cb (BaseAPI): The CBAPI object reference. + sensor_id (int): The ID of the sensor being used. + result_queue (Queue): The queue where results are placed. + """ super(JobWorker, self).__init__() self.cb = cb self.sensor_id = sensor_id @@ -664,6 +832,7 @@ def __init__(self, cb, sensor_id, result_queue): self.result_queue = result_queue def run(self): + """Execute the job worker.""" try: self.lr_session = self.cb.live_response.request_session(self.sensor_id) self.result_queue.put(WorkerStatus(self.sensor_id, status="ready")) @@ -685,6 +854,12 @@ def run(self): self.result_queue.put(WorkerStatus(self.sensor_id, status="exiting")) def run_job(self, work_item): + """ + Execute an individual WorkItem. + + Args: + work_item (WorkItem): The work item to execute. + """ try: work_item.future.set_result(work_item.fn(self.lr_session)) except Exception as e: @@ -692,9 +867,18 @@ def run_job(self, work_item): class LiveResponseJobScheduler(threading.Thread): + """Thread that schedules Live Response jobs.""" + daemon = True def __init__(self, cb, max_workers=10): + """ + Initialize the LiveResponseJobScheduler. + + Args: + cb (BaseAPI): The CBAPI object reference. + max_workers (int): Maximum number of JobWorker threads to use. + """ super(LiveResponseJobScheduler, self).__init__() self._cb = cb self._job_workers = {} @@ -704,6 +888,7 @@ def __init__(self, cb, max_workers=10): self.schedule_queue = Queue() def run(self): + """Execute the job scheduler.""" log.debug("Starting Live Response Job Scheduler") while True: @@ -786,6 +971,12 @@ def _cleanup_unscheduled_jobs(self): del self._unscheduled_jobs[k] def submit_job(self, work_item): + """ + Submit a new job to be processed. + + Args: + work_item (WorkItem): New job to be processed. + """ self.schedule_queue.put(work_item) def _spawn_new_workers(self): @@ -809,10 +1000,20 @@ def _spawn_new_workers(self): class CbLRManagerBase(object): + """Live Response manager object.""" + cblr_base = "" # override in subclass for each product cblr_session_cls = NotImplemented # override in subclass for each product def __init__(self, cb, timeout=30, keepalive_sessions=False): + """ + Initialize the CbLRManagerBase object. + + Args: + cb (BaseAPI): The CBAPI object reference. + timeout (int): Timeout to use for requesus. + keepalive_sessions (bool): If True, "ping" sessions occasionally to ensure they stay alive. + """ self._timeout = timeout self._cb = cb self._sessions = {} @@ -827,6 +1028,16 @@ def __init__(self, cb, timeout=30, keepalive_sessions=False): self._job_scheduler = None def submit_job(self, job, sensor): + """ + Submit a new job to be executed as a Live Response. + + Args: + job (object): The job to be scheduled. + sensor (int): ID of the sensor to use for job execution. + + Returns: + Future: A reference to the running job. + """ if self._job_scheduler is None: # spawn the scheduler thread self._job_scheduler = LiveResponseJobScheduler(self._cb) @@ -863,6 +1074,15 @@ def _session_keepalive_thread(self): del self._sessions[sensor_id] def request_session(self, sensor_id): + """ + Initiate a new Live Response session. + + Args: + sensor_id (int): The sensor ID to use. + + Returns: + CbLRSessionBase: The new Live Response session. + """ if self._keepalive_sessions: with self._session_lock: if sensor_id in self._sessions: @@ -879,6 +1099,13 @@ def request_session(self, sensor_id): return session def close_session(self, sensor_id, session_id): + """ + Close the specified Live Response session. + + Args: + sensor_id (int): ID of the sensor. + session_id (int): ID of the session. + """ if self._keepalive_sessions: with self._session_lock: try: @@ -894,15 +1121,48 @@ def _send_keepalive(self, session_id): class GetFileJob(object): + """Object that retrieves a file via Live Response.""" + def __init__(self, file_name): + """ + Initialize the GetFileJob. + + Args: + file_name (str): The name of the file to be fetched. + """ self._file_name = file_name def run(self, session): + """ + Execute the file transfer. + + Args: + session (CbLRSessionBase): The Live Response session being used. + + Returns: + TBD + """ return session.get_file(self._file_name) # TODO: adjust the polling interval and also provide a callback function to report progress def poll_status(cb, url, desired_status="complete", timeout=None, delay=None): + """ + Poll the status of a Live Response query. + + Args: + cb (BaseAPI): The CBAPI object reference. + url (str): The URL to poll. + desired_status (str): The status we're looking for. + timeout (int): The timeout value in seconds. + delay (float): The delay between attempts in seconds. + + Returns: + object: The result of the Live Response query that has the desired status. + + Raises: + LiveResponseError: If an error response was encountered. + """ start_time = time.time() status = None From c21ebf8273c27e8cc6c26b7328631029fd6163a6 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Tue, 14 Jul 2020 16:48:10 -0600 Subject: [PATCH 125/197] Add count_valid default --- src/cbapi/psc/base_query.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cbapi/psc/base_query.py b/src/cbapi/psc/base_query.py index 452d9fd9..7812b3d8 100755 --- a/src/cbapi/psc/base_query.py +++ b/src/cbapi/psc/base_query.py @@ -148,6 +148,7 @@ class PSCQueryBase: def __init__(self, doc_class, cb): self._doc_class = doc_class self._cb = cb + self._count_valid = False class QueryBuilderSupportMixin: From 4fc585763ffdcfc819b0863c7979b45461fd2240 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 14 Jul 2020 15:28:17 -0600 Subject: [PATCH 126/197] mark CBAPI as Release 1.7.0 --- README.md | 2 +- docs/changelog.rst | 23 ++++++++++++++++++++++- docs/conf.py | 6 +++--- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b358c580..68a596f6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.6.2** +**Latest Version: 1.7.0** These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. diff --git a/docs/changelog.rst b/docs/changelog.rst index 109b8baa..b856f2b1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,13 +2,34 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.7.0 - Released July 14, 2020 +------------------------------------ + +Updates + +* General + * Updates to pool defaults in base API. + * Changes to exception handling to better discriminate ConnectionErrors and queries with invalid syntax. + * Various minor bug fixes throughout. +* Carbon Black Cloud + * Bug fixes to query implementation. + * Live Response: Account for sensor queue depth when submitting jobs. +* CB Defense + * Added examples for Dell BIOS verification. +* CB ThreatHunter + * Bug fixes to query implementation. + * Update process and event searches to v2. + * examples/create_feed: Make report optional during feed creation + * examples/process_exporter: Add headers to CSV file writer + * examples/threat_intelligence: Simplify report validation, add severity conversion to percent + CbAPI 1.6.2 - Released April 08, 2020 ------------------------------------- Updates * CB Response - * Changes to align with limits placed on the sensor update function in CB Response 7.1.0. Release notes are available on User Exchange, the ID is `CB 28683 `_. + * Changes to align with limits placed on the sensor update function in CB Response 7.1.0. Release notes are available on User Exchange, the ID is `CB 28683 `_. CbAPI 1.6.1 - Released January 13, 2020 --------------------------------------- diff --git a/docs/conf.py b/docs/conf.py index 2ccc21c0..560f31fb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,7 @@ # General information about the project. project = u'cbapi' -copyright = u'2016-2019, Carbon Black Developer Network' +copyright = u'2016-2020, VMware Inc.' author = u'Carbon Black Developer Network' # The version info for the project you're documenting, acts as replacement for @@ -59,9 +59,9 @@ # built documents. # # The short X.Y version. -version = u'1.6' +version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.6.2' +release = u'1.7.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 1cb038a7..c6186335 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( name='cbapi', - version='1.6.2', + version='1.7.0', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index 99cda4a8..0f1a2dc9 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2019 VMware Carbon Black' -__version__ = '1.6.2' +__version__ = '1.7.0' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From e925367b327043a14819779c571fbb3b61532765 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 14 Jul 2020 15:35:00 -0600 Subject: [PATCH 127/197] minor copyright changes as per Alex --- docs/conf.py | 2 +- src/cbapi/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 560f31fb..3ab59e9e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,7 @@ # General information about the project. project = u'cbapi' -copyright = u'2016-2020, VMware Inc.' +copyright = u'2016-2020, VMware Carbon Black' author = u'Carbon Black Developer Network' # The version info for the project you're documenting, acts as replacement for diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index 0f1a2dc9..5083cc42 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -5,7 +5,7 @@ __title__ = 'cbapi' __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' -__copyright__ = 'Copyright 2018-2019 VMware Carbon Black' +__copyright__ = 'Copyright 2018-2020 VMware Carbon Black' __version__ = '1.7.0' # New API as of cbapi 0.9.0 From b141a60abbc7bd06ff64b454c973f2eee5edd097 Mon Sep 17 00:00:00 2001 From: Thomas Bouve Date: Wed, 15 Jul 2020 23:10:53 +0200 Subject: [PATCH 128/197] Update the default v2 search rows value to 10000 --- src/cbapi/psc/threathunter/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index c37d51ba..1b666d8e 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -469,7 +469,7 @@ def _count(self): return self._total_results - def _search(self, start=0, rows=0): + def _search(self, start=0, rows=10000): if not self._query_token: self._submit() From 6431d9a57e45c84a9c3a582b2b841ba886eb6c4d Mon Sep 17 00:00:00 2001 From: Thomas Bouve Date: Wed, 15 Jul 2020 23:19:07 +0200 Subject: [PATCH 129/197] Changed rows in wrong function. Correcting. --- src/cbapi/psc/threathunter/query.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index 1b666d8e..0731411f 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -415,6 +415,7 @@ def _submit(self): raise ApiError("Query already submitted: token {0}".format(self._query_token)) args = self._get_query_parameters() + args['rows'] = 10000 self._validate(args) url = "/api/investigate/v2/orgs/{}/processes/search_jobs".format(self._cb.credentials.org_key) @@ -469,7 +470,7 @@ def _count(self): return self._total_results - def _search(self, start=0, rows=10000): + def _search(self, start=0, rows=0): if not self._query_token: self._submit() From 1a600a3d38c8247d5f392269f14602a8375b0188 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Fri, 17 Jul 2020 11:10:20 -0600 Subject: [PATCH 130/197] Add support for fetch alert by id --- src/cbapi/response/models.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 2c82e377..fc3446b8 100644 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -279,9 +279,22 @@ def _query_implementation(cls, cb): def __init__(self, cb, alert_id, initial_data=None): super(Alert, self).__init__(cb, alert_id, initial_data) + if alert_id is not None and initial_data is None: + self.refresh() - def _refresh(self): + def refresh(self): # there is no GET method for an Alert. + # /api/v1/alert?cb.fq.unique_id=963d7168-8bb0-46fb-ba1f-e72e505f7056 + url = '{}?cb.fq.unique_id={}'.format(self.urlobject, self.unique_id) + resp = self._cb.get_object(url) + result = resp.get("results", []) + if len(result) > 1: + raise MoreThanOneResultError("More than one Alert matched the unique_id") + elif len(result) == 0: + raise ObjectNotFoundError("Alert could not be found by unique_id") + else: + self._info = result[0] + self._last_refresh_time = time.time() return True @property From 26a8222159c7acaa8a2ccc9a2b6b23276b53a793 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Fri, 17 Jul 2020 11:17:10 -0600 Subject: [PATCH 131/197] Add missing import for error --- src/cbapi/response/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index fc3446b8..4451666c 100644 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -17,7 +17,7 @@ import time from cbapi.utils import convert_query_params -from ..errors import InvalidObjectError, ApiError, TimeoutError +from ..errors import InvalidObjectError, ApiError, TimeoutError, MoreThanOneResultError from ..oldmodels import BaseModel, immutable from ..models import NewBaseModel, MutableBaseModel, CreatableModelMixin From 877661656c8b08dc850d25df62a7edfbbee07af2 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Fri, 17 Jul 2020 11:19:36 -0600 Subject: [PATCH 132/197] Remove outdated comments --- src/cbapi/response/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 4451666c..2820e9f0 100644 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -283,8 +283,6 @@ def __init__(self, cb, alert_id, initial_data=None): self.refresh() def refresh(self): - # there is no GET method for an Alert. - # /api/v1/alert?cb.fq.unique_id=963d7168-8bb0-46fb-ba1f-e72e505f7056 url = '{}?cb.fq.unique_id={}'.format(self.urlobject, self.unique_id) resp = self._cb.get_object(url) result = resp.get("results", []) From 4c3a15681a5bbf24fcf8f8c250a78a630e97e11f Mon Sep 17 00:00:00 2001 From: Lisa Hilmes <55513548+lhilmes-cb@users.noreply.github.com> Date: Mon, 20 Jul 2020 10:58:36 -0600 Subject: [PATCH 133/197] Update index.rst Updating product names and other language adjustments. --- docs/index.rst | 89 ++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index b7ef79ee..ab5dc24f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,4 +1,4 @@ -.. cbapi documentation master file, created by +.. CBAPI documentation master file, created by sphinx-quickstart on Thu Apr 28 09:52:29 2016. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. @@ -8,9 +8,8 @@ cbapi: Carbon Black API for Python Release v\ |release|. -cbapi provides a straightforward interface to the Carbon Black products: CB Protection, Response, and Defense. -This library provides a Pythonic layer to access the raw power of the REST APIs of all CB products, making it trivial -to do the easy stuff and handling all of the "sharp corners" behind the scenes for you. Take a look:: +CBAPI provides a straightforward interface to the VMware Carbon Black products: Carbon Black EDR, Carbon Black App Control, and Carbon Black Cloud Endpoint Standard (formerly CB Response, CB Protection, and CB Defense). +This library provides a Pythonic layer to access the raw power of the REST APIs of all Carbon Black products, making it easier to query data from any platform or on-premise APIs, combine data from multiple API calls, manage all API credentials in one place, and manipulate data as Python objects. Take a look:: >>> from cbapi.response import CbResponseAPI, Process, Binary, Sensor >>> # @@ -39,7 +38,7 @@ to do the easy stuff and handling all of the "sharp corners" behind the scenes f ... s.network_isolation_enabled = True ... s.save() -If you're more a CB Protection fellow, then you're in luck as well:: +If you're a Carbon Black App Control customer (formerly CB Protection), you may use:: >>> from cbapi.protection.models import FileInstance >>> from cbapi.protection import CbProtectionAPI @@ -62,7 +61,7 @@ If you're more a CB Protection fellow, then you're in luck as well:: >>> fi.computer.policyId = 3 >>> fi.computer.save() -As of version 1.2, cbapi now provides support for CB Defense too! +As of version 1.2, CBAPI also supports Carbon Black Cloud Endpoint Standard (formerly CB Defense): >>> from cbapi.psc.defense import * >>> # @@ -85,45 +84,45 @@ Major Features -------------- - **Enhanced Live Response API** - The new cbapi now provides a robust interface to the CB Response Live Response capability. + The new CBAPI now provides a robust interface to the Carbon Black EDR Live Response capability. Easily create Live Response sessions, initiate commands on remote hosts, and pull down data as necessary to make your Incident Response process much more efficient and automated. -- **Consistent API for CB Response, Protection and Defense platforms** - We now support CB Response, Protection and Defense users in the same API layer. Even better, - the object model is the same for both; if you know one API you can easily transition to the other. cbapi - hides all the differences between the three REST APIs behind a single, consistent Python-like interface. +- **Consistent API accross VMware Carbon Black platforms** + CBAPI supports Carbon Black EDR, Carbon Black App Control, and Carbon Black Cloud Endpoint Standard customers from a single API layer. Even better, + the object model is the same for all three, and if you know one API, you can easily transition to another. CBAPI + manages the differences between the three REST APIs behind a single, consistent Python-like interface. - **Enhanced Performance** - cbapi now provides a built in caching layer to reduce the query load on the Carbon Black server. This is especially - useful when taking advantage of cbapi's new "joining" features. You can transparently access, for example, the - binary associated with a given process in CB Response. Since many processes may be associated + CBAPI now provides a built in caching layer to reduce the query load on the Carbon Black server. This is especially + useful when taking advantage of CBAPI's new "joining" features. You can transparently access, for example, the + binary associated with a given process in Carbon Black EDR. Since many processes may be associated with the same binary, it does not make sense to repeatedly request the same binary information from the server - over and over again. Therefore cbapi now caches this information to avoid unnecessary requests. + over and over again. Therefore CBAPI now caches this information to avoid unnecessary requests. - **Reduce Complexity** - cbapi now provides a friendly - dare I say "fun" - interface to the data. This greatly improves developer + CBAPI provides a friendly interface for accessing Carbon Black data. This greatly improves developer productivity and lowers the bar to entry. - **Python 3 and Python 2 compatible** - Use all the new features and modules available in Python 3 with cbapi. This module is compatible with Python + Use all the new features and modules available in Python 3 with CBAPI. This module is compatible with Python versions 2.6.6 and above, 2.7.x, 3.4.x, and 3.5.x. - **Better support for multiple CB servers** - cbapi now introduces the concept of Credential Profiles; named collections of URL, API keys, and optional proxy - configuration for connecting to any number of CB Protection, Defense, or Response servers. + CBAPI introduces the concept of Credential Profiles; named collections of URL, API keys, and optional proxy + configuration for connecting to any number of Carbon Black EDR, Carbon Black App Control, or Carbon Black Cloud Endpoint Standard servers. API Credentials --------------- -The new cbapi as of version 0.9.0 enforces the use of credential files. +CBAPI version 0.9.0 enforces the use of credential files. In order to perform any queries via the API, you will need to get the API token for your CB user. See the documentation on the Developer Network website on how to acquire the API token for -`CB Response `_, -`CB Protection `_, or -`CB Defense `_. +`Carbon Black EDR (CB Response) `_, +`Carbon Black App Control (CB Protection) `_, or +`Carbon Black Cloud Endpoint Standard (CB Defense) `_. Once you acquire your API token, place it in one of the default credentials file locations: @@ -133,9 +132,9 @@ Once you acquire your API token, place it in one of the default credentials file For distinction between credentials of different Carbon Black products, use the following naming convention for your credentials files: -* ``credentials.psc`` for CB Defense, CB ThreatHunter, and CB LiveOps -* ``credentials.response`` for CB Response -* ``credentials.protection`` for CB Protection +* ``credentials.psc`` for Carbon Black Cloud Endpoint Standard, Audit & Remediation, and Enterprise EDR (CB Defense, CB LiveOps, and CB ThreatHunter) +* ``credentials.response`` for Carbon Black EDR (CB Response) +* ``credentials.protection`` for Carbon Black App Control (CB Protection) For example, if you use a Carbon Black Cloud product, you should have created a credentials file in one of these locations: @@ -165,26 +164,26 @@ by key-value pairs providing the necessary credential information:: The possible options for each credential profile are: -* **url**: The base URL of the CB server. This should include the protocol (https) and the hostname, and nothing else. +* **url**: The base URL of the Carbon Black server. This should include the protocol (https) and the hostname, and nothing else. * **token**: The API token for the user ID. More than one credential profile can be specified for a given server, with different tokens for each. * **ssl_verify**: True or False; controls whether the SSL/TLS certificate presented by the server is validated against the local trusted CA store. * **org_key**: The organization key. This is required to access the Carbon Black Cloud, and can be found in the console. The format is ``123ABC45``. -* **proxy**: A proxy specification that will be used when connecting to the CB server. The format is: +* **proxy**: A proxy specification that will be used when connecting to the Carbon Black server. The format is: ``http://myusername:mypassword@proxy.company.com:8001/`` where the hostname of the proxy is ``proxy.company.com``, port 8001, and using username/password ``myusername`` and ``mypassword`` respectively. -* **ignore_system_proxy**: If you have a system-wide proxy specified, setting this to True will force cbapi to bypass - the proxy and directly connect to the CB server. +* **ignore_system_proxy**: If you have a system-wide proxy specified, setting this to True will force CBAPI to bypass + the proxy and directly connect to the Carbon Black server. -Future versions of cbapi will also provide the ability to "pin" the TLS certificate so as to provide certificate +Future versions of CBAPI will also provide the ability to "pin" the TLS certificate so as to provide certificate verification on self-signed or internal CA signed certificates. Environment Variable Support -The latest cbapi for python supports specifying API credentials in the following three environment variables: +The latest CBAPI for Python supports specifying API credentials in the following three environment variables: -`CBAPI_TOKEN` the envar for holding the CbR/CbP api token or the ConnectorId/APIKEY combination for CB Defense/Carbon Black Cloud. +`CBAPI_TOKEN` the envar for holding the CbR/CbP api token or the ConnectorId/APIKEY combination for Endpoint Standard (CB Defense)/Carbon Black Cloud. The `CBAPI_URL` envar holds the FQDN of the target, a CbR , CBD, or CbD/Carbon Black Cloud server specified just as they are in the configuration file format specified above. @@ -195,29 +194,28 @@ not explicitly set by the user. Backwards & Forwards Compatibility ---------------------------------- -The previous versions (0.8.x and earlier) of cbapi and bit9Api are now deprecated and will no longer receive updates. -However, existing scripts will work without change as cbapi includes both in its legacy package. -The legacy package is imported by default and placed in the top level cbapi namespace when the cbapi module +The previous versions (0.8.x and earlier) of CBAPI and bit9Api are now deprecated and will no longer receive updates. +However, existing scripts will work without change as CBAPI includes both in its legacy package. +The legacy package is imported by default and placed in the top level CBAPI namespace when the CBAPI module is imported on a Python 2.x interpreter. Therefore, scripts that expect to import cbapi.CbApi will continue to work exactly as they had previously. Since the old API was not compatible with Python 3, the legacy package is not importable in Python 3.x and therefore legacy scripts cannot run under Python 3. -Once cbapi 1.0.0 is released, the old :py:mod:`cbapi.legacy.CbApi` will be deprecated and removed entirely no earlier +Once CBAPI 1.0.0 is released, the old :py:mod:`cbapi.legacy.CbApi` will be deprecated and removed entirely no earlier than January 2017. New scripts should use the :py:mod:`cbapi.response.rest_api.CbResponseAPI` -(for CB Response), :py:mod:`cbapi.protection.rest_api.CbProtectionAPI` -(for CB Protection), or :py:mod:`cbapi.defense.rest_api.CbDefenseAPI` API entry points. +(for Carbon Black EDR, or CB Response), :py:mod:`cbapi.protection.rest_api.CbProtectionAPI` +(for Carbon Black App Control, or CB Protection), or :py:mod:`cbapi.defense.rest_api.CbDefenseAPI` API entry points. -The API is frozen as of version 1.0; afterward, any changes in the 1.x version branch +The API is frozen as of version 1.0; any changes in the 1.x version branch will be additions/bug fixes only. Breaking changes to the API will increment the major version number (2.x). User Guide ---------- -Let's get started with cbapi. Once you've mastered the concepts here, then you can always hop over to the API -Documentation (below) for detailed information on the objects and methods exposed by cbapi. +Get started with CBAPI here. For detailed information on the objects and methods exposed by CBAPI, see the full API Documentation below. .. toctree:: :maxdepth: 2 @@ -235,10 +233,9 @@ Documentation (below) for detailed information on the objects and methods expose API Documentation ----------------- -Once you've taken a look at the User Guide, read through some of the -`examples on GitHub `_, -and maybe even written some code of your own, the API documentation can help you get the most out of cbapi by -documenting all of the methods available to you. +Once you have read the User Guide, youc an view `examples on GitHub `_ +or try writing code of your own. You can use the full API documentation below to see all the methods available in CBAPI +and unlock the full functionality of the SDK. .. toctree:: :maxdepth: 2 From 326e9f6c7ae1e4e99bc236f7a4ec28303964c6f7 Mon Sep 17 00:00:00 2001 From: Lisa Hilmes <55513548+lhilmes-cb@users.noreply.github.com> Date: Mon, 20 Jul 2020 16:17:16 -0500 Subject: [PATCH 134/197] Update index.rst Fixing two preexisting type-os --- docs/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ab5dc24f..2dc0f45a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -88,10 +88,10 @@ Major Features Easily create Live Response sessions, initiate commands on remote hosts, and pull down data as necessary to make your Incident Response process much more efficient and automated. -- **Consistent API accross VMware Carbon Black platforms** +- **Consistent API across VMware Carbon Black platforms** CBAPI supports Carbon Black EDR, Carbon Black App Control, and Carbon Black Cloud Endpoint Standard customers from a single API layer. Even better, the object model is the same for all three, and if you know one API, you can easily transition to another. CBAPI - manages the differences between the three REST APIs behind a single, consistent Python-like interface. + manages the differences among the three REST APIs behind a single, consistent Python-like interface. - **Enhanced Performance** CBAPI now provides a built in caching layer to reduce the query load on the Carbon Black server. This is especially @@ -233,7 +233,7 @@ Get started with CBAPI here. For detailed information on the objects and methods API Documentation ----------------- -Once you have read the User Guide, youc an view `examples on GitHub `_ +Once you have read the User Guide, you can view `examples on GitHub `_ or try writing code of your own. You can use the full API documentation below to see all the methods available in CBAPI and unlock the full functionality of the SDK. From 2b53115834613873b5fc8aa633d11010a6f72889 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Wed, 22 Jul 2020 10:48:25 -0600 Subject: [PATCH 135/197] Add 1.7.1 notes and version bump --- README.md | 2 +- docs/changelog.rst | 13 +++++++++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 68a596f6..1b9caba9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.7.0** +**Latest Version: 1.7.1** These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. diff --git a/docs/changelog.rst b/docs/changelog.rst index b856f2b1..87d78d99 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,19 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.7.1 - Released July 22, 2020 +------------------------------------ + +Updates + +* General + * Documentation updates to indicate changed product names +* Carbon Black Cloud + * Process Search v2 rows defaults to 10k to match UI behavior +* CB Response + * Add support for fetching alert by ID + + CbAPI 1.7.0 - Released July 14, 2020 ------------------------------------ diff --git a/docs/conf.py b/docs/conf.py index 3ab59e9e..fe2f4543 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.7.0' +release = u'1.7.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index c6186335..8223006a 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( name='cbapi', - version='1.7.0', + version='1.7.1', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index 5083cc42..ec0f6c2b 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2020 VMware Carbon Black' -__version__ = '1.7.0' +__version__ = '1.7.1' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From d6f2c302bd1429c80f4b3a4c30fbac8875e27523 Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Thu, 13 Aug 2020 15:12:03 +0000 Subject: [PATCH 136/197] Fix warnings regarding unclosed file object. --- src/cbapi/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cbapi/models.py b/src/cbapi/models.py index e24ca39c..539ef777 100644 --- a/src/cbapi/models.py +++ b/src/cbapi/models.py @@ -35,8 +35,9 @@ def __new__(mcs, name, bases, clsdict): swagger_meta_file = clsdict.pop("swagger_meta_file", None) model_data = {} if swagger_meta_file: - model_data = yaml.safe_load( - open(os.path.join(mcs.model_base_directory, swagger_meta_file), 'rb').read()) + with open(os.path.join(mcs.model_base_directory, swagger_meta_file), 'rb') as f: + model_data = yaml.safe_load(f.read()) + clsdict["__doc__"] = "Represents a %s object in the Carbon Black server.\n\n" % (name,) for field_name, field_info in iteritems(model_data.get("properties", {})): From a17bcac10bed29840f31e323aa3d74535651aa4c Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Thu, 13 Aug 2020 16:24:21 -0600 Subject: [PATCH 137/197] Add event_export tool --- examples/defense/event_export.py | 76 ++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 examples/defense/event_export.py diff --git a/examples/defense/event_export.py b/examples/defense/event_export.py new file mode 100644 index 00000000..55053b0a --- /dev/null +++ b/examples/defense/event_export.py @@ -0,0 +1,76 @@ +""" +Event Export Tool + +usage: event_export.py [-h] [--appName APPNAME] startTime endTime fileName +""" + +import requests +import argparse +import json +from datetime import datetime, timedelta + + +parser = argparse.ArgumentParser() +parser.add_argument("startTime", help="Start Time (2020-08-04T00:00:00.000Z)") +parser.add_argument("endTime", help="End Time (2020-08-05T00:00:00.000Z)") +parser.add_argument("fileName", help="The name of the json file ie. events.json") +parser.add_argument("--appName", "-a", help="The app name to limit events") +args = parser.parse_args() + +with open(args.fileName, "a") as file: + + hostname = "!!REPLACE WITH HOSTNAME!!" + + url_with_app = '{}/integrationServices/v3/event?startTime={}&endTime={}&applicationName={}&rows=10000' + url_without_app = '{}/integrationServices/v3/event?startTime={}&endTime={}&rows=10000' + + headers = {'x-auth-token': '!!REPLACE WITH API SECRET KEY!!/!!REPLACE WITH API ID!!'} # key/id + + orig_end = datetime.strptime(args.endTime, '%Y-%m-%dT%H:%M:%S.%fZ') + orig_start = datetime.strptime(args.startTime, '%Y-%m-%dT%H:%M:%S.%fZ') + start = orig_end - timedelta(days=1) + end = orig_end + triggerEnd = False + file.write('[') + + while True: + print("Next End Event Time: {}".format(end.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))) + if start == orig_start: + triggerEnd = True + + if args.appName: + resp = requests.get(url_with_app.format(hostname, + start.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), + end.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), + args.appName), headers=headers) + else: + resp = requests.get(url_without_app.format(hostname, + start.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), + end.strftime('%Y-%m-%dT%H:%M:%S.%fZ')), headers=headers) + + resp_json = resp.json() + if resp_json["success"]: + results = resp_json['results'] + + end = datetime.fromtimestamp(((results[-1]["eventTime"] + 1) / 1000)) + start = end - timedelta(days=1) + + if start < orig_start: + start = orig_start + + file.write(json.dumps(results)[1:-2]) + + if resp_json["totalResults"] >= 10000: + triggerEnd = False + elif triggerEnd or end < start: + print("Events have been exported") + file.write(']') + break + file.write(',') + + else: + breakpoint() + print("API Call Failed!") + print(resp.content) + break + file.close() From bc94ba9c24c203194ce72a41d9f8bb9f37bfe1b3 Mon Sep 17 00:00:00 2001 From: alacercogitatus Date: Tue, 18 Aug 2020 12:12:03 -0400 Subject: [PATCH 138/197] Update connection.py Enables the use of a proxy based on a non-file credential input. Additionally, allows for ssl verification based on a passed cert file parameter. --- src/cbapi/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/connection.py b/src/cbapi/connection.py index 6c6253d7..ad83f256 100644 --- a/src/cbapi/connection.py +++ b/src/cbapi/connection.py @@ -360,7 +360,7 @@ def __init__(self, *args, **kwargs): else: credentials = {"url": url, "token": token} - for k in ("ssl_verify",): + for k in ("ssl_verify", "proxy", "ssl_cert_file"): if k in kwargs: credentials[k] = kwargs.pop(k) self.credentials = Credentials(credentials) From 41acded1d5c0b91759946300dae0c54a450a1eb8 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Tue, 18 Aug 2020 10:59:40 -0600 Subject: [PATCH 139/197] Release 1.7.2 --- README.md | 2 +- docs/changelog.rst | 9 +++++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1b9caba9..f2e284e5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.7.1** +**Latest Version: 1.7.2** These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. diff --git a/docs/changelog.rst b/docs/changelog.rst index 87d78d99..352872b2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.7.2 - Released July 22, 2020 +------------------------------------ + +Updates + +* General + * Allow passing in proxy configuration as direct parameters during class instantiation of base API. + + CbAPI 1.7.1 - Released July 22, 2020 ------------------------------------ diff --git a/docs/conf.py b/docs/conf.py index fe2f4543..dc63206e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.7.1' +release = u'1.7.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 8223006a..3a327dac 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( name='cbapi', - version='1.7.1', + version='1.7.2', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index ec0f6c2b..d6337391 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2020 VMware Carbon Black' -__version__ = '1.7.1' +__version__ = '1.7.2' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From 46917f9e4dbb1ebafc78c3bd6c142c1e1b387621 Mon Sep 17 00:00:00 2001 From: Thomas Bouve Date: Thu, 24 Sep 2020 19:12:48 +0200 Subject: [PATCH 140/197] Fix typo in query field parameter and added fields (#256) --- src/cbapi/psc/threathunter/query.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index 0731411f..5990db96 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -267,8 +267,11 @@ def _get_query_parameters(self): "device_group", "device_internal_ip", "device_os", + "device_policy", "process_effective_reputation", - "process_reputation,ttp" + "process_reputation", + "process_start_time", + "ttp" ] return args From f16892399f3e58045dec611a234e9be2233071f9 Mon Sep 17 00:00:00 2001 From: Marcel da Silva Date: Wed, 14 Oct 2020 13:56:36 +0200 Subject: [PATCH 141/197] add sensor builds list and support sensor info for all OS --- src/cbapi/response/models.py | 50 +++++++++++++++++++- src/cbapi/response/models/group-modify.yaml | 11 +++++ src/cbapi/response/models/sensor-builds.yaml | 29 ++++++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) mode change 100644 => 100755 src/cbapi/response/models.py create mode 100644 src/cbapi/response/models/sensor-builds.yaml diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py old mode 100644 new mode 100755 index 2820e9f0..7f5fe3ae --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -884,7 +884,7 @@ def _update_object(self): class SensorGroup(MutableBaseModel, CreatableModelMixin): swagger_meta_file = "response/models/group-modify.yaml" - urlobject = '/api/group' + urlobject = '/api/v2/group' @classmethod def _query_implementation(cls, cb): @@ -914,6 +914,54 @@ def site(self): def site(self, new_site): self.site_id = new_site.id +class SensorBuilds(MutableBaseModel): + swagger_meta_file = "response/models/sensor-builds.yaml" + urlobject = '/api/v2/builds' + + def __init__(self, cb, initial_data=None): + super(SensorBuilds, self).__init__(cb) + if initial_data is not None: + temp_list = [] + for b in initial_data['Windows']: + temp_list.append(b['version_string']) + self.Windows = temp_list + temp_list = [] + for v in initial_data['Linux']: + temp_list.append(v['version_string']) + self.Linux = temp_list + temp_list = [] + for v in initial_data['OSX']: + temp_list.append(v['version_string']) + self.OSX = temp_list + + @classmethod + def new_object(self, cb, initial_data): + o = self(cb, initial_data) + return o + + @classmethod + def _query_implementation(self, cb): + return BuildsQuery(self, cb) + + def _refresh(self): + self._info = {'Windows': [], 'Linux': [], 'OSX': []} + return True + + +class BuildsQuery(SimpleQuery): + + def __init__(self, doc_class, cb): + super(BuildsQuery, self).__init__(doc_class, cb) + + @property + def results(self): + if not self._full_init: + api_results = self._cb.get_object(self._urlobject, default={}) + self._results = self._doc_class.new_object(self._cb, api_results) + self._full_init = True + + return self._results + class SensorQuery(SimpleQuery): valid_field_names = ['ip', 'hostname', 'groupid'] diff --git a/src/cbapi/response/models/group-modify.yaml b/src/cbapi/response/models/group-modify.yaml index ad12e971..ff9bae74 100644 --- a/src/cbapi/response/models/group-modify.yaml +++ b/src/cbapi/response/models/group-modify.yaml @@ -54,6 +54,17 @@ properties: type: "string" description: "The version of the sensor in this group, default Manual" default: "Manual" + sensor_version_windows: + type: "string" + description: "The version of the sensor for Windows hosts in this group, default Manual" + default: "Manual" + sensor_version_osx: + type: "string" + description: "The version of the sensor for OSX hosts in this group, default Manual" + default: "Manual" + sensor_version_linux: + type: "string" + description: "The version of the sensor for Linux hosts in this group, default Manual" datastore_server: type: "string" description: "Datastore server address if different from sensorbackend server, default null" diff --git a/src/cbapi/response/models/sensor-builds.yaml b/src/cbapi/response/models/sensor-builds.yaml new file mode 100644 index 00000000..3b31077b --- /dev/null +++ b/src/cbapi/response/models/sensor-builds.yaml @@ -0,0 +1,29 @@ +type: "object" +properties: + Windows: + type: "array" + description: 'Sensor versions for Windows' + items: + type: "object" + properties: + version_string: + type: "string" + description: "Sensor build version" + Linux: + type: "array" + description: 'Sensor versions for Linux' + items: + type: "object" + properties: + version_string: + type: "string" + description: "Sensor build version" + OSX: + type: "array" + description: 'Sensor versions for OSX' + items: + type: "object" + properties: + version_string: + type: "string" + description: "Sensor build version" From 010bb5d75cef829fd5a0784369e2c8599ca62d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Notin?= Date: Fri, 6 Nov 2020 16:46:58 +0100 Subject: [PATCH 142/197] PSC Alerts API: requests pages of 100 rows by default Closes #275 --- src/cbapi/psc/alerts_query.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cbapi/psc/alerts_query.py b/src/cbapi/psc/alerts_query.py index 9a350f1f..6a775c6d 100755 --- a/src/cbapi/psc/alerts_query.py +++ b/src/cbapi/psc/alerts_query.py @@ -365,6 +365,7 @@ def _build_request(self, from_row, max_rows, add_sort=True): """ request = {"criteria": self._build_criteria()} request["query"] = self._query_builder._collapse() + request["rows"] = 100 if from_row > 0: request["start"] = from_row if max_rows >= 0: From 3857b3e8a97af2df7acaadb80bea14f904c29226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Notin?= Date: Fri, 6 Nov 2020 18:51:25 +0100 Subject: [PATCH 143/197] small typo --- examples/threathunter/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/threathunter/search.py b/examples/threathunter/search.py index ee05c2f9..d757aaf7 100644 --- a/examples/threathunter/search.py +++ b/examples/threathunter/search.py @@ -15,7 +15,7 @@ def main(): parser.add_argument("-c", help="show children for query results", action="store_true", default=False) parser.add_argument("-p", help="show parents for query results", action="store_true", default=False) parser.add_argument("-t", help="show tree for query results", action="store_true", default=False) - parser.add_argument("-S", type=str, help="sory by this field", required=False) + parser.add_argument("-S", type=str, help="sort by this field", required=False) parser.add_argument("-D", help="return results in descending order", action="store_true") args = parser.parse_args() From d87d8103dd7a7651b5d81eb6accbd60c6e67d81e Mon Sep 17 00:00:00 2001 From: Marcel da Silva Date: Fri, 16 Oct 2020 08:44:39 +0200 Subject: [PATCH 144/197] remove newline --- src/cbapi/response/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 7f5fe3ae..1fcae69e 100755 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -947,7 +947,6 @@ def _refresh(self): self._info = {'Windows': [], 'Linux': [], 'OSX': []} return True - class BuildsQuery(SimpleQuery): def __init__(self, doc_class, cb): From 0fb652296427e48cb16ed67423a92eb9c100bc0e Mon Sep 17 00:00:00 2001 From: Luke Lyon <52218532+llyon-cb@users.noreply.github.com> Date: Mon, 16 Nov 2020 16:15:19 -0700 Subject: [PATCH 145/197] Update Response Models (#279) Add Alert.set_ignored(), Alert.assign(), and Alert.change_status() Co-authored-by: Jared Fagel --- src/cbapi/response/models.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 2820e9f0..aa454e7b 100644 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -254,8 +254,9 @@ def _bulk_update(self, payload): return None - def set_ignored(self, ignored_flag=True): - payload = {"updates": {"is_ignored": ignored_flag, "requested_status": "False Positive"}} + def set_ignored(self, ignored_flag=True, status="False Positive"): + """Ignore all future Alerts from the Report that triggered this Alert.""" + payload = {"set_ignored": ignored_flag, "requested_status": status} return self._bulk_update(payload) def assign(self, target): @@ -263,6 +264,9 @@ def assign(self, target): return self._bulk_update(payload) def change_status(self, new_status): + allowed_statuses = ["In Progress", "Unresolved", "Resolved", "False Positive"] + if new_status not in allowed_statuses: + raise ApiError("Alert status must be one of {0}".format(allowed_statuses)) payload = {"requested_status": new_status} return self._bulk_update(payload) @@ -295,6 +299,25 @@ def refresh(self): self._last_refresh_time = time.time() return True + def set_ignored(self, ignored_flag=True, status="False Positive"): + """Ignore all future Alerts from the Report that triggered this Alert.""" + payload = {"set_ignored": ignored_flag, "requested_status": status} + payload["alert_ids"] = [self.unique_id] + return self._cb.post_object("/api/v1/alerts", payload) + + def assign(self, target): + payload = {"assigned_to": target, "requested_status": "In Progress"} + payload["alert_ids"] = [self.unique_id] + return self._cb.post_object("/api/v1/alerts", payload) + + def change_status(self, new_status): + allowed_statuses = ["In Progress", "Unresolved", "Resolved", "False Positive"] + if new_status not in allowed_statuses: + raise ApiError("Alert status must be one of {0}".format(allowed_statuses)) + payload = {"status": new_status} + payload["unique_id"] = self.unique_id + return self._cb.post_object("/api/v1/alert/{0}".format(self.unique_id), payload) + @property def process(self): if 'process' in self.alert_type: From 25c347683bac9eb4d32018792957cbcc6ba4c1ae Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Wed, 18 Nov 2020 16:44:57 -0500 Subject: [PATCH 146/197] Wed Nov 18 16:44:57 EST 2020 VMW-71 Added Endpoint Standard call for audit logs. --- src/cbapi/psc/defense/rest_api.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/cbapi/psc/defense/rest_api.py b/src/cbapi/psc/defense/rest_api.py index 12db10f9..113ab96b 100644 --- a/src/cbapi/psc/defense/rest_api.py +++ b/src/cbapi/psc/defense/rest_api.py @@ -24,6 +24,7 @@ class CbDefenseAPI(CbPSCBaseAPI): >>> from cbapi import CbDefenseAPI >>> cb = CbDefenseAPI(profile="production") """ + def __init__(self, *args, **kwargs): super(CbDefenseAPI, self).__init__(*args, **kwargs) @@ -48,6 +49,14 @@ def get_notifications(self): res = self.get_object("/integrationServices/v3/notification") return res.get("notifications", []) + def get_auditlogs(self): + """Retrieve queued audit logs from the Carbon Black Cloud Endpoint Standard server. + + :returns: list of dictionary objects representing the audit logs, or an empty list if none available. + """ + res = self.get_object("/integrationServices/v3/auditlogs") + return res.get("notifications", []) + class Query(PaginatedQuery): """Represents a prepared query to the Cb Defense server. @@ -71,6 +80,7 @@ class Query(PaginatedQuery): - You can chain where clauses together to create AND queries; only objects that match all ``where`` clauses will be returned. """ + def __init__(self, doc_class, cb, query=None): super(Query, self).__init__(doc_class, cb, None) if query: @@ -164,7 +174,7 @@ def _search(self, start=0, rows=0): still_querying = False break - args['start'] = current + 1 # as of 6/2017, the indexing on the Cb Defense backend is still 1-based + args['start'] = current + 1 # as of 6/2017, the indexing on the Cb Defense backend is still 1-based if current >= self._total_results: break From 8356ac8982b077c59e20d4dbbea3c24603836e9a Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Wed, 18 Nov 2020 16:45:43 -0500 Subject: [PATCH 147/197] Wed Nov 18 16:45:43 EST 2020 VMW-71 Added API reference --- src/cbapi/psc/defense/rest_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/psc/defense/rest_api.py b/src/cbapi/psc/defense/rest_api.py index 113ab96b..37a628a2 100644 --- a/src/cbapi/psc/defense/rest_api.py +++ b/src/cbapi/psc/defense/rest_api.py @@ -51,7 +51,7 @@ def get_notifications(self): def get_auditlogs(self): """Retrieve queued audit logs from the Carbon Black Cloud Endpoint Standard server. - + Note that this can only be used with a 'API' key generated in the CBC console. :returns: list of dictionary objects representing the audit logs, or an empty list if none available. """ res = self.get_object("/integrationServices/v3/auditlogs") From 8aa0a91483e7d607f8c8d9fabfb286cf4c4c0183 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Jan 2021 23:21:35 +0000 Subject: [PATCH 148/197] Bump lxml in /examples/threathunter/threat_intelligence Bumps [lxml](https://github.com/lxml/lxml) from 4.4.1 to 4.6.2. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.4.1...lxml-4.6.2) Signed-off-by: dependabot[bot] --- examples/threathunter/threat_intelligence/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/threathunter/threat_intelligence/requirements.txt b/examples/threathunter/threat_intelligence/requirements.txt index 94b0c32b..19710ea8 100644 --- a/examples/threathunter/threat_intelligence/requirements.txt +++ b/examples/threathunter/threat_intelligence/requirements.txt @@ -2,7 +2,7 @@ cybox==2.1.0.18 dataclasses>=0.6 cabby==0.1.20 stix==1.2.0.7 -lxml==4.4.1 +lxml==4.6.2 urllib3>=1.24.2 cbapi>=1.5.6 python_dateutil==2.8.1 From 5f1257df94254e017eb8a1edb825422490b3465c Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Wed, 13 Jan 2021 11:59:06 -0700 Subject: [PATCH 149/197] Add notice to readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f2e284e5..be5bb47a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ **Latest Version: 1.7.2** +_Notice: The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI_ + These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. From 4490d40664f3fbc7b21571175a88b01db15f4507 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Wed, 13 Jan 2021 12:02:11 -0700 Subject: [PATCH 150/197] Bold notice --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index be5bb47a..c41e778c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **Latest Version: 1.7.2** -_Notice: The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI_ +_**Notice**: The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI_ These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. From f7d5e3bd20f97114ffafe9111289d00d514e10eb Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Thu, 14 Jan 2021 09:36:30 -0700 Subject: [PATCH 151/197] Remove new --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c41e778c..2cd5f6cb 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ _**Notice**: The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI_ -These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. +These are the Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. Please visit https://cbapi.readthedocs.io for detailed documentation on this API. Additionally, we have a slideshow From 60ad1a9140628f64f764818f0a207ac5a9f45c5b Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Fri, 15 Jan 2021 13:53:21 -0700 Subject: [PATCH 152/197] Version Bump --- LICENSE | 2 +- README.md | 2 +- docs/changelog.rst | 23 +++++++++++++++++++++++ docs/conf.py | 4 ++-- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 6 files changed, 29 insertions(+), 6 deletions(-) diff --git a/LICENSE b/LICENSE index 2b532005..89ad6804 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2016-2019 Carbon Black +Copyright (c) 2016-2021 Carbon Black Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.md b/README.md index 2cd5f6cb..9dc35ede 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.7.2** +**Latest Version: 1.7.3** _**Notice**: The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI_ diff --git a/docs/changelog.rst b/docs/changelog.rst index 352872b2..934826d9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,29 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.7.3 - Released January 15, 2021 +------------------------------------ + +Updates + +* General + * Fix resource warnings regarding unclosed file object + * Notice added to readme for Carbon Black Cloud features moving to Carbon Black Cloud SDK repo +* Carbon Black Cloud + * Increase default rows of alerts to 100 + * Add get_auditlogs function to API object +* CB Threathunter + * Fix typo in process query + * Bump lxml from 4.4.1 to 4.6.2 for Threat Intelligence example +* CB Response + * Add Sensor Builds + * Alert.set_ignored() and AlertQuery.set_ignored(): + * Added a docstring to specify what happens with this method + * Modified the payload keys based on manual testing + * Alert.change_status() and AlertQuery.change_status(): + * Added a status check to ensure it's a valid status + + CbAPI 1.7.2 - Released July 22, 2020 ------------------------------------ diff --git a/docs/conf.py b/docs/conf.py index dc63206e..8d948821 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,7 @@ # General information about the project. project = u'cbapi' -copyright = u'2016-2020, VMware Carbon Black' +copyright = u'2016-2021, VMware Carbon Black' author = u'Carbon Black Developer Network' # The version info for the project you're documenting, acts as replacement for @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.7.2' +release = u'1.7.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 3a327dac..372e2129 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( name='cbapi', - version='1.7.2', + version='1.7.3', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index d6337391..b4a9b4b9 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2020 VMware Carbon Black' -__version__ = '1.7.2' +__version__ = '1.7.3' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From 85c5f8c30407436f81609bdf0e968810d135b384 Mon Sep 17 00:00:00 2001 From: Chris Kuethe Date: Wed, 3 Feb 2021 13:51:48 -0800 Subject: [PATCH 153/197] make cbapi-python accept a pre-configured requests.Session() Some environment have highly restrictive outbound network filters and require uncommon proxy configurations, perhaps requiring certificate authentication or some library preloads. This diff allows users to set up an appropriately configured Session() to be used for access to the CB API. Example: ``` # do whatever is necessary to get a requests Session that # can access the internet from my_special_network_library import internet_proxy my_proxy = internet_proxy() # quickly check that this proxy is acceptable assert isinstance(my_proxy, requests.sessions.Session) assert my_proxy.get('http://www.example.com').ok # use the proxy session in the CB API from cbapi.response import CbEnterpriseResponseAPI, Sensor, SensorGroup cb = CbEnterpriseResponseAPI(proxy_session=my_proxy) ``` signed-off-by: chris.kuethe+github@gmail.com --- docs/index.rst | 5 +++++ src/cbapi/connection.py | 10 +++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 2dc0f45a..df0fa93f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -191,6 +191,11 @@ configuration file format specified above. The optional `CBAPI_SSL_VERIFY` envar can be used to control SSL validation(True/False or 0/1), which will default to ON when not explicitly set by the user. +For environments where complex outbound network filters and proxy configurations are used (eg. anything other than +an unauthenticated or basic password authenticated proxy) a prepared `requests.Session` object may be supplied as a +`proxy_session` parameter. This session will then be used for all communication with the API. Construction of such a +`Session` is beyond the scope of this document, consult your local network/security administrators for assistance. + Backwards & Forwards Compatibility ---------------------------------- diff --git a/src/cbapi/connection.py b/src/cbapi/connection.py index ad83f256..53c59243 100644 --- a/src/cbapi/connection.py +++ b/src/cbapi/connection.py @@ -143,7 +143,7 @@ def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool class Connection(object): """Object that encapsulates the HTTP connection to the CB server.""" - def __init__(self, credentials, integration_name=None, timeout=None, max_retries=None, **pool_kwargs): + def __init__(self, credentials, integration_name=None, timeout=None, max_retries=None, proxy_session=None, **pool_kwargs): """ Initialize the Connection object. @@ -184,7 +184,10 @@ def __init__(self, credentials, integration_name=None, timeout=None, max_retries self.token = credentials.token self.token_header = {'X-Auth-Token': self.token, 'User-Agent': user_agent} - self.session = requests.Session() + if proxy_session: + self.session = proxy_session + else: + self.session = requests.Session() self._timeout = timeout @@ -371,12 +374,13 @@ def __init__(self, *args, **kwargs): timeout = kwargs.pop("timeout", DEFAULT_POOL_TIMEOUT) max_retries = kwargs.pop("max_retries", DEFAULT_RETRIES) + proxy_session = kwargs.pop("proxy_session", None) pool_connections = kwargs.pop("pool_connections", 1) pool_maxsize = kwargs.pop("pool_maxsize", DEFAULT_POOLSIZE) pool_block = kwargs.pop("pool_block", DEFAULT_POOLBLOCK) self.session = Connection(self.credentials, integration_name=integration_name, timeout=timeout, - max_retries=max_retries, pool_connections=pool_connections, + max_retries=max_retries, proxy_session=proxy_session, pool_connections=pool_connections, pool_maxsize=pool_maxsize, pool_block=pool_block) def raise_unless_json(self, ret, expected): From 4a3e7eb64a5c0bcf53b85247f5eab47a3130f932 Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Thu, 18 Mar 2021 21:37:57 +0200 Subject: [PATCH 154/197] Fixing issues: 140, 272, 285, EA-17106 --- docs/response-examples.rst | 2 +- examples/response/sensor_group_operations.py | 7 +- src/cbapi/response/models.py | 3 +- src/cbapi/response/models/group-modify.yaml | 68 ++++++++++++++++++-- 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/docs/response-examples.rst b/docs/response-examples.rst index 6f39a207..40bd004e 100644 --- a/docs/response-examples.rst +++ b/docs/response-examples.rst @@ -371,7 +371,7 @@ pulls the most common process names for our sample host:: >>> def print_facet_histogram(facets): ... for entry in facets: - ... print("%15s: %5s%% %s" % (entry["name"][:15], entry["ratio"], u"\u25A0"*(int(entry["percent"])/2))) + ... print("%15s: %5s%% %s" % (entry["name"][:15], entry["ratio"], u"\u25A0"*int((entry["percent"])/2))) ... >>> facet_query = cb.select(Process).where("hostname:WIN-IA9NQ1GN8OI").and_("username:bit9rad") diff --git a/examples/response/sensor_group_operations.py b/examples/response/sensor_group_operations.py index 46798e53..d3af35e0 100755 --- a/examples/response/sensor_group_operations.py +++ b/examples/response/sensor_group_operations.py @@ -26,7 +26,9 @@ def list_sensors(cb, parser, args): print(" {0:40}{1:18}{2}".format("Hostname", "IP Address", "Last Checkin Time")) for sensor in group.sensors: ipaddrs = [iface.ipaddr for iface in sensor.network_interfaces if iface.ipaddr not in ("127.0.0.1", "0.0.0.0")] - print(" {0:40}{1:18}{2}".format(sensor.hostname, ipaddrs[0], sensor.last_checkin_time)) + print(" {0:40}{1:18}{2}".format(sensor.hostname, + ipaddrs[0] if len(ipaddrs) else '', + sensor.last_checkin_time)) def add_sensor_group(cb, parser, args): @@ -48,6 +50,7 @@ def add_sensor_group(cb, parser, args): g.name = args.new_group_name g.site = site + g.sensorbackend_server = args.sensorbackend_server try: g.save() @@ -101,6 +104,8 @@ def main(): site_group = add_command.add_mutually_exclusive_group(required=False) site_group.add_argument("-s", "--site", action="store", help="Site name", dest="site_name") site_group.add_argument("-i", "--site-id", action="store", help="Site ID", dest="site_id") + add_command.add_argument("-b", "--sensorbackend-server", action="store", help="Sensor backend server", required=True, + dest="sensorbackend_server") del_command = commands.add_parser("delete", help="Delete sensor groups") del_sensor_group_specifier = del_command.add_mutually_exclusive_group(required=True) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 007b8857..da4c7159 100755 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -297,6 +297,7 @@ def refresh(self): else: self._info = result[0] self._last_refresh_time = time.time() + self._full_init = True return True def set_ignored(self, ignored_flag=True, status="False Positive"): @@ -907,7 +908,7 @@ def _update_object(self): class SensorGroup(MutableBaseModel, CreatableModelMixin): swagger_meta_file = "response/models/group-modify.yaml" - urlobject = '/api/v2/group' + urlobject = '/api/v3/group' @classmethod def _query_implementation(cls, cb): diff --git a/src/cbapi/response/models/group-modify.yaml b/src/cbapi/response/models/group-modify.yaml index ff9bae74..200f5bbc 100644 --- a/src/cbapi/response/models/group-modify.yaml +++ b/src/cbapi/response/models/group-modify.yaml @@ -2,6 +2,7 @@ type: "object" required: - name - site_id + - sensorbackend_server properties: id: type: "integer" @@ -50,10 +51,6 @@ properties: format: "int32" description: "Set the tamper level, default 0" default: "0" - sensor_version: - type: "string" - description: "The version of the sensor in this group, default Manual" - default: "Manual" sensor_version_windows: type: "string" description: "The version of the sensor for Windows hosts in this group, default Manual" @@ -78,6 +75,10 @@ properties: type: "boolean" description: "Enable/Disable banning, default true" default: "true" + collect_amsi: + type: "boolean" + description: "Enable/Disable amsi collection, default true" + default: "false" collect_emet_events: type: "boolean" description: "Enable/Disable emet events collection, default true" @@ -130,6 +131,9 @@ properties: type: "boolean" description: "Enable/Disable filter known dll, default false" default: "false" + number_of_hosts: + type: "integer" + description: "Number of hosts" process_filter_level: type: "integer" format: "int32" @@ -155,6 +159,62 @@ properties: format: "int32" description: "Percent of the storefile, default 1" default: "1" + server_cert_id: + type: "integer" + description: "Server certificate ID to use" + default: "2" + isolation_exclusions: + type: "array" + description: "isolation exclusions" + items: + title: "modIsolationExclusions" + type: "object" + properties: + id: + type: "integer" + description: "id of exclusion" + group_id: + type: "integer" + description: "ID of groupy to add exclusions to" + name: + type: "string" + description: "name of exclusions" + ip_address: + type: "string" + description: "IP address of exclusion" + enabled: + type: "boolean" + description: "status of exclusion" + uploader_id: + type: "integer" + description: "id of uploader" + added_time: + type: "string" + format: "iso-date-time" + description: "time when added" + changed_time: + type: "string" + format: "iso-date-time" + description: "time when changed" + uploader_username: + type: "string" + description: "username of uploader" + event_exclusions: + type: "array" + description: "Configured set of events to exclude from event collection for a group of sensors" + items: + type: "object" + properties: + eventsToIgnore: + type: "array" + description: "Set of event strings, one or more of 'crossproc', 'filemod', 'filemod-non-binary', 'modload', 'netconn', 'process-info' or 'regmod'" + items: + type: "string" + paths: + type: "array" + description: 'Set of path strings (e.g. "c:\\Windows\\system32\\regsvr32.exe")' + items: + type: "string" vdi_enabled: type: "boolean" description: "Enable/Disable vdi, default false" From c3d2fa0bc82ac6e7479d5d453d660dc540c4fce0 Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Fri, 19 Mar 2021 08:50:20 +0200 Subject: [PATCH 155/197] Fixing EA-18370 by manually setting the value of sensor_version_xx. --- examples/response/sensor_group_operations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/response/sensor_group_operations.py b/examples/response/sensor_group_operations.py index d3af35e0..bedf9cae 100755 --- a/examples/response/sensor_group_operations.py +++ b/examples/response/sensor_group_operations.py @@ -51,6 +51,7 @@ def add_sensor_group(cb, parser, args): g.name = args.new_group_name g.site = site g.sensorbackend_server = args.sensorbackend_server + g.sensor_version_windows = g.sensor_version_linux = g.sensor_version_osx = 'Manual' try: g.save() From 16e91c895c0bd89483b2a81fec1e54028f07b7b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Mar 2021 23:21:24 +0000 Subject: [PATCH 156/197] Bump pyyaml in /examples/threathunter/threat_intelligence Bumps [pyyaml](https://github.com/yaml/pyyaml) from 5.1.2 to 5.4. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/5.1.2...5.4) Signed-off-by: dependabot[bot] --- examples/threathunter/threat_intelligence/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/threathunter/threat_intelligence/requirements.txt b/examples/threathunter/threat_intelligence/requirements.txt index 19710ea8..d360638c 100644 --- a/examples/threathunter/threat_intelligence/requirements.txt +++ b/examples/threathunter/threat_intelligence/requirements.txt @@ -6,5 +6,5 @@ lxml==4.6.2 urllib3>=1.24.2 cbapi>=1.5.6 python_dateutil==2.8.1 -PyYAML==5.1.2 +PyYAML==5.4 schema From 52d0de8893ec08f719e6fc1520f4ccd769eae2fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 Mar 2021 19:34:37 +0000 Subject: [PATCH 157/197] Bump lxml in /examples/threathunter/threat_intelligence Bumps [lxml](https://github.com/lxml/lxml) from 4.6.2 to 4.6.3. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.2...lxml-4.6.3) Signed-off-by: dependabot[bot] --- examples/threathunter/threat_intelligence/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/threathunter/threat_intelligence/requirements.txt b/examples/threathunter/threat_intelligence/requirements.txt index 19710ea8..72570994 100644 --- a/examples/threathunter/threat_intelligence/requirements.txt +++ b/examples/threathunter/threat_intelligence/requirements.txt @@ -2,7 +2,7 @@ cybox==2.1.0.18 dataclasses>=0.6 cabby==0.1.20 stix==1.2.0.7 -lxml==4.6.2 +lxml==4.6.3 urllib3>=1.24.2 cbapi>=1.5.6 python_dateutil==2.8.1 From 487a23ea8b45a4338280c1d73ed0c8fc9848d3c6 Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Fri, 2 Apr 2021 21:22:45 +0300 Subject: [PATCH 158/197] CBAPI-2348: API doesn't allow user to find blocked executions of banned hashes (CBAPI for EDR) --- src/cbapi/response/models.py | 62 ++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index da4c7159..42150bdb 100755 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -2160,6 +2160,22 @@ def _lookup_privilege(privilege_code): return CbCrossProcEvent(self.process_model, timestamp, seq, new_crossproc) + def parse_processblock(self, seq, processblock): + processblock_json = json.loads(processblock) + new_processblock = {} + timestamp = processblock_json.get("timestamp", None) + new_processblock["type"] = processblock_json.get("block_type", None) + new_processblock["event"] = processblock_json.get("block_event", None) + new_processblock["result"] = processblock_json.get("block_result", None) + new_processblock["error"] = processblock_json.get("block_error", None) + new_processblock["md5"] = processblock_json.get("blocked_md5", None) + new_processblock["path"] = processblock_json.get("blocked_path", None) + new_processblock["cmdline"] = processblock_json.get("blocked_cmdline", None) + new_processblock["uid"] = processblock_json.get("blocked_uid", None) + new_processblock["username"] = processblock_json.get("blocked_username", None) + + return CbProcessBlockEvent(self.process_model, timestamp, seq, new_processblock) + class ProcessV2Parser(ProcessV1Parser): def __init__(self, process_model): @@ -2541,7 +2557,7 @@ def end(self): def require_events(self): event_key_list = ['filemod_complete', 'regmod_complete', 'modload_complete', 'netconn_complete', - 'crossproc_complete', 'childproc_complete'] + 'crossproc_complete', 'childproc_complete', 'processblock_complete'] if not self.valid_process or self.suppressed_process: return @@ -2579,6 +2595,18 @@ def refresh(self): self._segments = [] super(Process, self).refresh() + @property + def processblocks(self): + """ + Generator that returns :py:class:`CbProcessBlockEvent` objects associated with this process + """ + self.require_events() + + i = 0 + for raw_processblock in self._events.get(self.current_segment, {}).get('processblock_complete', []): + yield self._event_parser.parse_processblock(i, raw_processblock) + i += 1 + @property def segment(self): log.debug("The .segment attribute will be deprecated in future versions of the cbapi Python module.") @@ -3000,7 +3028,8 @@ def require_all_events(self): 'modload_count': 'modload_complete', 'netconn_count': 'netconn_complete', 'crossproc_count': 'crossproc_complete', - 'childproc_count': 'childproc_complete' + 'childproc_count': 'childproc_complete', + 'processblock_count': 'processblock_complete' } if not self.valid_process or self.suppressed_process: @@ -3100,6 +3129,28 @@ def all_filemods(self): yield self._event_parser.parse_filemod(i, raw_filemod) i += 1 + def all_processblocks(self): + if self._cb.cb_server_version < LooseVersion('6.0.0'): + self.get_segments() + segments = self._segments + + i = 0 + for segment in segments: + self.current_segment = segment + self.require_events() + + for raw_processblock in self._events.get(self.current_segment, {}).get('processblock_complete', []): + yield self._event_parser.parse_processblock(i, raw_processblock) + i += 1 + else: + if not self.all_events_loaded: + self.require_all_events() + + i = 0 + for raw_processblock in self._events.get('all', {}).get('processblock_complete', []): + yield self._event_parser.parse_processblock(i, raw_processblock) + i += 1 + def all_regmods(self): if self._cb.cb_server_version < LooseVersion('6.0.0'): self.get_segments() @@ -3267,6 +3318,13 @@ def __init__(self, parent_process, timestamp, sequence, event_data, version=1): self.stat_titles.extend(['local_ip', 'local_port', 'proxy_ip', 'proxy_port']) +class CbProcessBlockEvent(CbEvent): + def __init__(self, parent_process, timestamp, sequence, event_data): + super(CbProcessBlockEvent, self).__init__(parent_process, timestamp, sequence, event_data) + self.event_type = u'Cb Process Block event' + self.stat_titles.extend(['type', 'event', 'result', 'error', 'md5', 'path', 'uid', 'username']) + + class CbChildProcEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data, is_suppressed=False, proc_data=None, max_children=15): super(CbChildProcEvent, self).__init__(parent_process, timestamp, sequence, event_data) From 8045cffa4eadaf86fd50bca022bda6555f02b359 Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Wed, 7 Apr 2021 18:23:58 +0300 Subject: [PATCH 159/197] Version bump --- README.md | 2 +- docs/changelog.rst | 14 ++++++++++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9dc35ede..ac28ac9c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.7.3** +**Latest Version: 1.7.4** _**Notice**: The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI_ diff --git a/docs/changelog.rst b/docs/changelog.rst index 934826d9..0f7cc362 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,20 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.7.4 - Released April 7, 2021 +------------------------------------ + +Updates + +* General + * Fix example code in the documentation for Facets +* CB Response + * Add missing fields for SensorGroup class and fix example script to properly create SensorGroup + * Fix example script sensor_group_operations.py to list groups without ipaddresses + * Fix alert.save() + * Allow blocked processes to be accessed through the Process (processblocks) + + CbAPI 1.7.3 - Released January 15, 2021 ------------------------------------ diff --git a/docs/conf.py b/docs/conf.py index 8d948821..d87119b6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.7.3' +release = u'1.7.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 372e2129..83a9147e 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( name='cbapi', - version='1.7.3', + version='1.7.4', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index b4a9b4b9..618958c7 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2020 VMware Carbon Black' -__version__ = '1.7.3' +__version__ = '1.7.4' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From 3297f2b9977384af87012e346bc39f892ab06e31 Mon Sep 17 00:00:00 2001 From: Kylie Ebringer <55465092+kebringer-cb@users.noreply.github.com> Date: Wed, 14 Apr 2021 16:41:37 -0600 Subject: [PATCH 160/197] Update changelog.rst fixed bullet points --- docs/changelog.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0f7cc362..bf86cdd2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,12 +8,12 @@ CbAPI 1.7.4 - Released April 7, 2021 Updates * General - * Fix example code in the documentation for Facets + * Fix example code in the documentation for Facets * CB Response - * Add missing fields for SensorGroup class and fix example script to properly create SensorGroup - * Fix example script sensor_group_operations.py to list groups without ipaddresses - * Fix alert.save() - * Allow blocked processes to be accessed through the Process (processblocks) + * Add missing fields for SensorGroup class and fix example script to properly create SensorGroup + * Fix example script sensor_group_operations.py to list groups without ipaddresses + * Fix alert.save() + * Allow blocked processes to be accessed through the Process (processblocks) CbAPI 1.7.3 - Released January 15, 2021 From 0a12dddc7add489d0f95338747f8827209e92ba4 Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Mon, 14 Jun 2021 20:43:28 +0300 Subject: [PATCH 161/197] [CBAPI-2626] Include credentials.use_custom_proxy_session --- src/cbapi/connection.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cbapi/connection.py b/src/cbapi/connection.py index 53c59243..523f1101 100644 --- a/src/cbapi/connection.py +++ b/src/cbapi/connection.py @@ -152,6 +152,7 @@ def __init__(self, credentials, integration_name=None, timeout=None, max_retries integration_name (str): The integration name being used. timeout (int): The timeout value to use for HTTP requests on this connection. max_retries (int): The maximum number of times to retry a request. + proxy_session (requests.Session) custom session to be used **pool_kwargs: Additional arguments to be used to initialize connection pooling. Raises: @@ -186,8 +187,10 @@ def __init__(self, credentials, integration_name=None, timeout=None, max_retries self.token_header = {'X-Auth-Token': self.token, 'User-Agent': user_agent} if proxy_session: self.session = proxy_session + credentials.use_custom_proxy_session = True else: self.session = requests.Session() + credentials.use_custom_proxy_session = False self._timeout = timeout @@ -207,7 +210,10 @@ def __init__(self, credentials, integration_name=None, timeout=None, max_retries self.session.mount(self.server, tls_adapter) self.proxies = {} - if credentials.ignore_system_proxy: # see https://github.com/kennethreitz/requests/issues/879 + if credentials.use_custom_proxy_session: + # get the custom session proxies + self.proxies = self.session.proxies + elif credentials.ignore_system_proxy: # see https://github.com/kennethreitz/requests/issues/879 # Unfortunately, requests will look for any proxy-related environment variables and use those anyway. The # only way to solve this without side effects, is passing in empty strings for 'http' and 'https': self.proxies = { From dbaa3ed25bf274f1eb123e4355f5605b9f30956b Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Wed, 16 Jun 2021 13:15:39 +0300 Subject: [PATCH 162/197] version bump for release 1.7.5 --- README.md | 2 +- docs/changelog.rst | 8 ++++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ac28ac9c..fe3819f0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.7.4** +**Latest Version: 1.7.5** _**Notice**: The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI_ diff --git a/docs/changelog.rst b/docs/changelog.rst index bf86cdd2..b7c492be 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.7.5 - Released June 16, 2021 +------------------------------------ + +Updates + +* General + * Allow the CbAPI to accept a pre-configured Session object to be used for access, to get around unusual configuration requirements. + CbAPI 1.7.4 - Released April 7, 2021 ------------------------------------ diff --git a/docs/conf.py b/docs/conf.py index d87119b6..3fdb0dc7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.7.4' +release = u'1.7.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 83a9147e..6f353565 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( name='cbapi', - version='1.7.4', + version='1.7.5', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index 618958c7..26373177 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2020 VMware Carbon Black' -__version__ = '1.7.4' +__version__ = '1.7.5' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From b3f9e073ea3f8a82cbf22500ce8cdb081186a602 Mon Sep 17 00:00:00 2001 From: Kylie Ebringer Date: Fri, 10 Dec 2021 13:33:02 -0700 Subject: [PATCH 163/197] Updated product names in 'docs' directory Updated Protection to App Control Updated Response to EDR --- README.md | 50 ++++++++++++++++++-------------------- docs/changelog.rst | 20 +++++++++------ docs/concepts.rst | 26 ++++++++++---------- docs/getting-started.rst | 8 +++--- docs/index.rst | 10 ++++---- docs/installation.rst | 7 +++--- docs/live-response.rst | 2 +- docs/protection-api.rst | 10 ++++---- docs/response-api.rst | 8 +++--- docs/response-examples.rst | 50 +++++++++++++++++++------------------- 10 files changed, 97 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index fe3819f0..cc19a97e 100644 --- a/README.md +++ b/README.md @@ -2,24 +2,25 @@ **Latest Version: 1.7.5** -_**Notice**: The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI_ +_**Notice**:_ +* The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI +* Carbon Black EDR (Endpoint Detection and Response) is the new name for the product formerly called CB Response. +* Carbon Black App Control is the new name for the product formerly called CB Protection. -These are the Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. +These are the Python bindings for the Carbon Black EDR and App Control REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. -Please visit https://cbapi.readthedocs.io for detailed documentation on this API. Additionally, we have a slideshow -available at https://developer.carbonblack.com/2016/07/presentation-on-the-new-carbon-black-python-api/ that provides -an overview of the concepts that underly this API binding. +Please visit https://cbapi.readthedocs.io for detailed documentation on this API. ## Support 1. View all API and integration offerings on the [Developer Network](https://developer.carbonblack.com/) along with reference documentation, video tutorials, and how-to guides. 2. Use the [Developer Community Forum](https://community.carbonblack.com/t5/Developer-Relations/bd-p/developer-relations) to discuss issues and get answers from other API developers in the Carbon Black Community. -3. Report bugs and change requests to [Carbon Black Support](http://carbonblack.com/resources/support/). +3. Report bugs and change requests to [Carbon Black Support](https://www.vmware.com/support/services.html). ## Requirements -The new cbapi is designed to work on Python 2.6.6 and above (including 3.x). If you're just starting out, +The cbapi is designed to work on Python 2.6.6 and above (including 3.x). If you're just starting out, we recommend using the latest version of Python 3.6.x or above. All requirements are installed as part of `pip install`. @@ -28,11 +29,10 @@ The legacy cbapi (`cbapi.CbApi`) and legacy bit9api (`cbapi.bit9Api`) are still ## Backwards Compatibility Backwards compatibility with old scripts is maintained through the `cbapi.legacy` module. Old scripts that import -`cbapi.CbApi` directly will continue to work. Once cbapi 2.0.0 is released, the old `CbApi` will be deprecated and -removed entirely no earlier than January 2017. +`cbapi.CbApi` directly will continue to work. -New scripts should use the `cbapi.CbResponseAPI` (for CB Response) and -`cbapi.CbProtectionAPI` (for CB Protection / former Bit9) API entry points. +New scripts should use the `cbapi.CbResponseAPI` (for EDR (CB Response)) and +`cbapi.CbProtectionAPI` (for App Control (CB Protection)) API entry points. ## Getting Started @@ -49,10 +49,10 @@ Clone this repository, cd into `cbapi-python` then run setup.py with the `develo ### Sample Code -There are several examples in the `examples` directory for both Carbon Black Enterprise Response and Protection. We -will be adding more samples over time. For a quick start, see the following code snippets: +There are several examples in the `examples` directory for both EDR and App Control. +For a quick start, see the following code snippets: -**Carbon Black Enterprise Response** +**Carbon Black EDR** from cbapi.response.models import Process, Binary, Sensor, Feed, Watchlist, Investigation from cbapi.response.rest_api import CbEnterpriseResponseAPI @@ -78,7 +78,7 @@ will be adding more samples over time. For a quick start, see the following code s.save() -**Carbon Black Enterprise Protection** +**Carbon Black App Control** from cbapi.protection.models import * from cbapi.protection.rest_api import CbEnterpriseProtectionAPI @@ -100,9 +100,8 @@ will be adding more samples over time. For a quick start, see the following code In order to perform any queries via the API, you will need to get the API token for your CB user. See the documentation on the Developer Network website on how to acquire the API token for -[CB Response](http://developer.carbonblack.com/reference/enterprise-response/authentication/), -[CB Protection](http://developer.carbonblack.com/reference/enterprise-protection/authentication/), or -[CB Defense](http://developer.carbonblack.com/reference/cb-defense/authentication/). +[CB Response](http://developer.carbonblack.com/reference/enterprise-response/authentication/) or +[CB Protection](http://developer.carbonblack.com/reference/enterprise-protection/authentication/). Once you acquire your API token, place it in one of the default credentials file locations: @@ -112,15 +111,14 @@ Once you acquire your API token, place it in one of the default credentials file For distinction between credentials of different Carbon Black products, use the following naming convention for your credentials files: -* ``credentials.psc`` for CB Defense, CB ThreatHunter, and CB LiveOps -* ``credentials.response`` for CB Response -* ``credentials.protection`` for CB Protection +* ``credentials.response`` for EDR (CB Response) +* ``credentials.protection`` for App Control (CB Protection) For example, if you use a Carbon Black Cloud product, you should have created a credentials file in one of these locations: -* ``/etc/carbonblack/credentials.psc`` -* ``~/.carbonblack/credentials.psc`` -* ``/current_working_directory/.carbonblack/credentials.psc`` +* ``/etc/carbonblack/credentials.response`` +* ``~/.carbonblack/credentials.response`` +* ``/current_working_directory/.carbonblack/credentials.response`` Credentials found in a later path will overwrite earlier ones. @@ -144,7 +142,7 @@ by key-value pairs providing the necessary credential information:: The possible options for each credential profile are: -* **url**: The base URL of the CB server. This should include the protocol (https) and the hostname, and nothing else. +* **url**: The base URL of the Carbon Black server. This should include the protocol (https) and the hostname, and nothing else. * **token**: The API token for the user ID. More than one credential profile can be specified for a given server, with different tokens for each. * **ssl_verify**: True or False; controls whether the SSL/TLS certificate presented by the server is validated against @@ -156,5 +154,5 @@ The possible options for each credential profile are: * **ignore_system_proxy**: If you have a system-wide proxy specified, setting this to True will force cbapi to bypass the proxy and directly connect to the CB server. -Future versions of cbapi will also provide the ability to "pin" the TLS certificate so as to provide certificate +Future versions of cbapi may provide the ability to "pin" the TLS certificate so as to provide certificate verification on self-signed or internal CA signed certificates. diff --git a/docs/changelog.rst b/docs/changelog.rst index b7c492be..a6369ca1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,13 +2,19 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +Updates Dec 13, 2021 +------------------------------------ + +Updates to documentation for product names and terminology. No coding changes. + + CbAPI 1.7.5 - Released June 16, 2021 ------------------------------------ Updates * General - * Allow the CbAPI to accept a pre-configured Session object to be used for access, to get around unusual configuration requirements. + * Allow the CbAPI to accept a pre-configured Session object to be used for access, to get around unusual configuration requirements. CbAPI 1.7.4 - Released April 7, 2021 ------------------------------------ @@ -17,7 +23,7 @@ Updates * General * Fix example code in the documentation for Facets -* CB Response +* EDR (CB Response) * Add missing fields for SensorGroup class and fix example script to properly create SensorGroup * Fix example script sensor_group_operations.py to list groups without ipaddresses * Fix alert.save() @@ -38,7 +44,7 @@ Updates * CB Threathunter * Fix typo in process query * Bump lxml from 4.4.1 to 4.6.2 for Threat Intelligence example -* CB Response +* EDR (CB Response) * Add Sensor Builds * Alert.set_ignored() and AlertQuery.set_ignored(): * Added a docstring to specify what happens with this method @@ -65,7 +71,7 @@ Updates * Documentation updates to indicate changed product names * Carbon Black Cloud * Process Search v2 rows defaults to 10k to match UI behavior -* CB Response +* EDR (CB Response) * Add support for fetching alert by ID @@ -358,7 +364,7 @@ CB Response: You can also use the special segment "0" to retrieve process events across all segments. * Fix ``cmdline_filters`` in the ``IngressFilter`` model object. -CB Protection: +App Control (CB Protection): * Tamper Protection can now be set and cleared in the ``Computer`` model object. @@ -378,7 +384,7 @@ Security fix: Bug fixes: * Add rule filename parameter to CB Defense ``policy_operations.py`` script's ``add-rule`` command. -* Add support for ``tamperProtectionActive`` attribute to CB Protection's ``Computer`` object. +* Add support for ``tamperProtectionActive`` attribute to App Control's (CB Protection) ``Computer`` object. * Work around CB Response issue- the ``/api/v1/sensor`` route incorrectly returns an HTTP 500 if no sensors match the provided query. CbAPI now catches this exception and will instead return an empty set back to the caller. @@ -401,7 +407,7 @@ Security changes: * Add the ability to "pin" a specific server certificate to a credential profile. - * You can now force TLS certificate verification on self-signed, on-premise installations of CB Response or Protection + * You can now force TLS certificate verification on self-signed, on-premise installations of EDR (CB Response) or App Control (Protection) through the ``ssl_cert_file`` option in the credential profile. * To "pin" a server certificate, save the PEM-formatted server certificate to a file, and put the full path to that PEM file in the ``ssl_cert_file`` option of that server's credential profile. diff --git a/docs/concepts.rst b/docs/concepts.rst index b8ef6855..051e8796 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -5,7 +5,7 @@ There are a few critical concepts that will make understanding and using the cba explained below, and also covered in a slide deck presented at the Carbon Black regional User Exchanges in 2016. You can see the slide deck `here `_. -At a high level, the cbapi tries to represent data in CB Response or CB Protection as Python objects. If you've worked +At a high level, the cbapi tries to represent data in EDR (CB Response) or App Control (CB Protection) as Python objects. If you've worked with SQL Object-relational Mapping (ORM) frameworks before, then this structure may seem familiar -- cbapi was designed to operate much like an ORM such as SQLAlchemy or Ruby's ActiveRecord. If you haven't worked with one of these libraries, don't worry! The concepts will become clear after a little practice. @@ -14,9 +14,9 @@ Model Objects ------------- Everything in cbapi is represented in terms of "Model Objects". A Model Object in cbapi represents a single instance -of a specific type of data in CB Response or Protection. For example, a process document from CB Response (as seen +of a specific type of data in EEDR (CB Response) or App Control (CB Protection). For example, a process document from EDR (as seen on an Analyze Process page in the Web UI) is represented as a :py:mod:`cbapi.response.models.Process` Model Object. -Similarly, a file instance in CB Protection is represented as a :py:mod:`cbapi.protection.models.FileInstance` +Similarly, a file instance in App Control (CB Protection) is represented as a :py:mod:`cbapi.protection.models.FileInstance` Model Object. Once you have an instance of a Model Object, you can access all of the data contained within as Python properties. @@ -27,7 +27,7 @@ in the ``cmdline`` property), you would write the code:: This would automatically retrieve the ``cmdline`` attribute of the process and print it out to your screen. -The data in CB Response and Protection may change rapidly, and so a comprehensive list of valid properties is difficult +The data in EDR (CB Response) or App Control (CB Protection) may change rapidly, and so a comprehensive list of valid properties is difficult to keep up-to-date. Therefore, if you are curious what properties are available on a specific Model Object, you can print that Model Object to the screen. It will dump all of the available properties and their current values. For example:: @@ -56,7 +56,7 @@ method to retrieve the property, and return a default value if the property does In summary, Model Objects contain all the data associated with a specific type of API call. In this example, the :py:mod:`cbapi.response.models.Binary` Model Object reflects all the data available via the -``/api/v1/binary`` API route on a CB Response server. +``/api/v1/binary`` API route on an EDR (CB Response) server. Joining Model Objects --------------------- @@ -90,13 +90,13 @@ Queries Now that we've covered how to get data out of a specific Model Object, we now need to learn how to obtain Model Objects in the first place! To do this, we have to create and execute a Query. cbapi Queries use the same query -syntax accepted by CB Response or Protection's APIs, and add a few little helpful features along the way. +syntax accepted by EDR (CB Response) or App Control (CB Protection) APIs, and add a few little helpful features along the way. To create a query in cbapi, use the ``.select()`` method on the CbResponseAPI or CbProtectionAPI object. Pass the Model Object type as a parameter to the ``.select()`` call and optionally add filtering criteria with ``.where()`` clauses. -Let's start with a simple query for CB Response:: +Let's start with a simple query for EDR (CB Response):: >>> from cbapi.response import * >>> cb = CbResponseAPI() @@ -152,15 +152,15 @@ will throw a :py:mod:`MoreThanOneResultError` exception if there are zero or mor second method is ``.first()``, which will return the first result from the result set, or None if there are no results. Every time you access a Query object, it will perform a REST API query to the Carbon Black server. For large result -sets, the results are retrieved in batches- by default, 100 results per API request on CB Response and 1,000 results -per API request on CB Protection. The search queries themselves are not cached, but the resulting Model Objects are. +sets, the results are retrieved in batches- by default, 100 results per API request on EDR (CB Response) and 1,000 results +per API request on App Control (CB Protection). The search queries themselves are not cached, but the resulting Model Objects are. Retrieving Objects by ID ------------------------ Every Model Object (and in fact any object addressable via the REST API) has a unique ID associated with it. If you already have a unique ID for a given Model Object, for example, a Process GUID for CB Response, or a Computer ID -for CB Protection, you can ask cbapi to give you the associated Model Object for that ID by passing that ID to the +for App Control (CB Protection), you can ask cbapi to give you the associated Model Object for that ID by passing that ID to the ``.select()`` call. For example:: >>> binary = cb.select(Binary, "CA4FAFFA957C71C006B59E29DFE3EB8B") @@ -178,8 +178,8 @@ object and if it does not exist, cbapi will throw a :py:mod:`ObjectNotFoundError Creating New Objects -------------------- -The CB Response and Protection REST APIs provide the ability to insert new data under certain circumstances. For -example, the CB Response REST API allows you to insert a new banned hash into its database. Model Objects that +The EDR (CB Response) and App Control (CB Protection) REST APIs provide the ability to insert new data under certain circumstances. For +example, the EDR REST API allows you to insert a new banned hash into its database. Model Objects that represent these data types can be "created" in cbapi by using the ``create()`` method:: >>> bh = cb.create(BannedHash) @@ -199,7 +199,7 @@ exception with a list of the properties that are required and not currently set. Once the ``.save()`` method is called, the appropriate REST API call is made to create the object. The Model Object is then updated to the current state returned by the API, which may include additional data properties initialized -by CB Response or Protection. +by EDR (CB Response) or App Control (CB Protection). Modifying Existing Objects -------------------------- diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 5ce22b15..7561d356 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -10,18 +10,18 @@ on the Developer Network website. API Authentication ------------------ -CB Response and CB Protection use a per-user API secret token to authenticate requests via the API. The API token +EDR (CB Response) and App Control (CB Protection) use a per-user API secret token to authenticate requests via the API. The API token confers the same permissions and authorization as the user it is associated with, so protect the API token with the same care as a password. To learn how to obtain the API token for a user, see the Developer Network website: there you will find instructions -for obtaining an API token for `CB Response `_ -and `CB Protection `_. +for obtaining an API token for `EDR (CB Response) `_ +and `App Control (CB Protection) `_. Once you have the API token, cbapi helps keep your credentials secret by enforcing the use of a credential file. To encourage sharing of scripts across the community while at the same time protecting the security of our customers, cbapi strongly discourages embedding credentials in individual scripts. Instead, you can place credentials for several -CB Response or CB Protection servers inside the API credential file and select which "profile" you would like to use +EDR (CB Response) or App Control (CB Protection) servers inside the API credential file and select which "profile" you would like to use at runtime. To create the initial credential file, a simple-to-use script is provided. Just run the ``cbapi-response``, diff --git a/docs/index.rst b/docs/index.rst index df0fa93f..644141c5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,7 +8,7 @@ cbapi: Carbon Black API for Python Release v\ |release|. -CBAPI provides a straightforward interface to the VMware Carbon Black products: Carbon Black EDR, Carbon Black App Control, and Carbon Black Cloud Endpoint Standard (formerly CB Response, CB Protection, and CB Defense). +CBAPI provides a straightforward interface to the VMware Carbon Black products: Carbon Black EDR, Carbon Black App Control, and Carbon Black Cloud Endpoint Standard(formerly CB Response, CB Protection, and CB Defense). This library provides a Pythonic layer to access the raw power of the REST APIs of all Carbon Black products, making it easier to query data from any platform or on-premise APIs, combine data from multiple API calls, manage all API credentials in one place, and manipulate data as Python objects. Take a look:: >>> from cbapi.response import CbResponseAPI, Process, Binary, Sensor @@ -183,9 +183,9 @@ Environment Variable Support The latest CBAPI for Python supports specifying API credentials in the following three environment variables: -`CBAPI_TOKEN` the envar for holding the CbR/CbP api token or the ConnectorId/APIKEY combination for Endpoint Standard (CB Defense)/Carbon Black Cloud. +`CBAPI_TOKEN` the envar for holding the EDR (CbR) or App Control (CbP) api token or the ConnectorId/APIKEY combination for Endpoint Standard (CB Defense)/Carbon Black Cloud. -The `CBAPI_URL` envar holds the FQDN of the target, a CbR , CBD, or CbD/Carbon Black Cloud server specified just as they are in the +The `CBAPI_URL` envar holds the FQDN of the target, an EDR (CbR), CBD, or CbD/Carbon Black Cloud server specified just as they are in the configuration file format specified above. The optional `CBAPI_SSL_VERIFY` envar can be used to control SSL validation(True/False or 0/1), which will default to ON when @@ -211,8 +211,8 @@ legacy scripts cannot run under Python 3. Once CBAPI 1.0.0 is released, the old :py:mod:`cbapi.legacy.CbApi` will be deprecated and removed entirely no earlier than January 2017. New scripts should use the :py:mod:`cbapi.response.rest_api.CbResponseAPI` -(for Carbon Black EDR, or CB Response), :py:mod:`cbapi.protection.rest_api.CbProtectionAPI` -(for Carbon Black App Control, or CB Protection), or :py:mod:`cbapi.defense.rest_api.CbDefenseAPI` API entry points. +(for Carbon Black EDR (CB Response)), :py:mod:`cbapi.protection.rest_api.CbProtectionAPI` +(for Carbon Black App Control (CB Protection)), or :py:mod:`cbapi.defense.rest_api.CbDefenseAPI` API entry points. The API is frozen as of version 1.0; any changes in the 1.x version branch will be additions/bug fixes only. Breaking changes to the API will increment the major version number (2.x). diff --git a/docs/installation.rst b/docs/installation.rst index 41e79b93..7491f46b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1,12 +1,11 @@ Installation ============ -Before installing cbapi, make sure that you have access to a working CB Response or CB Protection server. The server -can be either on-premise or in the cloud. CB Response clusters are also supported. Once you have access to a working +Before installing cbapi, make sure that you have access to a working EDR (CB Response) or App Control (CB Protection) server. The server +can be either on-premise or in the cloud. EDR (CB Response) clusters are also supported. Once you have access to a working can use the standard Python packaging tools to install cbapi on your local machine. -Feel free to follow along with this document or watch the `Development Environment Setup video `_ -on the Developer Network website. +Documentation is also available on the `Developer Network `. If you already have Python installed, you can skip right down to "Using Pip". diff --git a/docs/live-response.rst b/docs/live-response.rst index 7c657561..3da7e256 100644 --- a/docs/live-response.rst +++ b/docs/live-response.rst @@ -30,7 +30,7 @@ Since the Live Response API is synchronous, the script will not continue until e established and the file contents are retrieved, or an exception occurs (in this case, either a timeout error or an error reading the file). -As seen in the example above, the ``.lr_session()`` method is context-aware. CB Response has a limited number of +As seen in the example above, the ``.lr_session()`` method is context-aware. EDR (CB Response) has a limited number of concurrent Live Response session slots (by default, only ten). By wrapping the ``.lr_session()`` call within a ``with`` context, the session is automatically closed at the end of the block and frees that slot for another concurrent Live Response session in another script or user context. diff --git a/docs/protection-api.rst b/docs/protection-api.rst index d5631e3c..b7dd44bf 100644 --- a/docs/protection-api.rst +++ b/docs/protection-api.rst @@ -1,15 +1,15 @@ .. _protection_api: -CB Protection API -================= +Carbon Black App Control (CB Protection) API +=========================================== -This page documents the public interfaces exposed by cbapi when communicating with a Carbon Black Enterprise -Protection server. +This page documents the public interfaces exposed by cbapi when communicating with a Carbon Black App Control (Enterprise +Protection) server. Main Interface -------------- -To use cbapi with Carbon Black Protection, you will be using the CbProtectionAPI. +To use cbapi with Carbon Black App Control (CB Protection), you will be using the CbProtectionAPI. The CbProtectionAPI object then exposes two main methods to select data on the Carbon Black server: .. autoclass:: cbapi.protection.rest_api.CbProtectionAPI diff --git a/docs/response-api.rst b/docs/response-api.rst index 441ee260..bca619b8 100644 --- a/docs/response-api.rst +++ b/docs/response-api.rst @@ -1,15 +1,15 @@ .. _response_api: -CB Response API +EDR (CB Response) API =============== -This page documents the public interfaces exposed by cbapi when communicating with a Carbon Black Enterprise -Response server. +This page documents the public interfaces exposed by cbapi when communicating with a Carbon Black EDR (Enterprise +Response) server. Main Interface -------------- -To use cbapi with Carbon Black Response, you will be using the CbResponseAPI. +To use cbapi with Carbon Black EDR (Response), you will be using the CbResponseAPI. The CbResponseAPI object then exposes two main methods to access data on the Carbon Black server: ``select`` and ``create``. .. autoclass:: cbapi.response.rest_api.CbResponseAPI diff --git a/docs/response-examples.rst b/docs/response-examples.rst index 40bd004e..2ea709b0 100644 --- a/docs/response-examples.rst +++ b/docs/response-examples.rst @@ -1,9 +1,9 @@ -CB Response API Examples -======================== +EDR (CB Response) API Examples +============================== -Now that we've covered the basics, let's step through a few examples using the CB Response API. In these examples, +Now that we've covered the basics, let's step through a few examples using the EDR (CB Response) API. In these examples, we will assume the following boilerplate code to enable logging and establish a connection to the "default" -CB Response server in our credential file:: +EDR (CB Response) server in our credential file:: >>> import logging >>> root = logging.getLogger() @@ -15,10 +15,10 @@ CB Response server in our credential file:: With that boilerplate out of the way, let's take a look at a few examples. -Download a Binary from CB Response ----------------------------------- +Download a Binary from EDR (CB Response) +---------------------------------------- -Let's grab a binary that CB Response has collected from one of the endpoints. This can be useful if you want to +Let's grab a binary that EDR (CB Response) has collected from one of the endpoints. This can be useful if you want to send this binary for further automated analysis or pull it down for manual reverse engineering. You can see a full example with command line options in the examples directory: ``binary_download.py``. @@ -58,13 +58,13 @@ Now let's take this binary and add a Banning rule for it. To do this, we create Received response: {u'result': u'success'} HTTP GET /api/v1/banning/blacklist/7FB55F5A62E78AF9B58D08AAEEAEF848 took 0.039s (response 200) -Note that if the hash is already banned in CB Response, then you will receive a `ServerError` exception with the message that +Note that if the hash is already banned in EDR (CB Response), then you will receive a `ServerError` exception with the message that the banned hash already exists. Isolate a Sensor ---------------- -Switching gears, let's take a Sensor and quarantine it from the network. The CB Response network isolation +Switching gears, let's take a Sensor and quarantine it from the network. The EDR (CB Response) network isolation functionality allows administrators to isolate endpoints that may be actively involved in an incident, while preserving access to perform Live Response on that endpoint and collect further endpoint telemetry. @@ -82,7 +82,7 @@ This will select the first sensor that matches the hostname ``HOSTNAME``. Now we ... True -The ``.isolate()`` method will keep polling the CB Response server until the sensor has confirmed that it is now +The ``.isolate()`` method will keep polling the EDR (CB Response) server until the sensor has confirmed that it is now isolated from the network. If the sensor is offline or otherwise unreachable, this call could never return. Therefore, there is also a ``timeout=`` keyword parameter that can be used to set an optional timeout that, if reached, will throw a ``TimeoutError`` exception. The ``.isolate()`` function returns True when the sensor is successfully @@ -104,7 +104,7 @@ you can optionally specify a timeout using the ``timeout=`` keyword parameter. Querying Processes and Events ----------------------------- -Now, let's do some queries into the CB Response database. The true power of CB Response is its continuous recording +Now, let's do some queries into the EDR (CB Response) database. The true power of EDR (CB Response) is its continuous recording and powerful query language that allows you to go back in time and track the root cause of any security incident on your endpoints. Let's start with a simple query to find instances of a specific behavioral IOC, where our attacker used the built-in Windows tool ``net.exe`` to mount an internal network share. We will iterate over all uses @@ -175,42 +175,42 @@ New Filters: Group By, Time Restrictions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In the query above, there is an extra ``.group_by()`` method. This method is new in cbapi 1.1.0 and is part of five -new query filters available when communicating with a CB Response 6.1 server. These filters are accessible via methods +new query filters available when communicating with a EDR (CB Response) 6.1 server. These filters are accessible via methods on the ``Process`` Query object. These new methods are: * ``.group_by()`` - Group the result set by a field in the response. Typically you will want to group by ``id``, which will ensure that the result set only has one result per *process* rather than one result per *event segment*. For - more information on processes, process segments, and how segments are stored in CB Response 6.0, see the - `Process API Changes for CB Response 6.0 `_ + more information on processes, process segments, and how segments are stored in EDR (CB Response) 6.0, see the + `Process API Changes for EDR (CB Response) 6.0 `_ page on the Developer Network website. * ``.min_last_update()`` - Only return processes that have events after a given date/time stamp (relative to the individual sensor's clock) * ``.max_last_update()`` - Only return processes that have events before a given date/time stamp (relative to the individual sensor's clock) * ``.min_last_server_update()`` - Only return processes that have events after a given date/time stamp (relative to the - CB Response server's clock) + EDR (CB Response) server's clock) * ``.max_last_server_update()`` - Only return processes that have events before a given date/time stamp (relative to the - CB Response server's clock) + EDR (CB Response) server's clock) -CB Response 6.1 uses a new way of recording process events that greatly increases the speed and scale of collection, +EDR (CB Response) 6.1 uses a new way of recording process events that greatly increases the speed and scale of collection, allowing you to store and search data for more endpoints on the same hardware. Details on the new database format -can be found on the Developer Network website at the `Process API Changes for CB Response 6.0 +can be found on the Developer Network website at the `Process API Changes for EDR (CB Response) 6.0 `_ page. The ``Process`` Model Object traditionally referred to a single "segment" of events in the CB Response database. In -CB Response versions prior to 6.0, a single segment will include up to 10,000 individual endpoint events, enough to +EDR (CB Response) versions prior to 6.0, a single segment will include up to 10,000 individual endpoint events, enough to handle over 95% of the typical event activity for a given process. Therefore, even though a ``Process`` Model Object technically refers to a single *segment* in a process, since most processes had less than 10,000 events and therefore were only comprised of a single segment, this distinction wasn't necessary. However, now that processes are split across many segments, a better way of handling this is necessary. Therefore, -CB Response 6.0 introduces the new ``.group_by()`` method. +EDR (CB Response) 6.0 introduces the new ``.group_by()`` method. More on Filters ~~~~~~~~~~~~~~~ Querying for a process will return *all* segments that match. For example, if you search for ``process_name:cmd.exe``, -the result set will include *all* segments of *all* ``cmd.exe`` processes. Therefore, CB Response 6.1 introduced +the result set will include *all* segments of *all* ``cmd.exe`` processes. Therefore, EDR (CB Response) 6.1 introduced the ability to "group" result sets by a field in the result. Typically you will want to group by the internal process id (the ``id`` field), and this is what we did in the query above. Grouping by the ``id`` field will ensure that only one result is returned per *process* rather than per *segment*. @@ -230,7 +230,7 @@ Let's take a look at an example:: 00000001-0000-0e48-01d2-c2a397f4cfe0 1495463176570 00000001-0000-0e48-01d2-c2a397f4cfe0 1495463243492 -Notice that the "same" process ID is returned seven times, but with seven different segment IDs. CB Response will +Notice that the "same" process ID is returned seven times, but with seven different segment IDs. EDR (CB Response) will return *every* process event segment that matches a given query, in this case, any event segment that contains the process command name ``cmd.exe``. @@ -248,7 +248,7 @@ Feed and Watchlist Maintenance The cbapi provides several helper functions to assist in creating watchlists and -Watchlists are simply saved Queries that are automatically run on the CB Response server on a periodic basis. Results +Watchlists are simply saved Queries that are automatically run on the EDR (CB Response) server on a periodic basis. Results of the watchlist are tagged in the database and optionally trigger alerts. Therefore, a cbapi Query can easily be converted into a watchlist through the Query ``.create_watchlist()`` function:: @@ -359,7 +359,7 @@ Those few lines of Python above are jam-packed with functionality. Now for each contextual information on the source host, the group that host is part of, and details about the signing status of the binary that was executed. The magic is performed behind the scenes when we use the ``.binary`` and ``.sensor`` properties on the Process Model Object. Just like our previous example, cbapi's caching layer ensures that we do not overload -the CB Response server with duplicate requests for the same data. In this example, multiple redundant requests for sensor, +the EDR (CB Response) server with duplicate requests for the same data. In this example, multiple redundant requests for sensor, sensor group, and binary data are all eliminated by cbapi's cache. Facets @@ -395,7 +395,7 @@ Administrative Tasks In addition to querying data, you can also perform various administrative tasks using cbapi. -Let's create a user on our CB Response server:: +Let's create a user on our EDR (CB Response) server:: >>> user = cb.create(User) >>> user.username = "jgarman" From df357c7c9ef606fada4f42fb4760c88d9f899072 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 20:30:27 +0000 Subject: [PATCH 164/197] Bump lxml in /examples/threathunter/threat_intelligence Bumps [lxml](https://github.com/lxml/lxml) from 4.6.3 to 4.6.5. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.3...lxml-4.6.5) --- updated-dependencies: - dependency-name: lxml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- examples/threathunter/threat_intelligence/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/threathunter/threat_intelligence/requirements.txt b/examples/threathunter/threat_intelligence/requirements.txt index 50835151..eb1beb53 100644 --- a/examples/threathunter/threat_intelligence/requirements.txt +++ b/examples/threathunter/threat_intelligence/requirements.txt @@ -2,7 +2,7 @@ cybox==2.1.0.18 dataclasses>=0.6 cabby==0.1.20 stix==1.2.0.7 -lxml==4.6.3 +lxml==4.6.5 urllib3>=1.24.2 cbapi>=1.5.6 python_dateutil==2.8.1 From 71fbc04774722accac6884099d29f15c61b01d7b Mon Sep 17 00:00:00 2001 From: Kylie Ebringer Date: Wed, 15 Dec 2021 16:50:25 -0700 Subject: [PATCH 165/197] Small updates from review. --- README.md | 2 +- docs/changelog.rst | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/README.md b/README.md index cc19a97e..ffaa36a0 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Please visit https://cbapi.readthedocs.io for detailed documentation on this API ## Requirements -The cbapi is designed to work on Python 2.6.6 and above (including 3.x). If you're just starting out, +The cbapi package is designed to work on Python 2.6.6 and above (including 3.x). If you're just starting out, we recommend using the latest version of Python 3.6.x or above. All requirements are installed as part of `pip install`. diff --git a/docs/changelog.rst b/docs/changelog.rst index a6369ca1..5d20226b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,12 +2,6 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) -Updates Dec 13, 2021 ------------------------------------- - -Updates to documentation for product names and terminology. No coding changes. - - CbAPI 1.7.5 - Released June 16, 2021 ------------------------------------ From 5cae382b0335755eda68dab4f657d4037ca02182 Mon Sep 17 00:00:00 2001 From: Filipe Spencer Date: Thu, 16 Dec 2021 18:27:25 +0000 Subject: [PATCH 166/197] Wrap has_legacy_partition check with try/except --- src/cbapi/response/rest_api.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/cbapi/response/rest_api.py b/src/cbapi/response/rest_api.py index 408bdc78..fd8143c8 100644 --- a/src/cbapi/response/rest_api.py +++ b/src/cbapi/response/rest_api.py @@ -7,7 +7,7 @@ from distutils.version import LooseVersion from ..connection import BaseAPI from .models import Process, Binary, Watchlist, Investigation, Alert, ThreatReport, StoragePartition -from ..errors import UnauthorizedError, ApiError +from ..errors import UnauthorizedError, ApiError, ClientError from .cblr import LiveResponseSessionManager from .query import Query @@ -49,10 +49,19 @@ def __init__(self, *args, **kwargs): raise ApiError("CbEnterpriseResponseAPI only supports Cb servers version >= 5.0.0") self._has_legacy_partitions = False - if self.cb_server_version >= LooseVersion('6.0'): - legacy_partitions = [p for p in self.select(StoragePartition) if p.info.get("isLegacy", False)] - if legacy_partitions: - self._has_legacy_partitions = True + try: + if self.cb_server_version >= LooseVersion('6.0'): + legacy_partitions = [p for p in self.select(StoragePartition) if p.info.get("isLegacy", False)] + if legacy_partitions: + self._has_legacy_partitions = True + except ClientError as ce: + # If we get a 403 on this endpoint, ignore during init, + # as we will not be able to work with StoragePartitions regardless + # https://github.com/carbonblack/cbapi-python/issues/303 + if ce.error_code == 403: + pass + else: + raise ce # no intervention self._lr_scheduler = None From fdf161677a25b64e3d41e093a8cd5a739da127bf Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Wed, 10 Feb 2021 16:07:16 -0700 Subject: [PATCH 167/197] CBAPI-1821 - initial pass on .rst files --- docs/concepts.rst | 10 +++++----- docs/defense-api.rst | 8 ++++---- docs/getting-started.rst | 6 ++++-- docs/index.rst | 14 ++++++++------ docs/installation.rst | 4 +--- docs/live-response.rst | 2 +- docs/livequery-api.rst | 10 ++++------ docs/livequery-examples.rst | 19 ++++++++++++------- docs/protection-api.rst | 3 --- docs/psc-api.rst | 20 +++++++++----------- docs/response-api.rst | 6 ------ docs/response-examples.rst | 14 +++++--------- docs/threathunter-api.rst | 13 ++++++------- 13 files changed, 59 insertions(+), 70 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 051e8796..ca093db9 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -62,9 +62,9 @@ Joining Model Objects --------------------- Many times, there are relationships between different Model Objects. To make navigating these relationships easy, -cbapi provides special properties to "join" Model Objects together. For example, a :py:mod:`cbapi.response.models.Process` -Model Object can reference the :py:mod:`cbapi.response.models.Sensor` or :py:mod:`cbapi.response.models.Binary` -associated with this Process. +cbapi provides special properties to "join" Model Objects together. For example, a +:py:mod:`cbapi.response.models.Process` Model Object can reference the :py:mod:`cbapi.response.models.Sensor` or +:py:mod:`cbapi.response.models.Binary` associated with this Process. In this case, special "join" properties are provided for you. When you use one of these properties, cbapi will automatically retrieve the associated Model Object, if necessary. @@ -186,8 +186,8 @@ represent these data types can be "created" in cbapi by using the ``create()`` m If you attempt to create a Model Object that cannot be created, you will receive a :py:mod:`ApiError` exception. -Once a Model Object is created, it's blank (it has no data). You will need to set the required properties and then call the -``.save()`` method:: +Once a Model Object is created, it's blank (it has no data). You will need to set the required properties and then call +the ``.save()`` method:: >>> bh = cb.create(BannedHash) >>> bh.text = "Banned from API" diff --git a/docs/defense-api.rst b/docs/defense-api.rst index cbf012ab..88b99d50 100644 --- a/docs/defense-api.rst +++ b/docs/defense-api.rst @@ -1,14 +1,14 @@ .. _defense_api: -CB Defense API -============== +Cloud Endpoint Standard API +=========================== -This page documents the public interfaces exposed by cbapi when communicating with a CB Defense server. +This page documents the public interfaces exposed by cbapi when communicating with a Cloud Endpoint Standard server. Main Interface -------------- -To use cbapi with Carbon Black Defense, you will be using the CBDefenseAPI. +To use cbapi with VMware Carbon Black Cloud Endpoint Standard, you will be using the CBDefenseAPI. The CBDefenseAPI object then exposes two main methods to select data on the Carbon Black server: .. autoclass:: cbapi.psc.defense.rest_api.CbDefenseAPI diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 7561d356..5586cd42 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -4,7 +4,8 @@ Getting Started First, let's make sure that your API authentication tokens have been imported into cbapi. Once that's done, then read on for the key concepts that will explain how to interact with Carbon Black APIs via cbapi. -Feel free to follow along with this document or watch the `Development Environment Setup video `_ +Feel free to follow along with this document or watch the +`Development Environment Setup video `_ on the Developer Network website. API Authentication @@ -36,7 +37,8 @@ Alternatively, if you're using Windows (change ``c:\python27`` if Python is inst This configuration script will walk you through entering your API credentials and will save them to your current user's credential file location, which is located in the ``.carbonblack`` directory in your user's home directory. -If using cbapi-psc, you will also be asked to provide an org key. An org key is required to access the Carbon Black Cloud, and can be found in the console under Settings -> API Keys. +If using cbapi-psc, you will also be asked to provide an org key. An org key is required to access the Carbon Black +Cloud, and can be found in the console under Settings -> API Keys. Your First Query ---------------- diff --git a/docs/index.rst b/docs/index.rst index 644141c5..e0403713 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -116,7 +116,7 @@ Major Features API Credentials --------------- -CBAPI version 0.9.0 enforces the use of credential files. +CBAPI version 0.9.0 enforces the use of credential files. In order to perform any queries via the API, you will need to get the API token for your CB user. See the documentation on the Developer Network website on how to acquire the API token for @@ -130,13 +130,15 @@ Once you acquire your API token, place it in one of the default credentials file * ``~/.carbonblack/`` * ``/current_working_directory/.carbonblack/`` -For distinction between credentials of different Carbon Black products, use the following naming convention for your credentials files: +For distinction between credentials of different Carbon Black products, use the following naming convention for your +credentials files: * ``credentials.psc`` for Carbon Black Cloud Endpoint Standard, Audit & Remediation, and Enterprise EDR (CB Defense, CB LiveOps, and CB ThreatHunter) * ``credentials.response`` for Carbon Black EDR (CB Response) * ``credentials.protection`` for Carbon Black App Control (CB Protection) -For example, if you use a Carbon Black Cloud product, you should have created a credentials file in one of these locations: +For example, if you use a Carbon Black Cloud product, you should have created a credentials file in one of these +locations: * ``/etc/carbonblack/credentials.psc`` * ``~/.carbonblack/credentials.psc`` @@ -188,8 +190,8 @@ The latest CBAPI for Python supports specifying API credentials in the following The `CBAPI_URL` envar holds the FQDN of the target, an EDR (CbR), CBD, or CbD/Carbon Black Cloud server specified just as they are in the configuration file format specified above. -The optional `CBAPI_SSL_VERIFY` envar can be used to control SSL validation(True/False or 0/1), which will default to ON when -not explicitly set by the user. +The optional `CBAPI_SSL_VERIFY` envar can be used to control SSL validation(True/False or 0/1), which will default to +ON when not explicitly set by the user. For environments where complex outbound network filters and proxy configurations are used (eg. anything other than an unauthenticated or basic password authenticated proxy) a prepared `requests.Session` object may be supplied as a @@ -239,7 +241,7 @@ API Documentation ----------------- Once you have read the User Guide, you can view `examples on GitHub `_ -or try writing code of your own. You can use the full API documentation below to see all the methods available in CBAPI +or try writing code of your own. You can use the full API documentation below to see all the methods available in CBAPI and unlock the full functionality of the SDK. .. toctree:: diff --git a/docs/installation.rst b/docs/installation.rst index 7491f46b..2168e8dc 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -39,8 +39,7 @@ https://www.python.org/ftp/python/3.6.4/python-3.6.4-amd64.exe. :alt: Windows installation options showing "Add python.exe to path" :align: right -Ensure that the "Add Python to PATH" option is -checked. +Ensure that the "Add Python to PATH" option is checked. If for some reason you do not have pip installed, follow the instructions at this `handy guide `_. @@ -75,4 +74,3 @@ Once you have a copy of the source, you can install it in "development" mode int This will link the version of cbapi-python you checked out into your Python site-packages directory. Any changes you make to the checked out version of cbapi will be reflected in your local Python installation. This is a good choice if you are thinking of changing or developing on cbapi itself. - diff --git a/docs/live-response.rst b/docs/live-response.rst index 3da7e256..ddc08fd5 100644 --- a/docs/live-response.rst +++ b/docs/live-response.rst @@ -1,7 +1,7 @@ CbAPI and Live Response ======================= -Working with the CB Live Response REST API directly can be difficult. Thankfully, just like the rest of Carbon +Working with the Live Response REST API directly can be difficult. Thankfully, just like the rest of Carbon Black's REST APIs, cbapi provides Pythonic APIs to make working with the Live Response API much easier. In addition to easy-to-use APIs to call into Live Response, cbapi also provides a "job-based" interface that allows diff --git a/docs/livequery-api.rst b/docs/livequery-api.rst index d80bb872..faec5929 100644 --- a/docs/livequery-api.rst +++ b/docs/livequery-api.rst @@ -1,10 +1,9 @@ .. _livequery_api: CB LiveQuery API -=================== +================ -This page documents the public interfaces exposed by cbapi when communicating with -Carbon Black LiveQuery devices. +This page documents the public interfaces exposed by cbapi when communicating with Carbon Black LiveQuery devices. Main Interface -------------- @@ -20,9 +19,8 @@ The LiveQuery API is used in two stages: run submission and result retrieval. Queries ------- -The LiveQuery API uses QueryBuilder instances to construct structured -or unstructured (i.e., raw string) queries. You can either construct these -instances manually, or allow ``CbLiveQueryAPI.select()`` to do it for you: +The LiveQuery API uses QueryBuilder instances to construct structured or unstructured (i.e., raw string) queries. +You can either construct these instances manually, or allow ``CbLiveQueryAPI.select()`` to do it for you: .. autoclass:: cbapi.psc.livequery.query.QueryBuilder :members: diff --git a/docs/livequery-examples.rst b/docs/livequery-examples.rst index a786889f..f3f88262 100644 --- a/docs/livequery-examples.rst +++ b/docs/livequery-examples.rst @@ -14,14 +14,14 @@ Now that we've imported the necessary libraries, we can perform some queries on Create a Query Run ---------------------------------- -Let's create a Query Run. First, we specify which profile to use for authentication from our credentials.psc file and create the LiveQuery object. +Let's create a Query Run. First, we specify which profile to use for authentication from our credentials.psc file and +create the LiveQuery object. >>> profile = "default' >>> cb = CbLiveQueryAPI(profile=profile) Now, we specify the SQL query that we want to run, name of the run, device IDs, and device types. - >>> sql = 'select * from logged_in_users;' >>> name_of_run = 'Selecting all logged in users' >>> device_ids = '1234567' @@ -41,7 +41,8 @@ Finally, we submit the query and print the results. This query should return all logged in Windows endpoints with a ``device_id`` of ``1234567``. -The same query can be executed with the example script `manage_run.py `_. :: +The same query can be executed with the example script +`manage_run.py `_. :: python manage_run.py --profile default create --sql 'select * from logged_in_users;' --name 'Selecting all logged in users' --device_ids '1234567' --device_types 'WINDOWS' @@ -50,7 +51,8 @@ Other possible arguments to ``manage_run.py`` include ``--notify`` and ``--polic Get Query Run Status --------------------- -Now that we've created a Query Run, let's check the status. If we haven't already authenticated with a credentials profile, we begin by specifying which profile to authenticate with. +Now that we've created a Query Run, let's check the status. If we haven't already authenticated with a credentials +profile, we begin by specifying which profile to authenticate with. >>> profile = 'default' >>> cb = CbLiveQueryAPI(profile=profile) @@ -61,11 +63,13 @@ Next, we select the run with the unique run ID. >>> run = cb.select(Run, run_id) >>> print(run) -This can also be accomplished with the example script `manage_run.py `_:: +This can also be accomplished with the example script +`manage_run.py `_:: python manage_run.py --profile default --id a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz -In addition, you can specify which order you want results returned. To change from the default ascending order, use the flag ``-d`` or ``--descending_results``:: +In addition, you can specify which order you want results returned. To change from the default ascending order, use +the flag ``-d`` or ``--descending_results``:: python manage_run.py --profile default --id a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz --descending_results @@ -105,6 +109,7 @@ Finally, we print the results. ... print(result) -You can also retrieve run results with the example script `run_search.py `_:: +You can also retrieve run results with the example script +`run_search.py `_:: python run_search.py --profile default --id a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz --device_ids '1234567' --status 'matched' diff --git a/docs/protection-api.rst b/docs/protection-api.rst index b7dd44bf..1799bb5a 100644 --- a/docs/protection-api.rst +++ b/docs/protection-api.rst @@ -3,9 +3,6 @@ Carbon Black App Control (CB Protection) API =========================================== -This page documents the public interfaces exposed by cbapi when communicating with a Carbon Black App Control (Enterprise -Protection) server. - Main Interface -------------- diff --git a/docs/psc-api.rst b/docs/psc-api.rst index 891f3563..25ca80cf 100755 --- a/docs/psc-api.rst +++ b/docs/psc-api.rst @@ -1,15 +1,14 @@ .. _psc_api: -CB PSC API -========== +VMware Carbon Black Cloud API +============================= -This page documents the public interfaces exposed by cbapi when communicating with -the Carbon Black Predictive Security Cloud (PSC). +This page documents the public interfaces exposed by cbapi when communicating with the VMware Carbon Black Cloud. Main Interface -------------- -To use cbapi with the Carbon Black PSC, you use CbPSCBaseAPI objects. +To use cbapi with the VMware Carbon Black Cloud, you use CbPSCBaseAPI objects. .. autoclass:: cbapi.psc.rest_api.CbPSCBaseAPI :members: @@ -18,7 +17,7 @@ To use cbapi with the Carbon Black PSC, you use CbPSCBaseAPI objects. Device API ---------- -The PSC can be used to enumerate devices within your organization, and change their +The Carbon Black Cloud can be used to enumerate devices within your organization, and change their status via a control request. You can use the select() method on the CbPSCBaseAPI to create a query object for @@ -45,12 +44,11 @@ Selects all devices running Linux from the current organization. Alerts API ---------- -Using the API, you can search for alerts within your organization, and dismiss or -undismiss them, either individually or in bulk. +Using the API, you can search for alerts within your organization, and dismiss or undismiss them, either individually +or in bulk. -You can use the select() method on the CbPSCBaseAPI to create a query object for -BaseAlert objects, which can be used to locate a list of alerts. You can also -search for more specialized alert types: +You can use the select() method on the CbPSCBaseAPI to create a query object for BaseAlert objects, which can be used +to locate a list of alerts. You can also search for more specialized alert types: * CBAnalyticsAlert - Alerts from CB Analytics * VMwareAlert - Alerts from VMware diff --git a/docs/response-api.rst b/docs/response-api.rst index bca619b8..24761824 100644 --- a/docs/response-api.rst +++ b/docs/response-api.rst @@ -3,9 +3,6 @@ EDR (CB Response) API =============== -This page documents the public interfaces exposed by cbapi when communicating with a Carbon Black EDR (Enterprise -Response) server. - Main Interface -------------- @@ -77,6 +74,3 @@ Process Operations .. automethod:: cbapi.live_response_api.CbLRSessionBase.kill_process .. automethod:: cbapi.live_response_api.CbLRSessionBase.create_process .. automethod:: cbapi.live_response_api.CbLRSessionBase.list_processes - - - diff --git a/docs/response-examples.rst b/docs/response-examples.rst index 2ea709b0..6cbd0da1 100644 --- a/docs/response-examples.rst +++ b/docs/response-examples.rst @@ -1,10 +1,6 @@ EDR (CB Response) API Examples ============================== -Now that we've covered the basics, let's step through a few examples using the EDR (CB Response) API. In these examples, -we will assume the following boilerplate code to enable logging and establish a connection to the "default" -EDR (CB Response) server in our credential file:: - >>> import logging >>> root = logging.getLogger() >>> root.addHandler(logging.StreamHandler()) @@ -246,7 +242,7 @@ the command name ``cmd.exe``. Just add the ``.group_by("id")`` filter to your qu Feed and Watchlist Maintenance ------------------------------ -The cbapi provides several helper functions to assist in creating watchlists and +The cbapi provides several helper functions to assist in creating watchlists and feeds. Watchlists are simply saved Queries that are automatically run on the EDR (CB Response) server on a periodic basis. Results of the watchlist are tagged in the database and optionally trigger alerts. Therefore, a cbapi Query can easily be @@ -295,9 +291,10 @@ The cbapi provides helper functions to manage alerts and threat reports in bulk. the ThreatReport and Alert Model Objects provide a few bulk operations to help manage large numbers of Threat Reports and Alerts, respectively. -To mark a large number of Threat Reports as false positives, create a query that matches the Reports you're interested in. -For example, if every Report from the Feed named "SOC" that contains the word "FUZZYWOMBAT" in the report title should be -considered a false positive (and no longer trigger Alerts), you can write the following code to do so:: +To mark a large number of Threat Reports as false positives, create a query that matches the Reports you're +interested in. For example, if every Report from the Feed named "SOC" that contains the word "FUZZYWOMBAT" in the +report title should be considered a false positive (and no longer trigger Alerts), you can write the following code +to do so:: >>> feed = c.select(Feed).where("name:SOC").one() >>> report_query = feed.reports.where("title:FUZZYWOMBAT") @@ -429,4 +426,3 @@ How about moving a sensor to a new Sensor Group:: Sending HTTP PUT /api/v1/sensor/3 with {"boot_id": "2", "build_id": 2, "build_version_string": "005.002.000.60922", ... HTTP PUT /api/v1/sensor/3 took 0.087s (response 204) HTTP GET /api/v1/sensor/3 took 0.030s (response 200) - diff --git a/docs/threathunter-api.rst b/docs/threathunter-api.rst index 25a399d5..858aa1ef 100644 --- a/docs/threathunter-api.rst +++ b/docs/threathunter-api.rst @@ -1,17 +1,16 @@ .. _threathunter_api: -CB ThreatHunter API -=================== +VMware Carbon Black Cloud Enterprise EDR API +============================================ This page documents the public interfaces exposed by cbapi when communicating with a -Carbon Black Cloud ThreatHunter server. +VMware Carbon Black Cloud Enterprise EDR server. Main Interface -------------- -To use cbapi with Carbon Black ThreatHunter, you use CbThreatHunterAPI objects. -These objects expose two main methods to access data on the -ThreatHunter server: ``select`` and ``create``. +To use cbapi with Enterprise EDR, you use CbThreatHunterAPI objects. +These objects expose two main methods to access data on the Enterprise EDR server: ``select`` and ``create``. .. autoclass:: cbapi.psc.threathunter.rest_api.CbThreatHunterAPI :members: @@ -20,7 +19,7 @@ ThreatHunter server: ``select`` and ``create``. Queries ------- -The ThreatHunter API uses QueryBuilder instances to construct structured +The Enterprise EDR API uses QueryBuilder instances to construct structured or unstructured (i.e., raw string) queries. You can either construct these instances manually, or allow ``CbThreatHunterAPI.select()`` to do it for you: From d5af680a65b0c16fdac719454b7db3a1573b6e3f Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Thu, 11 Feb 2021 11:40:59 -0700 Subject: [PATCH 168/197] CBAPI-1821: additional terminology removal and change, including in docstrings --- docs/index.rst | 6 +-- docs/live-response.rst | 1 - src/cbapi/protection/rest_api.py | 32 +++++++------- src/cbapi/psc/defense/rest_api.py | 15 +++---- src/cbapi/psc/threathunter/models.py | 10 ++--- src/cbapi/psc/threathunter/query.py | 8 ++-- src/cbapi/psc/threathunter/rest_api.py | 6 +-- src/cbapi/response/models.py | 58 +++++++++++++------------- src/cbapi/response/query.py | 8 ++-- src/cbapi/response/rest_api.py | 21 +++++----- 10 files changed, 82 insertions(+), 83 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index e0403713..4fbc9f57 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,7 +13,7 @@ This library provides a Pythonic layer to access the raw power of the REST APIs >>> from cbapi.response import CbResponseAPI, Process, Binary, Sensor >>> # - >>> # Create our CbAPI object + >>> # Create our EDR API object >>> # >>> c = CbResponseAPI() >>> # @@ -43,7 +43,7 @@ If you're a Carbon Black App Control customer (formerly CB Protection), you may >>> from cbapi.protection.models import FileInstance >>> from cbapi.protection import CbProtectionAPI >>> # - >>> # Create our CB Protection API object + >>> # Create our App Control API object >>> # >>> p = CbProtectionAPI() >>> # @@ -65,7 +65,7 @@ As of version 1.2, CBAPI also supports Carbon Black Cloud Endpoint Standard (for >>> from cbapi.psc.defense import * >>> # - >>> # Create our CB Defense API object + >>> # Create our Cloud Endpoint Standard API object >>> # >>> p = CbDefenseAPI() >>> # diff --git a/docs/live-response.rst b/docs/live-response.rst index ddc08fd5..69c87fb7 100644 --- a/docs/live-response.rst +++ b/docs/live-response.rst @@ -92,4 +92,3 @@ back from the endpoint, and submit the ``.run()`` method to the Live Response Jo Your script resumes execution immediately after the call to ``.submit_job()``. The job(s) that you've submitted will be executed in a set of background threads managed by cbapi. - diff --git a/src/cbapi/protection/rest_api.py b/src/cbapi/protection/rest_api.py index f3fa6c0a..3ee1f166 100644 --- a/src/cbapi/protection/rest_api.py +++ b/src/cbapi/protection/rest_api.py @@ -11,15 +11,15 @@ class CbProtectionAPI(BaseAPI): - """The main entry point into the Carbon Black Enterprise Protection API. + """The main entry point into the Carbon Black App Control API. :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. Uses the profile named 'default' when not specified. Usage:: - >>> from cbapi import CbEnterpriseProtectionAPI - >>> cb = CbEnterpriseProtectionAPI(profile="production") + >>> from cbapi import CbProtectionAPI + >>> cb = CbProtectionAPI(profile="production") """ def __init__(self, *args, **kwargs): super(CbProtectionAPI, self).__init__(product_name="protection", *args, **kwargs) @@ -76,11 +76,11 @@ class CbEnterpriseProtectionAPI(CbProtectionAPI): class Query(PaginatedQuery): - """Represents a prepared query to the Carbon Black Enterprise Protection server. + """Represents a prepared query to the Carbon Black App Control server. - This object is returned as part of a :py:meth:`CbEnterpriseProtectionAPI.select` + This object is returned as part of a :py:meth:`CbProtectionAPI.select` operation on models requested from the Carbon Black - Enterprise Protection server. You should not have to create this class yourself. + App Control server. You should not have to create this class yourself. The query is not executed on the server until it's accessed, either as an iterator (where it will generate values on demand as they're requested) or as a list (where it will retrieve the entire result set and save to a list). @@ -88,12 +88,12 @@ class Query(PaginatedQuery): the query. The syntax for query :py:meth:where and :py:meth:sort methods can be found in the - `Enterprise Protection API reference`_ posted on the Carbon Black Developer Network website. + `App Control REST API reference`_ posted on the Carbon Black Developer Network website. Examples:: - >>> from cbapi.protection import CbEnterpriseProtectionAPI, Computer - >>> cb = CbEnterpriseProtectionAPI() + >>> from cbapi.protection import CbProtectionAPI, Computer + >>> cb = CbProtectionAPI() >>> query = cb.select(Computer) # returns a Query object matching all Computers >>> query = query.where("ipAddress:10.201.2.*") # add a filter to this Query >>> query = query.sort("processorSpeed DESC") # sort by computer processor speed, descending @@ -108,7 +108,7 @@ class Query(PaginatedQuery): - You can chain where clauses together to create AND queries; only objects that match all ``where`` clauses will be returned. - .. _Enterprise Protection API reference: + .. _App Control REST API reference: https://developer.carbonblack.com/reference/enterprise-protection/8.0/rest-api/ """ def __init__(self, doc_class, cb, query=None): @@ -134,11 +134,11 @@ def _clone(self): def where(self, q): """Add a filter to this query. - :param str q: Query string - see the `Enterprise Protection API reference`_. + :param str q: Query string - see the `App Control REST API reference`_. :return: Query object :rtype: :py:class:`Query` - .. _Enterprise Protection API reference: + .. _App Control REST API reference: https://developer.carbonblack.com/reference/enterprise-protection/8.0/rest-api/ """ nq = self._clone() @@ -148,11 +148,11 @@ def where(self, q): def and_(self, q): """Add a filter to this query. Equivalent to calling :py:meth:`where` on this object. - :param str q: Query string - see the `Enterprise Protection API reference`_. + :param str q: Query string - see the `App Control REST API reference`_. :return: Query object :rtype: :py:class:`Query` - .. _Enterprise Protection API reference: + .. _App Control REST API reference: https://developer.carbonblack.com/reference/enterprise-protection/8.0/rest-api/ """ return self.where(q) @@ -160,11 +160,11 @@ def and_(self, q): def sort(self, new_sort): """Set the sort order for this query. - :param str new_sort: Sort order - see the `Enterprise Protection API reference`_. + :param str new_sort: Sort order - see the `App Control REST API reference`_. :return: Query object :rtype: :py:class:`Query` - .. _Enterprise Protection API reference: + .. _App Control REST API reference: https://developer.carbonblack.com/reference/enterprise-protection/8.0/rest-api/ """ new_sort = new_sort.strip() diff --git a/src/cbapi/psc/defense/rest_api.py b/src/cbapi/psc/defense/rest_api.py index 37a628a2..2368e754 100644 --- a/src/cbapi/psc/defense/rest_api.py +++ b/src/cbapi/psc/defense/rest_api.py @@ -14,7 +14,7 @@ def convert_to_kv_pairs(q): class CbDefenseAPI(CbPSCBaseAPI): - """The main entry point into the Cb Defense API. + """The main entry point into the Carbon Black Cloud Endpoint Standard Defense API. :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. Uses the profile named 'default' when not specified. @@ -32,8 +32,8 @@ def _perform_query(self, cls, query_string=None): return Query(cls, self, query_string) def notification_listener(self, interval=60): - """Generator to continually poll the Cb Defense server for notifications (alerts). Note that this can only - be used with a 'SIEM' key generated in the Cb Defense console. + """Generator to continually poll the Cloud Endpoint Standard server for notifications (alerts). Note that + this can only be used with a 'SIEM' key generated in the Carbon Black Cloud console. """ while True: for notification in self.get_notifications(): @@ -41,8 +41,8 @@ def notification_listener(self, interval=60): time.sleep(interval) def get_notifications(self): - """Retrieve queued notifications (alerts) from the Cb Defense server. Note that this can only be used - with a 'SIEM' key generated in the Cb Defense console. + """Retrieve queued notifications (alerts) from the Cloud Endpoint Standard server. Note that this can only be + used with a 'SIEM' key generated in the Carbon Black Cloud console. :returns: list of dictionary objects representing the notifications, or an empty list if none available. """ @@ -59,10 +59,11 @@ def get_auditlogs(self): class Query(PaginatedQuery): - """Represents a prepared query to the Cb Defense server. + """Represents a prepared query to the Cloud Endpoint Standard server. This object is returned as part of a :py:meth:`CbDefenseAPI.select` - operation on models requested from the Cb Defense server. You should not have to create this class yourself. + operation on models requested from the Cloud Endpoint Standardserver. You should not have to create + this class yourself. The query is not executed on the server until it's accessed, either as an iterator (where it will generate values on demand as they're requested) or as a list (where it will retrieve the entire result set and save to a list). diff --git a/src/cbapi/psc/threathunter/models.py b/src/cbapi/psc/threathunter/models.py index de2aac50..1ac3f31d 100644 --- a/src/cbapi/psc/threathunter/models.py +++ b/src/cbapi/psc/threathunter/models.py @@ -254,7 +254,7 @@ def __init__(self, cb, model_unique_id=None, initial_data=None): self._reports = [Report(cb, initial_data=report, feed_id=feed_id) for report in reports] def save(self, public=False): - """Saves this feed on the ThreatHunter server. + """Saves this feed on the Enterprise EDR server. :param public: Whether to make the feed publicly available :return: The saved feed @@ -294,7 +294,7 @@ def validate(self): report.validate() def delete(self): - """Deletes this feed from the ThreatHunter server. + """Deletes this feed from the Enterprise EDR server. :raise InvalidObjectError: if `id` is missing """ @@ -501,7 +501,7 @@ def update(self, **kwargs): return self def delete(self): - """Deletes this report from the ThreatHunter server. + """Deletes this report from the Enterprise EDR server. >>> report.delete() @@ -822,7 +822,7 @@ def __init__(self, cb, model_unique_id=None, initial_data=None): force_init=False, full_doc=True) def save(self): - """Saves this watchlist on the ThreatHunter server. + """Saves this watchlist on the Enterprise EDR server. :return: The saved watchlist :rtype: :py:class:`Watchlist` @@ -888,7 +888,7 @@ def classifier_(self): return (classifier_dict["key"], classifier_dict["value"]) def delete(self): - """Deletes this watchlist from the ThreatHunter server. + """Deletes this watchlist from the Enterprise EDR server. :raise InvalidObjectError: if `id` is missing """ diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py index 5990db96..ae2b1516 100644 --- a/src/cbapi/psc/threathunter/query.py +++ b/src/cbapi/psc/threathunter/query.py @@ -12,8 +12,8 @@ class QueryBuilder(object): """ - Provides a flexible interface for building prepared queries for the CB - ThreatHunter backend. + Provides a flexible interface for building prepared queries for the Carbon Black + Enterprise EDR backend. This object can be instantiated directly, or can be managed implicitly through the :py:meth:`CbThreatHunterAPI.select` API. @@ -157,10 +157,10 @@ def _collapse(self): class Query(PaginatedQuery): - """Represents a prepared query to the Cb ThreatHunter backend. + """Represents a prepared query to the Carbon Black Enterprise EDR backend. This object is returned as part of a :py:meth:`CbThreatHunterPI.select` - operation on models requested from the Cb ThreatHunter backend. You should not have to create this class yourself. + operation on models requested from the Enterprise EDR backend. You should not have to create this class yourself. The query is not executed on the server until it's accessed, either as an iterator (where it will generate values on demand as they're requested) or as a list (where it will retrieve the entire result set and save to a list). diff --git a/src/cbapi/psc/threathunter/rest_api.py b/src/cbapi/psc/threathunter/rest_api.py index 8de417bc..b163d6ba 100644 --- a/src/cbapi/psc/threathunter/rest_api.py +++ b/src/cbapi/psc/threathunter/rest_api.py @@ -8,7 +8,7 @@ class CbThreatHunterAPI(CbPSCBaseAPI): - """The main entry point into the Carbon Black Cloud ThreatHunter API. + """The main entry point into the Carbon Black Cloud Enterprise EDR API. :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. Uses the profile named 'default' when not specified. @@ -60,7 +60,7 @@ def validate_query(self, query): return resp.get("valid", False) def convert_query(self, query): - """Converts a legacy CB Response query to a ThreatHunter query. + """Converts a legacy Carbon Black EDR query to an Enterprise EDR query. :param str query: the query to convert :return: the converted query @@ -87,7 +87,7 @@ def custom_severities(self): def queries(self): """Retrieves a list of queries, active or complete, known by - the ThreatHunter server. + the Enterprise EDR server. :return: a list of query ids :rtype: list(str) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 42150bdb..06e07217 100755 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -714,21 +714,21 @@ def os(self): @property def registration_time(self): """ - Returns the time the sensor registered with the Cb Response Server + Returns the time the sensor registered with the EDR Server """ return convert_from_cb(getattr(self, 'registration_time', -1)) @property def sid(self): """ - Security Identifier being used by the Cb Response Sensor + Security Identifier being used by the EDR Sensor """ return getattr(self, 'computer_sid') @property def webui_link(self): """ - Returns the Cb Response Web UI link associated with this Sensor + Returns the Carbon Black EDR Web UI link associated with this Sensor """ return '{0:s}/#/host/{1}'.format(self._cb.url, self._model_unique_id) @@ -736,7 +736,7 @@ def webui_link(self): @property def queued_stats(self): """ - Returns a list of status and size of the queued event logs from the associated Cb Response Sensor + Returns a list of status and size of the queued event logs from the associated EDR Sensor :example: @@ -755,14 +755,14 @@ def queued_stats(self): @property def activity_stats(self): """ - Returns a list of activity statistics from the associated Cb Response Sensor + Returns a list of activity statistics from the associated EDR Sensor """ return self._cb.get_object("{0}/activity".format(self._build_api_request_uri()), default=[]) @property def resource_status(self): """ - Returns a list of memory statistics used by the Cb Response Sensor + Returns a list of memory statistics used by the EDR Sensor """ return self._cb.get_object("{0}/resourcestatus".format(self._build_api_request_uri()), default=[]) @@ -782,9 +782,9 @@ def lr_session(self): def flush_events(self): """ - Performs a flush of events for this Cb Response Sensor + Performs a flush of events for this EDR Sensor - :warning: This may cause a significant amount of network traffic from this sensor to the Cb Response Server + :warning: This may cause a significant amount of network traffic from this sensor to the EDR Server """ # Note that Cb Response 6 requires the date/time stamp to be sent in RFC822 format (not ISO 8601). @@ -796,7 +796,7 @@ def restart_sensor(self): """ Restarts the Carbon Black sensor (*not* the underlying endpoint operating system). - This simply sets the flag to ask the sensor to restart the next time it checks into the Cb Response server, + This simply sets the flag to ask the sensor to restart the next time it checks into the EDR server, it does not wait for the sensor to restart. """ self.restart_queued = True @@ -804,7 +804,7 @@ def restart_sensor(self): def isolate(self, timeout=None): """ - Turn on network isolation for this Cb Response Sensor. + Turn on network isolation for this EDR Sensor. This function will block and only return when the isolation is complete, or if a timeout is reached. By default, there is no timeout. You can specify a timeout period (in seconds) in the "timeout" parameter to this @@ -829,7 +829,7 @@ def isolate(self, timeout=None): def unisolate(self, timeout=None): """ - Turn off network isolation for this Cb Response Sensor. + Turn off network isolation for this EDR Sensor. This function will block and only return when the isolation is removed, or if a timeout is reached. By default, there is no timeout. You can specify a timeout period (in seconds) in the "timeout" parameter to this @@ -1515,8 +1515,8 @@ def group_by(self, field_name): """Set the group-by field name for this query. Typically, you will want to set this to 'id' if you only want one result per process. - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :param str field_name: Field name to group the result set by. :return: Query object @@ -1533,8 +1533,8 @@ def group_by(self, field_name): def max_children(self, num_children): """Sets the number of children to fetch with the process - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :default: 15 :param int num_children: Number of children to fetch with process @@ -1564,8 +1564,8 @@ def min_last_update(self, v): This option will limit the number of Solr cores that need to be searched for events that match the query. - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :param str v: Timestamp (either string or datetime object). :return: Query object @@ -1590,8 +1590,8 @@ def min_last_server_update(self, v): This option will limit the number of Solr cores that need to be searched for events that match the query. - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :param str v: Timestamp (either string or datetime object). :return: Query object @@ -1616,8 +1616,8 @@ def max_last_update(self, v): This option will limit the number of Solr cores that need to be searched for events that match the query. - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :param str v: Timestamp (either string or datetime object). :return: Query object @@ -1642,8 +1642,8 @@ def max_last_server_update(self, v): This option will limit the number of Solr cores that need to be searched for events that match the query. - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :param str v: Timestamp (either string or datetime object). :return: Query object @@ -1754,7 +1754,7 @@ def _build_api_request_uri(self): @property def webui_link(self): """ - Returns the Cb Response Web UI link associated with this Binary object + Returns the Carbon Black EDR Web UI link associated with this Binary object """ return '{0:s}/#binary/{1:s}'.format(self._cb.url, self.md5sum) @@ -1943,7 +1943,7 @@ def icon(self): @property def banned(self): """ - Returns *BannedHash* object if this Binary's hash has been whitelisted (Banned), otherwise returns *False* + Returns *BannedHash* object if this Binary's hash has been banned, otherwise returns *False* """ try: bh = self._cb.select(BannedHash, self.md5sum.lower()) @@ -2527,7 +2527,7 @@ def walk_children(self, callback, max_depth=0, depth=0): @property def parent_md5(self): """ - Workaround since parent_md5 silently disappeared in ~Cb Response 6.x + Workaround since parent_md5 silently disappeared in EDR 6.x """ return self.parent.process_md5 @@ -2862,7 +2862,7 @@ def binary(self): @property def comms_ip(self): """ - Returns ascii representation of the ip address used to communicate with the Cb Response Server + Returns ascii representation of the ip address used to communicate with the EDR Server """ try: ip_address = socket.inet_ntoa(struct.pack('>i', self._attribute('comms_ip', 0))) @@ -2874,7 +2874,7 @@ def comms_ip(self): @property def interface_ip(self): """ - Returns ascii representation of the ip address of the interface used to communicate with the Cb Response server. + Returns ascii representation of the ip address of the interface used to communicate with the EDR server. If using NAT, this will be the "internal" IP address of the sensor. """ try: @@ -2953,7 +2953,7 @@ def sensor(self): @property def webui_link(self): """ - Returns the Cb Response Web UI link associated with this process + Returns the Carbon Black EDR Web UI link associated with this process """ if not self.suppressed_process: return '%s/#analyze/%s/%s' % (self._cb.url, self.id, self.current_segment) diff --git a/src/cbapi/response/query.py b/src/cbapi/response/query.py index 8d71221d..9d96a629 100644 --- a/src/cbapi/response/query.py +++ b/src/cbapi/response/query.py @@ -11,11 +11,11 @@ class Query(PaginatedQuery): - """Represents a prepared query to the Carbon Black Enterprise Response server. + """Represents a prepared query to the Carbon Black EDR server. - This object is returned as part of a :py:meth:`CbEnterpriseResponseAPI.select` + This object is returned as part of a :py:meth:`CbResponseAPI.select` operation on Process and Binary objects from the Carbon Black - Enterprise Response server. You should not have to create this class yourself. + EDR server. You should not have to create this class yourself. The query is not executed on the server until it's accessed, either as an iterator (where it will generate values on demand as they're requested) or as a list (where it will retrieve the entire result set and save to a list). @@ -28,7 +28,7 @@ class Query(PaginatedQuery): Examples:: - >>> cb = CbEnterpriseResponseAPI() + >>> cb = CbResponseAPI() >>> query = cb.select(Process) # returns a Query object matching all Processes >>> query = query.where("process_name:notepad.exe") # add a filter to this Query >>> query = query.sort("last_update desc") # sort by last update time, most recent first diff --git a/src/cbapi/response/rest_api.py b/src/cbapi/response/rest_api.py index 408bdc78..a19ed8d5 100644 --- a/src/cbapi/response/rest_api.py +++ b/src/cbapi/response/rest_api.py @@ -17,7 +17,7 @@ class CbResponseAPI(BaseAPI): - """The main entry point into the Carbon Black Enterprise Response API. + """The main entry point into the Carbon Black EDR API. Note that calling this will automatically connect to the Carbon Black server in order to verify connectivity and get the server version. @@ -30,8 +30,8 @@ class CbResponseAPI(BaseAPI): Usage:: - >>> from cbapi import CbEnterpriseResponseAPI - >>> cb = CbEnterpriseResponseAPI(profile="production") + >>> from cbapi import CbResponseAPI + >>> cb = CbResponseAPI(profile="production") """ def __init__(self, *args, **kwargs): timeout = kwargs.pop("timeout", 120) # set default timeout period to two minutes, 2x the default nginx timeout @@ -66,7 +66,7 @@ def live_response(self): return self._lr_scheduler def info(self): - """Retrieve basic version information from the Carbon Black Enterprise Response server. + """Retrieve basic version information from the Carbon Black DER server. :return: Dictionary with information retrieved from the ``/api/info`` API route :rtype: dict @@ -75,7 +75,7 @@ def info(self): return r.json() def dashboard_statistics(self): - """Retrieve dashboard statistics from the Carbon Black Enterprise Response server. + """Retrieve dashboard statistics from the Carbon Black EDR server. :return: Dictionary with information retrieved from the ``/api/v1/dashboard/statistics`` API route :rtype: dict @@ -84,7 +84,7 @@ def dashboard_statistics(self): return r.json() def license_request(self): - """Retrieve license request block from the Carbon Black Enterprise Response server. + """Retrieve license request block from the Carbon Black EDR server. :return: License request block :rtype: str @@ -93,7 +93,7 @@ def license_request(self): return r.json().get("license_request_block", "") def update_license(self, license_block): - """Upload new license to the Carbon Black Enterprise Response server. + """Upload new license to the Carbon Black EDR server. :param str license_block: Licence block provided by Carbon Black support :raises ServerError: if the license is not accepted by the Carbon Black server @@ -108,14 +108,13 @@ def _perform_query(self, cls, **kwargs): return Query(cls, self, **kwargs) def from_ui(self, uri): - """Retrieve a Carbon Black Enterprise Response object based on URL from the Carbon Black Enterprise Response - web user interface. + """Retrieve a Carbon Black EDR object based on URL from the Carbon Black EDR web user interface. For example, calling this function with ``https://server/#/analyze/00000001-0000-0554-01d1-3bc4553b8c9f/1`` as the ``uri`` argument will return a new :py:class: cbapi.response.models.Process class initialized with the process GUID from the URL. - :param str uri: Web browser URL from the Cb web interface + :param str uri: Web browser URL from the CB web interface :return: the appropriate model object for the URL provided :raises ApiError: if the URL does not correspond to a recognized model object """ @@ -166,7 +165,7 @@ def _request_lr_session(self, sensor_id): return self.live_response.request_session(sensor_id) def create_new_partition(self): - """Create a new Solr time partition for event storage. Available in Cb Response 6.1 and above. + """Create a new Solr time partition for event storage. Available in Carbon Black EDR 6.1 and above. This will force roll-over current hot partition into warm partition (by renaming it to a time-stamped name) and create a new hot partition ("writer"). From 6b1f13ed9ae6d619eef38f19f8b4ece6bd9eabf0 Mon Sep 17 00:00:00 2001 From: Amy Bowersox Date: Thu, 11 Feb 2021 13:03:17 -0700 Subject: [PATCH 169/197] added in a missing space (thanks Lisa!) --- src/cbapi/psc/defense/rest_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/psc/defense/rest_api.py b/src/cbapi/psc/defense/rest_api.py index 2368e754..68c5abcd 100644 --- a/src/cbapi/psc/defense/rest_api.py +++ b/src/cbapi/psc/defense/rest_api.py @@ -62,7 +62,7 @@ class Query(PaginatedQuery): """Represents a prepared query to the Cloud Endpoint Standard server. This object is returned as part of a :py:meth:`CbDefenseAPI.select` - operation on models requested from the Cloud Endpoint Standardserver. You should not have to create + operation on models requested from the Cloud Endpoint Standard server. You should not have to create this class yourself. The query is not executed on the server until it's accessed, either as an iterator (where it will generate values From 273c311ebea2fa4d9451c4ac71de68815914d528 Mon Sep 17 00:00:00 2001 From: Kylie Ebringer Date: Fri, 17 Dec 2021 09:55:16 -0700 Subject: [PATCH 170/197] Changing sensor to return paginated query per suggestion on 301 --- src/cbapi/response/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 06e07217..c64bbff8 100755 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -658,7 +658,7 @@ def _query_implementation(cls, cb): # return SensorPaginatedQuery(cls, cb) # else: # return SensorQuery(cls, cb) - return SensorQuery(cls, cb) + return SensorPaginatedQuery(cls, cb) @property def group(self): From b801ee391b424326c5cca2b7cf76e7b19a32c497 Mon Sep 17 00:00:00 2001 From: Kylie Ebringer Date: Fri, 17 Dec 2021 10:14:09 -0700 Subject: [PATCH 171/197] Release notes in preparation of next CBAPI release --- docs/changelog.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d20226b..78013232 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,17 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.7.65- Planned Release Dec 17, 2021 +------------------------------------ + +Bug Fixes + * Removed the requirement for an admin token to connect + * Added sensor paginated query + +General + * Updated version of lxml library + + CbAPI 1.7.5 - Released June 16, 2021 ------------------------------------ From 1aa2a09c2c5a16259052122ec64a8a8ae2bf97da Mon Sep 17 00:00:00 2001 From: Kylie Ebringer Date: Fri, 17 Dec 2021 12:41:37 -0700 Subject: [PATCH 172/197] Fixed indentation --- src/cbapi/response/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index c64bbff8..d336acf3 100755 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -658,7 +658,7 @@ def _query_implementation(cls, cb): # return SensorPaginatedQuery(cls, cb) # else: # return SensorQuery(cls, cb) - return SensorPaginatedQuery(cls, cb) + return SensorPaginatedQuery(cls, cb) @property def group(self): From 7721b1628143aeb72d205cd05abae10f3e0aaa97 Mon Sep 17 00:00:00 2001 From: Kylie Ebringer Date: Fri, 17 Dec 2021 16:23:55 -0700 Subject: [PATCH 173/197] Fixed release version. --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 78013232..7b9b4af5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) -CbAPI 1.7.65- Planned Release Dec 17, 2021 +CbAPI 1.7.6- Release Dec 188888888, 2021 ------------------------------------ Bug Fixes From 22f7b88ad868cd691aaeb052735a8e4b3dbdf460 Mon Sep 17 00:00:00 2001 From: Kylie Ebringer Date: Mon, 20 Dec 2021 08:17:26 -0700 Subject: [PATCH 174/197] fixed date on changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7b9b4af5..54af41e4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) -CbAPI 1.7.6- Release Dec 188888888, 2021 +CbAPI 1.7.6 - Release Dec 20, 2021 ------------------------------------ Bug Fixes From 1e059749e94116e150d7dc5fd0790c12ca531467 Mon Sep 17 00:00:00 2001 From: Kylie Ebringer Date: Mon, 20 Dec 2021 13:25:45 -0700 Subject: [PATCH 175/197] Updated version to 1.7.6 --- README.md | 2 +- docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ffaa36a0..a75558f4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.7.5** +**Latest Version: 1.7.6** _**Notice**:_ * The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI diff --git a/docs/conf.py b/docs/conf.py index 3fdb0dc7..1f538938 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.7.5' +release = u'1.7.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 6f353565..745195b0 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( name='cbapi', - version='1.7.5', + version='1.7.6', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index 26373177..99d52b26 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2020 VMware Carbon Black' -__version__ = '1.7.5' +__version__ = '1.7.6' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From 5f5914ab7885e84230b1e1bc826f6df8b823f34d Mon Sep 17 00:00:00 2001 From: Zachary Estep Date: Fri, 28 Jan 2022 08:54:06 -0500 Subject: [PATCH 176/197] CB-38082: Fix order for paginated sensor search --- src/cbapi/response/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index d336acf3..535d385e 100755 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -590,6 +590,8 @@ def _search(self, start=0, rows=0): else: args = {} + args.update({"sort.col":"computer_name", "sort.dir":"asc"}) + args['start'] = start if rows: From 126af2f1fd8295af59389d1c6004bb34a1086d52 Mon Sep 17 00:00:00 2001 From: Kylie Ebringer Date: Fri, 28 Jan 2022 09:23:22 -0700 Subject: [PATCH 177/197] Updated release version, date and changelog for v1.7.7 --- README.md | 2 +- docs/changelog.rst | 7 +++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a75558f4..fc89025a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.7.6** +**Latest Version: 1.7.7** _**Notice**:_ * The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI diff --git a/docs/changelog.rst b/docs/changelog.rst index 54af41e4..728f7255 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.7.7 - Release Jan 28, 2022 +------------------------------------ + +Bug Fixes + * Changed the sort order for EDR sensor searches from 'last_checkin_time' (default when none provided explicitly) to 'hostname' to make the sort stable as sensors checkin during paging + + CbAPI 1.7.6 - Release Dec 20, 2021 ------------------------------------ diff --git a/docs/conf.py b/docs/conf.py index 1f538938..10361fba 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.7.6' +release = u'1.7.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 745195b0..e318464a 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( name='cbapi', - version='1.7.6', + version='1.7.7', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index 99d52b26..a9ebcc1e 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2020 VMware Carbon Black' -__version__ = '1.7.6' +__version__ = '1.7.7' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From 8339f1142982964fc67a6efe61061d01e2e574c4 Mon Sep 17 00:00:00 2001 From: Jared Fagel Date: Fri, 18 Feb 2022 15:30:23 -0600 Subject: [PATCH 178/197] Added additional export data. Export should include the created date (for retention purposes), the description, the action configuration, and the enabled status. --- examples/response/watchlist_exporter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/response/watchlist_exporter.py b/examples/response/watchlist_exporter.py index ae521789..728ed015 100755 --- a/examples/response/watchlist_exporter.py +++ b/examples/response/watchlist_exporter.py @@ -50,13 +50,18 @@ def export_watchlists(cb, args): if not confirm(watchlist.name): continue + watchlist_actions = cb.get_object('/api/v1/watchlist/{0}/action_type'.format(watchlist.id)) + exported_watchlists.append( { + "Created": str(watchlist.date_added), "Name": watchlist.name, "URL": watchlist.search_query, "Type": watchlist.index_type, "SearchString": watchlist.query, - "Description": "Please fill in if you intend to share this." + "Description": watchlist.description, + "Actions": watchlist_actions, + "Enabled": watchlist.enabled } ) From 2fc2f1dce4aed5717946b32f4e0dc173fb128a61 Mon Sep 17 00:00:00 2001 From: Dimitar Ganev Date: Tue, 14 Jun 2022 10:04:02 +0300 Subject: [PATCH 179/197] Change the attrdict builtin into attrdict3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e318464a..657f708f 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ install_requires = [ 'requests', - 'attrdict', + 'attrdict3', 'cachetools', 'pyyaml', 'pika', From db507d2b6b88a147e268653ebd0eebb3c09f4b64 Mon Sep 17 00:00:00 2001 From: Dimitar Ganev Date: Wed, 15 Jun 2022 09:54:23 +0300 Subject: [PATCH 180/197] Vendor the attrdict module --- setup.py | 1 - src/cbapi/attrdict.py | 273 ++++++++++++++++++++++++++++++++++++++++++ src/cbapi/auth.py | 2 +- 3 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 src/cbapi/attrdict.py diff --git a/setup.py b/setup.py index 657f708f..da033d9c 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,6 @@ install_requires = [ 'requests', - 'attrdict3', 'cachetools', 'pyyaml', 'pika', diff --git a/src/cbapi/attrdict.py b/src/cbapi/attrdict.py new file mode 100644 index 00000000..c551397b --- /dev/null +++ b/src/cbapi/attrdict.py @@ -0,0 +1,273 @@ +# Copyright (c) 2013 Brendan Curran-Johnson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +"""Functions and Classes from attrdict, modified for Python3""" + +from abc import ABCMeta, abstractmethod +from collections.abc import Mapping, MutableMapping, Sequence +import re + +"""A right-favoring Mapping merge.""" + + +def merge(left, right): + """ + Merge two mappings objects together, combining overlapping Mappings, favoring right-values + left: The left Mapping object. + right: The right (favored) Mapping object. + NOTE: This is not commutative (merge(a,b) != merge(b,a)). + """ + merged = {} + + left_keys = frozenset(left) + right_keys = frozenset(right) + + # Items only in the left Mapping + for key in left_keys - right_keys: + merged[key] = left[key] + + # Items only in the right Mapping + for key in right_keys - left_keys: + merged[key] = right[key] + + # in both + for key in left_keys & right_keys: + left_value = left[key] + right_value = right[key] + + if (isinstance(left_value, Mapping) and isinstance(right_value, Mapping)): # recursive merge + merged[key] = merge(left_value, right_value) + else: # overwrite with right value + merged[key] = right_value + + return merged + + +""" +Mixin Classes for Attr-support. +""" + + +class Attr(Mapping): + """ + A mixin class for a mapping that allows for attribute-style access of values. + A key may be used as an attribute if: + * It is a string + * It matches /^[A-Za-z][A-Za-z0-9_]*$/ (i.e., a public attribute) + * The key doesn't overlap with any class attributes (for Attr, + those would be 'get', 'items', 'keys', 'values', 'mro', and + 'register'). + If a values which is accessed as an attribute is a Sequence-type + (and is not a string/bytes), it will be converted to a + _sequence_type with any mappings within it converted to Attrs. + NOTE: This means that if _sequence_type is not None, then a + sequence accessed as an attribute will be a different object + than if accessed as an attribute than if it is accessed as an + item. + """ + @abstractmethod + def _configuration(self): + """All required state for building a new instance with the same settings as the current object.""" + + @classmethod + def _constructor(cls, mapping, configuration): + """ + A standardized constructor used internally by Attr. + mapping: A mapping of key-value pairs. It is HIGHLY recommended + that you use this as the internal key-value pair mapping, as + that will allow nested assignment (e.g., attr.foo.bar = baz) + configuration: The return value of Attr._configuration + """ + raise NotImplementedError("You need to implement this") + + def __call__(self, key): + """ + Dynamically access a key-value pair. + key: A key associated with a value in the mapping. + This differs from __getitem__, because it returns a new instance + of an Attr (if the value is a Mapping object). + """ + if key not in self: + raise AttributeError( + "'{cls} instance has no attribute '{name}'".format( + cls=self.__class__.__name__, name=key + ) + ) + + return self._build(self[key]) + + def __getattr__(self, key): + """Access an item as an attribute.""" + if key not in self or not self._valid_name(key): + raise AttributeError( + "'{cls}' instance has no attribute '{name}'".format( + cls=self.__class__.__name__, name=key + ) + ) + + return self._build(self[key]) + + def __add__(self, other): + """ + Add a mapping to this Attr, creating a new, merged Attr. + other: A mapping. + NOTE: Addition is not commutative. a + b != b + a. + """ + if not isinstance(other, Mapping): + return NotImplemented + + return self._constructor(merge(self, other), self._configuration()) + + def __radd__(self, other): + """ + Add this Attr to a mapping, creating a new, merged Attr. + other: A mapping. + NOTE: Addition is not commutative. a + b != b + a. + """ + if not isinstance(other, Mapping): + return NotImplemented + + return self._constructor(merge(other, self), self._configuration()) + + def _build(self, obj): + """Conditionally convert an object to allow for recursive mapping access. + obj: An object that was a key-value pair in the mapping. If obj + is a mapping, self._constructor(obj, self._configuration()) + will be called. If obj is a non-string/bytes sequence, and + self._sequence_type is not None, the obj will be converted + to type _sequence_type and build will be called on its + elements. + """ + if isinstance(obj, Mapping): + obj = self._constructor(obj, self._configuration()) + elif (isinstance(obj, Sequence) and not isinstance(obj, (str, bytes))): + sequence_type = getattr(self, '_sequence_type', None) + + if sequence_type: + obj = sequence_type(self._build(element) for element in obj) + + return obj + + @classmethod + def _valid_name(cls, key): + """ + Check whether a key is a valid attribute name. + A key may be used as an attribute if: + * It is a string + * It matches /^[A-Za-z][A-Za-z0-9_]*$/ (i.e., a public attribute) + * The key doesn't overlap with any class attributes (for Attr, + those would be 'get', 'items', 'keys', 'values', 'mro', and + 'register'). + """ + return ( + isinstance(key, str) and + re.match('^[A-Za-z][A-Za-z0-9_]*$', key) and + not hasattr(cls, key) + ) + + +class MutableAttr(Attr, MutableMapping, metaclass=ABCMeta): + """A mixin class for a mapping that allows for attribute-style access of values.""" + def _setattr(self, key, value): + """Add an attribute to the object, without attempting to add it as a key to the mapping.""" + super(MutableAttr, self).__setattr__(key, value) + + def __setattr__(self, key, value): + """ + Add an attribute. + key: The name of the attribute + value: The attributes contents + """ + if self._valid_name(key): + self[key] = value + elif getattr(self, '_allow_invalid_attributes', True): + super(MutableAttr, self).__setattr__(key, value) + else: + raise TypeError( + "'{cls}' does not allow attribute creation.".format( + cls=self.__class__.__name__ + ) + ) + + def _delattr(self, key): + """Delete an attribute from the object, without attempting to remove it from the mapping.""" + super(MutableAttr, self).__delattr__(key) + + def __delattr__(self, key, force=False): + """ + Delete an attribute. + key: The name of the attribute + """ + if self._valid_name(key): + del self[key] + elif getattr(self, '_allow_invalid_attributes', True): + super(MutableAttr, self).__delattr__(key) + else: + raise TypeError( + "'{cls}' does not allow attribute deletion.".format( + cls=self.__class__.__name__ + ) + ) + + +""" +A dict that implements MutableAttr. +""" + + +class AttrDict(dict, MutableAttr): + """A dict that implements MutableAttr.""" + def __init__(self, *args, **kwargs): + """Initilize the AttrDict""" + super(AttrDict, self).__init__(*args, **kwargs) + + self._setattr('_sequence_type', tuple) + self._setattr('_allow_invalid_attributes', False) + + def _configuration(self): + """The configuration for an attrmap instance.""" + return self._sequence_type + + def __getstate__(self): + """Serialize the object.""" + return ( + self.copy(), + self._sequence_type, + self._allow_invalid_attributes + ) + + def __setstate__(self, state): + """Deserialize the object.""" + mapping, sequence_type, allow_invalid_attributes = state + self.update(mapping) + self._setattr('_sequence_type', sequence_type) + self._setattr('_allow_invalid_attributes', allow_invalid_attributes) + + def __repr__(self): + """Override offical string representation.""" + return f'AttrDict({super(AttrDict, self).__repr__()})' + + @classmethod + def _constructor(cls, mapping, configuration): + """A standardized constructor.""" + attr = cls(mapping) + attr._setattr('_sequence_type', configuration) + + return attr \ No newline at end of file diff --git a/src/cbapi/auth.py b/src/cbapi/auth.py index baba7f8f..60cce5a8 100644 --- a/src/cbapi/auth.py +++ b/src/cbapi/auth.py @@ -1,6 +1,6 @@ from cbapi.six.moves.configparser import RawConfigParser import os -import attrdict +from cbapi import attrdict import logging import cbapi.six as six From dd65d6fee75be6d8bf12efe884ff1d737b1c1435 Mon Sep 17 00:00:00 2001 From: Dimtiar Ganev Date: Fri, 17 Jun 2022 09:45:52 +0300 Subject: [PATCH 181/197] Upgrade version to 1.7.8 --- README.md | 2 +- docs/changelog.rst | 6 ++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fc89025a..29187db0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.7.7** +**Latest Version: 1.7.8** _**Notice**:_ * The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI diff --git a/docs/changelog.rst b/docs/changelog.rst index 728f7255..10748d66 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,12 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.7.8 - Release Jun 17, 2022 +------------------------------------ + +Bug Fixes + * Vendor the `attrdict` module because of `ImportError` for Python3.10 + CbAPI 1.7.7 - Release Jan 28, 2022 ------------------------------------ diff --git a/docs/conf.py b/docs/conf.py index 10361fba..101852e4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.7.7' +release = u'1.7.8' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index da033d9c..0888a67e 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setup( name='cbapi', - version='1.7.7', + version='1.7.8', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index a9ebcc1e..e34a1a95 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2020 VMware Carbon Black' -__version__ = '1.7.7' +__version__ = '1.7.8' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From c47702b142a4cb6eb6ac16d2b413dfc5859dc583 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Mon, 12 Sep 2022 15:00:28 -0600 Subject: [PATCH 182/197] Add more performant sensor spawner --- src/cbapi/live_response_api.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/cbapi/live_response_api.py b/src/cbapi/live_response_api.py index 04632411..3009b5e8 100644 --- a/src/cbapi/live_response_api.py +++ b/src/cbapi/live_response_api.py @@ -985,9 +985,21 @@ def _spawn_new_workers(self): schedule_max = self._max_workers - len(self._job_workers) - sensors = [s for s in self._cb.select(Sensor) if s.id in self._unscheduled_jobs - and s.id not in self._job_workers - and s.status == "Online"] + sensors = [] + + for sensor_id in self._unscheduled_jobs: + sensor = self._cb.select(Sensor, sensor_id) + if sensor_id in self._job_workers: + log.debug("Skipping sensor {} already has a worker created".format(sensor_id)) + elif sensor.status == "Online": + sensors.append() + else: + log.warn("Sensor {} could not be found or is not Online".format(sensor_id)) + + # Non performant original code + # sensors = [s for s in self._cb.select(Sensor) if s.id in self._unscheduled_jobs + # and s.id not in self._job_workers + # and s.status == "Online"] sensors_to_schedule = sorted(sensors, key=lambda x: ( int(x.num_storefiles_bytes) + int(x.num_eventlog_bytes), x.next_checkin_time ))[:schedule_max] From 1074daba91f361807411785af755af2e1000950e Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Mon, 12 Sep 2022 15:16:10 -0600 Subject: [PATCH 183/197] Fix missing append parameter --- src/cbapi/live_response_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/live_response_api.py b/src/cbapi/live_response_api.py index 3009b5e8..7c05932c 100644 --- a/src/cbapi/live_response_api.py +++ b/src/cbapi/live_response_api.py @@ -992,7 +992,7 @@ def _spawn_new_workers(self): if sensor_id in self._job_workers: log.debug("Skipping sensor {} already has a worker created".format(sensor_id)) elif sensor.status == "Online": - sensors.append() + sensors.append(sensor) else: log.warn("Sensor {} could not be found or is not Online".format(sensor_id)) From f9ec4df68367753521382e35effcd4c0d3493656 Mon Sep 17 00:00:00 2001 From: Alex Van Brunt Date: Thu, 29 Sep 2022 13:48:48 -0600 Subject: [PATCH 184/197] Version bump for 1.7.9 --- README.md | 8 ++++---- docs/changelog.rst | 6 ++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 4 ++-- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 29187db0..18c0226d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.7.8** +**Latest Version: 1.7.9** -_**Notice**:_ +_**Notice**:_ * The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI -* Carbon Black EDR (Endpoint Detection and Response) is the new name for the product formerly called CB Response. +* Carbon Black EDR (Endpoint Detection and Response) is the new name for the product formerly called CB Response. * Carbon Black App Control is the new name for the product formerly called CB Protection. These are the Python bindings for the Carbon Black EDR and App Control REST APIs. @@ -49,7 +49,7 @@ Clone this repository, cd into `cbapi-python` then run setup.py with the `develo ### Sample Code -There are several examples in the `examples` directory for both EDR and App Control. +There are several examples in the `examples` directory for both EDR and App Control. For a quick start, see the following code snippets: **Carbon Black EDR** diff --git a/docs/changelog.rst b/docs/changelog.rst index 10748d66..28ee5882 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,12 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.7.9 - Release Sept 29, 2022 +------------------------------------ + +Bug Fixes + * Adjust Live Response Worker creation for EDR sensors to optimize for sensor specific jobs + CbAPI 1.7.8 - Release Jun 17, 2022 ------------------------------------ diff --git a/docs/conf.py b/docs/conf.py index 101852e4..061d816d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.7.8' +release = u'1.7.9' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 0888a67e..42503e25 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setup( name='cbapi', - version='1.7.8', + version='1.7.9', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index e34a1a95..39b2e873 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -5,8 +5,8 @@ __title__ = 'cbapi' __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' -__copyright__ = 'Copyright 2018-2020 VMware Carbon Black' -__version__ = '1.7.8' +__copyright__ = 'Copyright 2018-2022 VMware Carbon Black' +__version__ = '1.7.9' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From 2d2fc87d0838ec9e36758e73675c9cbbff3d91bc Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Thu, 26 Jan 2023 20:52:30 +0200 Subject: [PATCH 185/197] Using packaging for python3.7+ --- examples/response/partition_operations.py | 7 ++- setup.py | 3 +- src/cbapi/protection/models.py | 24 ++++++----- src/cbapi/protection/rest_api.py | 8 +++- src/cbapi/response/models.py | 52 ++++++++++++----------- src/cbapi/response/query.py | 8 +++- src/cbapi/response/rest_api.py | 12 ++++-- 7 files changed, 69 insertions(+), 45 deletions(-) diff --git a/examples/response/partition_operations.py b/examples/response/partition_operations.py index 07a87bad..bd896907 100755 --- a/examples/response/partition_operations.py +++ b/examples/response/partition_operations.py @@ -1,5 +1,8 @@ import sys -from distutils.version import LooseVersion +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from cbapi.response.models import StoragePartition from cbapi.example_helpers import build_cli_parser, get_cb_response_object @@ -73,7 +76,7 @@ def main(): args = parser.parse_args() cb = get_cb_response_object(args) - if cb.cb_server_version < LooseVersion("6.1.0"): + if cb.cb_server_version < parse("6.1.0"): parser.error("This script can only work with server versions >= 6.1.0; {0} is running {1}" .format(cb.url, cb.cb_server_version)) return 1 diff --git a/setup.py b/setup.py index 42503e25..d9a5a598 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,8 @@ install_requires.extend(['simplejson', 'total-ordering', 'ordereddict']) if sys.version_info < (3, 0): install_requires.extend(['futures']) - +if sys.version_info > (3, 6): + install_requires.extend(['packaging']) setup( name='cbapi', version='1.7.9', diff --git a/src/cbapi/protection/models.py b/src/cbapi/protection/models.py index 1e282d33..5b982767 100644 --- a/src/cbapi/protection/models.py +++ b/src/cbapi/protection/models.py @@ -3,7 +3,11 @@ from ..oldmodels import BaseModel, immutable, MutableModel from ..models import MutableBaseModel, CreatableModelMixin, NewBaseModel from contextlib import closing -from distutils.version import LooseVersion +import sys +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from zipfile import ZipFile import cbapi.six as six @@ -146,7 +150,7 @@ class DriftReport(NewBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class DriftReportContents(NewBaseModel): @@ -154,7 +158,7 @@ class DriftReportContents(NewBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class Event(NewBaseModel): @@ -280,7 +284,7 @@ class GrantedUserPolicyPermission(NewBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") @immutable @@ -374,7 +378,7 @@ class PublisherCertificate(NewBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class ScriptRule(MutableBaseModel): @@ -382,7 +386,7 @@ class ScriptRule(MutableBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") @immutable @@ -413,7 +417,7 @@ class TrustedDirectory(MutableBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class TrustedUser(MutableBaseModel, CreatableModelMixin): @@ -422,7 +426,7 @@ class TrustedUser(MutableBaseModel, CreatableModelMixin): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class User(MutableBaseModel, CreatableModelMixin): @@ -431,7 +435,7 @@ class User(MutableBaseModel, CreatableModelMixin): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class UserGroup(MutableBaseModel, CreatableModelMixin): @@ -440,4 +444,4 @@ class UserGroup(MutableBaseModel, CreatableModelMixin): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") diff --git a/src/cbapi/protection/rest_api.py b/src/cbapi/protection/rest_api.py index 3ee1f166..628c0016 100644 --- a/src/cbapi/protection/rest_api.py +++ b/src/cbapi/protection/rest_api.py @@ -1,7 +1,11 @@ from ..utils import convert_query_params from ..query import PaginatedQuery from ..errors import UnauthorizedError, ApiError -from distutils.version import LooseVersion +import sys +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from cbapi.connection import BaseAPI @@ -32,7 +36,7 @@ def __init__(self, *args, **kwargs): log.debug('Connected to Cb server version %s at %s' % (self._server_info['ParityServerVersion'], self.session.server)) - self.cb_server_version = LooseVersion(self._server_info['ParityServerVersion']) + self.cb_server_version = parse(self._server_info['ParityServerVersion']) def _perform_query(self, cls, **kwargs): if hasattr(cls, "_query_implementation"): diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py index 535d385e..0f177f13 100755 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -3,7 +3,11 @@ import copy import json -from distutils.version import LooseVersion +import sys +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from collections import namedtuple, defaultdict import base64 from datetime import datetime, timedelta @@ -656,7 +660,7 @@ def __init__(self, *args, **kwargs): def _query_implementation(cls, cb): # ** Disable the paginated query implementation for now ** - # if cb.cb_server_version >= LooseVersion("5.2.0"): + # if cb.cb_server_version >= parse("5.2.0"): # return SensorPaginatedQuery(cls, cb) # else: # return SensorQuery(cls, cb) @@ -869,7 +873,7 @@ def _update_object(self): if "event_log_flush_time" in self._dirty_attributes and self._info.get("event_log_flush_time", None) is not None: - if self._cb.cb_server_version > LooseVersion("6.0.0"): + if self._cb.cb_server_version > parse("6.0.0"): # since the date/time stamp just needs to be far in the future, we just fake a GMT timezone. try: self._info["event_log_flush_time"] = self.event_log_flush_time.strftime("%a, %d %b %Y %H:%M:%S GMT") @@ -1070,7 +1074,7 @@ def _retrieve_cb_info(self): return info def _update_object(self): - if self._cb.cb_server_version < LooseVersion("6.1.0") or self._info.get("id", None) is None: + if self._cb.cb_server_version < parse("6.1.0") or self._info.get("id", None) is None: # only include IDs of the teams and not the entire dictionary # - applies to Cb Response server < 6.0 as well as Cb Response servers >= 6.0 where the user hasn't # been created yet. @@ -1394,7 +1398,7 @@ class ThreatReport(MutableBaseModel): @classmethod def _query_implementation(cls, cb): - if cb.cb_server_version >= LooseVersion('5.1.0'): + if cb.cb_server_version >= parse('5.1.0'): return ThreatReportQuery(cls, cb) else: return Query(cls, cb) @@ -1524,7 +1528,7 @@ def group_by(self, field_name): :return: Query object :rtype: :py:class:`ProcessQuery` """ - if self._cb.cb_server_version >= LooseVersion('6.0.0'): + if self._cb.cb_server_version >= parse('6.0.0'): nq = self._clone() nq._default_args["cb.group"] = field_name return nq @@ -1573,7 +1577,7 @@ def min_last_update(self, v): :return: Query object :rtype: :py:class:`ProcessQuery` """ - if self._cb.cb_server_version >= LooseVersion('6.0.0'): + if self._cb.cb_server_version >= parse('6.0.0'): nq = self._clone() try: v = v.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -1599,7 +1603,7 @@ def min_last_server_update(self, v): :return: Query object :rtype: :py:class:`ProcessQuery` """ - if self._cb.cb_server_version >= LooseVersion('6.0.0'): + if self._cb.cb_server_version >= parse('6.0.0'): nq = self._clone() try: v = v.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -1625,7 +1629,7 @@ def max_last_update(self, v): :return: Query object :rtype: :py:class:`ProcessQuery` """ - if self._cb.cb_server_version >= LooseVersion('6.0.0'): + if self._cb.cb_server_version >= parse('6.0.0'): nq = self._clone() try: v = v.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -1651,7 +1655,7 @@ def max_last_server_update(self, v): :return: Query object :rtype: :py:class:`ProcessQuery` """ - if self._cb.cb_server_version >= LooseVersion('6.0.0'): + if self._cb.cb_server_version >= parse('6.0.0'): nq = self._clone() try: v = v.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -2285,7 +2289,7 @@ def parse_guid(self, procguid): # new 5.x process IDs are hex strings with optional segment IDs. if len(procguid) == 45: return procguid[:36], int(procguid[38:], 16) - elif len(procguid) == 49 and self._cb.cb_server_version >= LooseVersion('6.0.0'): + elif len(procguid) == 49 and self._cb.cb_server_version >= parse('6.0.0'): return procguid[:36], int(procguid[38:], 16) else: return None, None @@ -2309,7 +2313,7 @@ def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=Non self.__children_info = None self.__sibling_info = None - if cb.cb_server_version < LooseVersion('6.0.0'): + if cb.cb_server_version < parse('6.0.0'): self._default_segment = 1 else: self._default_segment = 0 @@ -2322,7 +2326,7 @@ def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=Non if len(procguid) == 45: self.id = procguid[:36] self.current_segment = int(procguid[38:], 16) - elif len(procguid) == 49 and cb.cb_server_version >= LooseVersion('6.0.0'): + elif len(procguid) == 49 and cb.cb_server_version >= parse('6.0.0'): self.id = procguid[:36] self.current_segment = int(procguid[38:], 16) else: @@ -2339,14 +2343,14 @@ def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=Non self._process_summary_api = 'v1' - if cb.cb_server_version >= LooseVersion('6.0.0'): + if cb.cb_server_version >= parse('6.0.0'): self._process_summary_api = 'v2' self._process_event_api = 'v4' self._event_parser = ProcessV4Parser(self) - elif cb.cb_server_version >= LooseVersion('5.2.0'): + elif cb.cb_server_version >= parse('5.2.0'): self._process_event_api = 'v3' self._event_parser = ProcessV3Parser(self) - elif cb.cb_server_version >= LooseVersion('5.1.0'): + elif cb.cb_server_version >= parse('5.1.0'): # CbER 5.1.0 introduced an extended event API self._process_event_api = 'v2' self._event_parser = ProcessV2Parser(self) @@ -2755,7 +2759,7 @@ def all_events_segment(self): def get_segments(self): if not self._segments: - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): log.debug("using process_id search for cb response server < 6.0") segment_query = Query(Process, self._cb, query="process_id:{0}".format(self.id)).sort("") proclist = sorted([res["segment_id"] for res in segment_query._search()]) @@ -3066,7 +3070,7 @@ def require_all_events(self): self.all_events_loaded = True def all_childprocs(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -3088,7 +3092,7 @@ def all_childprocs(self): i += 1 def all_modloads(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -3110,7 +3114,7 @@ def all_modloads(self): i += 1 def all_filemods(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -3132,7 +3136,7 @@ def all_filemods(self): i += 1 def all_processblocks(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -3154,7 +3158,7 @@ def all_processblocks(self): i += 1 def all_regmods(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -3176,7 +3180,7 @@ def all_regmods(self): i += 1 def all_crossprocs(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -3198,7 +3202,7 @@ def all_crossprocs(self): i += 1 def all_netconns(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments diff --git a/src/cbapi/response/query.py b/src/cbapi/response/query.py index 9d96a629..e02b11df 100644 --- a/src/cbapi/response/query.py +++ b/src/cbapi/response/query.py @@ -1,7 +1,11 @@ from ..utils import convert_query_params from ..query import PaginatedQuery from cbapi.six.moves import urllib -from distutils.version import LooseVersion +import sys +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from ..errors import ApiError import copy @@ -56,7 +60,7 @@ def __init__(self, doc_class, cb, query=None, raw_query=None): # FIX: Cb Response server version 5.1.0-3 throws an exception after returning HTTP 504 when facet=false in the # HTTP request. Work around this by only setting facet=false on 5.1.1 and above server versions. - if self._cb.cb_server_version >= LooseVersion('5.1.1'): + if self._cb.cb_server_version >= parse('5.1.1'): self._default_args["facet"] = "false" def _clone(self): diff --git a/src/cbapi/response/rest_api.py b/src/cbapi/response/rest_api.py index 8407bb7a..e172ab90 100644 --- a/src/cbapi/response/rest_api.py +++ b/src/cbapi/response/rest_api.py @@ -4,7 +4,11 @@ from cbapi.six.moves import urllib -from distutils.version import LooseVersion +import sys +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from ..connection import BaseAPI from .models import Process, Binary, Watchlist, Investigation, Alert, ThreatReport, StoragePartition from ..errors import UnauthorizedError, ApiError, ClientError @@ -44,13 +48,13 @@ def __init__(self, *args, **kwargs): raise UnauthorizedError(uri=self.url, message="Invalid API token for server {0:s}.".format(self.url)) log.debug('Connected to Cb server version %s at %s' % (self.server_info['version'], self.session.server)) - self.cb_server_version = LooseVersion(self.server_info['version']) - if self.cb_server_version < LooseVersion('5.0'): + self.cb_server_version = parse(self.server_info['version']) + if self.cb_server_version < parse('5.0'): raise ApiError("CbEnterpriseResponseAPI only supports Cb servers version >= 5.0.0") self._has_legacy_partitions = False try: - if self.cb_server_version >= LooseVersion('6.0'): + if self.cb_server_version >= parse('6.0'): legacy_partitions = [p for p in self.select(StoragePartition) if p.info.get("isLegacy", False)] if legacy_partitions: self._has_legacy_partitions = True From d6ac00004dc8e339509e2e2977d0bd9cd4f5b8e6 Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Wed, 1 Feb 2023 20:10:34 +0200 Subject: [PATCH 186/197] Version bump --- README.md | 2 +- docs/changelog.rst | 6 ++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 18c0226d..26625acc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.7.9** +**Latest Version: 1.7.10** _**Notice**:_ * The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI diff --git a/docs/changelog.rst b/docs/changelog.rst index 28ee5882..39c826f5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,12 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +CbAPI 1.7.10 - Release Feb 1, 2023 +------------------------------------ + +Bug Fixes + * Update CbAPI to use packaging instead of distutils for python3.7+ + CbAPI 1.7.9 - Release Sept 29, 2022 ------------------------------------ diff --git a/docs/conf.py b/docs/conf.py index 061d816d..d7d3f338 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.7.9' +release = u'1.7.10' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index d9a5a598..e8eb37dc 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ install_requires.extend(['packaging']) setup( name='cbapi', - version='1.7.9', + version='1.7.10', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index 39b2e873..f4acafd8 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2022 VMware Carbon Black' -__version__ = '1.7.9' +__version__ = '1.7.10' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI From 8a174a14e5b09bec61464c940d2587bf99bbebff Mon Sep 17 00:00:00 2001 From: Harshitha P Date: Thu, 28 Sep 2023 11:41:27 +0530 Subject: [PATCH 187/197] [CB-42564] Added current_password to user.yaml to change user password --- docs/concepts.rst | 3 ++- src/cbapi/response/models/user.yaml | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index ca093db9..639fdd60 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -207,9 +207,10 @@ Modifying Existing Objects The same ``.save()`` method can be used to modify existing Model Objects if the REST API provides that capability. If you attempt to modify a Model Object that cannot be changed, you will receive a :py:mod:`ApiError` exception. -For example, if you want to change the "jgarman" user's password to "cbisawesome":: +For example, if you want to change the "jgarman" user's password from the "cb" to "cbisawesome":: >>> user = cb.select(User, "jgarman") + >>> user.current_password = "cb" >>> user.password = "cbisawesome" >>> user.save() diff --git a/src/cbapi/response/models/user.yaml b/src/cbapi/response/models/user.yaml index 6758453e..33915583 100644 --- a/src/cbapi/response/models/user.yaml +++ b/src/cbapi/response/models/user.yaml @@ -41,3 +41,6 @@ properties: email: type: string description: '' + current_password: + type: string + description: '' From 16ff4eb3a9fc4098242e1b9cffcb128f3b603d46 Mon Sep 17 00:00:00 2001 From: Kylie Ebringer Date: Fri, 1 Dec 2023 14:47:28 -0700 Subject: [PATCH 188/197] Removed references to Carbon Black Cloud. CBAPI does not support this. Carbon Black Cloud users should be using the Python SDK here: https://github.com/carbonblack/carbon-black-cloud-sdk-python --- docs/index.rst | 47 +++++++++++++---------------------------------- 1 file changed, 13 insertions(+), 34 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4fbc9f57..73b5662f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,8 +8,8 @@ cbapi: Carbon Black API for Python Release v\ |release|. -CBAPI provides a straightforward interface to the VMware Carbon Black products: Carbon Black EDR, Carbon Black App Control, and Carbon Black Cloud Endpoint Standard(formerly CB Response, CB Protection, and CB Defense). -This library provides a Pythonic layer to access the raw power of the REST APIs of all Carbon Black products, making it easier to query data from any platform or on-premise APIs, combine data from multiple API calls, manage all API credentials in one place, and manipulate data as Python objects. Take a look:: +CBAPI provides a straightforward interface to the VMware Carbon Black products Carbon Black EDR and Carbon Black App Control. +This library provides a Pythonic layer to access the raw power of the REST APIs of these Carbon Black products, making it easier to query data from on-premise APIs, combine data from multiple API calls, manage all API credentials in one place, and manipulate data as Python objects. Take a look:: >>> from cbapi.response import CbResponseAPI, Process, Binary, Sensor >>> # @@ -61,24 +61,6 @@ If you're a Carbon Black App Control customer (formerly CB Protection), you may >>> fi.computer.policyId = 3 >>> fi.computer.save() -As of version 1.2, CBAPI also supports Carbon Black Cloud Endpoint Standard (formerly CB Defense): - - >>> from cbapi.psc.defense import * - >>> # - >>> # Create our Cloud Endpoint Standard API object - >>> # - >>> p = CbDefenseAPI() - >>> # - >>> # Select any devices that have the hostname WIN-IA9NQ1GN8OI and an internal IP address of 192.168.215.150 - >>> # - >>> devices = c.select(Device).where('hostNameExact:WIN-IA9NQ1GN8OI').and_("ipAddress:192.168.215.150").first() - >>> # - >>> # Change those devices' policy into the Windows_Restrictive_Workstation policy. - >>> # - >>> for dev in devices: - >>> dev.policyName = "Restrictive_Windows_Workstation" - >>> dev.save() - Major Features -------------- @@ -89,9 +71,9 @@ Major Features necessary to make your Incident Response process much more efficient and automated. - **Consistent API across VMware Carbon Black platforms** - CBAPI supports Carbon Black EDR, Carbon Black App Control, and Carbon Black Cloud Endpoint Standard customers from a single API layer. Even better, + CBAPI supports Carbon Black EDR and Carbon Black App Control customers from a single API layer. Even better, the object model is the same for all three, and if you know one API, you can easily transition to another. CBAPI - manages the differences among the three REST APIs behind a single, consistent Python-like interface. + manages the differences among the two REST APIs behind a single, consistent Python-like interface. - **Enhanced Performance** CBAPI now provides a built in caching layer to reduce the query load on the Carbon Black server. This is especially @@ -110,7 +92,7 @@ Major Features - **Better support for multiple CB servers** CBAPI introduces the concept of Credential Profiles; named collections of URL, API keys, and optional proxy - configuration for connecting to any number of Carbon Black EDR, Carbon Black App Control, or Carbon Black Cloud Endpoint Standard servers. + configuration for connecting to any number of Carbon Black EDR or Carbon Black App Control servers. API Credentials @@ -120,9 +102,8 @@ CBAPI version 0.9.0 enforces the use of credential files. In order to perform any queries via the API, you will need to get the API token for your CB user. See the documentation on the Developer Network website on how to acquire the API token for -`Carbon Black EDR (CB Response) `_, -`Carbon Black App Control (CB Protection) `_, or -`Carbon Black Cloud Endpoint Standard (CB Defense) `_. +`Carbon Black EDR (CB Response) `_, or +`Carbon Black App Control (CB Protection) `_. Once you acquire your API token, place it in one of the default credentials file locations: @@ -133,16 +114,15 @@ Once you acquire your API token, place it in one of the default credentials file For distinction between credentials of different Carbon Black products, use the following naming convention for your credentials files: -* ``credentials.psc`` for Carbon Black Cloud Endpoint Standard, Audit & Remediation, and Enterprise EDR (CB Defense, CB LiveOps, and CB ThreatHunter) * ``credentials.response`` for Carbon Black EDR (CB Response) * ``credentials.protection`` for Carbon Black App Control (CB Protection) -For example, if you use a Carbon Black Cloud product, you should have created a credentials file in one of these +For example, if you use Carbon Black EDR, you should have created a credentials file in one of these locations: -* ``/etc/carbonblack/credentials.psc`` -* ``~/.carbonblack/credentials.psc`` -* ``/current_working_directory/.carbonblack/credentials.psc`` +* ``/etc/carbonblack/credentials.response`` +* ``~/.carbonblack/credentials.response`` +* ``/current_working_directory/.carbonblack/credentials.response`` Credentials found in a later path will overwrite earlier ones. @@ -171,7 +151,6 @@ The possible options for each credential profile are: different tokens for each. * **ssl_verify**: True or False; controls whether the SSL/TLS certificate presented by the server is validated against the local trusted CA store. -* **org_key**: The organization key. This is required to access the Carbon Black Cloud, and can be found in the console. The format is ``123ABC45``. * **proxy**: A proxy specification that will be used when connecting to the Carbon Black server. The format is: ``http://myusername:mypassword@proxy.company.com:8001/`` where the hostname of the proxy is ``proxy.company.com``, port 8001, and using username/password ``myusername`` and ``mypassword`` respectively. @@ -185,9 +164,9 @@ Environment Variable Support The latest CBAPI for Python supports specifying API credentials in the following three environment variables: -`CBAPI_TOKEN` the envar for holding the EDR (CbR) or App Control (CbP) api token or the ConnectorId/APIKEY combination for Endpoint Standard (CB Defense)/Carbon Black Cloud. +`CBAPI_TOKEN` the envar for holding the EDR (CbR) or App Control (CbP) api token. -The `CBAPI_URL` envar holds the FQDN of the target, an EDR (CbR), CBD, or CbD/Carbon Black Cloud server specified just as they are in the +The `CBAPI_URL` envar holds the FQDN of the target, an EDR (CbR) server specified just as they are in the configuration file format specified above. The optional `CBAPI_SSL_VERIFY` envar can be used to control SSL validation(True/False or 0/1), which will default to From 77c8575c484f350b94e73760b6c2c84642fa9b22 Mon Sep 17 00:00:00 2001 From: Kylie Ebringer <55465092+kebringer-cb@users.noreply.github.com> Date: Fri, 16 Feb 2024 10:23:42 -0700 Subject: [PATCH 189/197] Remove references to Carbon Black Cloud (#325) * Marked references to CBC or PSC as deprecated. * Removing doc table of content * added changelog for the doc update --- .readthedocs.yaml | 30 +++++ docs/changelog.rst | 10 ++ docs/conf.py | 4 +- docs/defense-api.rst | 35 +---- docs/getting-started.rst | 7 +- docs/index.rst | 4 - docs/livequery-api.rst | 49 +------ docs/livequery-examples.rst | 119 +---------------- docs/psc-api.rst | 122 +----------------- docs/requirements.txt | 7 + docs/threathunter-api.rst | 96 +------------- .../DellBiosVerification/BiosVerification.py | 0 .../cblr/DellBiosVerification/README.md | 0 .../cblr/DellBiosVerification/dellbios.bat | 0 .../cblr/examplejob.py | 0 .../cblr/jobrunner.py | 0 .../cblr_cli.py | 0 .../event_export.py | 0 .../list_devices.py | 0 .../list_events.py | 0 .../list_events_with_cmdline_csv.py | 0 .../move_device.py | 0 .../notifications.py | 0 .../policy_operations.py | 0 .../alert_search_suggestions.py | 0 .../bulk_update_alerts.py | 0 .../bulk_update_cbanalytics_alerts.py | 0 .../bulk_update_threat_alerts.py | 0 .../bulk_update_vmware_alerts.py | 0 .../bulk_update_watchlist_alerts.py | 0 .../{psc => DEPRECATED_psc}/device_control.py | 0 .../download_device_list.py | 0 .../helpers/alertsv6.py | 0 .../list_alert_facets.py | 0 .../{psc => DEPRECATED_psc}/list_alerts.py | 0 .../list_cbanalytics_alert_facets.py | 0 .../list_cbanalytics_alerts.py | 0 .../{psc => DEPRECATED_psc}/list_devices.py | 0 .../list_vmware_alert_facets.py | 0 .../list_vmware_alerts.py | 0 .../list_watchlist_alert_facets.py | 0 .../list_watchlist_alerts.py | 0 .../create_feed.py | 0 .../events.py | 0 .../events_exporter.py | 0 .../feed_operations.py | 0 .../import_response_feeds.py | 0 .../modify_feed.py | 0 .../process_exporter.py | 0 .../process_query.py | 0 .../process_tree.py | 0 .../process_tree_exporter.py | 0 .../search.py | 0 .../threat_intelligence/README.md | 0 .../threat_intelligence/Taxii_README.md | 0 .../threat_intelligence/config.yml | 0 .../threat_intelligence/feed_helper.py | 0 .../threat_intelligence/get_feed_ids.py | 0 .../threat_intelligence/requirements.txt | 0 .../threat_intelligence/results.py | 0 .../threat_intelligence/schemas.py | 0 .../threat_intelligence/stix_parse.py | 0 .../threat_intelligence/stix_taxii.py | 0 .../threat_intelligence/threatintel.py | 0 .../watchlist_operations.py | 0 examples/README.md | 8 ++ src/cbapi/psc/defense/rest_api.py | 12 +- 67 files changed, 103 insertions(+), 400 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 docs/requirements.txt rename examples/{defense => DEPRECATED_defense}/cblr/DellBiosVerification/BiosVerification.py (100%) rename examples/{defense => DEPRECATED_defense}/cblr/DellBiosVerification/README.md (100%) rename examples/{defense => DEPRECATED_defense}/cblr/DellBiosVerification/dellbios.bat (100%) rename examples/{defense => DEPRECATED_defense}/cblr/examplejob.py (100%) rename examples/{defense => DEPRECATED_defense}/cblr/jobrunner.py (100%) rename examples/{defense => DEPRECATED_defense}/cblr_cli.py (100%) rename examples/{defense => DEPRECATED_defense}/event_export.py (100%) rename examples/{defense => DEPRECATED_defense}/list_devices.py (100%) rename examples/{defense => DEPRECATED_defense}/list_events.py (100%) rename examples/{defense => DEPRECATED_defense}/list_events_with_cmdline_csv.py (100%) rename examples/{defense => DEPRECATED_defense}/move_device.py (100%) rename examples/{defense => DEPRECATED_defense}/notifications.py (100%) rename examples/{defense => DEPRECATED_defense}/policy_operations.py (100%) rename examples/{psc => DEPRECATED_psc}/alert_search_suggestions.py (100%) rename examples/{psc => DEPRECATED_psc}/bulk_update_alerts.py (100%) rename examples/{psc => DEPRECATED_psc}/bulk_update_cbanalytics_alerts.py (100%) rename examples/{psc => DEPRECATED_psc}/bulk_update_threat_alerts.py (100%) rename examples/{psc => DEPRECATED_psc}/bulk_update_vmware_alerts.py (100%) rename examples/{psc => DEPRECATED_psc}/bulk_update_watchlist_alerts.py (100%) rename examples/{psc => DEPRECATED_psc}/device_control.py (100%) rename examples/{psc => DEPRECATED_psc}/download_device_list.py (100%) rename examples/{psc => DEPRECATED_psc}/helpers/alertsv6.py (100%) rename examples/{psc => DEPRECATED_psc}/list_alert_facets.py (100%) rename examples/{psc => DEPRECATED_psc}/list_alerts.py (100%) rename examples/{psc => DEPRECATED_psc}/list_cbanalytics_alert_facets.py (100%) rename examples/{psc => DEPRECATED_psc}/list_cbanalytics_alerts.py (100%) rename examples/{psc => DEPRECATED_psc}/list_devices.py (100%) rename examples/{psc => DEPRECATED_psc}/list_vmware_alert_facets.py (100%) rename examples/{psc => DEPRECATED_psc}/list_vmware_alerts.py (100%) rename examples/{psc => DEPRECATED_psc}/list_watchlist_alert_facets.py (100%) rename examples/{psc => DEPRECATED_psc}/list_watchlist_alerts.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/create_feed.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/events.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/events_exporter.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/feed_operations.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/import_response_feeds.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/modify_feed.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/process_exporter.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/process_query.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/process_tree.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/process_tree_exporter.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/search.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/threat_intelligence/README.md (100%) rename examples/{threathunter => DEPRECATED_threathunter}/threat_intelligence/Taxii_README.md (100%) rename examples/{threathunter => DEPRECATED_threathunter}/threat_intelligence/config.yml (100%) rename examples/{threathunter => DEPRECATED_threathunter}/threat_intelligence/feed_helper.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/threat_intelligence/get_feed_ids.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/threat_intelligence/requirements.txt (100%) rename examples/{threathunter => DEPRECATED_threathunter}/threat_intelligence/results.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/threat_intelligence/schemas.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/threat_intelligence/stix_parse.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/threat_intelligence/stix_taxii.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/threat_intelligence/threatintel.py (100%) rename examples/{threathunter => DEPRECATED_threathunter}/watchlist_operations.py (100%) diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..ac48e9db --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,30 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + builder: dirhtml +# fail_on_warning: true + +# If using Sphinx, optionally build your docs in additional formats, such as PDF +formats: + - pdf + - epub + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt + - method: setuptools + path: . diff --git a/docs/changelog.rst b/docs/changelog.rst index 39c826f5..6de9a8bd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,16 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) +Documentation - Release Feb 14, 2023 +------------------------------------ + +Updates + * Removed references to and documentation about Carbon Black Cloud. CBAPI (this SDK) is not maintained for Carbon Black Cloud. + + Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. Please see + `Carbon Black Cloud Python SDK on the Developer Network `_ + for details. + CbAPI 1.7.10 - Release Feb 1, 2023 ------------------------------------ diff --git a/docs/conf.py b/docs/conf.py index d7d3f338..52251114 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -68,7 +68,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -299,4 +299,4 @@ def setup(app): - app.add_stylesheet('css/custom.css') + app.add_css_file('css/custom.css') diff --git a/docs/defense-api.rst b/docs/defense-api.rst index 88b99d50..1c983a39 100644 --- a/docs/defense-api.rst +++ b/docs/defense-api.rst @@ -1,33 +1,12 @@ .. _defense_api: -Cloud Endpoint Standard API -=========================== +Cloud Endpoint Standard API - DEPRECATED +======================================== -This page documents the public interfaces exposed by cbapi when communicating with a Cloud Endpoint Standard server. +Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. -Main Interface --------------- +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. -To use cbapi with VMware Carbon Black Cloud Endpoint Standard, you will be using the CBDefenseAPI. -The CBDefenseAPI object then exposes two main methods to select data on the Carbon Black server: - -.. autoclass:: cbapi.psc.defense.rest_api.CbDefenseAPI - :members: - :inherited-members: - - .. :automethod:: select - .. :automethod:: create - -Queries -------- - -.. autoclass:: cbapi.psc.defense.rest_api.Query - :members: - - -Models ------- - -.. automodule:: cbapi.psc.defense.models - :members: - :undoc-members: +CBAPI is not maintained for Carbon Black Cloud. diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 5586cd42..3f0d7a0e 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -25,8 +25,8 @@ cbapi strongly discourages embedding credentials in individual scripts. Instead, EDR (CB Response) or App Control (CB Protection) servers inside the API credential file and select which "profile" you would like to use at runtime. -To create the initial credential file, a simple-to-use script is provided. Just run the ``cbapi-response``, -``cbapi-protection``, or ``cbapi-psc`` script with the ``configure`` argument. On Mac OS X and Linux:: +To create the initial credential file, a simple-to-use script is provided. Just run the ``cbapi-response`` or +``cbapi-protection`` script with the ``configure`` argument. On Mac OS X and Linux:: $ cbapi-response configure @@ -37,9 +37,6 @@ Alternatively, if you're using Windows (change ``c:\python27`` if Python is inst This configuration script will walk you through entering your API credentials and will save them to your current user's credential file location, which is located in the ``.carbonblack`` directory in your user's home directory. -If using cbapi-psc, you will also be asked to provide an org key. An org key is required to access the Carbon Black -Cloud, and can be found in the console under Settings -> API Keys. - Your First Query ---------------- diff --git a/docs/index.rst b/docs/index.rst index 73b5662f..7e1fdf1b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -228,10 +228,6 @@ and unlock the full functionality of the SDK. response-api protection-api - defense-api - threathunter-api - psc-api - livequery-api exceptions Indices and tables diff --git a/docs/livequery-api.rst b/docs/livequery-api.rst index faec5929..8224bbc5 100644 --- a/docs/livequery-api.rst +++ b/docs/livequery-api.rst @@ -1,47 +1,12 @@ .. _livequery_api: -CB LiveQuery API -================ +CB LiveQuery API - DEPRECATED +============================= -This page documents the public interfaces exposed by cbapi when communicating with Carbon Black LiveQuery devices. +Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. -Main Interface --------------- - -To use cbapi with Carbon Black LiveQuery, you use CbLiveQueryAPI objects. - -The LiveQuery API is used in two stages: run submission and result retrieval. - -.. autoclass:: cbapi.psc.livequery.rest_api.CbLiveQueryAPI - :members: - :inherited-members: - -Queries -------- - -The LiveQuery API uses QueryBuilder instances to construct structured or unstructured (i.e., raw string) queries. -You can either construct these instances manually, or allow ``CbLiveQueryAPI.select()`` to do it for you: - -.. autoclass:: cbapi.psc.livequery.query.QueryBuilder - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.livequery.query.RunQuery - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.livequery.models.ResultQuery - :members: - :inherited-members: - -Models ------- - -.. autoclass:: cbapi.psc.livequery.models.Run - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.livequery.models.Result - :members: - :inherited-members: +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. +CBAPI is not maintained for Carbon Black Cloud. diff --git a/docs/livequery-examples.rst b/docs/livequery-examples.rst index f3f88262..ac32dea4 100644 --- a/docs/livequery-examples.rst +++ b/docs/livequery-examples.rst @@ -1,115 +1,10 @@ -CB LiveQuery API Examples -========================= +CB LiveQuery API Examples - DEPRECATED +====================================== -Let's cover a few example functions that our LiveQuery Python bindings enable. To begin, we need to import the -relevant libraries:: +Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. - >>> import sys - >>> from cbapi.psc.livequery import CbLiveQueryAPI - >>> from cbapi.psc.livequery.models import Run, Result +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. - -Now that we've imported the necessary libraries, we can perform some queries on our endpoints. - -Create a Query Run ----------------------------------- - -Let's create a Query Run. First, we specify which profile to use for authentication from our credentials.psc file and -create the LiveQuery object. - - >>> profile = "default' - >>> cb = CbLiveQueryAPI(profile=profile) - -Now, we specify the SQL query that we want to run, name of the run, device IDs, and device types. - - >>> sql = 'select * from logged_in_users;' - >>> name_of_run = 'Selecting all logged in users' - >>> device_ids = '1234567' - >>> device_types = 'WINDOWS' - -Now, we create a query and add these values to it. - - >>> query = cb.query(sql) - >>> query.name(name_of_run) - >>> query.device_ids(device_ids) - >>> query.device_types(device_types) - -Finally, we submit the query and print the results. - - >>> run = query.submit() - >>> print(run) - -This query should return all logged in Windows endpoints with a ``device_id`` of ``1234567``. - -The same query can be executed with the example script -`manage_run.py `_. :: - - python manage_run.py --profile default create --sql 'select * from logged_in_users;' --name 'Selecting all logged in users' --device_ids '1234567' --device_types 'WINDOWS' - -Other possible arguments to ``manage_run.py`` include ``--notify`` and ``--policy_ids``. - -Get Query Run Status ---------------------- - -Now that we've created a Query Run, let's check the status. If we haven't already authenticated with a credentials -profile, we begin by specifying which profile to authenticate with. - - >>> profile = 'default' - >>> cb = CbLiveQueryAPI(profile=profile) - -Next, we select the run with the unique run ID. - - >>> run_id = 'a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz' - >>> run = cb.select(Run, run_id) - >>> print(run) - -This can also be accomplished with the example script -`manage_run.py `_:: - - python manage_run.py --profile default --id a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz - -In addition, you can specify which order you want results returned. To change from the default ascending order, use -the flag ``-d`` or ``--descending_results``:: - - python manage_run.py --profile default --id a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz --descending_results - -Get Query Run Results ---------------------- - -Let's view the results of a run. If we haven't already authenticated, we must start with that. - - >>> profile = 'default' - >>> cb = CbLiveQueryAPI(profile=profile) - -To view the results of a run, we must specify the run ID. - - >>> run_id = 'a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz' - >>> results = cb.select(Result).run_id(run_id) - -Finally, we print the results. - - >>> for result in results: - ... print(result) - -Results can be narrowed down with the following criteria:: - - device_ids - status - -Examples of using these criteria are below:: - - >>> device_id = '1234567' - >>> results.criteria(device_id=device_id) - >>> status = 'matched' - >>> results.criteria(status=status) - -Finally, we print the results. - - >>> for result in results: - ... print(result) - - -You can also retrieve run results with the example script -`run_search.py `_:: - - python run_search.py --profile default --id a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz --device_ids '1234567' --status 'matched' +CBAPI is not maintained for Carbon Black Cloud. diff --git a/docs/psc-api.rst b/docs/psc-api.rst index 25ca80cf..712ad928 100755 --- a/docs/psc-api.rst +++ b/docs/psc-api.rst @@ -1,120 +1,12 @@ .. _psc_api: -VMware Carbon Black Cloud API -============================= +VMware Carbon Black Cloud API - DEPRECATED +========================================== -This page documents the public interfaces exposed by cbapi when communicating with the VMware Carbon Black Cloud. +Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. -Main Interface --------------- +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. -To use cbapi with the VMware Carbon Black Cloud, you use CbPSCBaseAPI objects. - -.. autoclass:: cbapi.psc.rest_api.CbPSCBaseAPI - :members: - :inherited-members: - -Device API ----------- - -The Carbon Black Cloud can be used to enumerate devices within your organization, and change their -status via a control request. - -You can use the select() method on the CbPSCBaseAPI to create a query object for -Device objects, which can be used to locate a list of Devices. - -*Example:* - - >>> cbapi = CbPSCBaseAPI(...) - >>> devices = cbapi.select(Device).set_os("LINUX").status("ALL") - -Selects all devices running Linux from the current organization. - -**Query Object:** - -.. autoclass:: cbapi.psc.devices_query.DeviceSearchQuery - :members: - -**Model Object:** - -.. autoclass:: cbapi.psc.models.Device - :members: - :undoc-members: - -Alerts API ----------- - -Using the API, you can search for alerts within your organization, and dismiss or undismiss them, either individually -or in bulk. - -You can use the select() method on the CbPSCBaseAPI to create a query object for BaseAlert objects, which can be used -to locate a list of alerts. You can also search for more specialized alert types: - -* CBAnalyticsAlert - Alerts from CB Analytics -* VMwareAlert - Alerts from VMware -* WatchlistAlert - Alerts from watch lists - -*Example:* - - >>> cbapi = CbPSCBaseAPI(...) - >>> alerts = cbapi.select(BaseAlert).set_device_os(["WINDOWS"]).set_process_name(["IEXPLORE.EXE"]) - -Selects all alerts on a Windows device running the Internet Explorer process. - -Individual alerts may have their status changed using the dismiss() or update() -methods on the BaseAlert object. To dismiss multiple alerts at once, you can use -the dismiss() or update() methods on the standard query, after adding criteria to it. -This method returns a request ID, which can be used to create a WorkflowStatus object; -querying this object's "finished" property will let you know when the operation is -finished. - -*Example:* - - >>> cbapi = CbPSCBaseAPI(...) - >>> query = cbapi.select(BaseAlert).set_process_name(["IEXPLORE.EXE"]) - >>> reqid = query.dismiss("Using Chrome") - >>> stat = cbapi.select(WorkflowStatus, reqid) - >>> while not stat.finished: - >>> # wait for it to finish - -This dismisses all alerts which reference the Internet Explorer process. - -**Query Objects:** - -.. autoclass:: cbapi.psc.alerts_query.BaseAlertSearchQuery - :members: - -.. autoclass:: cbapi.psc.alerts_query.CBAnalyticsAlertSearchQuery - :members: - -.. autoclass:: cbapi.psc.alerts_query.VMwareAlertSearchQuery - :members: - -.. autoclass:: cbapi.psc.alerts_query.WatchlistAlertSearchQuery - :members: - -**Model Objects:** - -.. autoclass:: cbapi.psc.models.Workflow - :members: - :undoc-members: - -.. autoclass:: cbapi.psc.models.BaseAlert - :members: - :undoc-members: - -.. autoclass:: cbapi.psc.models.CBAnalyticsAlert - :members: - :undoc-members: - -.. autoclass:: cbapi.psc.models.VMwareAlert - :members: - :undoc-members: - -.. autoclass:: cbapi.psc.models.WatchlistAlert - :members: - :undoc-members: - -.. autoclass:: cbapi.psc.models.WorkflowStatus - :members: - :undoc-members: +CBAPI is not maintained for Carbon Black Cloud. diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..a662ecf9 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,7 @@ +# Defining the exact version will make sure things don't break +sphinx==6.2.1 +sphinx-copybutton==0.4.0 +sphinx-rtd-theme==1.2.2 +sphinxcontrib-apidoc +sphinx-copybutton==0.4.0 +pygments diff --git a/docs/threathunter-api.rst b/docs/threathunter-api.rst index 858aa1ef..82d58bc7 100644 --- a/docs/threathunter-api.rst +++ b/docs/threathunter-api.rst @@ -1,94 +1,12 @@ .. _threathunter_api: -VMware Carbon Black Cloud Enterprise EDR API -============================================ +VMware Carbon Black Cloud Enterprise EDR API - DEPRECATED +========================================================= -This page documents the public interfaces exposed by cbapi when communicating with a -VMware Carbon Black Cloud Enterprise EDR server. +Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. -Main Interface --------------- +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. -To use cbapi with Enterprise EDR, you use CbThreatHunterAPI objects. -These objects expose two main methods to access data on the Enterprise EDR server: ``select`` and ``create``. - -.. autoclass:: cbapi.psc.threathunter.rest_api.CbThreatHunterAPI - :members: - :inherited-members: - -Queries -------- - -The Enterprise EDR API uses QueryBuilder instances to construct structured -or unstructured (i.e., raw string) queries. You can either construct these -instances manually, or allow ``CbThreatHunterAPI.select()`` to do it for you: - -.. autoclass:: cbapi.psc.threathunter.query.QueryBuilder - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.query.Query - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.AsyncProcessQuery - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.query.FeedQuery - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.query.ReportQuery - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.query.WatchlistQuery - :members: - :inherited-members: - -Models ------- - -.. autoclass:: cbapi.psc.threathunter.models.Process - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Event - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Tree - :members: - -.. autoclass:: cbapi.psc.threathunter.models.Feed - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Report - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.IOC - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.IOC_V2 - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Watchlist - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.ReportSeverity - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Binary - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Downloads - :members: - :inherited-members: +CBAPI is not maintained for Carbon Black Cloud. diff --git a/examples/defense/cblr/DellBiosVerification/BiosVerification.py b/examples/DEPRECATED_defense/cblr/DellBiosVerification/BiosVerification.py similarity index 100% rename from examples/defense/cblr/DellBiosVerification/BiosVerification.py rename to examples/DEPRECATED_defense/cblr/DellBiosVerification/BiosVerification.py diff --git a/examples/defense/cblr/DellBiosVerification/README.md b/examples/DEPRECATED_defense/cblr/DellBiosVerification/README.md similarity index 100% rename from examples/defense/cblr/DellBiosVerification/README.md rename to examples/DEPRECATED_defense/cblr/DellBiosVerification/README.md diff --git a/examples/defense/cblr/DellBiosVerification/dellbios.bat b/examples/DEPRECATED_defense/cblr/DellBiosVerification/dellbios.bat similarity index 100% rename from examples/defense/cblr/DellBiosVerification/dellbios.bat rename to examples/DEPRECATED_defense/cblr/DellBiosVerification/dellbios.bat diff --git a/examples/defense/cblr/examplejob.py b/examples/DEPRECATED_defense/cblr/examplejob.py similarity index 100% rename from examples/defense/cblr/examplejob.py rename to examples/DEPRECATED_defense/cblr/examplejob.py diff --git a/examples/defense/cblr/jobrunner.py b/examples/DEPRECATED_defense/cblr/jobrunner.py similarity index 100% rename from examples/defense/cblr/jobrunner.py rename to examples/DEPRECATED_defense/cblr/jobrunner.py diff --git a/examples/defense/cblr_cli.py b/examples/DEPRECATED_defense/cblr_cli.py similarity index 100% rename from examples/defense/cblr_cli.py rename to examples/DEPRECATED_defense/cblr_cli.py diff --git a/examples/defense/event_export.py b/examples/DEPRECATED_defense/event_export.py similarity index 100% rename from examples/defense/event_export.py rename to examples/DEPRECATED_defense/event_export.py diff --git a/examples/defense/list_devices.py b/examples/DEPRECATED_defense/list_devices.py similarity index 100% rename from examples/defense/list_devices.py rename to examples/DEPRECATED_defense/list_devices.py diff --git a/examples/defense/list_events.py b/examples/DEPRECATED_defense/list_events.py similarity index 100% rename from examples/defense/list_events.py rename to examples/DEPRECATED_defense/list_events.py diff --git a/examples/defense/list_events_with_cmdline_csv.py b/examples/DEPRECATED_defense/list_events_with_cmdline_csv.py similarity index 100% rename from examples/defense/list_events_with_cmdline_csv.py rename to examples/DEPRECATED_defense/list_events_with_cmdline_csv.py diff --git a/examples/defense/move_device.py b/examples/DEPRECATED_defense/move_device.py similarity index 100% rename from examples/defense/move_device.py rename to examples/DEPRECATED_defense/move_device.py diff --git a/examples/defense/notifications.py b/examples/DEPRECATED_defense/notifications.py similarity index 100% rename from examples/defense/notifications.py rename to examples/DEPRECATED_defense/notifications.py diff --git a/examples/defense/policy_operations.py b/examples/DEPRECATED_defense/policy_operations.py similarity index 100% rename from examples/defense/policy_operations.py rename to examples/DEPRECATED_defense/policy_operations.py diff --git a/examples/psc/alert_search_suggestions.py b/examples/DEPRECATED_psc/alert_search_suggestions.py similarity index 100% rename from examples/psc/alert_search_suggestions.py rename to examples/DEPRECATED_psc/alert_search_suggestions.py diff --git a/examples/psc/bulk_update_alerts.py b/examples/DEPRECATED_psc/bulk_update_alerts.py similarity index 100% rename from examples/psc/bulk_update_alerts.py rename to examples/DEPRECATED_psc/bulk_update_alerts.py diff --git a/examples/psc/bulk_update_cbanalytics_alerts.py b/examples/DEPRECATED_psc/bulk_update_cbanalytics_alerts.py similarity index 100% rename from examples/psc/bulk_update_cbanalytics_alerts.py rename to examples/DEPRECATED_psc/bulk_update_cbanalytics_alerts.py diff --git a/examples/psc/bulk_update_threat_alerts.py b/examples/DEPRECATED_psc/bulk_update_threat_alerts.py similarity index 100% rename from examples/psc/bulk_update_threat_alerts.py rename to examples/DEPRECATED_psc/bulk_update_threat_alerts.py diff --git a/examples/psc/bulk_update_vmware_alerts.py b/examples/DEPRECATED_psc/bulk_update_vmware_alerts.py similarity index 100% rename from examples/psc/bulk_update_vmware_alerts.py rename to examples/DEPRECATED_psc/bulk_update_vmware_alerts.py diff --git a/examples/psc/bulk_update_watchlist_alerts.py b/examples/DEPRECATED_psc/bulk_update_watchlist_alerts.py similarity index 100% rename from examples/psc/bulk_update_watchlist_alerts.py rename to examples/DEPRECATED_psc/bulk_update_watchlist_alerts.py diff --git a/examples/psc/device_control.py b/examples/DEPRECATED_psc/device_control.py similarity index 100% rename from examples/psc/device_control.py rename to examples/DEPRECATED_psc/device_control.py diff --git a/examples/psc/download_device_list.py b/examples/DEPRECATED_psc/download_device_list.py similarity index 100% rename from examples/psc/download_device_list.py rename to examples/DEPRECATED_psc/download_device_list.py diff --git a/examples/psc/helpers/alertsv6.py b/examples/DEPRECATED_psc/helpers/alertsv6.py similarity index 100% rename from examples/psc/helpers/alertsv6.py rename to examples/DEPRECATED_psc/helpers/alertsv6.py diff --git a/examples/psc/list_alert_facets.py b/examples/DEPRECATED_psc/list_alert_facets.py similarity index 100% rename from examples/psc/list_alert_facets.py rename to examples/DEPRECATED_psc/list_alert_facets.py diff --git a/examples/psc/list_alerts.py b/examples/DEPRECATED_psc/list_alerts.py similarity index 100% rename from examples/psc/list_alerts.py rename to examples/DEPRECATED_psc/list_alerts.py diff --git a/examples/psc/list_cbanalytics_alert_facets.py b/examples/DEPRECATED_psc/list_cbanalytics_alert_facets.py similarity index 100% rename from examples/psc/list_cbanalytics_alert_facets.py rename to examples/DEPRECATED_psc/list_cbanalytics_alert_facets.py diff --git a/examples/psc/list_cbanalytics_alerts.py b/examples/DEPRECATED_psc/list_cbanalytics_alerts.py similarity index 100% rename from examples/psc/list_cbanalytics_alerts.py rename to examples/DEPRECATED_psc/list_cbanalytics_alerts.py diff --git a/examples/psc/list_devices.py b/examples/DEPRECATED_psc/list_devices.py similarity index 100% rename from examples/psc/list_devices.py rename to examples/DEPRECATED_psc/list_devices.py diff --git a/examples/psc/list_vmware_alert_facets.py b/examples/DEPRECATED_psc/list_vmware_alert_facets.py similarity index 100% rename from examples/psc/list_vmware_alert_facets.py rename to examples/DEPRECATED_psc/list_vmware_alert_facets.py diff --git a/examples/psc/list_vmware_alerts.py b/examples/DEPRECATED_psc/list_vmware_alerts.py similarity index 100% rename from examples/psc/list_vmware_alerts.py rename to examples/DEPRECATED_psc/list_vmware_alerts.py diff --git a/examples/psc/list_watchlist_alert_facets.py b/examples/DEPRECATED_psc/list_watchlist_alert_facets.py similarity index 100% rename from examples/psc/list_watchlist_alert_facets.py rename to examples/DEPRECATED_psc/list_watchlist_alert_facets.py diff --git a/examples/psc/list_watchlist_alerts.py b/examples/DEPRECATED_psc/list_watchlist_alerts.py similarity index 100% rename from examples/psc/list_watchlist_alerts.py rename to examples/DEPRECATED_psc/list_watchlist_alerts.py diff --git a/examples/threathunter/create_feed.py b/examples/DEPRECATED_threathunter/create_feed.py similarity index 100% rename from examples/threathunter/create_feed.py rename to examples/DEPRECATED_threathunter/create_feed.py diff --git a/examples/threathunter/events.py b/examples/DEPRECATED_threathunter/events.py similarity index 100% rename from examples/threathunter/events.py rename to examples/DEPRECATED_threathunter/events.py diff --git a/examples/threathunter/events_exporter.py b/examples/DEPRECATED_threathunter/events_exporter.py similarity index 100% rename from examples/threathunter/events_exporter.py rename to examples/DEPRECATED_threathunter/events_exporter.py diff --git a/examples/threathunter/feed_operations.py b/examples/DEPRECATED_threathunter/feed_operations.py similarity index 100% rename from examples/threathunter/feed_operations.py rename to examples/DEPRECATED_threathunter/feed_operations.py diff --git a/examples/threathunter/import_response_feeds.py b/examples/DEPRECATED_threathunter/import_response_feeds.py similarity index 100% rename from examples/threathunter/import_response_feeds.py rename to examples/DEPRECATED_threathunter/import_response_feeds.py diff --git a/examples/threathunter/modify_feed.py b/examples/DEPRECATED_threathunter/modify_feed.py similarity index 100% rename from examples/threathunter/modify_feed.py rename to examples/DEPRECATED_threathunter/modify_feed.py diff --git a/examples/threathunter/process_exporter.py b/examples/DEPRECATED_threathunter/process_exporter.py similarity index 100% rename from examples/threathunter/process_exporter.py rename to examples/DEPRECATED_threathunter/process_exporter.py diff --git a/examples/threathunter/process_query.py b/examples/DEPRECATED_threathunter/process_query.py similarity index 100% rename from examples/threathunter/process_query.py rename to examples/DEPRECATED_threathunter/process_query.py diff --git a/examples/threathunter/process_tree.py b/examples/DEPRECATED_threathunter/process_tree.py similarity index 100% rename from examples/threathunter/process_tree.py rename to examples/DEPRECATED_threathunter/process_tree.py diff --git a/examples/threathunter/process_tree_exporter.py b/examples/DEPRECATED_threathunter/process_tree_exporter.py similarity index 100% rename from examples/threathunter/process_tree_exporter.py rename to examples/DEPRECATED_threathunter/process_tree_exporter.py diff --git a/examples/threathunter/search.py b/examples/DEPRECATED_threathunter/search.py similarity index 100% rename from examples/threathunter/search.py rename to examples/DEPRECATED_threathunter/search.py diff --git a/examples/threathunter/threat_intelligence/README.md b/examples/DEPRECATED_threathunter/threat_intelligence/README.md similarity index 100% rename from examples/threathunter/threat_intelligence/README.md rename to examples/DEPRECATED_threathunter/threat_intelligence/README.md diff --git a/examples/threathunter/threat_intelligence/Taxii_README.md b/examples/DEPRECATED_threathunter/threat_intelligence/Taxii_README.md similarity index 100% rename from examples/threathunter/threat_intelligence/Taxii_README.md rename to examples/DEPRECATED_threathunter/threat_intelligence/Taxii_README.md diff --git a/examples/threathunter/threat_intelligence/config.yml b/examples/DEPRECATED_threathunter/threat_intelligence/config.yml similarity index 100% rename from examples/threathunter/threat_intelligence/config.yml rename to examples/DEPRECATED_threathunter/threat_intelligence/config.yml diff --git a/examples/threathunter/threat_intelligence/feed_helper.py b/examples/DEPRECATED_threathunter/threat_intelligence/feed_helper.py similarity index 100% rename from examples/threathunter/threat_intelligence/feed_helper.py rename to examples/DEPRECATED_threathunter/threat_intelligence/feed_helper.py diff --git a/examples/threathunter/threat_intelligence/get_feed_ids.py b/examples/DEPRECATED_threathunter/threat_intelligence/get_feed_ids.py similarity index 100% rename from examples/threathunter/threat_intelligence/get_feed_ids.py rename to examples/DEPRECATED_threathunter/threat_intelligence/get_feed_ids.py diff --git a/examples/threathunter/threat_intelligence/requirements.txt b/examples/DEPRECATED_threathunter/threat_intelligence/requirements.txt similarity index 100% rename from examples/threathunter/threat_intelligence/requirements.txt rename to examples/DEPRECATED_threathunter/threat_intelligence/requirements.txt diff --git a/examples/threathunter/threat_intelligence/results.py b/examples/DEPRECATED_threathunter/threat_intelligence/results.py similarity index 100% rename from examples/threathunter/threat_intelligence/results.py rename to examples/DEPRECATED_threathunter/threat_intelligence/results.py diff --git a/examples/threathunter/threat_intelligence/schemas.py b/examples/DEPRECATED_threathunter/threat_intelligence/schemas.py similarity index 100% rename from examples/threathunter/threat_intelligence/schemas.py rename to examples/DEPRECATED_threathunter/threat_intelligence/schemas.py diff --git a/examples/threathunter/threat_intelligence/stix_parse.py b/examples/DEPRECATED_threathunter/threat_intelligence/stix_parse.py similarity index 100% rename from examples/threathunter/threat_intelligence/stix_parse.py rename to examples/DEPRECATED_threathunter/threat_intelligence/stix_parse.py diff --git a/examples/threathunter/threat_intelligence/stix_taxii.py b/examples/DEPRECATED_threathunter/threat_intelligence/stix_taxii.py similarity index 100% rename from examples/threathunter/threat_intelligence/stix_taxii.py rename to examples/DEPRECATED_threathunter/threat_intelligence/stix_taxii.py diff --git a/examples/threathunter/threat_intelligence/threatintel.py b/examples/DEPRECATED_threathunter/threat_intelligence/threatintel.py similarity index 100% rename from examples/threathunter/threat_intelligence/threatintel.py rename to examples/DEPRECATED_threathunter/threat_intelligence/threatintel.py diff --git a/examples/threathunter/watchlist_operations.py b/examples/DEPRECATED_threathunter/watchlist_operations.py similarity index 100% rename from examples/threathunter/watchlist_operations.py rename to examples/DEPRECATED_threathunter/watchlist_operations.py diff --git a/examples/README.md b/examples/README.md index cd2e34e6..bd8e5b14 100755 --- a/examples/README.md +++ b/examples/README.md @@ -14,3 +14,11 @@ Once you have done so, you should be able to run any example script with the com Executing any script with the `--help` argument should give you a detailed message about the arguments that can be supplied to the script when it is executed. + +### Note: Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. + +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. + +CBAPI is not maintained for Carbon Black Cloud. \ No newline at end of file diff --git a/src/cbapi/psc/defense/rest_api.py b/src/cbapi/psc/defense/rest_api.py index 68c5abcd..3e719473 100644 --- a/src/cbapi/psc/defense/rest_api.py +++ b/src/cbapi/psc/defense/rest_api.py @@ -14,7 +14,13 @@ def convert_to_kv_pairs(q): class CbDefenseAPI(CbPSCBaseAPI): - """The main entry point into the Carbon Black Cloud Endpoint Standard Defense API. + """THIS SDK IS DEPRECATED FOR CARBON BLACK CLOUD + + Please see + `Carbon Black Cloud Python SDK on the Developer Network `_ + for details on the replacement Carbon Black Cloud Python SDK. + + The main entry point into the Carbon Black Cloud Endpoint Standard Defense API. :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. Uses the profile named 'default' when not specified. @@ -41,7 +47,7 @@ def notification_listener(self, interval=60): time.sleep(interval) def get_notifications(self): - """Retrieve queued notifications (alerts) from the Cloud Endpoint Standard server. Note that this can only be + """DEPRECATED: Retrieve queued notifications (alerts) from the Cloud Endpoint Standard server. Note that this can only be used with a 'SIEM' key generated in the Carbon Black Cloud console. :returns: list of dictionary objects representing the notifications, or an empty list if none available. @@ -50,7 +56,7 @@ def get_notifications(self): return res.get("notifications", []) def get_auditlogs(self): - """Retrieve queued audit logs from the Carbon Black Cloud Endpoint Standard server. + """DEPRECATED: Retrieve queued audit logs from the Carbon Black Cloud Endpoint Standard server. Note that this can only be used with a 'API' key generated in the CBC console. :returns: list of dictionary objects representing the audit logs, or an empty list if none available. """ From 545eadf878da765582b61e1b7186548aab56bcbe Mon Sep 17 00:00:00 2001 From: Kylie Ebringer <55465092+kebringer-cb@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:23:07 -0600 Subject: [PATCH 190/197] Date correction --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6de9a8bd..37ba3b0c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ CbAPI Changelog =============== .. top-of-changelog (DO NOT REMOVE THIS COMMENT) -Documentation - Release Feb 14, 2023 +Documentation - Release Feb 14, 2024 ------------------------------------ Updates From 18226835a730f69132bc2ce8919686e7c0270761 Mon Sep 17 00:00:00 2001 From: Kunal Deshpande Date: Mon, 8 Apr 2024 16:40:34 +0530 Subject: [PATCH 191/197] CB-43532: Add ARH support in cbapi for group API --- src/cbapi/response/models/group-modify.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cbapi/response/models/group-modify.yaml b/src/cbapi/response/models/group-modify.yaml index 200f5bbc..0d57430e 100644 --- a/src/cbapi/response/models/group-modify.yaml +++ b/src/cbapi/response/models/group-modify.yaml @@ -22,6 +22,12 @@ properties: type: "integer" description: "Site id to add group to" default: "1" + address_resolution_hint: + type: string + enum: + - 'ipv4' + - 'ipv6' + - '' team_access: type: "array" description: "Team access array" From 5f1e39583bffd349ec354fe8126df9cb0b29cf86 Mon Sep 17 00:00:00 2001 From: averma-cb Date: Thu, 30 May 2024 13:16:06 -0400 Subject: [PATCH 192/197] six.py replaced with 1.16 version --- src/cbapi/six.py | 210 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 169 insertions(+), 41 deletions(-) diff --git a/src/cbapi/six.py b/src/cbapi/six.py index 7769534c..1b0eaf56 100644 --- a/src/cbapi/six.py +++ b/src/cbapi/six.py @@ -1,6 +1,4 @@ -"""Utilities for writing code that runs on Python 2 and 3""" - -# Copyright (c) 2010-2015 Benjamin Peterson +# Copyright (c) 2010-2020 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -20,6 +18,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +"""Utilities for writing code that runs on Python 2 and 3""" + from __future__ import absolute_import import functools @@ -29,7 +29,7 @@ import types __author__ = "Benjamin Peterson " -__version__ = "1.10.0" +__version__ = "1.16.0" # Useful for very coarse version differentiation. @@ -39,17 +39,17 @@ if PY3: string_types = str, - integer_types = (int,), + integer_types = int, class_types = type, text_type = str binary_type = bytes MAXSIZE = sys.maxsize else: - string_types = basestring, # noqa: F821 - integer_types = (int, long) # noqa: F821 + string_types = basestring, # noqa: F821 + integer_types = (int, long) # noqa: F821 class_types = (type, types.ClassType) - text_type = unicode # noqa: F821 + text_type = unicode # noqa: F821 binary_type = str if sys.platform.startswith("java"): @@ -71,6 +71,11 @@ def __len__(self): MAXSIZE = int((1 << 63) - 1) del X +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + def _add_doc(func, doc): """Add documentation to a function.""" @@ -186,6 +191,11 @@ def find_module(self, fullname, path=None): return self return None + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + def __get_module(self, fullname): try: return self.known_modules[fullname] @@ -223,6 +233,11 @@ def get_code(self, fullname): return None get_source = get_code # same as get_code + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass _importer = _SixMetaPathImporter(__name__) @@ -242,12 +257,13 @@ class _MovedItems(_LazyModule): MovedAttribute("map", "itertools", "builtins", "imap", "map"), MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), MovedAttribute("reduce", "__builtin__", "functools"), MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), MovedAttribute("StringIO", "StringIO", "io"), - MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserDict", "UserDict", "collections", "IterableUserDict", "UserDict"), MovedAttribute("UserList", "UserList", "collections"), MovedAttribute("UserString", "UserString", "collections"), MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), @@ -255,18 +271,21 @@ class _MovedItems(_LazyModule): MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), + MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"), MovedModule("copyreg", "copy_reg"), MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"), MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), MovedModule("http_cookies", "Cookie", "http.cookies"), MovedModule("html_entities", "htmlentitydefs", "html.entities"), MovedModule("html_parser", "HTMLParser", "html.parser"), MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), - MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), @@ -338,10 +357,12 @@ class Module_six_moves_urllib_parse(_LazyModule): MovedAttribute("quote_plus", "urllib", "urllib.parse"), MovedAttribute("unquote", "urllib", "urllib.parse"), MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), MovedAttribute("urlencode", "urllib", "urllib.parse"), MovedAttribute("splitquery", "urllib", "urllib.parse"), MovedAttribute("splittag", "urllib", "urllib.parse"), MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), MovedAttribute("uses_params", "urlparse", "urllib.parse"), @@ -417,6 +438,8 @@ class Module_six_moves_urllib_request(_LazyModule): MovedAttribute("URLopener", "urllib", "urllib.request"), MovedAttribute("FancyURLopener", "urllib", "urllib.request"), MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), ] for attr in _urllib_request_moved_attributes: setattr(Module_six_moves_urllib_request, attr.name, attr) @@ -480,7 +503,6 @@ class Module_six_moves_urllib(types.ModuleType): def __dir__(self): return ['parse', 'error', 'request', 'response', 'robotparser'] - _importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib") @@ -633,13 +655,16 @@ def u(s): import io StringIO = io.StringIO BytesIO = io.BytesIO + del io _assertCountEqual = "assertCountEqual" if sys.version_info[1] <= 1: _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" else: _assertRaisesRegex = "assertRaisesRegex" _assertRegex = "assertRegex" + _assertNotRegex = "assertNotRegex" else: def b(s): return s @@ -661,6 +686,7 @@ def indexbytes(buf, i): _assertCountEqual = "assertItemsEqual" _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" _add_doc(b, """Byte literal""") _add_doc(u, """Text literal""") @@ -677,15 +703,23 @@ def assertRegex(self, *args, **kwargs): return getattr(self, _assertRegex)(*args, **kwargs) +def assertNotRegex(self, *args, **kwargs): + return getattr(self, _assertNotRegex)(*args, **kwargs) + + if PY3: exec_ = getattr(moves.builtins, "exec") def reraise(tp, value, tb=None): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None else: def exec_(_code_, _globs_=None, _locs_=None): @@ -701,19 +735,19 @@ def exec_(_code_, _globs_=None, _locs_=None): exec("""exec _code_ in _globs_, _locs_""") exec_("""def reraise(tp, value, tb=None): - raise tp, value, tb + try: + raise tp, value, tb + finally: + tb = None """) -if sys.version_info[:2] == (3, 2): +if sys.version_info[:2] > (3,): exec_("""def raise_from(value, from_value): - if from_value is None: - raise value - raise value from from_value -""") -elif sys.version_info[:2] > (3, 2): - exec_("""def raise_from(value, from_value): - raise value from from_value + try: + raise value from from_value + finally: + value = None """) else: def raise_from(value, from_value): @@ -732,8 +766,8 @@ def write(data): if not isinstance(data, basestring): # noqa: F821 data = str(data) # If the file has an encoding, encode unicode with it. - if (isinstance(fp, file) and # noqa: F821 - isinstance(data, unicode) and # noqa: F821 + if (isinstance(fp, file) and # noqa: F821 + isinstance(data, unicode) and # noqa: F821 fp.encoding is not None): errors = getattr(fp, "errors", None) if errors is None: @@ -743,13 +777,13 @@ def write(data): want_unicode = False sep = kwargs.pop("sep", None) if sep is not None: - if isinstance(sep, unicode): # noqa: F821 + if isinstance(sep, unicode): # noqa: F821 want_unicode = True elif not isinstance(sep, str): raise TypeError("sep must be None or a string") end = kwargs.pop("end", None) if end is not None: - if isinstance(end, unicode): # noqa: F821 + if isinstance(end, unicode): # noqa: F821 want_unicode = True elif not isinstance(end, str): raise TypeError("end must be None or a string") @@ -757,12 +791,12 @@ def write(data): raise TypeError("invalid keyword arguments to print()") if not want_unicode: for arg in args: - if isinstance(arg, unicode): # noqa: F821 + if isinstance(arg, unicode): # noqa: F821 want_unicode = True break if want_unicode: - newline = unicode("\n") # noqa: F821 - space = unicode(" ") # noqa: F821 + newline = unicode("\n") # noqa: F821 + space = unicode(" ") # noqa: F821 else: newline = "\n" space = " " @@ -788,13 +822,33 @@ def print_(*args, **kwargs): _add_doc(reraise, """Reraise an exception.""") if sys.version_info[0:2] < (3, 4): + # This does exactly the same what the :func:`py3:functools.update_wrapper` + # function does on Python versions after 3.2. It sets the ``__wrapped__`` + # attribute on ``wrapper`` object and it doesn't raise an error if any of + # the attributes mentioned in ``assigned`` and ``updated`` are missing on + # ``wrapped`` object. + def _update_wrapper(wrapper, wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + wrapper.__wrapped__ = wrapped + return wrapper + _update_wrapper.__doc__ = functools.update_wrapper.__doc__ + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, updated=functools.WRAPPER_UPDATES): - def wrapper(f): - f = functools.wraps(wrapped, assigned, updated)(f) - f.__wrapped__ = wrapped - return f - return wrapper + return functools.partial(_update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) + wraps.__doc__ = functools.wraps.__doc__ + else: wraps = functools.wraps @@ -804,10 +858,22 @@ def with_metaclass(meta, *bases): # This requires a bit of explanation: the basic idea is to make a dummy # metaclass for one level of class instantiation that replaces itself with # the actual metaclass. - class metaclass(meta): + class metaclass(type): def __new__(cls, name, this_bases, d): - return meta(name, bases, d) + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d['__orig_bases__'] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) return type.__new__(metaclass, 'temporary_class', (), {}) @@ -823,13 +889,75 @@ def wrapper(cls): orig_vars.pop(slots_var) orig_vars.pop('__dict__', None) orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, binary_type): + return s + if isinstance(s, text_type): + return s.encode(encoding, errors) + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + # Optimization: Fast return for the common case. + if type(s) is str: + return s + if PY2 and isinstance(s, text_type): + return s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + return s.decode(encoding, errors) + elif not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + def python_2_unicode_compatible(klass): """ - A decorator that defines __unicode__ and __str__ methods under Python 2. + A class decorator that defines __unicode__ and __str__ methods under Python 2. Under Python 3 it does nothing. To support Python 2 and 3 with a single code base, define a __str__ method From 68a388d0128bf59592d7915eebd6fdf83d3904cf Mon Sep 17 00:00:00 2001 From: averma-cb Date: Thu, 30 May 2024 14:28:31 -0400 Subject: [PATCH 193/197] replacing readfp with read_file --- bin/cbapi-defense | 2 +- bin/cbapi-protection | 2 +- bin/cbapi-psc | 2 +- bin/cbapi-response | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/cbapi-defense b/bin/cbapi-defense index 8e6c2443..d5b0b398 100644 --- a/bin/cbapi-defense +++ b/bin/cbapi-defense @@ -48,7 +48,7 @@ def configure(opts): token = input("API key: ") config = RawConfigParser() - config.readfp(StringIO('[default]')) + config.read_file(StringIO('[default]')) config.set("default", "url", url) config.set("default", "token", "{0}/{1}".format(token, connector_id)) config.set("default", "ssl_verify", ssl_verify) diff --git a/bin/cbapi-protection b/bin/cbapi-protection index e8773543..d45ccd3a 100644 --- a/bin/cbapi-protection +++ b/bin/cbapi-protection @@ -55,7 +55,7 @@ def configure(opts): token = input("API token: ") config = RawConfigParser() - config.readfp(StringIO('[default]')) + config.read_file(StringIO('[default]')) config.set("default", "url", url) config.set("default", "token", token) config.set("default", "ssl_verify", ssl_verify) diff --git a/bin/cbapi-psc b/bin/cbapi-psc index 18bbc6e6..af7b1764 100644 --- a/bin/cbapi-psc +++ b/bin/cbapi-psc @@ -50,7 +50,7 @@ def configure(opts): org_key = input("Org Key: ") config = RawConfigParser() - config.readfp(StringIO('[default]')) + config.read_file(StringIO('[default]')) config.set("default", "url", url) config.set("default", "token", "{0}/{1}".format(token, connector_id)) config.set("default", "org_key", org_key) diff --git a/bin/cbapi-response b/bin/cbapi-response index 84b45fa0..b04c4f44 100644 --- a/bin/cbapi-response +++ b/bin/cbapi-response @@ -70,7 +70,7 @@ def configure(opts): return 1 config = RawConfigParser() - config.readfp(StringIO('[default]')) + config.read_file(StringIO('[default]')) config.set("default", "url", url) config.set("default", "token", token) config.set("default", "ssl_verify", ssl_verify) From f41b22935d01ad367e87de45705b2ee5b0f0b5d0 Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Thu, 25 Jul 2024 15:52:48 +0300 Subject: [PATCH 194/197] remove dead code --- bin/cbapi-defense | 94 -- bin/cbapi-psc | 97 -- .../DellBiosVerification/BiosVerification.py | 97 -- .../cblr/DellBiosVerification/README.md | 82 -- .../cblr/DellBiosVerification/dellbios.bat | 4 - .../DEPRECATED_defense/cblr/examplejob.py | 10 - examples/DEPRECATED_defense/cblr/jobrunner.py | 70 -- examples/DEPRECATED_defense/cblr_cli.py | 44 - examples/DEPRECATED_defense/event_export.py | 76 -- examples/DEPRECATED_defense/list_devices.py | 32 - examples/DEPRECATED_defense/list_events.py | 71 -- .../list_events_with_cmdline_csv.py | 86 -- examples/DEPRECATED_defense/move_device.py | 40 - examples/DEPRECATED_defense/notifications.py | 21 - .../DEPRECATED_defense/policy_operations.py | 206 --- .../alert_search_suggestions.py | 22 - examples/DEPRECATED_psc/bulk_update_alerts.py | 50 - .../bulk_update_cbanalytics_alerts.py | 50 - .../bulk_update_threat_alerts.py | 47 - .../bulk_update_vmware_alerts.py | 50 - .../bulk_update_watchlist_alerts.py | 50 - examples/DEPRECATED_psc/device_control.py | 73 -- .../DEPRECATED_psc/download_device_list.py | 50 - examples/DEPRECATED_psc/helpers/alertsv6.py | 159 --- examples/DEPRECATED_psc/list_alert_facets.py | 34 - examples/DEPRECATED_psc/list_alerts.py | 33 - .../list_cbanalytics_alert_facets.py | 34 - .../DEPRECATED_psc/list_cbanalytics_alerts.py | 33 - examples/DEPRECATED_psc/list_devices.py | 45 - .../list_vmware_alert_facets.py | 34 - examples/DEPRECATED_psc/list_vmware_alerts.py | 33 - .../list_watchlist_alert_facets.py | 34 - .../DEPRECATED_psc/list_watchlist_alerts.py | 33 - .../DEPRECATED_threathunter/create_feed.py | 78 -- examples/DEPRECATED_threathunter/events.py | 33 - .../events_exporter.py | 50 - .../feed_operations.py | 268 ---- .../import_response_feeds.py | 144 --- .../DEPRECATED_threathunter/modify_feed.py | 51 - .../process_exporter.py | 60 - .../DEPRECATED_threathunter/process_query.py | 40 - .../DEPRECATED_threathunter/process_tree.py | 29 - .../process_tree_exporter.py | 44 - examples/DEPRECATED_threathunter/search.py | 78 -- .../threat_intelligence/README.md | 129 -- .../threat_intelligence/Taxii_README.md | 41 - .../threat_intelligence/config.yml | 78 -- .../threat_intelligence/feed_helper.py | 45 - .../threat_intelligence/get_feed_ids.py | 21 - .../threat_intelligence/requirements.txt | 10 - .../threat_intelligence/results.py | 81 -- .../threat_intelligence/schemas.py | 44 - .../threat_intelligence/stix_parse.py | 466 ------- .../threat_intelligence/stix_taxii.py | 392 ------ .../threat_intelligence/threatintel.py | 82 -- .../watchlist_operations.py | 327 ----- setup.py | 8 +- src/cbapi/__init__.py | 5 - src/cbapi/defense.py | 2 - src/cbapi/example_helpers.py | 30 - src/cbapi/psc/__init__.py | 2 +- src/cbapi/psc/alerts_query.py | 704 ----------- src/cbapi/psc/cblr.py | 250 ---- src/cbapi/psc/defense/__init__.py | 6 - src/cbapi/psc/defense/models.py | 164 --- src/cbapi/psc/defense/models/deviceInfo.yaml | 221 ---- src/cbapi/psc/defense/models/policyInfo.yaml | 25 - src/cbapi/psc/defense/rest_api.py | 194 --- src/cbapi/psc/devices_query.py | 2 +- src/cbapi/psc/models.py | 190 --- src/cbapi/psc/models/base_alert.yaml | 139 -- src/cbapi/psc/models/workflow.yaml | 23 - src/cbapi/psc/models/workflow_status.yaml | 56 - src/cbapi/psc/rest_api.py | 12 - src/cbapi/psc/threathunter/__init__.py | 9 - src/cbapi/psc/threathunter/models.py | 1117 ----------------- src/cbapi/psc/threathunter/models/binary.yaml | 79 -- src/cbapi/psc/threathunter/models/feed.yaml | 33 - src/cbapi/psc/threathunter/models/ioc_v2.yaml | 23 - src/cbapi/psc/threathunter/models/iocs.yaml | 32 - src/cbapi/psc/threathunter/models/report.yaml | 45 - .../threathunter/models/report_severity.yaml | 12 - .../psc/threathunter/models/watchlist.yaml | 43 - src/cbapi/psc/threathunter/query.py | 654 ---------- src/cbapi/psc/threathunter/rest_api.py | 115 -- test/cbapi/psc/test_alertsv6_api.py | 535 -------- test/cbapi/psc/test_models.py | 136 +- tests/test_defense_policy.py | 53 - 88 files changed, 5 insertions(+), 9399 deletions(-) delete mode 100644 bin/cbapi-defense delete mode 100644 bin/cbapi-psc delete mode 100755 examples/DEPRECATED_defense/cblr/DellBiosVerification/BiosVerification.py delete mode 100644 examples/DEPRECATED_defense/cblr/DellBiosVerification/README.md delete mode 100644 examples/DEPRECATED_defense/cblr/DellBiosVerification/dellbios.bat delete mode 100755 examples/DEPRECATED_defense/cblr/examplejob.py delete mode 100755 examples/DEPRECATED_defense/cblr/jobrunner.py delete mode 100644 examples/DEPRECATED_defense/cblr_cli.py delete mode 100644 examples/DEPRECATED_defense/event_export.py delete mode 100644 examples/DEPRECATED_defense/list_devices.py delete mode 100644 examples/DEPRECATED_defense/list_events.py delete mode 100644 examples/DEPRECATED_defense/list_events_with_cmdline_csv.py delete mode 100644 examples/DEPRECATED_defense/move_device.py delete mode 100644 examples/DEPRECATED_defense/notifications.py delete mode 100644 examples/DEPRECATED_defense/policy_operations.py delete mode 100755 examples/DEPRECATED_psc/alert_search_suggestions.py delete mode 100755 examples/DEPRECATED_psc/bulk_update_alerts.py delete mode 100755 examples/DEPRECATED_psc/bulk_update_cbanalytics_alerts.py delete mode 100755 examples/DEPRECATED_psc/bulk_update_threat_alerts.py delete mode 100755 examples/DEPRECATED_psc/bulk_update_vmware_alerts.py delete mode 100755 examples/DEPRECATED_psc/bulk_update_watchlist_alerts.py delete mode 100755 examples/DEPRECATED_psc/device_control.py delete mode 100755 examples/DEPRECATED_psc/download_device_list.py delete mode 100755 examples/DEPRECATED_psc/helpers/alertsv6.py delete mode 100755 examples/DEPRECATED_psc/list_alert_facets.py delete mode 100755 examples/DEPRECATED_psc/list_alerts.py delete mode 100755 examples/DEPRECATED_psc/list_cbanalytics_alert_facets.py delete mode 100755 examples/DEPRECATED_psc/list_cbanalytics_alerts.py delete mode 100755 examples/DEPRECATED_psc/list_devices.py delete mode 100755 examples/DEPRECATED_psc/list_vmware_alert_facets.py delete mode 100755 examples/DEPRECATED_psc/list_vmware_alerts.py delete mode 100755 examples/DEPRECATED_psc/list_watchlist_alert_facets.py delete mode 100755 examples/DEPRECATED_psc/list_watchlist_alerts.py delete mode 100644 examples/DEPRECATED_threathunter/create_feed.py delete mode 100644 examples/DEPRECATED_threathunter/events.py delete mode 100644 examples/DEPRECATED_threathunter/events_exporter.py delete mode 100644 examples/DEPRECATED_threathunter/feed_operations.py delete mode 100644 examples/DEPRECATED_threathunter/import_response_feeds.py delete mode 100644 examples/DEPRECATED_threathunter/modify_feed.py delete mode 100644 examples/DEPRECATED_threathunter/process_exporter.py delete mode 100644 examples/DEPRECATED_threathunter/process_query.py delete mode 100644 examples/DEPRECATED_threathunter/process_tree.py delete mode 100644 examples/DEPRECATED_threathunter/process_tree_exporter.py delete mode 100644 examples/DEPRECATED_threathunter/search.py delete mode 100644 examples/DEPRECATED_threathunter/threat_intelligence/README.md delete mode 100644 examples/DEPRECATED_threathunter/threat_intelligence/Taxii_README.md delete mode 100644 examples/DEPRECATED_threathunter/threat_intelligence/config.yml delete mode 100644 examples/DEPRECATED_threathunter/threat_intelligence/feed_helper.py delete mode 100644 examples/DEPRECATED_threathunter/threat_intelligence/get_feed_ids.py delete mode 100644 examples/DEPRECATED_threathunter/threat_intelligence/requirements.txt delete mode 100644 examples/DEPRECATED_threathunter/threat_intelligence/results.py delete mode 100644 examples/DEPRECATED_threathunter/threat_intelligence/schemas.py delete mode 100644 examples/DEPRECATED_threathunter/threat_intelligence/stix_parse.py delete mode 100644 examples/DEPRECATED_threathunter/threat_intelligence/stix_taxii.py delete mode 100644 examples/DEPRECATED_threathunter/threat_intelligence/threatintel.py delete mode 100644 examples/DEPRECATED_threathunter/watchlist_operations.py delete mode 100644 src/cbapi/defense.py delete mode 100755 src/cbapi/psc/alerts_query.py delete mode 100644 src/cbapi/psc/cblr.py delete mode 100644 src/cbapi/psc/defense/__init__.py delete mode 100644 src/cbapi/psc/defense/models.py delete mode 100644 src/cbapi/psc/defense/models/deviceInfo.yaml delete mode 100644 src/cbapi/psc/defense/models/policyInfo.yaml delete mode 100644 src/cbapi/psc/defense/rest_api.py delete mode 100755 src/cbapi/psc/models/base_alert.yaml delete mode 100755 src/cbapi/psc/models/workflow.yaml delete mode 100755 src/cbapi/psc/models/workflow_status.yaml delete mode 100644 src/cbapi/psc/threathunter/__init__.py delete mode 100644 src/cbapi/psc/threathunter/models.py delete mode 100644 src/cbapi/psc/threathunter/models/binary.yaml delete mode 100644 src/cbapi/psc/threathunter/models/feed.yaml delete mode 100644 src/cbapi/psc/threathunter/models/ioc_v2.yaml delete mode 100644 src/cbapi/psc/threathunter/models/iocs.yaml delete mode 100644 src/cbapi/psc/threathunter/models/report.yaml delete mode 100644 src/cbapi/psc/threathunter/models/report_severity.yaml delete mode 100644 src/cbapi/psc/threathunter/models/watchlist.yaml delete mode 100644 src/cbapi/psc/threathunter/query.py delete mode 100644 src/cbapi/psc/threathunter/rest_api.py delete mode 100755 test/cbapi/psc/test_alertsv6_api.py delete mode 100644 tests/test_defense_policy.py diff --git a/bin/cbapi-defense b/bin/cbapi-defense deleted file mode 100644 index d5b0b398..00000000 --- a/bin/cbapi-defense +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python - -import argparse -import contextlib - -from cbapi.six import iteritems -from cbapi.six.moves import input -import os -import sys -import cbapi.six as six -if six.PY3: - from io import StringIO as StringIO -else: - from cStringIO import StringIO - -from cbapi.six.moves.configparser import RawConfigParser - - -@contextlib.contextmanager -def temp_umask(umask): - oldmask = os.umask(umask) - try: - yield - finally: - os.umask(oldmask) - - -def configure(opts): - credential_path = os.path.join(os.path.expanduser("~"), ".carbonblack") - credential_file = os.path.join(credential_path, "credentials.defense") - - print("Welcome to the CbAPI.") - if os.path.exists(credential_file): - print("An existing credential file exists at {0}.".format(credential_file)) - resp = input("Do you want to continue and overwrite the existing configuration? [Y/N] ") - if resp.strip().upper() != "Y": - print("Exiting.") - return 1 - - if not os.path.exists(credential_path): - os.makedirs(credential_path, 0o700) - - url = input("URL to the Cb Defense API server (do not include '/integrationServices') [https://hostname]: ") - - ssl_verify = True - - connector_id = input("Connector ID: ") - token = input("API key: ") - - config = RawConfigParser() - config.read_file(StringIO('[default]')) - config.set("default", "url", url) - config.set("default", "token", "{0}/{1}".format(token, connector_id)) - config.set("default", "ssl_verify", ssl_verify) - with temp_umask(0): - with os.fdopen(os.open(credential_file, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0o600), 'w') as fp: - os.chmod(credential_file, 0o600) - config.write(fp) - print("Successfully wrote credentials to {0}.".format(credential_file)) - - -command_map = { - "configure": { - "extra_args": {}, - "help": "Configure CbAPI", - "method": configure - } -} - - -def main(args): - parser = argparse.ArgumentParser() - commands = parser.add_subparsers(dest="command_name", help="CbAPI subcommand") - - for cmd_name, cmd_config in iteritems(command_map): - cmd_parser = commands.add_parser(cmd_name, help=cmd_config.get("help", None)) - for cmd_arg_name, cmd_arg_config in iteritems(cmd_config.get("extra_args", {})): - cmd_parser.add_argument(cmd_arg_name, **cmd_arg_config) - - opts = parser.parse_args(args) - command = command_map.get(opts.command_name) - if not command: - parser.print_usage() - return - - command_method = command.get("method", None) - if command_method: - return command_method(opts) - else: - parser.print_usage() - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/bin/cbapi-psc b/bin/cbapi-psc deleted file mode 100644 index af7b1764..00000000 --- a/bin/cbapi-psc +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python - -import argparse -import contextlib - -from cbapi.six import iteritems -from cbapi.six.moves import input -import os -import sys -import cbapi.six as six -if six.PY3: - from io import StringIO as StringIO -else: - from cStringIO import StringIO - -from cbapi.six.moves.configparser import RawConfigParser - - -@contextlib.contextmanager -def temp_umask(umask): - oldmask = os.umask(umask) - try: - yield - finally: - os.umask(oldmask) - - -def configure(opts): - credential_path = os.path.join(os.path.expanduser("~"), ".carbonblack") - credential_file = os.path.join(credential_path, "credentials.psc") - - print("Welcome to the CbAPI.") - if os.path.exists(credential_file): - print("An existing credential file exists at {0}.".format(credential_file)) - resp = input("Do you want to continue and overwrite the existing configuration? [Y/N] ") - if resp.strip().upper() != "Y": - print("Exiting.") - return 1 - - if not os.path.exists(credential_path): - os.makedirs(credential_path, 0o700) - - url = input("URL to the Carbon Black Cloud API server (do not include '/integrationServices') [https://hostname]: ") - - ssl_verify = True - - connector_id = input("Connector ID: ") - token = input("API key: ") - - org_key = input("Org Key: ") - - config = RawConfigParser() - config.read_file(StringIO('[default]')) - config.set("default", "url", url) - config.set("default", "token", "{0}/{1}".format(token, connector_id)) - config.set("default", "org_key", org_key) - config.set("default", "ssl_verify", ssl_verify) - with temp_umask(0): - with os.fdopen(os.open(credential_file, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0o600), 'w') as fp: - os.chmod(credential_file, 0o600) - config.write(fp) - print("Successfully wrote credentials to {0}.".format(credential_file)) - - -command_map = { - "configure": { - "extra_args": {}, - "help": "Configure CbAPI", - "method": configure - } -} - - -def main(args): - parser = argparse.ArgumentParser() - commands = parser.add_subparsers(dest="command_name", help="CbAPI subcommand") - - for cmd_name, cmd_config in iteritems(command_map): - cmd_parser = commands.add_parser(cmd_name, help=cmd_config.get("help", None)) - for cmd_arg_name, cmd_arg_config in iteritems(cmd_config.get("extra_args", {})): - cmd_parser.add_argument(cmd_arg_name, **cmd_arg_config) - - opts = parser.parse_args(args) - command = command_map.get(opts.command_name) - if not command: - parser.print_usage() - return - - command_method = command.get("method", None) - if command_method: - return command_method(opts) - else: - parser.print_usage() - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/examples/DEPRECATED_defense/cblr/DellBiosVerification/BiosVerification.py b/examples/DEPRECATED_defense/cblr/DellBiosVerification/BiosVerification.py deleted file mode 100755 index 5d55ca73..00000000 --- a/examples/DEPRECATED_defense/cblr/DellBiosVerification/BiosVerification.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 - -# Carbon Black Cloud -Dell Bios Verification LiveResponse -# Copyright VMware 2020 -# May 2020 -# Version 0.1 -# pdrapeau [at] vmware . com -# -# usage: BiosVerification.py [-h] [-m MACHINENAME] [-g] [-o ORGPROFILE] -# -# optional arguments: -# -h, --help show this help message and exit -# -m MACHINENAME, --machinename MACHINENAME -# machinename to run host bios forensics on -# -g, --get Get BIOS images -# -# -o ORGPROFILE, --orgprofile ORGPROFILE -# Select your cbapi credential profile - -import os, sys, time, argparse -from cbapi.defense import * - -def live_response(cb, host=None, response=None): - - print ("") - - #Select the device you want to gather forensic data from - query_hostname = "hostNameExact:%s" % host - print ("[ * ] Establishing LiveResponse Session with Remote Host:") - - #Create a new device object to launch LR on - device = cb.select(Device).where(query_hostname).first() - print(" - Hostname: {}".format(device.name)) - print(" - OS Version: {}".format(device.osVersion)) - print(" - Sensor Version: {}".format(device.sensorVersion)) - print(" - AntiVirus Status: {}".format(device.avStatus)) - print(" - Internal IP Address: {}".format(device.lastInternalIpAddress)) - print(" - External IP Address: {}".format(device.lastExternalIpAddress)) - print ("") - - #Execute our LR session - with device.lr_session() as lr_session: - print ("[ * ] Uploading scripts to the remote host") - lr_session.put_file(open("dellbios.bat", "rb"), "C:\\Program Files\\Confer\\temp\\dellbios.bat") - - if response == "get": - print ("[ * ] Getting the images") - result = lr_session.create_process("cmd.exe /c .\\dellbios.bat", wait_for_output=True, remote_output_file_name=None, working_directory="C:\\Program Files\\Confer\\temp\\", wait_timeout=120, wait_for_completion=True).decode("utf-8") - print ("") - print("{}".format(result)) - - print ("[ * ] Removing scripts") - lr_session.create_process("powershell.exe del .\\dellbios.bat", wait_for_output=False, remote_output_file_name=None, working_directory="C:\\Program Files\\Confer\\temp\\", wait_timeout=30, wait_for_completion=False) - - - print ("[ * ] Downloading images") - zipdata = lr_session.get_file("C:\\Program Files\\Confer\\temp\\BiosImages.zip") - - print ("[ * ] Writing out " + host + "-BiosImages.zip") - zipfile = open(host + "-BiosImages.zip","wb") - zipfile.write(zipdata) - - print ("") - - - - else: - print ("[ * ] Nothing to do") - - - print ("[ * ] Cleaning up") - lr_session.create_process("powershell.exe del .\\BiosImages.zip", wait_for_output=False, remote_output_file_name=None, working_directory="C:\\Program Files\\Confer\\temp\\", wait_timeout=30, wait_for_completion=False) - lr_session.create_process("powershell.exe del C:\\tmpbios\\*.*", wait_for_output=False, remote_output_file_name=None, working_directory="C:\\Program Files\\Confer\\temp\\", wait_timeout=30, wait_for_completion=False) - - - print ("") - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-m", "--machinename", help = "machinename to run host forensics recon on") - parser.add_argument("-g", "--get", help = "Get the Dell BIOS Verification images", action = "store_true") - parser.add_argument('-o', '--orgprofile', help = "Select your cbapi credential profile", dest = "orgprofile", default = "default") - args = parser.parse_args() - - #Create the CbD LR API object - cb = CbDefenseAPI(profile="{}".format(args.orgprofile)) - - if args.machinename: - if args.get: - live_response(cb, host=args.machinename, response="get") - else: - print ("Nothing to do...") - else: - print ("[ ! ] You must specify a machinename with a --machinename parameter. IE ./BiosVerification.py --machinename cheese") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/examples/DEPRECATED_defense/cblr/DellBiosVerification/README.md b/examples/DEPRECATED_defense/cblr/DellBiosVerification/README.md deleted file mode 100644 index 1ef0fa35..00000000 --- a/examples/DEPRECATED_defense/cblr/DellBiosVerification/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Dell BiosVerification.py Live Response API Script - -## References - -Dell Trusted Device Product Information: https://www.delltechnologies.com/endpointsecurity - -Dell Trusted Device Installation Instructions: https://www.dell.com/support/manuals/us/en/04/trusted-device/trusted_device/installation?guid=guid-b9217d4f-6932-47d2-8db5-50633eb47691&lang=en-us - -Troubleshooting: https://www.dell.com/support/manuals/us/en/04/trusted-device/trusted_device/results-troubleshooting-and-remediation?guid=guid-240f1964-167a-41b0-9fb3-687dddbdb71f&lang=en-us - - - -## Summary - -This set of tools uses the VMware Carbon Black Security Cloud Live Response APIs to retrieve -artifacts generated by the Dell Trusted Device SafeBIOS verification service. The Dell Trusted -Device agent saves BIOS image files to the filesystem when a verification failure event is -detected. - -Incident responders can use this set of scripts to retrieve the BIOS image files for forensic -analysis. - - -## Instructions - -Usage: - -To retrieve the BIOS image files from a device in a failed verification state via the Live Response API: - - -1. Copy the BiosVerification.py and dellbios.bat files to the same directory on the administrator system. -2. Install the cbapi Python bindings: https://github.com/carbonblack/cbapi-python -3. Create a Live Response API key https://developer.carbonblack.com/reference/carbon-black-cloud/authentication/ -4. Configure credentials on the administrator system: https://cbapi.readthedocs.io/en/latest/getting-started.html -5. Run the provided BiosVerification.py utility with the following command line to target the failed system: -``` -BiosVerification.py --get --machinename -``` - -If failed BIOS image files are found the script will retrieve the image files to the local administrator system in a compressed archive named -``` --BiosImages.zip -``` - -## Example - -``` -$ ./BiosVerification.py --get --machinename "x\LT-7400" - -[ * ] Establishing LiveResponse Session with Remote Host: - - Hostname: x\LT-7400 - - OS Version: Windows 10 x64 - - Sensor Version: 3.6.0.1201 - - AntiVirus Status: ['AV_ACTIVE', 'ONDEMAND_SCAN_DISABLED'] - - Internal IP Address: 172.16.0.196 - - External IP Address: x.x.x.x - -[ * ] Uploading scripts to the remote host -[ * ] Getting the images - - -c:\program files\confer\temp>mkdir c:\tmpbios -A subdirectory or file c:\tmpbios already exists. - -c:\program files\confer\temp>del BiosImages.zip -Could Not Find c:\program files\confer\temp\BiosImages.zip - -c:\program files\confer\temp>"C:\Program Files\Dell\BiosVerification\Dell.TrustedDevice.Service.Console.exe" -exportall -export c:\tmpbios -Wrote image to c:\tmpbios\BIOSImageCaptureBVS06092020_120255.bv - -c:\program files\confer\temp>powershell.exe -ExecutionPolicy Bypass Compress-Archive -Path c:\tmpbios\*.* -DestinationPath BiosImages.zip -Force - -[ * ] Removing scripts -[ * ] Downloading images -[ * ] Writing out x\LT-7400-BiosImages.zip - -[ * ] Cleaning up - -``` - - -This script is compatible with the full VMware Carbon Black Cloud API and requires the python cbapi. \ No newline at end of file diff --git a/examples/DEPRECATED_defense/cblr/DellBiosVerification/dellbios.bat b/examples/DEPRECATED_defense/cblr/DellBiosVerification/dellbios.bat deleted file mode 100644 index b4d8e57a..00000000 --- a/examples/DEPRECATED_defense/cblr/DellBiosVerification/dellbios.bat +++ /dev/null @@ -1,4 +0,0 @@ -mkdir c:\tmpbios -del BiosImages.zip -"C:\Program Files\Dell\BiosVerification\Dell.TrustedDevice.Service.Console.exe" -exportall -export c:\tmpbios -powershell.exe -ExecutionPolicy Bypass Compress-Archive -Path c:\tmpbios\*.* -DestinationPath BiosImages.zip -Force \ No newline at end of file diff --git a/examples/DEPRECATED_defense/cblr/examplejob.py b/examples/DEPRECATED_defense/cblr/examplejob.py deleted file mode 100755 index b5bf0eec..00000000 --- a/examples/DEPRECATED_defense/cblr/examplejob.py +++ /dev/null @@ -1,10 +0,0 @@ -class GetFileJob(object): - def __init__(self, file_name): - self._file_name = file_name - - def run(self, session): - return session.get_file(self._file_name) - - -def getjob(): - return GetFileJob("c:\\test.txt") diff --git a/examples/DEPRECATED_defense/cblr/jobrunner.py b/examples/DEPRECATED_defense/cblr/jobrunner.py deleted file mode 100755 index 6ec0c062..00000000 --- a/examples/DEPRECATED_defense/cblr/jobrunner.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python - -from cbapi.defense import Device -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object -from concurrent.futures import as_completed -import sys -from datetime import datetime, timedelta - - -def main(): - parser = build_cli_parser() - parser.add_argument("--job", action="store", default="examplejob", required=True) - - args = parser.parse_args() - - cb = get_cb_defense_object(args) - - sensor_query = cb.select(Device) - - # Retrieve the list of sensors that are online - # calculate based on sensors that have checked in during the last five minutes - now = datetime.utcnow() - delta = timedelta(minutes=5) - - online_sensors = [] - offline_sensors = [] - for sensor in sensor_query: - if now - sensor.lastContact < delta: - online_sensors.append(sensor) - else: - offline_sensors.append(sensor) - - print("The following sensors are offline and will not be queried:") - for sensor in offline_sensors: - print(" {0}: {1}".format(sensor.deviceId, sensor.name)) - - print("The following sensors are online and WILL be queried:") - for sensor in online_sensors: - print(" {0}: {1}".format(sensor.deviceId, sensor.name)) - - # import our job object from the jobfile - job = __import__(args.job) - jobobject = job.getjob() - - completed_sensors = [] - futures = {} - - # collect 'future' objects for all jobs - for sensor in online_sensors: - f = cb.live_response.submit_job(jobobject.run, sensor) - futures[f] = sensor.deviceId - - # iterate over all the futures - for f in as_completed(futures.keys(), timeout=100): - if f.exception() is None: - print("Sensor {0} had result:".format(futures[f])) - print(f.result()) - completed_sensors.append(futures[f]) - else: - print("Sensor {0} had error:".format(futures[f])) - print(f.exception()) - - still_to_do = set([s.deviceId for s in online_sensors]) - set(completed_sensors) - print("The following sensors were attempted but not completed or errored out:") - for sensor in still_to_do: - print(" {0}".format(still_to_do)) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/examples/DEPRECATED_defense/cblr_cli.py b/examples/DEPRECATED_defense/cblr_cli.py deleted file mode 100644 index 5e2594e7..00000000 --- a/examples/DEPRECATED_defense/cblr_cli.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -import sys - -import logging - -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object, CblrCli -from cbapi.psc.defense import Device - -log = logging.getLogger(__name__) - - -def connect_callback(cb, line): - try: - sensor_id = int(line) - except ValueError: - sensor_id = None - - if not sensor_id: - q = cb.select(Device).where("hostNameExact:{0}".format(line)) - sensor = q.first() - else: - sensor = cb.select(Device, sensor_id) - - return sensor - - -def main(): - parser = build_cli_parser("Cb Defense Live Response CLI") - parser.add_argument("--log", help="Log activity to a file", default='') - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.log: - file_handler = logging.FileHandler(args.log) - file_handler.setLevel(logging.DEBUG) - log.addHandler(file_handler) - - cli = CblrCli(cb, connect_callback) - cli.cmdloop() - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_defense/event_export.py b/examples/DEPRECATED_defense/event_export.py deleted file mode 100644 index 55053b0a..00000000 --- a/examples/DEPRECATED_defense/event_export.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Event Export Tool - -usage: event_export.py [-h] [--appName APPNAME] startTime endTime fileName -""" - -import requests -import argparse -import json -from datetime import datetime, timedelta - - -parser = argparse.ArgumentParser() -parser.add_argument("startTime", help="Start Time (2020-08-04T00:00:00.000Z)") -parser.add_argument("endTime", help="End Time (2020-08-05T00:00:00.000Z)") -parser.add_argument("fileName", help="The name of the json file ie. events.json") -parser.add_argument("--appName", "-a", help="The app name to limit events") -args = parser.parse_args() - -with open(args.fileName, "a") as file: - - hostname = "!!REPLACE WITH HOSTNAME!!" - - url_with_app = '{}/integrationServices/v3/event?startTime={}&endTime={}&applicationName={}&rows=10000' - url_without_app = '{}/integrationServices/v3/event?startTime={}&endTime={}&rows=10000' - - headers = {'x-auth-token': '!!REPLACE WITH API SECRET KEY!!/!!REPLACE WITH API ID!!'} # key/id - - orig_end = datetime.strptime(args.endTime, '%Y-%m-%dT%H:%M:%S.%fZ') - orig_start = datetime.strptime(args.startTime, '%Y-%m-%dT%H:%M:%S.%fZ') - start = orig_end - timedelta(days=1) - end = orig_end - triggerEnd = False - file.write('[') - - while True: - print("Next End Event Time: {}".format(end.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))) - if start == orig_start: - triggerEnd = True - - if args.appName: - resp = requests.get(url_with_app.format(hostname, - start.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), - end.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), - args.appName), headers=headers) - else: - resp = requests.get(url_without_app.format(hostname, - start.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), - end.strftime('%Y-%m-%dT%H:%M:%S.%fZ')), headers=headers) - - resp_json = resp.json() - if resp_json["success"]: - results = resp_json['results'] - - end = datetime.fromtimestamp(((results[-1]["eventTime"] + 1) / 1000)) - start = end - timedelta(days=1) - - if start < orig_start: - start = orig_start - - file.write(json.dumps(results)[1:-2]) - - if resp_json["totalResults"] >= 10000: - triggerEnd = False - elif triggerEnd or end < start: - print("Events have been exported") - file.write(']') - break - file.write(',') - - else: - breakpoint() - print("API Call Failed!") - print(resp.content) - break - file.close() diff --git a/examples/DEPRECATED_defense/list_devices.py b/examples/DEPRECATED_defense/list_devices.py deleted file mode 100644 index 1c8a5406..00000000 --- a/examples/DEPRECATED_defense/list_devices.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object -from cbapi.psc.defense import Device - - -def main(): - parser = build_cli_parser("List devices") - device_options = parser.add_mutually_exclusive_group(required=False) - device_options.add_argument("-i", "--id", type=int, help="Device ID of sensor") - device_options.add_argument("-n", "--hostname", help="Hostname") - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.id: - devices = [cb.select(Device, args.id)] - elif args.hostname: - devices = list(cb.select(Device).where("hostNameExact:{0}".format(args.hostname))) - else: - devices = list(cb.select(Device)) - - print("{0:9} {1:40}{2:18}{3}".format("ID", "Hostname", "IP Address", "Last Checkin Time")) - for device in devices: - print("{0:9} {1:40s}{2:18s}{3}".format(device.deviceId, device.name or "None", - device.lastInternalIpAddress or "Unknown", device.lastContact)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_defense/list_events.py b/examples/DEPRECATED_defense/list_events.py deleted file mode 100644 index 882650c0..00000000 --- a/examples/DEPRECATED_defense/list_events.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python -# Example of using cbapi to get event data -# usage: -# python list_events.py --hostname --start --end - - -# Notes on this script: -# - script can only pull up to 2 weeks of events at one time ( this is an API limitation) -# - if no data exists between the start and end range, the script will pull no data. -# - script can be run with no arguments, it will return events from all endpoints for the past 2 weeks - -import sys -import re -from datetime import datetime -from cbapi.psc.defense.models import Event -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object - - -# Function to format epoch time -def convert_time(epoch_time): - converted_time = datetime.fromtimestamp(int(epoch_time / 1000.0)).strftime(' %b %d %Y %H:%M:%S') - return converted_time - - -# Function to strip HTML from a string. -def strip_html(string): - p = re.compile(r'<.*?>') - return p.sub('', string) - - -def main(): - parser = build_cli_parser("List Events for a device") - event_options = parser.add_mutually_exclusive_group(required=False) - event_date_options = parser.add_argument_group("Date Range Arguments") - event_date_options.add_argument("--start", help="start time") - event_date_options.add_argument("--end", help="end time") - event_options.add_argument("-n", "--hostname", help="Hostname") - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.hostname: - events = list(cb.select(Event).where("hostNameExact:{0}".format(args.hostname))) - elif args.start and args.end: - # flipped the start and end arguments around so script can be called with the start date being - # the earliest date. it's just easier on the eyes for most folks. - - events = list(cb.select(Event).where("startTime:{0}".format(args.end))) and ( - cb.select(Event).where("endTime:{0}".format(args.start))) - else: - events = list(cb.select(Event)) - - for event in events: - # convert event and create times - event_time = str(convert_time(event.createTime)) - create_time = str(convert_time(event.eventTime)) - - # stripping HTML tags out of the long description - long_description = strip_html(event.longDescription) - - # format and print out the event time, Event ID, Creation time, Event type and Description - print("{0:^25}{1:^25}{2:^32}{3}".format("Event Time", "Event ID", "Create Time", "Event Type")) - print("{0} | {1} | {2} | {3}".format(event_time, event.eventId, create_time, event.eventType)) - print("{0:50}".format(" ")) - print("{0} {1}".format("Description: ", long_description)) - print("{0:50}".format("------------------------------------")) - print("{0:50}".format(" ")) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_defense/list_events_with_cmdline_csv.py b/examples/DEPRECATED_defense/list_events_with_cmdline_csv.py deleted file mode 100644 index ab0f13fa..00000000 --- a/examples/DEPRECATED_defense/list_events_with_cmdline_csv.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Example of using cbapi to get event data -# usage: -# python list_events.py --hostname --start --end - - -# Notes on this script: -# - based on https://github.com/carbonblack/cbapi-python/blob/master/examples/defense/list_events.py -# with 2 primary changes -# 1. this script outputs the command line of the main process process -# 2. this script places a '|' delimiter between fields so it can be read into a spreadsheet -# - can only pull up to 2 weeks of events at one time ( this is an API limitation) -# - if no data exists between the start and end range, the script will pull no data. -# - script can be run with no arguments, it will return events from all endpoints for the past 2 weeks - -import sys -import re -import unicodedata -from datetime import datetime -from cbapi.psc.defense.models import Event -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object - - -# Function to format epoch time -def convert_time(epoch_time): - converted_time = datetime.fromtimestamp(int(epoch_time / 1000.0)).strftime(' %b %d %Y %H:%M:%S') - return converted_time - - -# Function to strip HTML from a string. -def strip_html(string): - p = re.compile(r'<.*?>') - return p.sub('', string) - - -def main(): - parser = build_cli_parser("List Events for a device") - event_options = parser.add_mutually_exclusive_group(required=False) - event_date_options = parser.add_argument_group("Date Range Arguments") - event_date_options.add_argument("--start", help="start time") - event_date_options.add_argument("--end", help="end time") - event_options.add_argument("-n", "--hostname", help="Hostname") - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.hostname: - events = list(cb.select(Event).where("hostNameExact:{0}".format(args.hostname))) - elif args.start and args.end: - # flipped the start and end arguments around so script can be called with the start date - # being the earliest date. it's just easier on the eyes for most folks. - - events = list(cb.select(Event).where("startTime:{0}".format(args.end))) and ( - cb.select(Event).where("endTime:{0}".format(args.start))) - else: - events = list(cb.select(Event)) - - # print the column headers - print("Event Time|Event ID|Create Time|Event Type|Description|Command Line") - - for event in events: - # convert event and create times - event_time = str(convert_time(event.createTime)) - create_time = str(convert_time(event.eventTime)) - - # stripping HTML tags out of the long description - long_description = unicodedata.normalize('NFD', strip_html(event.longDescription)) - - if event.processDetails: - # stripping out the command line arguments from the processDetails field - processDetails = str(event.processDetails) - start_cmdline = processDetails.find("u'commandLine'") - end_cmdline = processDetails.find(", u'parentName'") - commandline = processDetails[start_cmdline + 18: end_cmdline - 1] - print("{0}|{1}|{2}|{3}|{4}|{5}".format(event_time, event.eventId, create_time, event.eventType, - long_description, commandline)) - else: - print("{0}|{1}|{2}|{3}|{4}".format(event_time, event.eventId, create_time, event.eventType, - long_description)) - # format and print out the event time, Event ID, Creation time, Event type and Description - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_defense/move_device.py b/examples/DEPRECATED_defense/move_device.py deleted file mode 100644 index f28289bb..00000000 --- a/examples/DEPRECATED_defense/move_device.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object -from cbapi.psc.defense import Device - - -def main(): - parser = build_cli_parser("Move a device into a new security policy") - device_options = parser.add_mutually_exclusive_group(required=True) - device_options.add_argument("-i", "--id", type=int, help="Device ID of sensor to move") - device_options.add_argument("-n", "--hostname", help="Hostname to move") - - policy_options = parser.add_mutually_exclusive_group(required=True) - policy_options.add_argument("--policyid", type=int, help="Policy ID") - policy_options.add_argument("--policyname", help="Policy name") - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.id: - devices = [cb.select(Device, args.id)] - else: - devices = list(cb.select(Device).where("hostNameExact:{0}".format(args.hostname))) - - for device in devices: - if args.policyid: - destpolicy = int(args.policyid) - device.policyId = int(args.policyid) - else: - destpolicy = args.policyname - device.policyName = args.policyname - - device.save() - print("Moved device id {0} (hostname {1}) into policy {2}".format(device.deviceId, device.name, destpolicy)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_defense/notifications.py b/examples/DEPRECATED_defense/notifications.py deleted file mode 100644 index e414b05f..00000000 --- a/examples/DEPRECATED_defense/notifications.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object -import json - - -def main(): - parser = build_cli_parser("Listen to real-time notifications") - parser.add_argument("-s", type=int, help="# of seconds to sleep between polls", default=30) - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - while True: - for notification in cb.notification_listener(args.s): - print(json.dumps(notification, indent=2)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_defense/policy_operations.py b/examples/DEPRECATED_defense/policy_operations.py deleted file mode 100644 index 6f5b2199..00000000 --- a/examples/DEPRECATED_defense/policy_operations.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python -# - -import sys - -import json -import logging - -from cbapi.errors import ServerError -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object -from cbapi.psc.defense import Policy - -log = logging.getLogger(__name__) - - -def get_policy_by_name_or_id(cb, policy_id=None, name=None, return_all_if_none=False): - policies = [] - - try: - if policy_id: - if isinstance(policy_id, list): - attempted_to_find = "IDs of {0}".format(", ".join([str(pid) for pid in policy_id])) - policies = [p for p in cb.select(Policy) if p.id in policy_id] - else: - attempted_to_find = "ID of {0}".format(policy_id) - policies = [cb.select(Policy, policy_id, force_init=True)] - elif name: - if isinstance(name, list): - attempted_to_find = "names of {0}".format(", ".join(name)) - policies = [p for p in cb.select(Policy) if p.name in name] - else: - attempted_to_find = "name {0}".format(name) - policies = [p for p in cb.select(Policy) if p.name == name] - elif return_all_if_none: - attempted_to_find = "all policies" - policies = list(cb.select(Policy)) - except Exception as e: - print("Could not find policy with {0}: {1}".format(attempted_to_find, str(e))) - - return policies - - -def list_policies(cb, parser, args): - for p in cb.select(Policy): - print(u"Policy id {0}: {1} {2}".format(p.id, p.name, "({0})".format(p.description) if p.description else "")) - print("Rules:") - for r in p.rules.values(): - print(" {0}: {1} when {2} {3} is {4}".format(r.get('id'), r.get("action"), - r.get("application", {}).get("type"), - r.get("application", {}).get("value"), r.get("operation"))) - - -def import_policy(cb, parser, args): - p = cb.create(Policy) - - p.policy = json.load(open(args.policyfile, "r")) - p.description = args.description - p.name = args.name - p.priorityLevel = args.prioritylevel - p.version = 2 - - try: - p.save() - except ServerError as se: - print("Could not add policy: {0}".format(str(se))) - except Exception as e: - print("Could not add policy: {0}".format(str(e))) - else: - print("Added policy. New policy ID is {0}".format(p.id)) - - -def delete_policy(cb, parser, args): - policies = get_policy_by_name_or_id(cb, args.id, args.name) - if len(policies) == 0: - return - - num_matching_policies = len(policies) - if num_matching_policies > 1 and not args.force: - print("{0:d} policies match and --force not specified. No action taken.".format(num_matching_policies)) - return - - for p in policies: - try: - p.delete() - except Exception as e: - print("Could not delete policy: {0}".format(str(e))) - else: - print("Deleted policy id {0} with name {1}".format(p.id, p.name)) - - -def export_policy(cb, parser, args): - policies = get_policy_by_name_or_id(cb, args.id, args.name, return_all_if_none=True) - - for p in policies: - json.dump(p.policy, open("policy-{0}.json".format(p.id), "w"), indent=2) - print("Wrote policy {0} {1} to file policy-{0}.json".format(p.id, p.name)) - - -def add_rule(cb, parser, args): - policies = get_policy_by_name_or_id(cb, args.id, args.name) - - num_matching_policies = len(policies) - if num_matching_policies < 1: - print("No policies match. No action taken.".format(num_matching_policies)) - - for policy in policies: - policy.add_rule(json.load(open(args.rulefile, "r"))) - print("Added rule from {0} to policy {1}.".format(args.rulefile, policy.name)) - - -def del_rule(cb, parser, args): - policies = get_policy_by_name_or_id(cb, args.id, args.name) - - num_matching_policies = len(policies) - if num_matching_policies != 1: - print("{0:d} policies match. No action taken.".format(num_matching_policies)) - - policy = policies[0] - policy.delete_rule(args.ruleid) - - print("Removed rule id {0} from policy {1}.".format(args.ruleid, policy.name)) - - -def replace_rule(cb, parser, args): - policies = get_policy_by_name_or_id(cb, args.id, args.name) - - num_matching_policies = len(policies) - if num_matching_policies != 1: - print("{0:d} policies match. No action taken.".format(num_matching_policies)) - - policy = policies[0] - policy.replace_rule(args.ruleid, json.load(open(args.rulefile, "r"))) - - print("Replaced rule id {0} from policy {1} with rule from file {2}.".format(args.ruleid, policy.name, - args.rulefile)) - - -def main(): - parser = build_cli_parser("Policy operations") - commands = parser.add_subparsers(help="Policy commands", dest="command_name") - - commands.add_parser("list", help="List all configured policies") - - import_policy_command = commands.add_parser("import", help="Import policy from JSON file") - import_policy_command.add_argument("-N", "--name", help="Name of new policy", required=True) - import_policy_command.add_argument("-d", "--description", help="Description of new policy", required=True) - import_policy_command.add_argument("-p", "--prioritylevel", help="Priority level (HIGH, MEDIUM, LOW)", - default="LOW") - import_policy_command.add_argument("-f", "--policyfile", help="Filename containing the JSON policy description", - required=True) - - export_policy_command = commands.add_parser("export", help="Export policy to JSON file") - export_policy_specifier = export_policy_command.add_mutually_exclusive_group(required=False) - export_policy_specifier.add_argument("-i", "--id", type=int, help="ID of policy") - export_policy_specifier.add_argument("-N", "--name", help="Name of policy") - - del_command = commands.add_parser("delete", help="Delete policies") - del_policy_specifier = del_command.add_mutually_exclusive_group(required=True) - del_policy_specifier.add_argument("-i", "--id", type=int, help="ID of policy to delete") - del_policy_specifier.add_argument("-N", "--name", help="Name of policy to delete. Specify --force to delete" - " multiple policies that have the same name") - del_command.add_argument("--force", help="If NAME matches multiple policies, delete all matching policies", - action="store_true", default=False) - - add_rule_command = commands.add_parser("add-rule", help="Add rule to existing policy from JSON rule file") - add_rule_specifier = add_rule_command.add_mutually_exclusive_group(required=True) - add_rule_specifier.add_argument("-i", "--id", type=int, help="ID of policy (can specify multiple)", - action="append", metavar="POLICYID") - add_rule_specifier.add_argument("-N", "--name", help="Name of policy (can specify multiple)", - action="append", metavar="POLICYNAME") - add_rule_command.add_argument("-f", "--rulefile", help="Filename containing the JSON rule", required=True) - - del_rule_command = commands.add_parser("del-rule", help="Delete rule from existing policy") - del_rule_specifier = del_rule_command.add_mutually_exclusive_group(required=True) - del_rule_specifier.add_argument("-i", "--id", type=int, help="ID of policy") - del_rule_specifier.add_argument("-N", "--name", help="Name of policy") - del_rule_command.add_argument("-r", "--ruleid", type=int, help="ID of rule", required=True) - - replace_rule_command = commands.add_parser("replace-rule", help="Replace existing rule with a new one") - replace_rule_specifier = replace_rule_command.add_mutually_exclusive_group(required=True) - replace_rule_specifier.add_argument("-i", "--id", type=int, help="ID of policy") - replace_rule_specifier.add_argument("-N", "--name", help="Name of policy") - replace_rule_command.add_argument("-r", "--ruleid", type=int, help="ID of rule", required=True) - replace_rule_command.add_argument("-f", "--rulefile", help="Filename containing the JSON rule", required=True) - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.command_name == "list": - return list_policies(cb, parser, args) - elif args.command_name == "import": - return import_policy(cb, parser, args) - elif args.command_name == "export": - return export_policy(cb, parser, args) - elif args.command_name == "delete": - return delete_policy(cb, parser, args) - elif args.command_name == "add-rule": - return add_rule(cb, parser, args) - elif args.command_name == "del-rule": - return del_rule(cb, parser, args) - elif args.command_name == "replace-rule": - return replace_rule(cb, parser, args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/alert_search_suggestions.py b/examples/DEPRECATED_psc/alert_search_suggestions.py deleted file mode 100755 index cd871e19..00000000 --- a/examples/DEPRECATED_psc/alert_search_suggestions.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object - - -def main(): - parser = build_cli_parser("Get suggestions for searching alerts") - parser.add_argument("-q", "--query", default="", help="Query string for looking for alerts") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - suggestions = cb.alert_search_suggestions(args.query) - for suggestion in suggestions: - print("Search term: '{0}'".format(suggestion["term"])) - print("\tWeight: {0}".format(suggestion["weight"])) - print("\tAvailable with products: {0}".format(", ".join(suggestion["required_skus_some"]))) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/bulk_update_alerts.py b/examples/DEPRECATED_psc/bulk_update_alerts.py deleted file mode 100755 index 2c3ec81c..00000000 --- a/examples/DEPRECATED_psc/bulk_update_alerts.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -import sys -from time import sleep -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import BaseAlert, WorkflowStatus -from helpers.alertsv6 import setup_parser_with_basic_criteria, load_basic_criteria - - -def main(): - parser = build_cli_parser("Bulk update the status of alerts") - setup_parser_with_basic_criteria(parser) - parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") - parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") - operation = parser.add_mutually_exclusive_group(required=True) - operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") - operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(BaseAlert) - load_basic_criteria(query, args) - - if args.dismiss: - reqid = query.dismiss(args.remediation, args.comment) - elif args.undismiss: - reqid = query.update(args.remediation, args.comment) - else: - raise NotImplementedError("one of --dismiss or --undismiss must be specified") - - print("Submitted query with ID {0}".format(reqid)) - statobj = cb.select(WorkflowStatus, reqid) - while not statobj.finished: - print("Waiting...") - sleep(1) - if statobj.errors: - print("Errors encountered:") - for err in statobj.errors: - print("\t{0}".format(err)) - if statobj.failed_ids: - print("Failed alert IDs:") - for i in statobj.failed_ids: - print("\t{0}".format(err)) - print("{0} total alert(s) found, of which {1} were successfully changed" - .format(statobj.num_hits, statobj.num_success)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/bulk_update_cbanalytics_alerts.py b/examples/DEPRECATED_psc/bulk_update_cbanalytics_alerts.py deleted file mode 100755 index 147558c5..00000000 --- a/examples/DEPRECATED_psc/bulk_update_cbanalytics_alerts.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -import sys -from time import sleep -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import CBAnalyticsAlert, WorkflowStatus -from helpers.alertsv6 import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria - - -def main(): - parser = build_cli_parser("Bulk update the status of CB Analytics alerts") - setup_parser_with_cbanalytics_criteria(parser) - parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") - parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") - operation = parser.add_mutually_exclusive_group(required=True) - operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") - operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(CBAnalyticsAlert) - load_cbanalytics_criteria(query, args) - - if args.dismiss: - reqid = query.dismiss(args.remediation, args.comment) - elif args.undismiss: - reqid = query.update(args.remediation, args.comment) - else: - raise NotImplementedError("one of --dismiss or --undismiss must be specified") - - print("Submitted query with ID {0}".format(reqid)) - statobj = cb.select(WorkflowStatus, reqid) - while not statobj.finished: - print("Waiting...") - sleep(1) - if statobj.errors: - print("Errors encountered:") - for err in statobj.errors: - print("\t{0}".format(err)) - if statobj.failed_ids: - print("Failed alert IDs:") - for i in statobj.failed_ids: - print("\t{0}".format(err)) - print("{0} total alert(s) found, of which {1} were successfully changed" - .format(statobj.num_hits, statobj.num_success)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/bulk_update_threat_alerts.py b/examples/DEPRECATED_psc/bulk_update_threat_alerts.py deleted file mode 100755 index b3922390..00000000 --- a/examples/DEPRECATED_psc/bulk_update_threat_alerts.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python - -import sys -from time import sleep -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import WorkflowStatus - - -def main(): - parser = build_cli_parser("Bulk update the status of alerts by threat ID") - parser.add_argument("-T", "--threatid", action="append", type=str, required=True, - help="Threat IDs to update the alerts for") - parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") - parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") - operation = parser.add_mutually_exclusive_group(required=True) - operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") - operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - if args.dismiss: - reqid = cb.bulk_threat_dismiss(args.threatid, args.remediation, args.comment) - elif args.undismiss: - reqid = cb.bulk_threat_update(args.threatid, args.remediation, args.comment) - else: - raise NotImplementedError("one of --dismiss or --undismiss must be specified") - - print("Submitted query with ID {0}".format(reqid)) - statobj = cb.select(WorkflowStatus, reqid) - while not statobj.finished: - print("Waiting...") - sleep(1) - if statobj.errors: - print("Errors encountered:") - for err in statobj.errors: - print("\t{0}".format(err)) - if statobj.failed_ids: - print("Failed alert IDs:") - for i in statobj.failed_ids: - print("\t{0}".format(err)) - print("{0} total alert(s) found, of which {1} were successfully changed" - .format(statobj.num_hits, statobj.num_success)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/bulk_update_vmware_alerts.py b/examples/DEPRECATED_psc/bulk_update_vmware_alerts.py deleted file mode 100755 index a1cb0b58..00000000 --- a/examples/DEPRECATED_psc/bulk_update_vmware_alerts.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -import sys -from time import sleep -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import VMwareAlert, WorkflowStatus -from helpers.alertsv6 import setup_parser_with_vmware_criteria, load_vmware_criteria - - -def main(): - parser = build_cli_parser("Bulk update the status of VMware alerts") - setup_parser_with_vmware_criteria(parser) - parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") - parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") - operation = parser.add_mutually_exclusive_group(required=True) - operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") - operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(VMwareAlert) - load_vmware_criteria(query, args) - - if args.dismiss: - reqid = query.dismiss(args.remediation, args.comment) - elif args.undismiss: - reqid = query.update(args.remediation, args.comment) - else: - raise NotImplementedError("one of --dismiss or --undismiss must be specified") - - print("Submitted query with ID {0}".format(reqid)) - statobj = cb.select(WorkflowStatus, reqid) - while not statobj.finished: - print("Waiting...") - sleep(1) - if statobj.errors: - print("Errors encountered:") - for err in statobj.errors: - print("\t{0}".format(err)) - if statobj.failed_ids: - print("Failed alert IDs:") - for i in statobj.failed_ids: - print("\t{0}".format(err)) - print("{0} total alert(s) found, of which {1} were successfully changed" - .format(statobj.num_hits, statobj.num_success)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/bulk_update_watchlist_alerts.py b/examples/DEPRECATED_psc/bulk_update_watchlist_alerts.py deleted file mode 100755 index bef036c5..00000000 --- a/examples/DEPRECATED_psc/bulk_update_watchlist_alerts.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -import sys -from time import sleep -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import WatchlistAlert, WorkflowStatus -from helpers.alertsv6 import setup_parser_with_watchlist_criteria, load_watchlist_criteria - - -def main(): - parser = build_cli_parser("Bulk update the status of watchlist alerts") - setup_parser_with_watchlist_criteria(parser) - parser.add_argument("-R", "--remediation", help="Remediation message to store for the selected alerts") - parser.add_argument("-C", "--comment", help="Comment message to store for the selected alerts") - operation = parser.add_mutually_exclusive_group(required=True) - operation.add_argument("--dismiss", action="store_true", help="Dismiss all selected alerts") - operation.add_argument("--undismiss", action="store_true", help="Undismiss all selected alerts") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(WatchlistAlert) - load_watchlist_criteria(query, args) - - if args.dismiss: - reqid = query.dismiss(args.remediation, args.comment) - elif args.undismiss: - reqid = query.update(args.remediation, args.comment) - else: - raise NotImplementedError("one of --dismiss or --undismiss must be specified") - - print("Submitted query with ID {0}".format(reqid)) - statobj = cb.select(WorkflowStatus, reqid) - while not statobj.finished: - print("Waiting...") - sleep(1) - if statobj.errors: - print("Errors encountered:") - for err in statobj.errors: - print("\t{0}".format(err)) - if statobj.failed_ids: - print("Failed alert IDs:") - for i in statobj.failed_ids: - print("\t{0}".format(err)) - print("{0} total alert(s) found, of which {1} were successfully changed" - .format(statobj.num_hits, statobj.num_success)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/device_control.py b/examples/DEPRECATED_psc/device_control.py deleted file mode 100755 index bb07228a..00000000 --- a/examples/DEPRECATED_psc/device_control.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc import Device - - -def toggle_value(args): - if args.on: - return True - if args.off: - return False - raise Exception("Unknown toggle value") - - -def main(): - parser = build_cli_parser("Send control messages to device") - parser.add_argument("-d", "--device_id", type=int, required=True, help="The ID of the device to be controlled") - subparsers = parser.add_subparsers(dest="command", help="Device command help") - - bgscan_p = subparsers.add_parser("background_scan", help="Set background scanning status") - toggle = bgscan_p.add_mutually_exclusive_group(required=True) - toggle.add_argument("--on", action="store_true", help="Turn background scanning on") - toggle.add_argument("--off", action="store_true", help="Turn background scanning off") - - bypass_p = subparsers.add_parser("bypass", help="Set bypass mode") - toggle = bypass_p.add_mutually_exclusive_group(required=True) - toggle.add_argument("--on", action="store_true", help="Enable bypass mode") - toggle.add_argument("--off", action="store_true", help="Disable bypass mode") - - subparsers.add_parser("delete", help="Delete sensor") - subparsers.add_parser("uninstall", help="Uninstall sensor") - - quarantine_p = subparsers.add_parser("quarantine", help="Set quarantine mode") - toggle = quarantine_p.add_mutually_exclusive_group(required=True) - toggle.add_argument("--on", action="store_true", help="Enable quarantine mode") - toggle.add_argument("--off", action="store_true", help="Disable quarantine mode") - - policy_p = subparsers.add_parser("policy", help="Update policy for node") - policy_p.add_argument("-p", "--policy_id", type=int, required=True, help="New policy ID to set for node") - - sensorv_p = subparsers.add_parser("sensor_version", help="Update sensor version for node") - sensorv_p.add_argument("-o", "--os", required=True, help="Operating system for sensor") - sensorv_p.add_argument("-V", "--version", required=True, help="Version number of sensor") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - dev = cb.select(Device, args.device_id) - - if args.command: - if args.command == "background_scan": - dev.background_scan(toggle_value(args)) - elif args.command == "bypass": - dev.bypass(toggle_value(args)) - elif args.command == "delete": - dev.delete_sensor() - elif args.command == "uninstall": - dev.uninstall_sensor() - elif args.command == "quarantine": - dev.quarantine(toggle_value(args)) - elif args.command == "policy": - dev.update_policy(args.policy_id) - elif args.command == "sensor_version": - dev.update_sensor_version({args.os: args.version}) - else: - raise NotImplementedError("Unknown command") - print("OK") - else: - print(dev) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/download_device_list.py b/examples/DEPRECATED_psc/download_device_list.py deleted file mode 100755 index 25be3a43..00000000 --- a/examples/DEPRECATED_psc/download_device_list.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc import Device - -import logging -logging.basicConfig(level=logging.DEBUG) - - -def main(): - parser = build_cli_parser("Download device list in CSV format") - parser.add_argument("-q", "--query", help="Query string for looking for devices") - parser.add_argument("-A", "--ad_group_id", action="append", type=int, help="Active Directory Group ID") - parser.add_argument("-p", "--policy_id", action="append", type=int, help="Policy ID") - parser.add_argument("-s", "--status", action="append", help="Status of device") - parser.add_argument("-P", "--priority", action="append", help="Target priority of device") - parser.add_argument("-S", "--sort_by", help="Field to sort the output by") - parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") - parser.add_argument("-O", "--output", help="File to save output to (default stdout)") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(Device) - if args.query: - query = query.where(args.query) - if args.ad_group_id: - query = query.set_ad_group_ids(args.ad_group_id) - if args.policy_id: - query = query.set_policy_ids(args.policy_id) - if args.status: - query = query.set_status(args.status) - if args.priority: - query = query.set_target_priorities(args.priority) - if args.sort_by: - direction = "DESC" if args.reverse else "ASC" - query = query.sort_by(args.sort_by, direction) - - data = query.download() - if args.output: - file = open(args.output, "w") - file.write(data) - file.close() - else: - print(data) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/helpers/alertsv6.py b/examples/DEPRECATED_psc/helpers/alertsv6.py deleted file mode 100755 index 3d789669..00000000 --- a/examples/DEPRECATED_psc/helpers/alertsv6.py +++ /dev/null @@ -1,159 +0,0 @@ -def setup_parser_with_basic_criteria(parser): - parser.add_argument("-q", "--query", help="Query string for looking for alerts") - parser.add_argument("--category", action="append", choices=["THREAT", "MONITORED", "INFO", - "MINOR", "SERIOUS", "CRITICAL"], - help="Restrict search to the specified categories") - parser.add_argument("--deviceid", action="append", type=int, help="Restrict search to the specified device IDs") - parser.add_argument("--devicename", action="append", type=str, help="Restrict search to the specified device names") - parser.add_argument("--os", action="append", choices=["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"], - help="Restrict search to the specified device operating systems") - parser.add_argument("--osversion", action="append", type=str, - help="Restrict search to the specified device operating system versions") - parser.add_argument("--username", action="append", type=str, help="Restrict search to the specified user names") - parser.add_argument("--group", action="store_true", help="Group results") - parser.add_argument("--alertid", action="append", type=str, help="Restrict search to the specified alert IDs") - parser.add_argument("--legacyalertid", action="append", type=str, - help="Restrict search to the specified legacy alert IDs") - parser.add_argument("--severity", type=int, help="Restrict search to the specified minimum severity level") - parser.add_argument("--policyid", action="append", type=int, help="Restrict search to the specified policy IDs") - parser.add_argument("--policyname", action="append", type=str, help="Restrict search to the specified policy names") - parser.add_argument("--processname", action="append", type=str, - help="Restrict search to the specified process names") - parser.add_argument("--processhash", action="append", type=str, - help="Restrict search to the specified process SHA-256 hash values") - parser.add_argument("--reputation", action="append", choices=["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", - "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", - "COMPANY_BLACK_LIST"], - help="Restrict search to the specified reputation values") - parser.add_argument("--tag", action="append", type=str, help="Restrict search to the specified tag values") - parser.add_argument("--priority", action="append", choices=["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"], - help="Restrict search to the specified priority values") - parser.add_argument("--threatid", action="append", type=str, help="Restrict search to the specified threat IDs") - parser.add_argument("--type", action="append", choices=["CB_ANALYTICS", "VMWARE", "WATCHLIST"], - help="Restrict search to the specified alert types") - parser.add_argument("--workflow", action="append", choices=["OPEN", "DISMISSED"], - help="Restrict search to the specified workflow statuses") - - -def setup_parser_with_cbanalytics_criteria(parser): - setup_parser_with_basic_criteria(parser) - parser.add_argument("--blockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", - "KNOWN_MALWARE", "RISKY_PROGRAM"], - help="Restrict search to the specified threat categories that were blocked") - parser.add_argument("--location", action="append", choices=["ONSITE", "OFFSITE", "UNKNOWN"], - help="Restrict search to the specified device locations") - parser.add_argument("--killchain", action="append", choices=["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", - "INSTALL_RUN", "COMMAND_AND_CONTROL", "EXECUTE_GOAL", - "BREACH"], - help="Restrict search to the specified kill chain status values") - parser.add_argument("--notblockedthreat", action="append", choices=["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", - "KNOWN_MALWARE", "RISKY_PROGRAM"], - help="Restrict search to the specified threat categories that were NOT blocked") - parser.add_argument("--policyapplied", action="append", choices=["APPLIED", "NOT_APPLIED"], - help="Restrict search to the specified policy-application status values") - parser.add_argument("--reason", action="append", type=str, help="Restrict search to the specified reason codes") - parser.add_argument("--runstate", action="append", choices=["DID_NOT_RUN", "RAN", "UNKNOWN"], - help="Restrict search to the specified run states") - parser.add_argument("--sensoraction", action="append", choices=["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", - "TERMINATE", "DENY"], - help="Restrict search to the specified sensor actions") - parser.add_argument("--vector", action="append", choices=["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", - "REMOTE_DRIVE", "REMOVABLE_MEDIA", "UNKNOWN", - "APP_STORE", "THIRD_PARTY"], - help="Restrict search to the specified threat cause vectors") - - -def setup_parser_with_vmware_criteria(parser): - setup_parser_with_basic_criteria(parser) - parser.add_argument("--groupid", action="append", type=int, - help="Restrict search to the specified AppDefense alarm group IDs") - - -def setup_parser_with_watchlist_criteria(parser): - setup_parser_with_basic_criteria(parser) - parser.add_argument("--watchlistid", action="append", type=str, - help="Restrict search to the specified watchlists by ID") - parser.add_argument("--watchlistname", action="append", type=str, - help="Restrict search to the specified watchlists by name") - - -def load_basic_criteria(query, args): - if args.query: - query = query.where(args.query) - if args.category: - query = query.set_categories(args.category) - if args.deviceid: - query = query.set_device_ids(args.deviceid) - if args.devicename: - query = query.set_device_names(args.devicename) - if args.os: - query = query.set_device_os(args.os) - if args.osversion: - query = query.set_device_os_versions(args.osversion) - if args.username: - query = query.set_device_username(args.username) - if args.group: - query = query.set_group_results(True) - if args.alertid: - query = query.set_alert_ids(args.alertid) - if args.legacyalertid: - query = query.set_legacy_alert_ids(args.legacyalertid) - if args.severity: - query = query.set_minimum_severity(args.severity) - if args.policyid: - query = query.set_policy_ids(args.policyid) - if args.policyname: - query = query.set_policy_names(args.policyname) - if args.processname: - query = query.set_process_names(args.processname) - if args.processhash: - query = query.set_process_sha256(args.processhash) - if args.reputation: - query = query.set_reputations(args.reputation) - if args.tag: - query = query.set_tags(args.tag) - if args.priority: - query = query.set_target_priorities(args.priority) - if args.threatid: - query = query.set_threat_ids(args.threatid) - if args.type: - query = query.set_types(args.type) - if args.workflow: - query = query.set_workflows(args.workflow) - - -def load_cbanalytics_criteria(query, args): - load_basic_criteria(query, args) - if args.blockedthreat: - query = query.set_blocked_threat_categories(args.blockedthreat) - if args.location: - query = query.set_device_locations(args.location) - if args.killchain: - query = query.set_kill_chain_statuses(args.killchain) - if args.notblockedthreat: - query = query.set_not_blocked_threat_categories(args.notblockedthreat) - if args.policyapplied: - query = query.set_policy_applied(args.policyapplied) - if args.reason: - query = query.set_reason_code(args.reason) - if args.runstate: - query = query.set_run_states(args.runstate) - if args.sensoraction: - query = query.set_sensor_actions(args.sensoraction) - if args.vector: - query = query.set_threat_cause_vectors(args.vector) - - -def load_vmware_criteria(query, args): - load_basic_criteria(query, args) - if args.groupid: - query = query.set_group_ids(args.groupid) - - -def load_watchlist_criteria(query, args): - load_basic_criteria(query, args) - if args.watchlistid: - query = query.set_watchlist_ids(args.watchlistid) - if args.watchlistname: - query = query.set_watchlist_names(args.watchlistname) diff --git a/examples/DEPRECATED_psc/list_alert_facets.py b/examples/DEPRECATED_psc/list_alert_facets.py deleted file mode 100755 index 957e92da..00000000 --- a/examples/DEPRECATED_psc/list_alert_facets.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import BaseAlert -from helpers.alertsv6 import setup_parser_with_basic_criteria, load_basic_criteria - - -def main(): - parser = build_cli_parser("List alert facets") - setup_parser_with_basic_criteria(parser) - parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", - "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", - "DEVICE_NAME", "APPLICATION_HASH", - "APPLICATION_NAME", "STATUS", "RUN_STATE", - "POLICY_APPLIED_STATE", "POLICY_APPLIED", - "SENSOR_ACTION"], - required=True, help="Retrieve these fields as facet information") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(BaseAlert) - load_basic_criteria(query, args) - - facetinfos = query.facets(args.facet) - for facetinfo in facetinfos: - print("For field '{0}':".format(facetinfo["field"])) - for facetval in facetinfo["values"]: - print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/list_alerts.py b/examples/DEPRECATED_psc/list_alerts.py deleted file mode 100755 index ff2fd079..00000000 --- a/examples/DEPRECATED_psc/list_alerts.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import BaseAlert -from helpers.alertsv6 import setup_parser_with_basic_criteria, load_basic_criteria - - -def main(): - parser = build_cli_parser("List alerts") - setup_parser_with_basic_criteria(parser) - parser.add_argument("-S", "--sort_by", help="Field to sort the output by") - parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(BaseAlert) - load_basic_criteria(query, args) - if args.sort_by: - direction = "DESC" if args.reverse else "ASC" - query = query.sort_by(args.sort_by, direction) - - alerts = list(query) - print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) - for alert in alerts: - print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", - alert.threat_id or "Unknown", - alert.last_update_time)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/list_cbanalytics_alert_facets.py b/examples/DEPRECATED_psc/list_cbanalytics_alert_facets.py deleted file mode 100755 index d654671b..00000000 --- a/examples/DEPRECATED_psc/list_cbanalytics_alert_facets.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import CBAnalyticsAlert -from helpers.alertsv6 import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria - - -def main(): - parser = build_cli_parser("List CB Analytics alert facets") - setup_parser_with_cbanalytics_criteria(parser) - parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", - "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", - "DEVICE_NAME", "APPLICATION_HASH", - "APPLICATION_NAME", "STATUS", "RUN_STATE", - "POLICY_APPLIED_STATE", "POLICY_APPLIED", - "SENSOR_ACTION"], - required=True, help="Retrieve these fields as facet information") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(CBAnalyticsAlert) - load_cbanalytics_criteria(query, args) - - facetinfos = query.facets(args.facet) - for facetinfo in facetinfos: - print("For field '{0}':".format(facetinfo["field"])) - for facetval in facetinfo["values"]: - print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/list_cbanalytics_alerts.py b/examples/DEPRECATED_psc/list_cbanalytics_alerts.py deleted file mode 100755 index a45c62d8..00000000 --- a/examples/DEPRECATED_psc/list_cbanalytics_alerts.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import CBAnalyticsAlert -from helpers.alertsv6 import setup_parser_with_cbanalytics_criteria, load_cbanalytics_criteria - - -def main(): - parser = build_cli_parser("List CB Analytics alerts") - setup_parser_with_cbanalytics_criteria(parser) - parser.add_argument("-S", "--sort_by", help="Field to sort the output by") - parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(CBAnalyticsAlert) - load_cbanalytics_criteria(query, args) - if args.sort_by: - direction = "DESC" if args.reverse else "ASC" - query = query.sort_by(args.sort_by, direction) - - alerts = list(query) - print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) - for alert in alerts: - print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", - alert.threat_id or "Unknown", - alert.last_update_time)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/list_devices.py b/examples/DEPRECATED_psc/list_devices.py deleted file mode 100755 index 24a5bb18..00000000 --- a/examples/DEPRECATED_psc/list_devices.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc import Device - - -def main(): - parser = build_cli_parser("List devices") - parser.add_argument("-q", "--query", help="Query string for looking for devices") - parser.add_argument("-A", "--ad_group_id", action="append", type=int, help="Active Directory Group ID") - parser.add_argument("-p", "--policy_id", action="append", type=int, help="Policy ID") - parser.add_argument("-s", "--status", action="append", help="Status of device") - parser.add_argument("-P", "--priority", action="append", help="Target priority of device") - parser.add_argument("-S", "--sort_by", help="Field to sort the output by") - parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(Device) - if args.query: - query = query.where(args.query) - if args.ad_group_id: - query = query.set_ad_group_ids(args.ad_group_id) - if args.policy_id: - query = query.set_policy_ids(args.policy_id) - if args.status: - query = query.set_status(args.status) - if args.priority: - query = query.set_target_priorities(args.priority) - if args.sort_by: - direction = "DESC" if args.reverse else "ASC" - query = query.sort_by(args.sort_by, direction) - - devices = list(query) - print("{0:9} {1:40}{2:18}{3}".format("ID", "Hostname", "IP Address", "Last Checkin Time")) - for device in devices: - print("{0:9} {1:40s}{2:18s}{3}".format(device.id, device.name or "None", - device.last_internal_ip_address or "Unknown", - device.last_contact_time)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/list_vmware_alert_facets.py b/examples/DEPRECATED_psc/list_vmware_alert_facets.py deleted file mode 100755 index c8420037..00000000 --- a/examples/DEPRECATED_psc/list_vmware_alert_facets.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import VMwareAlert -from helpers.alertsv6 import setup_parser_with_vmware_criteria, load_vmware_criteria - - -def main(): - parser = build_cli_parser("List VMware alert facets") - setup_parser_with_vmware_criteria(parser) - parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", - "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", - "DEVICE_NAME", "APPLICATION_HASH", - "APPLICATION_NAME", "STATUS", "RUN_STATE", - "POLICY_APPLIED_STATE", "POLICY_APPLIED", - "SENSOR_ACTION"], - required=True, help="Retrieve these fields as facet information") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(VMwareAlert) - load_vmware_criteria(query, args) - - facetinfos = query.facets(args.facet) - for facetinfo in facetinfos: - print("For field '{0}':".format(facetinfo["field"])) - for facetval in facetinfo["values"]: - print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/list_vmware_alerts.py b/examples/DEPRECATED_psc/list_vmware_alerts.py deleted file mode 100755 index ee0fb44e..00000000 --- a/examples/DEPRECATED_psc/list_vmware_alerts.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import VMwareAlert -from helpers.alertsv6 import setup_parser_with_vmware_criteria, load_vmware_criteria - - -def main(): - parser = build_cli_parser("List VMware alerts") - setup_parser_with_vmware_criteria(parser) - parser.add_argument("-S", "--sort_by", help="Field to sort the output by") - parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(VMwareAlert) - load_vmware_criteria(query, args) - if args.sort_by: - direction = "DESC" if args.reverse else "ASC" - query = query.sort_by(args.sort_by, direction) - - alerts = list(query) - print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) - for alert in alerts: - print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", - alert.threat_id or "Unknown", - alert.last_update_time)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/list_watchlist_alert_facets.py b/examples/DEPRECATED_psc/list_watchlist_alert_facets.py deleted file mode 100755 index 35776ef1..00000000 --- a/examples/DEPRECATED_psc/list_watchlist_alert_facets.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import WatchlistAlert -from helpers.alertsv6 import setup_parser_with_watchlist_criteria, load_watchlist_criteria - - -def main(): - parser = build_cli_parser("List watchlist alert facets") - setup_parser_with_watchlist_criteria(parser) - parser.add_argument("-F", "--facet", action="append", choices=["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", - "TAG", "POLICY_ID", "POLICY_NAME", "DEVICE_ID", - "DEVICE_NAME", "APPLICATION_HASH", - "APPLICATION_NAME", "STATUS", "RUN_STATE", - "POLICY_APPLIED_STATE", "POLICY_APPLIED", - "SENSOR_ACTION"], - required=True, help="Retrieve these fields as facet information") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(WatchlistAlert) - load_watchlist_criteria(query, args) - - facetinfos = query.facets(args.facet) - for facetinfo in facetinfos: - print("For field '{0}':".format(facetinfo["field"])) - for facetval in facetinfo["values"]: - print("\tValue {0}: {1} occurrences".format(facetval["id"], facetval["total"])) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_psc/list_watchlist_alerts.py b/examples/DEPRECATED_psc/list_watchlist_alerts.py deleted file mode 100755 index 708efa07..00000000 --- a/examples/DEPRECATED_psc/list_watchlist_alerts.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_psc_object -from cbapi.psc.models import WatchlistAlert -from helpers.alertsv6 import setup_parser_with_watchlist_criteria, load_watchlist_criteria - - -def main(): - parser = build_cli_parser("List watchlist alerts") - setup_parser_with_watchlist_criteria(parser) - parser.add_argument("-S", "--sort_by", help="Field to sort the output by") - parser.add_argument("-R", "--reverse", action="store_true", help="Reverse order of sort") - - args = parser.parse_args() - cb = get_cb_psc_object(args) - - query = cb.select(WatchlistAlert) - load_watchlist_criteria(query, args) - if args.sort_by: - direction = "DESC" if args.reverse else "ASC" - query = query.sort_by(args.sort_by, direction) - - alerts = list(query) - print("{0:40} {1:40s} {2:40s} {3}".format("ID", "Hostname", "Threat ID", "Last Updated")) - for alert in alerts: - print("{0:40} {1:40s} {2:40s} {3}".format(alert.id, alert.device_name or "None", - alert.threat_id or "Unknown", - alert.last_update_time)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/create_feed.py b/examples/DEPRECATED_threathunter/create_feed.py deleted file mode 100644 index 619c7166..00000000 --- a/examples/DEPRECATED_threathunter/create_feed.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python - -import sys -import time - -from cbapi.example_helpers import read_iocs, build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Feed - - -def main(): - parser = build_cli_parser("Create a CbTH feed and, optionally, a report from a stream of IOCs") - - # Feed metadata arguments. - parser.add_argument("--name", type=str, help="Feed name", required=True) - parser.add_argument("--owner", type=str, help="Feed owner", required=True) - parser.add_argument("--url", type=str, help="Feed provider url", required=True) - parser.add_argument("--summary", type=str, help="Feed summary", required=True) - parser.add_argument("--category", type=str, help="Feed category", required=True) - parser.add_argument("--source_label", type=str, help="Feed source label", default=None) - parser.add_argument("--access", type=str, help="Feed access scope", default="private") - - # Report metadata arguments. - parser.add_argument("--read_report", action="store_true", help="Read a report from stdin") - parser.add_argument("--rep_timestamp", type=int, help="Report timestamp", default=int(time.time())) - parser.add_argument("--rep_title", type=str, help="Report title") - parser.add_argument("--rep_desc", type=str, help="Report description") - parser.add_argument("--rep_severity", type=int, help="Report severity", default=1) - parser.add_argument("--rep_link", type=str, help="Report link") - parser.add_argument("--rep_tags", type=str, help="Report tags, comma separated") - parser.add_argument("--rep_visibility", type=str, help="Report visibility") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - feed_info = { - "name": args.name, - "owner": args.owner, - "provider_url": args.url, - "summary": args.summary, - "category": args.category, - "access": args.access, - } - - reports = [] - if args.read_report: - rep_tags = [] - if args.rep_tags: - rep_tags = args.rep_tags.split(",") - - report = { - "timestamp": args.rep_timestamp, - "title": args.rep_title, - "description": args.rep_desc, - "severity": args.rep_severity, - "link": args.rep_link, - "tags": rep_tags, - "iocs_v2": [], # NOTE(ww): The feed server will convert IOCs to v2s for us. - } - - report_id, iocs = read_iocs(cb) - - report["id"] = report_id - report["iocs"] = iocs - reports.append(report) - - feed = { - "feedinfo": feed_info, - "reports": reports - } - - feed = cb.create(Feed, feed) - feed.save() - - print(feed) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/events.py b/examples/DEPRECATED_threathunter/events.py deleted file mode 100644 index 244fd7cd..00000000 --- a/examples/DEPRECATED_threathunter/events.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Event - - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-p", type=str, help="process guid", default=None) - parser.add_argument("-n", type=int, help="only output N events", default=None) - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if not args.p: - print("Error: Missing Process GUID to search for events with") - sys.exit(1) - - events = cb.select(Event).where(process_guid=args.p) - - if args.n: - events = events[0:args.n] - - for event in events: - print("Event type: {}".format(event.event_type)) - print("\tEvent GUID: {}".format(event.event_guid)) - print("\tEvent Timestamp: {}".format(event.event_timestamp)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/events_exporter.py b/examples/DEPRECATED_threathunter/events_exporter.py deleted file mode 100644 index 2f31c9f4..00000000 --- a/examples/DEPRECATED_threathunter/events_exporter.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Event -import json -import csv - - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-p", type=str, help="process guid", default=None) - parser.add_argument("-s", type=bool, help="silent mode", default=False) - parser.add_argument("-n", type=int, help="only output N events", default=None) - parser.add_argument("-f", type=str, help="output file name", default=None) - parser.add_argument("-of", type=str, help="output file format: csv or json", default="json") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if not args.p: - print("Error: Missing Process GUID to search for events with") - sys.exit(1) - - events = cb.select(Event).where(process_guid=args.p) - - if args.n: - events = events[0:args.n] - - if not args.s: - for event in events: - print("Event type: {}".format(event.event_type)) - print("\tEvent GUID: {}".format(event.event_guid)) - print("\tEvent Timestamp: {}".format(event.event_timestamp)) - - if args.f is not None: - if args.of == "json": - with open(args.f, 'w') as outfile: - for event in events: - json.dump(event.original_document, outfile) - else: - with open(args.f, 'w') as outfile: - csvwriter = csv.writer(outfile) - for event in events: - csvwriter.writerows(event) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/feed_operations.py b/examples/DEPRECATED_threathunter/feed_operations.py deleted file mode 100644 index 9ab4469c..00000000 --- a/examples/DEPRECATED_threathunter/feed_operations.py +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python -# - -import sys -from cbapi.psc.threathunter.models import Feed, Report -from cbapi.example_helpers import eprint, build_cli_parser, get_cb_threathunter_object -import logging -import json - -log = logging.getLogger(__name__) - - -def get_feed(cb, feed_id=None, feed_name=None): - if feed_id: - return cb.select(Feed, feed_id) - elif feed_name: - feeds = [feed for feed in cb.select(Feed) if feed.name == feed_name] - - if not feeds: - eprint("No feeds named '{}'".format(feed_name)) - sys.exit(1) - elif len(feeds) > 1: - eprint("More than one feed named '{}'".format(feed_name)) - sys.exit(1) - - return feeds[0] - else: - raise ValueError("expected either feed_id or feed_name") - - -def get_report(feed, report_id=None, report_name=None): - if report_id: - reports = [report for report in feed.reports if report.id == report_id] - - if not reports: - eprint("No reports with ID '{}'".format(report_id)) - sys.exit(1) - elif len(reports) > 1: - eprint("More than one report with ID '{}'".format(report_id)) - sys.exit(1) - elif report_name: - reports = [report for report in feed.reports if report.title == report_name] - - if not reports: - eprint("No reports named '{}'".format(report_name)) - sys.exit(1) - elif len(reports) > 1: - eprint("More than one report named '{}'".format(report_name)) - sys.exit(1) - else: - raise ValueError("expected either report_id or report_name") - - return reports[0] - - -def list_feeds(cb, parser, args): - if args.iocs and not args.reports: - eprint("--iocs specified without --reports") - sys.exit(1) - - feeds = cb.select(Feed).where(include_public=args.public) - - for feed in feeds: - print(feed) - if args.reports: - for report in feed.reports: - print(report) - if args.iocs: - for ioc in report.iocs_: - print(ioc) - - -def list_iocs(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - - for report in feed.reports: - for ioc in report.iocs_: - print(ioc) - - -def export_feed(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - - exported = {} - - exported["feedinfo"] = feed._info - exported["reports"] = [report._info for report in feed.reports] - print(json.dumps(exported)) - - -def import_feed(cb, parser, args): - imported = json.loads(sys.stdin.read()) - - if args.feedname: - imported["feedinfo"]["name"] = args.feedname - - feed = cb.create(Feed, imported) - feed.save(public=args.public) - - -def delete_feed(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - feed.delete() - - -def export_report(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - report = get_report(feed, report_id=args.reportid, report_name=args.reportname) - - print(json.dumps(report._info)) - - -def import_report(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - - imp_dict = json.loads(sys.stdin.read()) - - reports = feed.reports - existing_report = next( - (report for report in reports if imp_dict["id"] == report.id), None - ) - - if existing_report: - eprint("Report already exists; use replace-report.") - sys.exit(1) - else: - imp_report = cb.create(Report, imp_dict) - feed.append_reports([imp_report]) - - -def delete_report(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - report = get_report(feed, report_id=args.reportid, report_name=args.reportname) - report.delete() - - -def replace_report(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - - imported = json.loads(sys.stdin.read()) - - reports = feed.reports - existing_report = next( - (report for report in reports if imported["id"] == report.id), None - ) - - if existing_report: - existing_report.update(**imported) - else: - eprint("No existing report to replace") - sys.exit(1) - - -def main(): - parser = build_cli_parser() - commands = parser.add_subparsers(help="Feed commands", dest="command_name") - - list_command = commands.add_parser("list", help="List all configured feeds") - list_command.add_argument( - "-P", - "--public", - help="Include public feeds", - action="store_true", - default=False, - ) - list_command.add_argument( - "-r", - "--reports", - help="Include reports for each feed", - action="store_true", - default=False, - ) - list_command.add_argument( - "-i", - "--iocs", - help="Include IOCs for each feed's reports", - action="store_true", - default=False, - ) - - list_iocs_command = commands.add_parser( - "list-iocs", help="List all IOCs for a feed" - ) - specifier = list_iocs_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - - export_command = commands.add_parser( - "export", help="Export a feed into an importable format" - ) - specifier = export_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - - import_command = commands.add_parser( - "import", help="Import a previously exported feed" - ) - import_command.add_argument( - "-f", "--feedname", type=str, help="Renames the imported feed" - ) - import_command.add_argument( - "-P", "--public", help="Make the feed public", action="store_true" - ) - - del_command = commands.add_parser("delete", help="Delete feed") - specifier = del_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - - export_report_command = commands.add_parser( - "export-report", help="Export a feed's report into an importable format" - ) - specifier = export_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - specifier = export_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-I", "--reportid", type=str, help="Report ID") - specifier.add_argument("-r", "--reportname", type=str, help="Report Name") - - import_report_command = commands.add_parser( - "import-report", help="Import a previously exported report" - ) - specifier = import_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - - delete_report_command = commands.add_parser( - "delete-report", help="Delete a report from a feed" - ) - specifier = delete_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - specifier = delete_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-I", "--reportid", type=str, help="Report ID") - specifier.add_argument("-r", "--reportname", type=str, help="Report Name") - - replace_report_command = commands.add_parser( - "replace-report", help="Replace a feed's report" - ) - specifier = replace_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if args.command_name == "list": - return list_feeds(cb, parser, args) - elif args.command_name == "list-iocs": - return list_iocs(cb, parser, args) - elif args.command_name == "export": - return export_feed(cb, parser, args) - elif args.command_name == "import": - return import_feed(cb, parser, args) - elif args.command_name == "delete": - return delete_feed(cb, parser, args) - elif args.command_name == "export-report": - return export_report(cb, parser, args) - elif args.command_name == "import-report": - return import_report(cb, parser, args) - elif args.command_name == "delete-report": - return delete_report(cb, parser, args) - elif args.command_name == "replace-report": - return replace_report(cb, parser, args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/import_response_feeds.py b/examples/DEPRECATED_threathunter/import_response_feeds.py deleted file mode 100644 index 85146ea3..00000000 --- a/examples/DEPRECATED_threathunter/import_response_feeds.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python -# -import sys -from cbapi.psc.threathunter import CbThreatHunterAPI -from cbapi.psc.threathunter.models import Feed as FeedTH -from cbapi.response.models import Feed -from cbapi.example_helpers import build_cli_parser, get_cb_response_object -from cbapi.errors import ServerError -from urllib.parse import unquote -import logging - -log = logging.getLogger(__name__) - - -def list_feeds(cb, parser, args): - """ - Lists the feeds in CB Response - """ - for f in cb.select(Feed): - for fieldname in ["id", "category", "display_name", "enabled", "provider_url", "summary", "tech_data", - "feed_url", "use_proxy", "validate_server_cert"]: - print("%-20s: %s" % (fieldname, getattr(f, fieldname, ""))) - - if f.username: - for fieldname in ["username", "password"]: - print("%-20s: %s" % (fieldname, getattr(f, fieldname, ""))) - - if f.ssl_client_crt: - for fieldname in ["ssl_client_crt", "ssl_client_key"]: - print("%-20s: %s" % (fieldname, getattr(f, fieldname, ""))) - - print("\n") - - -def list_reports(cb, parser, args): - """ - Lists the reports in a feed from CB Response - :param: id - The ID of a feed - """ - feed = cb.select(Feed, args.id, force_init=True) - for report in feed.reports: - print(report) - print("\n") - - -def convert_feed(cb, cb_th, parser, args): - """ - Converts and copies a feed from CB Response to CB Threat Hunter - :param: id - The ID of a feed from CB Response - - Requires a credentials profile for both CB Response and CB Threat Hunter - Ensure that your credentials for CB Threat Hunter have permissions to the Feed Manager APIs - """ - th_feed = {"feedinfo": {}, "reports": []} - # Fetches the CB Response feed - feed = cb.select(Feed, args.id, force_init=True) - - th_feed["feedinfo"]["name"] = feed.name - th_feed["feedinfo"]["provider_url"] = feed.provider_url - th_feed["feedinfo"]["summary"] = feed.summary - th_feed["feedinfo"]["category"] = feed.category - th_feed["feedinfo"]["access"] = "private" - - # Temporary values until refresh - th_feed["feedinfo"]["owner"] = "org_key" - th_feed["feedinfo"]["id"] = "id" - - # Iterates the reports in the CB Response feed - for report in feed.reports: - th_report = {} - th_report["id"] = report.id - th_report["timestamp"] = report.timestamp - th_report["title"] = report.title - th_report["severity"] = (report.score % 10) + 1 - if hasattr(report, "description"): - th_report["description"] = report.description - else: - th_report["description"] = "" - if hasattr(report, "link"): - th_report["link"] = report.link - th_report["iocs"] = {} - if report.iocs: - if "md5" in report.iocs: - th_report["iocs"]["md5"] = report.iocs["md5"] - if "ipv4" in report.iocs: - th_report["iocs"]["ipv4"] = report.iocs["ipv4"] - if "ipv6" in report.iocs: - th_report["iocs"]["ipv6"] = report.iocs["ipv6"] - if "dns" in report.iocs: - th_report["iocs"]["dns"] = report.iocs["dns"] - - if "query" in report.iocs: - th_report["iocs"]["query"] = [] - for query in report.iocs.get("query", []): - try: - search = query.get('search_query', "") - if "q=" in search: - params = search.split('&') - for p in params: - if "q=" in p: - search = unquote(p[2:]) - # Converts the CB Response query to CB Threat Hunter - th_query = cb_th.convert_query(search) - if th_query: - query["search_query"] = th_query - th_report["iocs"]["query"].append(query) - except ServerError: - print('Invalid query {}'.format(query.get('search_query', ""))) - - th_feed["reports"].append(th_report) - - # Pushes the new feed to CB Threat Hunter - new_feed = cb_th.create(FeedTH, th_feed) - new_feed.save() - print("{}\n".format(new_feed)) - - -def main(): - parser = build_cli_parser() - parser.add_argument("-thp", "--threatprofile", help="Threat Hunter profile", default="default") - commands = parser.add_subparsers(help="Feed commands", dest="command_name") - - commands.add_parser("list", help="List all configured feeds") - - list_reports_command = commands.add_parser("list-reports", help="List all configured reports for a feed") - list_reports_command.add_argument("-i", "--id", type=str, help="Feed ID") - - convert_feed_command = commands.add_parser("convert", help="Convert feed from CB Response to CB Threat Hunter") - convert_feed_command.add_argument("-i", "--id", type=str, help="Feed ID") - - args = parser.parse_args() - cb = get_cb_response_object(args) - cb_th = CbThreatHunterAPI(profile=args.threatprofile) - - if args.command_name == "list": - return list_feeds(cb, parser, args) - if args.command_name == "list-reports": - return list_reports(cb, parser, args) - if args.command_name == "convert": - return convert_feed(cb, cb_th, parser, args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/modify_feed.py b/examples/DEPRECATED_threathunter/modify_feed.py deleted file mode 100644 index 8f543b8a..00000000 --- a/examples/DEPRECATED_threathunter/modify_feed.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_feed_object - - -def main(): - parser = build_cli_parser("Modify a CbTH feed") - parser.add_argument("--id", type=str, help="Feed ID", default=None) - parser.add_argument("--name", type=str, help="Feed name", default=None) - parser.add_argument("--owner", type=str, help="Feed owner", default=None) - parser.add_argument("--url", type=str, help="Feed provider url", default="https://example.com") - parser.add_argument("--summary", type=str, help="Feed summary", default=None) - parser.add_argument("--category", type=str, help="Feed category", default="Partner") - parser.add_argument("--access", type=str, help="Feed access scope", default="private") - - args = parser.parse_args() - cb = get_cb_threathunter_feed_object(args) - - feed = cb.feed(args.id) - - print("Before modification:") - print("=" * 80) - print(feed) - print("=" * 80) - - metadata = {} - if args.name: - metadata["name"] = args.name - if args.owner: - metadata["owner"] = args.owner - if args.url: - metadata["provider_url"] = args.url - if args.summary: - metadata["summary"] = args.summary - if args.category: - metadata["category"] = args.category - if args.access: - metadata["access"] = args.access - - feed.feedinfo.update(**metadata) - - print("After modification:") - print("=" * 80) - print(feed) - print("=" * 80) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/process_exporter.py b/examples/DEPRECATED_threathunter/process_exporter.py deleted file mode 100644 index b0d017e7..00000000 --- a/examples/DEPRECATED_threathunter/process_exporter.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Process -import json -import csv - - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-p", type=str, help="process guid", default=None) - parser.add_argument("-q", type=str, help="query string", default=None) - parser.add_argument("-s", type=bool, help="silent mode", default=False) - parser.add_argument("-n", type=int, help="only output N events", default=None) - parser.add_argument("-f", type=str, help="output file name", default=None) - parser.add_argument("-of", type=str, help="output file format: csv or json", default="json") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if not args.p and not args.q: - print("Error: Missing Process GUID to search for events with") - sys.exit(1) - - if args.q: - processes = cb.select(Process).where(args.q) - else: - processes = cb.select(Process).where(process_guid=args.p) - - if args.n: - processes = [p for p in processes[0:args.n]] - - if not args.s: - for process in processes: - print("Process: {}".format(process.process_name)) - print("\tPIDs: {}".format(process.process_pids)) - print("\tSHA256: {}".format(process.process_sha256)) - print("\tGUID: {}".format(process.process_guid)) - - if args.f is not None: - if args.of == "json": - with open(args.f, 'w') as outfile: - for p in processes: - json.dump(p.original_document, outfile) - print(p.original_document) - else: - headers = set() - headers.update(*(d.original_document.keys() for d in processes)) - with open(args.f, 'w') as outfile: - csvwriter = csv.DictWriter(outfile, fieldnames=headers) - csvwriter.writeheader() - for p in processes: - csvwriter.writerow(p.original_document) - - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/process_query.py b/examples/DEPRECATED_threathunter/process_query.py deleted file mode 100644 index 1b3bfbd8..00000000 --- a/examples/DEPRECATED_threathunter/process_query.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Process, Binary -from cbapi.errors import ObjectNotFoundError - - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-q", type=str, help="process query", default="process_name:notepad.exe") - parser.add_argument("-n", type=int, help="only output N processes", default=None) - parser.add_argument("-b", action="store_true", help="show binary information", default=False) - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - processes = cb.select(Process).where(args.q) - - if args.n: - processes = processes[0:args.n] - - for process in processes: - print("Process: {}".format(process.process_name)) - print("\tPIDs: {}".format(process.process_pids)) - print("\tSHA256: {}".format(process.process_sha256)) - print("\tGUID: {}".format(process.process_guid)) - - if args.b: - try: - binary = cb.select(Binary, process.process_sha256) - print(binary) - print(binary.summary) - except ObjectNotFoundError: - print("No binary found for process with hash: {}".format(process.process_sha256)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/process_tree.py b/examples/DEPRECATED_threathunter/process_tree.py deleted file mode 100644 index df2e64f2..00000000 --- a/examples/DEPRECATED_threathunter/process_tree.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Process - - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-p", type=str, help="process guid", default=None) - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if not args.p: - print("Error: Missing Process GUID to query the process tree with") - sys.exit(1) - - tree = cb.select(Process).where(process_guid=args.p)[0].tree() - for idx, child in enumerate(tree.children): - print("Child #{}".format(idx)) - print("\tName: {}".format(child.process_name)) - print("\tGUID: {}".format(child.process_guid)) - print("\tNumber of children: {}".format(len(child.children))) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/process_tree_exporter.py b/examples/DEPRECATED_threathunter/process_tree_exporter.py deleted file mode 100644 index 2eef7326..00000000 --- a/examples/DEPRECATED_threathunter/process_tree_exporter.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Process -import csv -import json - - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-p", type=str, help="process guid", default=None) - parser.add_argument("-f", type=str, help="output file name", default=None) - parser.add_argument("-of", type=str, help="output file format: csv or json", default="json") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if not args.p: - print("Error: Missing Process GUID to query the process tree with") - sys.exit(1) - - tree = cb.select(Process).where(process_guid=args.p)[0].tree() - - for idx, child in enumerate(tree.children): - print("Child #{}".format(idx)) - print("\tName: {}".format(child.process_name)) - print("\tNumber of children: {}".format(len(child.children))) - - if args.f is not None: - if args.of == "json": - with open(args.f, 'w') as outfile: - for idx, child in enumerate(tree.children): - json.dump(child.original_document, outfile) - else: - with open(args.f, 'w') as outfile: - csvwriter = csv.writer(outfile) - for idx, child in enumerate(tree.children): - csvwriter.writerow(child.original_document) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/search.py b/examples/DEPRECATED_threathunter/search.py deleted file mode 100644 index d757aaf7..00000000 --- a/examples/DEPRECATED_threathunter/search.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Process - - -def main(): - parser = build_cli_parser("Search processes") - parser.add_argument("-q", type=str, help="process query", default="process_name:notepad.exe") - parser.add_argument("-f", help="show full objects", action="store_true", default=False) - parser.add_argument("-n", type=int, help="only output N processes", default=None) - parser.add_argument("-e", help="show events for query results", action="store_true", default=False) - parser.add_argument("-c", help="show children for query results", action="store_true", default=False) - parser.add_argument("-p", help="show parents for query results", action="store_true", default=False) - parser.add_argument("-t", help="show tree for query results", action="store_true", default=False) - parser.add_argument("-S", type=str, help="sort by this field", required=False) - parser.add_argument("-D", help="return results in descending order", action="store_true") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - processes = cb.select(Process).where(args.q) - - direction = "ASC" - if args.D: - direction = "DESC" - - if args.S: - processes.sort_by(args.S, direction=direction) - - print("Number of processes: {}".format(len(processes))) - - if args.n: - processes = processes[0:args.n] - - for process in processes: - if args.f: - print(process) - else: - print("{} ({}): {}".format(process.process_name, process.process_guid, process.process_sha256)) - - if args.e: - print("=========== events ===========") - for event in process.events(): - if args.f: - print(event) - else: - print("\t{}".format(event.event_type)) - - if args.c: - print("========== children ==========") - for child in process.children: - if args.f: - print(child) - else: - print("\t{}: {}".format(child.process_name, child.process_sha256)) - - if args.p: - print("========== parents ==========") - for parent in process.parents: - if args.f: - print(parent) - else: - print("\t{}: {}".format(parent.process_name, parent.process_sha256)) - - if args.t: - print("=========== tree =============") - tree = process.tree() - print(tree) - print(tree.nodes) - - print("===========================") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/DEPRECATED_threathunter/threat_intelligence/README.md b/examples/DEPRECATED_threathunter/threat_intelligence/README.md deleted file mode 100644 index fa862284..00000000 --- a/examples/DEPRECATED_threathunter/threat_intelligence/README.md +++ /dev/null @@ -1,129 +0,0 @@ -# ThreatIntel Module -Python3 module that can be used in the development of Threat Intelligence Connectors for the Carbon Black Cloud. - -## Requirements - -The file `requirements.txt` contains a list of dependencies for this project. After cloning this repository, run the following command from the `examples/threathunter/threat_intelligence` directory: - -```python -pip3 install -r ./requirements.txt -``` - - -## Introduction -This document describes how to use the ThreatIntel Python3 module for development of connectors that retrieve Threat Intelligence and import it into a Carbon Black Cloud instance. - -Throughout this document, there are references to Carbon Black ThreatHunter Feed and Report formats. Documentation on Feed and Report definitions is [available here.](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/#definitions) - -## Example - -An example of implementing this ThreatIntel module is [available here](Taxii_README.md). The example uses cabby to connect to a TAXII server, collect threat intelligence, and send it to a ThreatHunter Feed. - - -## Usage - -`threatintel.py` has two main uses: - -1. Report Validation with `schemas.ReportSchema` -2. Pushing Reports to a Carbon Black ThreatHunter Feed with `threatintel.push_to_cb()` - -### Report validation - -Each Report to be sent to the Carbon Black Cloud should be validated -before sending. The [ThreatHunter Report format](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/#definitions) is a JSON object with -five required and five optional values. - -|Required|Type|Optional|Type| -|---|---|---|---| -|`id`|string|`link`|string| -|`timestamp`|integer|`[tags]`|[str]| -|`title`|string|`iocs`|[IOC Format](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/#definitions)| -|`description`|string|`[iocs_v2]`|[[IOCv2 Format](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/#definitions)]| -|`severity`|integer|`visibility`|string| - -The `push_to_cb` function checks for the existence and type of the five -required values, and (if applicable) checks the optional values, through a Schema. -See `schemas.py` for the definitions. - -### Pushing Reports to a Carbon Black ThreatHunter Feed - -The `push_to_cb` function takes a list of `AnalysisResult` objects (or objects of your own custom class) and a Carbon -Black ThreatHunter Feed ID as input, and writes output to the console. -The `AnalysisResult` class is defined in `results.py`, and requirements for a custom class are outlined in the Customization section below. - -`AnalysisResult` objects are expected to have the same properties as -ThreatHunter Reports (listed in the table above in Report Validation), with the addition of `iocs_v2`. The -`push_to_cb` function will convert `AnalysisResult` objects into -Report dictionaries, and then those dictionaries into ThreatHunter -Report objects. - -Any improperly formatted report dictionaries are saved to a file called `malformed_reports.json`. - -Upon successful sending of reports to a ThreatHunter Feed, you should -see something similar to the following INFO message in the console: - -`INFO:threatintel:Appended 1000 reports to ThreatHunter Feed AbCdEfGhIjKlMnOp` - - -### Using Validation and Pushing to ThreatHunter in your own code - -Import the module and supporting classes like any other python package, and instantiate a ThreatIntel object: - - ```python - from threatintel import ThreatIntel - from results import IOC_v2, AnalysisResult - ti = ThreatIntel() -``` - -Take the threat intelligence data from your source, and convert it into ``AnalysisResult`` objects. Then, attach the indicators of compromise, and store your data in a list. - -```python - myResults = [] - for intel in myThreatIntelligenceData: - result = AnalysisResult(analysis_name=intel.name, scan_time=intel.scan_time, score=intel.score, title=intel.title, description=intel.description) - #ioc_dict could be a collection of md5 hashes, dns values, file hashes, etc. - for ioc_key, ioc_val in intel.ioc_dict.items(): - result.attach_ioc_v2(values=ioc_val, field=ioc_key, link=link) - myResults.append(result) -``` - -Finally, push your threat intelligence data to a ThreatHunter Feed. -```python - ti.push_to_cb(feed_id='AbCdEfGhIjKlMnOp', results=myResults) -``` - -`ti.push_to_cb` automatically validates your input to ensure it has the values required for ThreatHunter. Validated reports will be sent to your specified ThreatHunter Feed, and any malformed reports will be available for review locally at `malformed_reports.json`. - - - -## Customization - -Although the `AnalysisResult` class is provided in `results.py` as an example, you may create your own custom class to use with `push_to_cb`. The class must have the following attributes to work with the provided `push_to_cb` function, as well as the ThreatHunter backend: - - -|Attribute|Type| -|---|---| -|`id`|string| -|`timestamp`|integer| -|`title`|string| -|`description`|string| -|`severity`|integer| -|`iocs_v2`|[[IOCv2 Format](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/#definitions)]| - -It is strongly recommended to use the provided `IOC_v2()` class from `results.py`. If you decide to use a custom `iocs_v2` class, that class must have a method called `as_dict` that returns `id`, `match_type`, `values`, `field`, and `link` as a dictionary. - - -## Writing a Custom Threat Intelligence Polling Connector - -An example of a custom Threat Intel connector that uses the `ThreatIntel` Python3 module is included in this repository as `stix_taxii.py`. Most use cases will warrant the use of the ThreatHunter `Report` attribute `iocs_v2`, so it is included in `ThreatIntel.push_to_cb()`. - -`ThreatIntel.push_to_cb()` and `AnalysisResult` can be adapted to include other ThreatHunter `Report` attributes like `link, tags, iocs, and visibility`. - - -## Troubleshooting - -### Credential Error -In order to use this code, you must have CBAPI installed and configured. If you receive an authentication error, visit the Developer Network Authentication Page for [instructions on setting up authentication](https://developer.carbonblack.com/reference/carbon-black-cloud/authentication/). See [ReadTheDocs](https://cbapi.readthedocs.io/en/latest/index.html#api-credentials) for instructions on configuring your credentials file. - -### 504 Gateway Timeout Error -The [Carbon Black ThreatHunter Feed Manager API](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/) is used in this code. When posting to a Feed, there is a 60 second limit before the gateway terminates your connection. The amount of reports you can POST to a Feed is limited by your connection speed. In this case, you will have to split your threat intelligence into smaller collections until the request takes less than 60 seconds, and send each smaller collection to an individual ThreatHunter Feed. diff --git a/examples/DEPRECATED_threathunter/threat_intelligence/Taxii_README.md b/examples/DEPRECATED_threathunter/threat_intelligence/Taxii_README.md deleted file mode 100644 index fff2e434..00000000 --- a/examples/DEPRECATED_threathunter/threat_intelligence/Taxii_README.md +++ /dev/null @@ -1,41 +0,0 @@ -# TAXII Connector -Connector for pulling and converting STIX information from TAXII Service Providers into CB Feeds. - -## Requirements/Installation - -The file `requirements.txt` contains a list of dependencies for this project. After cloning this repository, run the following command from the `examples/threathunter/threat_intelligence` directory: - -```python -pip3 install -r ./requirements.txt -``` - -## Introduction -This document describes how to configure the CB ThreatHunter TAXII connector. -This connector allows for the importing of STIX data by querying one or more TAXII services, retrieving that data and then converting it into CB feeds using the CB JSON format for IOCs. - -## Setup - TAXII Configuration File -The TAXII connector uses the configuration file `config.yml`. An example configuration file is available [here.](config.yml) An explanation of each entry in the configuration file is provided in the example. - - -## Running the Connector -The connector can be activated by running the Python3 file `stix_taxii.py`. The connector will attempt to connect to your TAXII service(s), poll the collection(s), retrieve the STIX data, and send it to the ThreatHunter Feed specified in your `config.yml` file. - -```python -python3 stix_taxii.py -``` - -This script supports updating each TAXII configuration's `start_date`, the date for which to start requesting data, via the command line with the argument `site_start_date`. To change the `stat_date` value for each site in your config file, you must supply the site name and desired `start_date` in `%Y-%m-%d %H:%M:%S` format. - -```python -python3 stix_taxii.py --site_start_date my_site_name_1 '2019-11-05 00:00:00' my_site_name_2 '2019-11-05 00:00:00' -``` - -This may be useful if the intention is to keep an up-to-date collection of STIX data in a ThreatHunter Feed. - -## Troubleshooting - -### Credential Error -In order to use this code, you must have CBAPI installed and configured. If you receive an authentication error, visit the Developer Network Authentication Page for [instructions on setting up authentication](https://developer.carbonblack.com/reference/carbon-black-cloud/authentication/). See [ReadTheDocs](https://cbapi.readthedocs.io/en/latest/index.html?highlight=credentials.psc#api-credentials) for instructions on configuring your credentials file. - -### 504 Gateway Timeout Error -The [Carbon Black ThreatHunter Feed Manager API](https://developer.carbonblack.com/reference/carbon-black-cloud/cb-threathunter/latest/feed-api/) is used in this code. When posting to a Feed, there is a 60 second limit before the gateway terminates your connection. The amount of reports you can POST to a Feed is limited by your connection speed. In this case, you will have to split your threat intelligence into smaller collections until the request takes less than 60 seconds, and send each smaller collection to an individual ThreatHunter Feed. diff --git a/examples/DEPRECATED_threathunter/threat_intelligence/config.yml b/examples/DEPRECATED_threathunter/threat_intelligence/config.yml deleted file mode 100644 index 121b204e..00000000 --- a/examples/DEPRECATED_threathunter/threat_intelligence/config.yml +++ /dev/null @@ -1,78 +0,0 @@ -sites: - my_site_name_1: - # the feed_id of the ThreatHunter Feed you want to send ThreatIntel to - # example: 7wP8BEc2QsS8ciEqaRv7Ad - feed_id: - - # the address of the site (only server ip or dns; don't put https:// or a trailing slash) - # example: limo.anomali.com - site: - - # the path of the site for discovering what services are available - # this is supplied by your taxii provider - # example: /api/v1/taxii/taxii-discovery-service/ - discovery_path: - - # the path of the site for listing what collections are available to you - # this is supplied by your taxii provider - # example: /api/v1/taxii/collection_management/ - collection_management_path: - - # the path of the site for polling a collection - # this is supplied by your taxii provider - # example: /api/v1/taxii/poll/ - poll_path: - - # if you require https for your TAXII service connection, set to true - # defaults to true - use_https: - - # by default, we validate SSL certificates. Change to false to turn off SSL verification - ssl_verify: - - # (optional) if you need SSL certificates for authentication, set the path of the - # certificate and key here. - cert_file: - key_file: - - # (optional) how to score each result. Accepts values [1,10], and defaults to 5 - default_score: - - # (optional) username for authorization with your taxii provider - username: - - # (optional) password for authorization with your taxii provider - password: - - # (optional) specify which collections to convert to feeds (comma-delimited) - # example: Abuse_ch_Ransomware_IPs_F135, DShield_Scanning_IPs_F150 - collections: - - # the start date for which to start requesting data. - # Use %y-%m-%d %H:%M:%S format - # example: 2019-01-01 00:00:00 - start_date: - - # (optional) the minutes to advance for each request. - # If you don't have a lot of data, you could advance your requests - # to every 60 minutes, or 1440 minutes for daily chunks - # defaults to 1440 - size_of_request_in_minutes: - - # (optional) path to a CA SSL certificate - ca_cert: - - # (optional) if you need requests to go through a proxy, specify an http URL here - http_proxy_url: - - # (optional) if you need requests to go through a proxy, specify an https URL here - https_proxy_url: - - # (optional) number of reports to collect from each site. - # Leave blank for no limit - reports_limit: - - # (optional) control the number of failed attempts per-collection before giving up - # trying to get (empty/malformed) STIX data out of a TAXII server. - # defaults to 10 - fail_limit: diff --git a/examples/DEPRECATED_threathunter/threat_intelligence/feed_helper.py b/examples/DEPRECATED_threathunter/threat_intelligence/feed_helper.py deleted file mode 100644 index 1484bce3..00000000 --- a/examples/DEPRECATED_threathunter/threat_intelligence/feed_helper.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Advances the `begin_date` and `end_date` fields while polling the TAXII server to iteratively get per-collection STIX content. - -This is tied to the `start_date` and `size_of_request_in_minutes` configuration options in your `config.yml`. -""" - -from datetime import datetime, timedelta, timezone -import logging -log = logging.getLogger(__name__) - - -class FeedHelper(): - def __init__(self, start_date, size_of_request_in_minutes): - self.size_of_request_in_minutes = size_of_request_in_minutes - if isinstance(start_date, datetime): - self.start_date = start_date.replace(tzinfo=timezone.utc) - elif isinstance(start_date, str): - self.start_date = datetime.strptime(start_date, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) - else: - log.error(f"Start_date must be a string or datetime object. Received a start_time config value with unsupported type: {type(start_date)}") - raise ValueError - self.end_date = self.start_date + \ - timedelta(minutes=self.size_of_request_in_minutes) - self.now = datetime.utcnow().replace(tzinfo=timezone.utc) - if self.end_date > self.now: - self.end_date = self.now - self.start = False - self.done = False - - def advance(self): - """Returns True if keep going, False if we already hit the end time and cannot advance.""" - if not self.start: - self.start = True - return True - - if self.done: - return False - - # continues shifting the time window by size_of_request_in_minutes until we hit current time, then stops - self.start_date = self.end_date - self.end_date += timedelta(minutes=self.size_of_request_in_minutes) - if self.end_date > self.now: - self.end_date = self.now - self.done = True - - return True diff --git a/examples/DEPRECATED_threathunter/threat_intelligence/get_feed_ids.py b/examples/DEPRECATED_threathunter/threat_intelligence/get_feed_ids.py deleted file mode 100644 index 488a29c9..00000000 --- a/examples/DEPRECATED_threathunter/threat_intelligence/get_feed_ids.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Lists ThreatHunter Feed IDs available for results dispatch.""" - -from cbapi.psc.threathunter import CbThreatHunterAPI -from cbapi.psc.threathunter.models import Feed -import logging - -log = logging.getLogger(__name__) - - -def get_feed_ids(): - cb = CbThreatHunterAPI() - feeds = cb.select(Feed) - if not feeds: - log.info("No feeds are available for the org key {}".format(cb.credentials.org_key)) - else: - for feed in feeds: - log.info("Feed name: {:<20} \t Feed ID: {:>20}".format(feed.name, feed.id)) - - -if __name__ == '__main__': - get_feed_ids() diff --git a/examples/DEPRECATED_threathunter/threat_intelligence/requirements.txt b/examples/DEPRECATED_threathunter/threat_intelligence/requirements.txt deleted file mode 100644 index eb1beb53..00000000 --- a/examples/DEPRECATED_threathunter/threat_intelligence/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -cybox==2.1.0.18 -dataclasses>=0.6 -cabby==0.1.20 -stix==1.2.0.7 -lxml==4.6.5 -urllib3>=1.24.2 -cbapi>=1.5.6 -python_dateutil==2.8.1 -PyYAML==5.4 -schema diff --git a/examples/DEPRECATED_threathunter/threat_intelligence/results.py b/examples/DEPRECATED_threathunter/threat_intelligence/results.py deleted file mode 100644 index e3d8d732..00000000 --- a/examples/DEPRECATED_threathunter/threat_intelligence/results.py +++ /dev/null @@ -1,81 +0,0 @@ -import enum -import logging - - -class IOC_v2(): - """Models an indicator of compromise detected during an analysis. - - Every IOC belongs to an AnalysisResult. - """ - - def __init__(self, analysis, match_type, values, field, link): - self.id = analysis - self.match_type = match_type - self.values = values - self.field = field - self.link = link - - class MatchType(str, enum.Enum): - """ - Represents the valid matching strategies for an IOC. - """ - - Equality: str = "equality" - Regex: str = "regex" - Query: str = "query" - - def as_dict(self): - return { - "id": str(self.id), - "match_type": self.match_type, - "values": list(self.values), - "field": self.field, - "link": self.link, - } - - -class AnalysisResult(): - """Models the result of an analysis performed by a connector.""" - - def __init__(self, analysis_name, scan_time, score, title, description): - self.id = str(analysis_name) - self.timestamp = scan_time - self.title = title - self.description = description - self.severity = score - self.iocs = [] - self.iocs_v2 = [] - self.link = None - self.tags = None - self.visibility = None - self.connector_name = "STIX_TAXII" - - def attach_ioc_v2(self, *, match_type=IOC_v2.MatchType.Equality, values, field, link): - self.iocs_v2.append(IOC_v2(analysis=self.id, match_type=match_type, values=values, field=field, link=link)) - - def normalize(self): - """Normalizes this result to make it palatable for the CbTH backend.""" - - if self.severity <= 0 or self.severity > 10: - logging.warning("normalizing OOB score: {}".format(self.severity)) - if self.severity > 10 and self.severity < 100: - #assume it's a percentage - self.severity = round(self.severity/10) - else: - # any severity above 10 becomes 10, or below 1 becomes 1 - # Report severity must be between 1 & 10, else CBAPI throws 400 error - self.severity = max(1, min(self.severity, 10)) - return self - - def as_dict(self): - return {"IOCs_v2": [ioc_v2.as_dict() for ioc_v2 in self.iocs_v2], **super().as_dict()} - - def as_dict_full(self): - return { - "id": self.id, - "timestamp": self.timestamp, - "title": self.title, - "description": self.description, - "severity": self.severity, - "iocs_v2": [iocv2.as_dict() for iocv2 in self.iocs_v2] - } diff --git a/examples/DEPRECATED_threathunter/threat_intelligence/schemas.py b/examples/DEPRECATED_threathunter/threat_intelligence/schemas.py deleted file mode 100644 index da361604..00000000 --- a/examples/DEPRECATED_threathunter/threat_intelligence/schemas.py +++ /dev/null @@ -1,44 +0,0 @@ -from schema import And, Or, Optional, Schema - - -IOCv2Schema = Schema( - { - "id": And(str, len), - "match_type": And(str, lambda type: type in ["query", "equality", "regex"]), - "values": And([str], len), - Optional("field"): str, - Optional("link"): str - } -) - -QueryIOCSchema = Schema( - { - "search_query": And(str, len), - Optional("index_type"): And(str, len) - } -) - -IOCSchema = Schema( - { - Optional("md5"): And([str], len), - Optional("ipv4"): And([str], len), - Optional("ipv6"): And([str], len), - Optional("dns"): And([str], len), - Optional("query"): [QueryIOCSchema] - } -) - -ReportSchema = Schema( - { - "id": And(str, len), - "timestamp": And(int, lambda n: n > 0), - "title": And(str, len), - "description": And(str, len), - "severity": And(int, lambda n: n > 0 and n < 11), - Optional("link"): str, - Optional("tags"): [str], - Optional("iocs_v2"): [IOCv2Schema], - Optional("iocs"): IOCSchema, - Optional("visibility"): str - } -) diff --git a/examples/DEPRECATED_threathunter/threat_intelligence/stix_parse.py b/examples/DEPRECATED_threathunter/threat_intelligence/stix_parse.py deleted file mode 100644 index 8663c549..00000000 --- a/examples/DEPRECATED_threathunter/threat_intelligence/stix_parse.py +++ /dev/null @@ -1,466 +0,0 @@ -"""Parses STIX observables from the XML data returned by the TAXII server. - -The following IOC types are extracted from STIX data: - -* MD5 Hashes -* Domain Names -* IP-Addresses -* IP-Address Ranges -""" - -from cybox.objects.domain_name_object import DomainName -from cybox.objects.address_object import Address -from cybox.objects.file_object import File -from cybox.objects.uri_object import URI -from lxml import etree -from io import BytesIO -from stix.core import STIXPackage - -import logging -import string -import socket -import uuid -import time -import datetime -import dateutil -import dateutil.tz -import re - -from cabby.constants import ( - CB_STIX_XML_111, CB_CAP_11, CB_SMIME, - CB_STIX_XML_10, CB_STIX_XML_101, CB_STIX_XML_11, CB_XENC_122002) - -CB_STIX_XML_12 = 'urn:stix.mitre.org:xml:1.2' - -BINDING_CHOICES = [CB_STIX_XML_111, CB_CAP_11, CB_SMIME, CB_STIX_XML_12, - CB_STIX_XML_10, CB_STIX_XML_101, CB_STIX_XML_11, - CB_XENC_122002] - - -logger = logging.getLogger(__name__) - - -domain_allowed_chars = string.printable[:-6] # Used by validate_domain_name function - - -def validate_domain_name(domain_name): - """Validates a domain name to ensure validity and saneness. - - Args: - domain_name: Domain name string to check. - - Returns: - True if checks pass, False otherwise. - """ - - if len(domain_name) > 255: - logger.warn( - "Excessively long domain name {} in IOC list".format(domain_name)) - return False - - if not all([c in domain_allowed_chars for c in domain_name]): - logger.warn("Malformed domain name {} in IOC list".format(domain_name)) - return False - - parts = domain_name.split('.') - if not parts: - logger.warn("Empty domain name found in IOC list") - return False - - for part in parts: - if len(part) < 1 or len(part) > 63: - logger.warn("Invalid label length {} in domain name {} for report %s".format( - part, domain_name)) - return False - - return True - - -def validate_md5sum(md5): - """Validates md5sum. - - Args: - md5sum: md5sum to check. - - Returns: - True if checks pass, False otherwise. - """ - - if 32 != len(md5): - logger.warn("Invalid md5 length for md5 {}".format(md5)) - return False - if not md5.isalnum(): - logger.warn("Malformed md5 {} in IOC list".format(md5)) - return False - for c in "ghijklmnopqrstuvwxyz": - if c in md5 or c.upper() in md5: - logger.warn("Malformed md5 {} in IOC list".format(md5)) - return False - - return True - - -def sanitize_id(id): - """Removes unallowed chars from an ID. - - Ids may only contain a-z, A-Z, 0-9, - and must have one character. - - Args: - id: the ID to be sanitized. - - Returns: - A sanitized ID. - """ - - return id.replace(':', '-') - - -def validate_ip_address(ip_address): - """Validates an IPv4 address.""" - - try: - socket.inet_aton(ip_address) - return True - except socket.error: - return False - - -def cybox_parse_observable(observable, indicator, timestamp, score): - """Parses a cybox observable and returns a list containing a report dictionary. - - cybox is a open standard language encoding info about cyber observables. - - Args: - observable: the cybox obserable to parse. - - Returns: - A report dictionary if the cybox observable has props of type: - - cybox.objects.address_object.Address, - cybox.objects.file_object.File, - cybox.objects.domain_name_object.DomainName, or - cybox.objects.uri_object.URI - - Otherwise it will return an empty list. - - """ - reports = [] - - if observable.object_ and observable.object_.properties: - props = observable.object_.properties - logger.debug("{0} has props type: {1}".format(indicator, type(props))) - else: - logger.debug("{} has no props; skipping".format(indicator)) - return reports - - # - # sometimes the description is None - # - description = '' - if observable.description and observable.description.value: - description = str(observable.description.value) - - # - # if description is an empty string, then use the indicator's description - # NOTE: This was added for RecordedFuture - # - - if not description and indicator and indicator.description and indicator.description.value: - description = str(indicator.description.value) - - # - # if description is still empty, use the indicator's title - # - if not description and indicator and indicator.title: - description = str(indicator.title) - - # - # use the first reference as a link - # This was added for RecordedFuture - # - link = '' - if indicator and indicator.producer and indicator.producer.references: - for reference in indicator.producer.references: - link = reference - break - else: - if indicator and indicator.title: - split_title = indicator.title.split() - title_found = True - elif observable and observable.title: - split_title = observable.title.split() - title_found = True - else: - title_found = False - - if title_found: - url_pattern = re.compile("^(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$") - for token in split_title: - if url_pattern.match(token): - link = token - break - - - # - # Sometimes the title is None, so generate a random UUID - # - - if observable and observable.title: - title = observable.title - else: - title = str(uuid.uuid4()) - - # ID must be unique. Collisions cause 500 error on Carbon Black backend - id = str(uuid.uuid4()) - - if type(props) == DomainName: - # go into domainname function - reports = parse_domain_name_observable(observable, props, id, description, title, timestamp, link, score) - - elif type(props) == Address: - reports = parse_address_observable(observable, props, id, description, title, timestamp, link, score) - - elif type(props) == File: - reports = parse_file_observable(observable, props, id, description, title, timestamp, link, score) - - elif type(props) == URI: - reports = parse_uri_observable(observable, props, id, description, title, timestamp, link, score) - - else: - return reports - - return reports - -def parse_uri_observable(observable, props, id, description, title, timestamp, link, score): - - reports = [] - - if props.value and props.value.value: - - iocs = {'netconn_domain': []} - # - # Sometimes props.value.value is a list - # - - if type(props.value.value) is list: - for domain_name in props.value.value: - if validate_domain_name(domain_name.strip()): - iocs['netconn_domain'].append(domain_name.strip()) - else: - domain_name = props.value.value.strip() - if validate_domain_name(domain_name): - iocs['netconn_domain'].append(domain_name) - - if len(iocs['netconn_domain']) > 0: - reports.append({'iocs_v2': iocs, - 'id': sanitize_id(id), - 'description': description, - 'title': title, - 'timestamp': timestamp, - 'link': link, - 'score': score}) - return reports - - -def parse_domain_name_observable(observable, props, id, description, title, timestamp, link, score): - - reports = [] - if props.value and props.value.value: - iocs = {'netconn_domain': []} - # - # Sometimes props.value.value is a list - # - - if type(props.value.value) is list: - for domain_name in props.value.value: - if validate_domain_name(domain_name.strip()): - iocs['netconn_domain'].append(domain_name.strip()) - else: - domain_name = props.value.value.strip() - if validate_domain_name(domain_name): - iocs['netconn_domain'].append(domain_name) - - if len(iocs['netconn_domain']) > 0: - reports.append({'iocs_v2': iocs, - 'id': sanitize_id(id), - 'description': description, - 'title': title, - 'timestamp': timestamp, - 'link': link, - 'score': score}) - return reports - - -def parse_address_observable(observable, props, id, description, title, timestamp, link, score): - - reports = [] - if props.category == 'ipv4-addr' and props.address_value: - iocs = {'netconn_ipv4': []} - - # - # Sometimes props.address_value.value is a list vs a string - # - if type(props.address_value.value) is list: - for ip in props.address_value.value: - if validate_ip_address(ip.strip()): - iocs['netconn_ipv4'].append(ip.strip()) - else: - ipv4 = props.address_value.value.strip() - if validate_ip_address(ipv4): - iocs['netconn_ipv4'].append(ipv4) - - if len(iocs['netconn_ipv4']) > 0: - reports.append({'iocs_v2': iocs, - 'id': sanitize_id(observable.id_), - 'description': description, - 'title': title, - 'timestamp': timestamp, - 'link': link, - 'score': score}) - - return reports - - -def parse_file_observable(observable, props, id, description, title, timestamp, link, score): - - reports = [] - iocs = {'hash': []} - if props.md5: - if type(props.md5) is list: - for hash in props.md5: - if validate_md5sum(hash.strip()): - iocs['hash'].append(hash.strip()) - else: - if hasattr(props.md5, 'value'): - hash = props.md5.value.strip() - else: - hash = props.md5.strip() - if validate_md5sum(hash): - iocs['hash'].append(hash) - - if len(iocs['hash']) > 0: - reports.append({'iocs_v2': iocs, - 'id': sanitize_id(id), - 'description': description, - 'title': title, - 'timestamp': timestamp, - 'link': link, - 'score': score}) - - return reports - - -def get_stix_indicator_score(indicator, default_score): - """Returns a digit representing the indicator score. - - Converts from "high", "medium", or "low" into a digit, if necessary. - """ - - if not indicator.confidence: - return default_score - - - confidence_val_str = indicator.confidence.value.__str__() - if confidence_val_str.isdigit(): - score = int(confidence_val_str) - return score - elif confidence_val_str.lower() == "high": - return 7 # 75 - elif confidence_val_str.lower() == "medium": - return 5 # 50 - elif confidence_val_str.lower() == "low": - return 2 # 25 - else: - return default_score - - -def get_stix_indicator_timestamp(indicator): - timestamp = 0 - if indicator.timestamp: - if indicator.timestamp.tzinfo: - timestamp = int((indicator.timestamp - - datetime.datetime(1970, 1, 1).replace( - tzinfo=dateutil.tz.tzutc())).total_seconds()) - else: - timestamp = int((indicator.timestamp - - datetime.datetime(1970, 1, 1)).total_seconds()) - return timestamp - - -def get_stix_package_timestamp(stix_package): - timestamp = 0 - if not stix_package or not stix_package.timestamp: - return timestamp - try: - timestamp = stix_package.timestamp - timestamp = int(time.mktime(timestamp.timetuple())) - except (TypeError, OverflowError, ValueError) as e: - logger.warning("Problem parsing stix timestamp: {}".format(e)) - return timestamp - - -def parse_stix_indicators(stix_package, default_score): - reports = [] - if not stix_package.indicators: - return reports - - for indicator in stix_package.indicators: - if not indicator or not indicator.observable: - continue - score = get_stix_indicator_score(indicator, default_score) - timestamp = get_stix_indicator_timestamp(indicator) - yield from cybox_parse_observable( - indicator.observable, indicator, timestamp, score) - - -def parse_stix_observables(stix_package, default_score): - reports = [] - if not stix_package.observables: - return reports - - timestamp = get_stix_package_timestamp(stix_package) - for observable in stix_package.observables: - if not observable: - continue - yield from cybox_parse_observable( # single element list - observable, None, timestamp, default_score) - - -def sanitize_stix(stix_xml): - ret_xml = b'' - try: - xml_root = etree.fromstring(stix_xml) - content = xml_root.find( - './/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content') - if content is not None and len(content) == 0 and len(list(content)) == 0: - # Content has no children. - # So lets make sure we parse the xml text for content and - # re-add it as valid XML so we can parse - _content = xml_root.find( - "{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content_Block/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content") - if _content: - new_stix_package = etree.fromstring(_content.text) - content.append(new_stix_package) - ret_xml = etree.tostring(xml_root) - except etree.ParseError as e: - logger.warning("Problem parsing stix: {}".format(e)) - return ret_xml - - -def parse_stix(stix_xml, default_score): - reports = [] - try: - stix_xml = sanitize_stix(stix_xml) - bio = BytesIO(stix_xml) - stix_package = STIXPackage.from_xml(bio) - if not stix_package: - logger.warning("Could not parse STIX xml") - return reports - if not stix_package.indicators and not stix_package.observables: - logger.info("No indicators or observables found in stix_xml") - return reports - yield from parse_stix_indicators(stix_package, default_score) - yield from parse_stix_observables(stix_package, default_score) - except etree.XMLSyntaxError as e: - logger.warning("Problem parsing stix: {}".format(e)) - return reports diff --git a/examples/DEPRECATED_threathunter/threat_intelligence/stix_taxii.py b/examples/DEPRECATED_threathunter/threat_intelligence/stix_taxii.py deleted file mode 100644 index 95358071..00000000 --- a/examples/DEPRECATED_threathunter/threat_intelligence/stix_taxii.py +++ /dev/null @@ -1,392 +0,0 @@ -"""Connects to TAXII servers via cabby and formats the data received for dispatching to a Carbon Black feed.""" - -import argparse -import logging -import traceback -from threatintel import ThreatIntel -from cabby.exceptions import NoURIProvidedError, ClientException -from requests.exceptions import ConnectionError -from cbapi.errors import ApiError -from cabby import create_client -from dataclasses import dataclass -import yaml -import os -from stix_parse import parse_stix, BINDING_CHOICES -from feed_helper import FeedHelper -from datetime import datetime -from results import AnalysisResult -from cbapi.psc.threathunter.models import Feed -import urllib3 -import copy - -# logging.basicConfig(filename='stix.log', filemode='w', level=logging.DEBUG) -logging.basicConfig(filename='stix.log', filemode='w', format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', - datefmt='%Y-%m-%d:%H:%M:%S', - level=logging.INFO) -handled_exceptions = (NoURIProvidedError, ClientException, ConnectionError) - - -def load_config_from_file(): - """Loads YAML formatted configuration from config.yml in working directory.""" - - logging.debug("loading config from file") - config_filename = os.path.join(os.path.dirname((os.path.abspath(__file__))), "config.yml") - with open(config_filename, "r") as config_file: - config_data = yaml.load(config_file, Loader=yaml.SafeLoader) - config_data_without_none_vals = copy.deepcopy(config_data) - for site_name, site_config_dict in config_data['sites'].items(): - for conf_key, conf_value in site_config_dict.items(): - if conf_value is None: - del config_data_without_none_vals['sites'][site_name][conf_key] - logging.info(f"loaded config data: {config_data_without_none_vals}") - return config_data_without_none_vals - - -@dataclass(eq=True, frozen=True) -class TaxiiSiteConfig: - """Contains information needed to interface with a TAXII server. - - These values are loaded in from config.yml for each entry in the configuration file. - Each TaxiiSiteConnector has its own TaxiiSiteConfig. - """ - - feed_id: str = '' - site: str = '' - discovery_path: str = '' - collection_management_path: str = '' - poll_path: str = '' - use_https: bool = True - ssl_verify: bool = True - cert_file: str = None - key_file: str = None - default_score: int = 5 # [1,10] - username: str = None - password: str = None - collections: str = '*' - start_date: str = None - size_of_request_in_minutes: int = 1440 - ca_cert: str = None - http_proxy_url: str = None - https_proxy_url: str = None - reports_limit: int = None - fail_limit: int = 10 # num attempts per collection for polling & parsing - - -class TaxiiSiteConnector(): - """Connects to and pulls data from a TAXII server.""" - - def __init__(self, site_conf): - self.config = TaxiiSiteConfig(**site_conf) - self.client = None - - def create_taxii_client(self): - """Connects to a TAXII server using cabby and configuration entries.""" - - conf = self.config - if not conf.start_date: - logging.error(f"A start_date is required for site {conf.site}. Exiting.") - return - if not conf.ssl_verify: - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - try: - client = create_client(conf.site, - use_https=conf.use_https, - discovery_path=conf.discovery_path) - client.set_auth(username=conf.username, - password=conf.password, - verify_ssl=conf.ssl_verify, - ca_cert=conf.ca_cert, - cert_file=conf.cert_file, - key_file=conf.key_file) - - proxy_dict = dict() - if conf.http_proxy_url: - proxy_dict['http'] = conf.http_proxy_url - if conf.https_proxy_url: - proxy_dict['https'] = conf.https_proxy_url - if proxy_dict: - client.set_proxies(proxy_dict) - - self.client = client - - except handled_exceptions as e: - logging.error(f"Error creating client: {e}") - - def create_uri(self, config_path): - """Formats a URI for discovery, collection, or polling of a TAXII server. - - Args: - config_path: A URI path to a TAXII server's discovery, collection, or polling service. Defined in config.yml configuration file. - - Returns: - A full URI to one of a TAXII server's service paths. - """ - - uri = None - if self.config.site and config_path: - if self.config.use_https: - uri = 'https://' - else: - uri = 'http://' - uri = uri + self.config.site + config_path - return uri - - def query_collections(self): - """Returns a list of STIX collections available to the user to poll.""" - - collections = [] - try: - uri = self.create_uri(self.config.collection_management_path) - collections = self.client.get_collections( - uri=uri) # autodetect if uri=None - for collection in collections: - logging.info(f"Collection: {collection.name}, {collection.type}") - except handled_exceptions as e: - logging.warning(f"Problem fetching collections from TAXII server. Check your TAXII Provider URL and username/password (if required to access TAXII server): {e}") - return collections - - def poll_server(self, collection, feed_helper): - """Returns a STIX content block for a specific TAXII collection. - - Args: - collection: Name of a TAXII collection to poll. - feed_helper: FeedHelper object. - """ - - content_blocks = [] - uri = self.create_uri(self.config.poll_path) - try: - logging.info(f"Polling Collection: {collection.name}") - content_blocks = self.client.poll( - uri=uri, - collection_name=collection.name, - begin_date=feed_helper.start_date, - end_date=feed_helper.end_date, - content_bindings=BINDING_CHOICES) - except handled_exceptions as e: - logging.warning(f"problem polling taxii server: {e}") - return content_blocks - - def parse_collection_content(self, content_blocks): - """Yields a formatted report dictionary for each STIX content_block. - - Args: - content_block: A chunk of STIX data from the TAXII collection being polled. - """ - - for block in content_blocks: - yield from parse_stix(block.content, self.config.default_score) - - def import_collection(self, collection): - """Polls a single TAXII server collection. - - Starting at the start_date set in config.yml, a FeedHelper object will continue to grab chunks - of data from a collection until the report limit is reached or we reach the current datetime. - - Args: - collection: Name of a TAXII collection to poll. - - Yields: - Formatted report dictionaries from parse_collection_content(content_blocks) - for each content_block pulled from a single TAXII collection. - """ - - num_times_empty_content_blocks = 0 - advance = True - reports_limit = self.config.reports_limit - if not self.config.size_of_request_in_minutes: - size_of_request_in_minutes = 1440 - else: - size_of_request_in_minutes = self.config.size_of_request_in_minutes - feed_helper = FeedHelper(self.config.start_date, - size_of_request_in_minutes) - # config parameters `start_date` and `size_of_request_in_minutes` tell this Feed Helper - # where to start polling in the collection, and then will advance polling in chunks of - # `size_of_request_in_minutes` until we hit the most current `content_block`, - # or reports_limit is reached. - while feed_helper.advance(): - num_reports = 0 - num_times_empty_content_blocks = 0 - content_blocks = self.poll_server(collection, feed_helper) - reports = self.parse_collection_content(content_blocks) - for report in reports: - yield report - num_reports += 1 - if reports_limit is not None and num_reports >= reports_limit: - logging.info(f"Reports limit of {self.config.reports_limit} reached") - advance = False - break - - if not advance: - break - if collection.type == 'DATA_SET': # data is unordered, not a feed - logging.info(f"collection:{collection}; type data_set; breaking") - break - if num_reports == 0: - num_times_empty_content_blocks += 1 - if num_times_empty_content_blocks > self.config.fail_limit: - logging.error('Max fail limit reached; Exiting.') - break - if reports_limit is not None: - reports_limit -= num_reports - - def import_collections(self, available_collections): - """Polls each desired collection specified in config.yml. - - Args: - available_collections: list of collections available to a TAXII server user. - - Yields: - From import_collection(self, collection) for each desired collection. - """ - - if not self.config.collections: - desired_collections = '*' - else: - desired_collections = self.config.collections - - desired_collections = [x.strip() - for x in desired_collections.lower().split(',')] - - want_all = True if '*' in desired_collections else False - - for collection in available_collections: - if collection.type != 'DATA_FEED' and collection.type != 'DATA_SET': - logging.debug(f"collection:{collection}; type not feed or data") - continue - if not collection.available: - logging.debug(f"collection:{collection} not available") - continue - if want_all or collection.name.lower() in desired_collections: - yield from self.import_collection(collection) - - def generate_reports(self): - """Returns a list of report dictionaries for each desired collection specified in config.yml.""" - - reports = [] - - self.create_taxii_client() - if not self.client: - logging.error('Unable to create taxii client.') - return reports - - available_collections = self.query_collections() - if not available_collections: - logging.warning('Unable to find any collections.') - return reports - - reports = self.import_collections(available_collections) - if not reports: - logging.warning('Unable to import collections.') - return reports - - return reports - - -class StixTaxii(): - """Allows for interfacing with multiple TAXII servers. - - Instantiates separate TaxiiSiteConnector objects for each site specified in config.yml. - Formats report dictionaries into AnalysisResult objects with formatted IOC_v2 attirbutes. - Sends AnalysisResult objects to ThreatIntel.push_to_cb for dispatching to a feed. - """ - - def __init__(self, site_confs): - self.config = site_confs - self.client = None - - def result(self, **kwargs): - """Returns a new AnalysisResult with the given fields populated.""" - - result = AnalysisResult(**kwargs).normalize() - return result - - def configure_sites(self): - """Creates a TaxiiSiteConnector for each site in config.yml""" - - self.sites = {} - try: - for site_name, site_conf in self.config['sites'].items(): - self.sites[site_name] = TaxiiSiteConnector(site_conf) - logging.info(f"loaded site {site_name}") - except handled_exceptions as e: - - logging.error(f"Error in parsing config file: {e}") - - def format_report(self, reports): - """Converts a dictionary into an AnalysisResult. - - Args: - reports: list of report dictionaries containing an id, title, description, timestamp, score, link, and iocs_v2. - - Yields: - An AnalysisResult for each report dictionary. - """ - - for report in reports: - try: - analysis_name = report['id'] - title = report['title'] - description = report['description'] - scan_time = datetime.fromtimestamp(report['timestamp']) - score = report['score'] - link = report['link'] - ioc_dict = report['iocs_v2'] - result = self.result( - analysis_name=analysis_name, - scan_time=scan_time, - score=score, - title=title, - description=description) - for ioc_key, ioc_val in ioc_dict.items(): - result.attach_ioc_v2(values=ioc_val, field=ioc_key, link=link) - except handled_exceptions as e: - logging.warning(f"Problem in report formatting: {e}") - result = self.result( - analysis_name="exception_format_report", error=True) - yield result - - def collect_and_send_reports(self): - """Collects and sends formatted reports to ThreatIntel.push_to_cb for validation and dispatching to a feed.""" - - self.configure_sites() - ti = ThreatIntel() - for site_name, site_conn in self.sites.items(): - logging.debug(f"Verifying Feed {site_conn.config.feed_id} exists") - try: - ti.verify_feed_exists(site_conn.config.feed_id) - except ApiError as e: - logging.error(f"Couldn't find CbTH Feed {site_conn.config.feed_id}. Skipping {site_name}: {e}") - continue - logging.info(f"Talking to {site_name} server") - reports = site_conn.generate_reports() - if not reports: - logging.error(f"No reports generated for {site_name}") - continue - else: - try: - ti.push_to_cb(feed_id=site_conn.config.feed_id, results=self.format_report(reports)) - except Exception as e: - logging.error(traceback.format_exc()) - logging.error(f"Failed to push reports to feed {site_conn.config.feed_id}: {e}") -if __name__ == '__main__': - - parser = argparse.ArgumentParser(description='Modify configuration values via command line.') - parser.add_argument('--site_start_date', metavar='s', nargs='+', - help='the site name and desired start date to begin polling from') - args = parser.parse_args() - - config = load_config_from_file() - - if args.site_start_date: - for index in range(len(args.site_start_date)): - arg = args.site_start_date[index] - if arg in config['sites']: # if we see a name that matches a site Name - try: - new_time = datetime.strptime(args.site_start_date[index+1], "%Y-%m-%d %H:%M:%S") - config['sites'][arg]['start_date'] = new_time - logging.info(f"Updated the start_date for {arg} to {new_time}") - except ValueError as e: - logging.error(f"Failed to update start_date for {arg}: {e}") - stix_taxii = StixTaxii(config) - stix_taxii.collect_and_send_reports() diff --git a/examples/DEPRECATED_threathunter/threat_intelligence/threatintel.py b/examples/DEPRECATED_threathunter/threat_intelligence/threatintel.py deleted file mode 100644 index c00672f9..00000000 --- a/examples/DEPRECATED_threathunter/threat_intelligence/threatintel.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Validates result dictionaries, creates ThreatHunter Reports, validates ThreatHunter Reports, and sends them to a ThreatHunter Feed. - -Also allows for conversion from result dictionaries into ThreatHunter `Report` objects. -""" - -import logging -import json -from cbapi.psc.threathunter import CbThreatHunterAPI, Report -from cbapi.errors import ApiError -from cbapi.psc.threathunter.models import Feed -from schemas import ReportSchema -from schema import SchemaError - -log = logging.getLogger(__name__) - - -class ThreatIntel: - def __init__(self): - self.cb = CbThreatHunterAPI(timeout=200) - - def verify_feed_exists(self, feed_id): - """Verify that a Feed exists.""" - try: - feed = self.cb.select(Feed, feed_id) - return feed - except ApiError: - raise ApiError - - def push_to_cb(self, feed_id, results): - feed = self.verify_feed_exists(feed_id) # will raise an ApiError if the feed cannot be found - if not feed: - return - report_list_to_send = [] - reports = [] - malformed_reports = [] - - for result in results: - try: - report_dict = { - "id": str(result.id), - "timestamp": int(result.timestamp.timestamp()), - "title": str(result.title), - "description": str(result.description), - "severity": int(result.severity), - "iocs_v2": [ioc_v2.as_dict() for ioc_v2 in result.iocs_v2] - } - try: - ReportSchema.validate(report_dict) - # create CB Report object - report = Report(self.cb, initial_data=report_dict, feed_id=feed_id) - report_list_to_send.append(report) - reports.append(report_dict) - except SchemaError as e: - log.warning("Report Validation failed. Saving report to file for reference.") - malformed_reports.append(report_dict) - except Exception as e: - log.error(f"Failed to create a report dictionary from result object. {e}") - - log.debug(f"Num Reports: {len(report_list_to_send)}") - try: - with open('reports.json', 'w') as f: - json.dump(reports, f, indent=4) - except Exception as e: - log.error(f"Failed to write reports to file: {e}") - - log.debug("Sending results to Carbon Black Cloud.") - - if report_list_to_send: - try: - feed.append_reports(report_list_to_send) - log.info(f"Appended {len(report_list_to_send)} reports to ThreatHunter Feed {feed_id}") - except Exception as e: - log.debug(f"Failed sending {len(report_list_to_send)} reports: {e}") - - if malformed_reports: - log.warning("Some report(s) failed validation. See malformed_reports.json for reports that failed.") - try: - with open('malformed_reports.json', 'w') as f: - json.dump(malformed_reports, f, indent=4) - except Exception as e: - log.error(f"Failed to write malformed_reports to file: {e}") - diff --git a/examples/DEPRECATED_threathunter/watchlist_operations.py b/examples/DEPRECATED_threathunter/watchlist_operations.py deleted file mode 100644 index 096690f1..00000000 --- a/examples/DEPRECATED_threathunter/watchlist_operations.py +++ /dev/null @@ -1,327 +0,0 @@ -#!/usr/bin/env python -# - -import sys -from cbapi.psc.threathunter.models import Watchlist, Report, Feed -from cbapi.example_helpers import eprint, read_iocs, build_cli_parser, get_cb_threathunter_object -from cbapi.errors import ObjectNotFoundError -import logging -import json -import time -import hashlib - -log = logging.getLogger(__name__) - - -def get_watchlist(cb, watchlist_id=None, watchlist_name=None): - if watchlist_id: - return cb.select(Watchlist, watchlist_id) - elif watchlist_name: - feeds = [feed for feed in cb.select(Watchlist) if feed.name == watchlist_name] - - if not feeds: - eprint("No watchlist named {}".format(watchlist_name)) - sys.exit(1) - elif len(feeds) > 1: - eprint("More than one feed named {}, not continuing".format(watchlist_name)) - sys.exit(1) - - return feeds[0] - else: - raise ValueError("expected either watchlist_id or watchlist_name") - - -def get_report(watchlist, report_id=None, report_name=None): - if report_id: - reports = [report for report in watchlist.reports if report.id == report_id] - elif report_name: - reports = [report for report in watchlist.reports if report.title == report_name] - else: - raise ValueError("expected either report_id or report_name") - - if not reports: - eprint("No matching reports found.") - sys.exit(1) - if len(reports) > 1: - eprint("More than one matching report found.") - sys.exit(1) - - return reports[0] - - -def get_report_feed(watchlist, report_id=None, report_name=None): - reports = watchlist.feed.reports - - if report_id: - reports = [report for report in reports if report.id == report_id] - elif report_name: - reports = [report for report in reports if report.title == report_name] - else: - raise ValueError("expected either report_id or report_name") - - if not reports: - eprint("No matching reports found.") - sys.exit(1) - if len(reports) > 1: - eprint("More than one matching report found.") - sys.exit(1) - - return reports[0] - - -def list_watchlists(cb, parser, args): - watchlists = cb.select(Watchlist) - - for watchlist in watchlists: - print(watchlist) - if args.reports: - for report in watchlist.reports: - print(report) - if watchlist.feed: - for report in watchlist.feed.reports: - print(report) - - -def subscribe_watchlist(cb, parser, args): - try: - cb.select(Feed, args.feed_id) - except ObjectNotFoundError: - eprint("Nonexistent or private feed: {}".format(args.feed_id)) - sys.exit(1) - - classifier = { - "key": "feed_id", - "value": args.feed_id, - } - - watchlist_dict = { - "name": args.watchlist_name, - "description": args.description, - "tags_enabled": args.tags, - "alerts_enabled": args.alerts, - "create_timestamp": args.timestamp, - "last_update_timestamp": args.last_update, - "report_ids": [], - "classifier": classifier, - } - - watchlist = cb.create(Watchlist, watchlist_dict) - watchlist.save() - - -def create_watchlist(cb, parser, args): - watchlist_dict = { - "name": args.watchlist_name, - "description": args.description, - "tags_enabled": args.tags, - "alerts_enabled": args.alerts, - "create_timestamp": args.timestamp, - "last_update_timestamp": args.last_update, - "report_ids": [], - "classifier": None, - } - - rep_tags = [] - if args.rep_tags: - rep_tags = args.rep_tags.split(",") - - report_dict = { - "timestamp": args.rep_timestamp, - "title": args.rep_title, - "description": args.rep_desc, - "severity": args.rep_severity, - "link": args.rep_link, - "tags": rep_tags, - "iocs_v2": [], # NOTE(ww): The feed server will convert IOCs to v2s for us. - } - - report_id, iocs = read_iocs(cb) - - report_dict["id"] = report_id - report_dict["iocs"] = iocs - - report = cb.create(Report, report_dict) - report.save_watchlist() - - watchlist_dict["report_ids"].append(report.id) - watchlist = cb.create(Watchlist, watchlist_dict) - watchlist.save() - - -def delete_watchlist(cb, parser, args): - watchlist = get_watchlist(cb, watchlist_id=args.watchlist_id, watchlist_name=args.watchlist_name) - - if args.reports: - [report.delete() for report in watchlist.reports] - - watchlist.delete() - - -def alter_report(cb, parser, args): - watchlist = get_watchlist(cb, watchlist_id=args.watchlist_id) - - if watchlist.reports: - report = get_report(watchlist, report_id=args.report_id) - else: - report = get_report_feed(watchlist, report_id=args.report_id) - - if args.severity: - if watchlist.reports: - report.update(severity=args.severity) - else: - report.custom_severity = args.severity - - if args.activate: - report.unignore() - elif args.deactivate: - report.ignore() - - -def alter_ioc(cb, parser, args): - watchlist = get_watchlist(cb, watchlist_id=args.watchlist_id) - report = get_report(watchlist, report_id=args.report_id) - - iocs = [ioc for ioc in report.iocs_ if ioc.id == args.ioc_id] - - if not iocs: - eprint("No IOC with ID {} found.".format(args.ioc_id)) - sys.exit(1) - elif len(iocs) > 1: - eprint("More than one IOC with ID {} found.".format(args.ioc_id)) - sys.exit(1) - - if args.activate: - iocs[0].unignore() - elif args.deactivate: - iocs[0].ignore() - - -def export_watchlist(cb, parser, args): - watchlist = get_watchlist(cb, watchlist_id=args.watchlist_id, watchlist_name=args.watchlist_name) - exported = { - 'watchlist': watchlist._info, - } - - exported['reports'] = [report._info for report in watchlist.reports] - - print(json.dumps(exported)) - - -def import_watchlist(cb, parser, args): - imported = json.loads(sys.stdin.read()) - - # clear any report IDs, since we'll regenerate them - imported["watchlist"]["report_ids"].clear() - - watchlist = cb.create(Watchlist, imported['watchlist']) - watchlist.save() - - # import each report and extract its new ID - report_ids = [] - for rep_dict in imported["reports"]: - - # NOTE(ww): Previous versions of the CbTH Watchlist API weren't - # generating IOC IDs on the server side. If they don't show up - # in our import, generate them manually. - for ioc in rep_dict["iocs_v2"]: - if not ioc["id"]: - ioc_id = hashlib.md5() - ioc_id.update(str(time.time()).encode("utf-8")) - [ioc_id.update(value.encode("utf-8")) for value in ioc["values"]] - ioc["id"] = ioc_id.hexdigest() - report = cb.create(Report, rep_dict) - report.save_watchlist() - report_ids.append(report.id) - - # finally, update our new watchlist with the imported reports - if report_ids: - watchlist.update(report_ids=report_ids) - - -def main(): - parser = build_cli_parser() - commands = parser.add_subparsers(help="Feed commands", dest="command_name") - - list_command = commands.add_parser("list", help="List all configured watchlists") - list_command.add_argument("-r", "--reports", action="store_true", help="List reports for each watchlist", - default=False) - - subscribe_command = commands.add_parser("subscribe", help="Create a watchlist with a feed") - subscribe_command.add_argument("-i", "--feed_id", type=str, help="The Feed ID", required=True) - subscribe_command.add_argument("-w", "--watchlist_name", type=str, help="Watchlist name", required=True) - subscribe_command.add_argument("-d", "--description", type=str, help="Watchlist description", required=True) - subscribe_command.add_argument("-t", "--tags", action="store_true", help="Enable tags", default=False) - subscribe_command.add_argument("-a", "--alerts", action="store_true", help="Enable alerts", default=False) - subscribe_command.add_argument("-T", "--timestamp", type=int, help="Creation timestamp", default=int(time.time())) - subscribe_command.add_argument("-U", "--last_update", type=int, help="Last update timestamp", - default=int(time.time())) - - create_command = commands.add_parser("create", help="Create a watchlist with a report") - create_command.add_argument("-w", "--watchlist_name", type=str, help="Watchlist name", required=True) - create_command.add_argument("-d", "--description", type=str, help="Watchlist description", required=True) - create_command.add_argument("-t", "--tags", action="store_true", help="Enable tags", default=False) - create_command.add_argument("-a", "--alerts", action="store_true", help="Enable alerts", default=False) - create_command.add_argument("-T", "--timestamp", type=int, help="Creation timestamp", default=int(time.time())) - create_command.add_argument("-U", "--last_update", type=int, help="Last update timestamp", default=int(time.time())) - # Report metadata arguments. - create_command.add_argument("--rep_timestamp", type=int, help="Report timestamp", default=int(time.time())) - create_command.add_argument("--rep_title", type=str, help="Report title", required=True) - create_command.add_argument("--rep_desc", type=str, help="Report description", required=True) - create_command.add_argument("--rep_severity", type=int, help="Report severity", default=1) - create_command.add_argument("--rep_link", type=str, help="Report link") - create_command.add_argument("--rep_tags", type=str, help="Report tags, comma separated") - create_command.add_argument("--rep_visibility", type=str, help="Report visibility") - - delete_command = commands.add_parser("delete", help="Delete a watchlist") - delete_command.add_argument("-R", "--reports", action="store_true", help="Delete all associated reports too", - default=False) - specifier = delete_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--watchlist_id", type=str, help="The watchlist ID") - specifier.add_argument("-w", "--watchlist_name", type=str, help="The watchlist name") - - alter_report_command = commands.add_parser("alter-report", help="Change the properties of a watchlist's report") - alter_report_command.add_argument("-i", "--watchlist_id", type=str, help="Watchlist ID", required=True) - alter_report_command.add_argument("-r", "--report_id", type=str, help="Report ID", required=True) - alter_report_command.add_argument("-s", "--severity", type=int, help="The report's severity", required=False) - specifier = alter_report_command.add_mutually_exclusive_group(required=False) - specifier.add_argument("-d", "--deactivate", action="store_true", help="Deactive alerts for this report") - specifier.add_argument("-a", "--activate", action="store_true", help="Activate alerts for this report") - - alter_ioc_command = commands.add_parser("alter-ioc", help="Change the properties of a watchlist's IOC") - alter_ioc_command.add_argument("-i", "--watchlist_id", type=str, help="Watchlist ID", required=True) - alter_ioc_command.add_argument("-r", "--report_id", type=str, help="Report ID", required=True) - alter_ioc_command.add_argument("-I", "--ioc_id", type=str, help="IOC ID", required=True) - specifier = alter_ioc_command.add_mutually_exclusive_group(required=False) - specifier.add_argument("-d", "--deactivate", action="store_true", help="Deactive alerts for this IOC") - specifier.add_argument("-a", "--activate", action="store_true", help="Activate alerts for this IOC") - - export_command = commands.add_parser("export", help="Export a watchlist into an importable format") - specifier = export_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--watchlist_id", type=str, help="Watchlist ID") - specifier.add_argument("-w", "--watchlist_name", type=str, help="Watchlist name") - - commands.add_parser("import", help="Import a previously exported watchlist") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if args.command_name == "list": - return list_watchlists(cb, parser, args) - elif args.command_name == "subscribe": - return subscribe_watchlist(cb, parser, args) - elif args.command_name == "create": - return create_watchlist(cb, parser, args) - elif args.command_name == "delete": - return delete_watchlist(cb, parser, args) - elif args.command_name == "alter-report": - return alter_report(cb, parser, args) - elif args.command_name == "alter-ioc": - return alter_ioc(cb, parser, args) - elif args.command_name == "export": - return export_watchlist(cb, parser, args) - elif args.command_name == "import": - return import_watchlist(cb, parser, args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/setup.py b/setup.py index e8eb37dc..50703cc8 100644 --- a/setup.py +++ b/setup.py @@ -9,11 +9,7 @@ 'cbapi', 'cbapi.protection', 'cbapi.response', - 'cbapi.cache', - 'cbapi.psc', - 'cbapi.psc.defense', - 'cbapi.psc.threathunter', - 'cbapi.psc.livequery' + 'cbapi.cache' ] install_requires = [ @@ -61,5 +57,5 @@ 'Programming Language :: Python', 'Topic :: Software Development :: Libraries :: Python Modules' ], - scripts=['bin/cbapi-response', 'bin/cbapi-protection', 'bin/cbapi-defense', 'bin/cbapi', 'bin/cbapi-psc'] + scripts=['bin/cbapi-response', 'bin/cbapi-protection', 'bin/cbapi'] ) diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index f4acafd8..163499f0 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -12,9 +12,4 @@ from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI from cbapi.protection.rest_api import CbEnterpriseProtectionAPI, CbProtectionAPI from cbapi.psc import CbPSCBaseAPI -from cbapi.psc.defense import CbDefenseAPI -from cbapi.psc.threathunter import CbThreatHunterAPI from cbapi.psc.livequery import CbLiveQueryAPI - -# for compatibility with Cb Defense code from cbapi < 1.4.0 -import cbapi.psc.defense as defense diff --git a/src/cbapi/defense.py b/src/cbapi/defense.py deleted file mode 100644 index e9a15cb6..00000000 --- a/src/cbapi/defense.py +++ /dev/null @@ -1,2 +0,0 @@ -# Compatibility with old Defense API code -from cbapi.psc.defense import * # noqa: F401, F403 diff --git a/src/cbapi/example_helpers.py b/src/cbapi/example_helpers.py index a237e35e..e0ee2731 100644 --- a/src/cbapi/example_helpers.py +++ b/src/cbapi/example_helpers.py @@ -16,8 +16,6 @@ from cbapi.protection import CbEnterpriseProtectionAPI from cbapi.psc import CbPSCBaseAPI -from cbapi.psc.defense import CbDefenseAPI -from cbapi.psc.threathunter import CbThreatHunterAPI from cbapi.psc.livequery import CbLiveQueryAPI from cbapi.response import CbEnterpriseResponseAPI @@ -94,34 +92,6 @@ def get_cb_psc_object(args): return cb -def get_cb_defense_object(args): - if args.verbose: - logging.basicConfig() - logging.getLogger("cbapi").setLevel(logging.DEBUG) - logging.getLogger("__main__").setLevel(logging.DEBUG) - - if args.cburl and args.apitoken: - cb = CbDefenseAPI(url=args.cburl, token=args.apitoken, ssl_verify=(not args.no_ssl_verify)) - else: - cb = CbDefenseAPI(profile=args.profile) - - return cb - - -def get_cb_threathunter_object(args): - if args.verbose: - logging.basicConfig() - logging.getLogger("cbapi").setLevel(logging.DEBUG) - logging.getLogger("__main__").setLevel(logging.DEBUG) - - if args.cburl and args.apitoken: - cb = CbThreatHunterAPI(url=args.cburl, token=args.apitoken, ssl_verify=(not args.no_ssl_verify)) - else: - cb = CbThreatHunterAPI(profile=args.profile) - - return cb - - def get_cb_livequery_object(args): if args.verbose: logging.basicConfig() diff --git a/src/cbapi/psc/__init__.py b/src/cbapi/psc/__init__.py index 3f3be5f0..239b1eb6 100644 --- a/src/cbapi/psc/__init__.py +++ b/src/cbapi/psc/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import from .rest_api import CbPSCBaseAPI -from .models import Device, Workflow, BaseAlert, WatchlistAlert, CBAnalyticsAlert, VMwareAlert, WorkflowStatus +from .models import Device diff --git a/src/cbapi/psc/alerts_query.py b/src/cbapi/psc/alerts_query.py deleted file mode 100755 index 6a775c6d..00000000 --- a/src/cbapi/psc/alerts_query.py +++ /dev/null @@ -1,704 +0,0 @@ -from cbapi.errors import ApiError -from .base_query import PSCQueryBase, QueryBuilder, QueryBuilderSupportMixin, IterableQueryMixin -from .devices_query import DeviceSearchQuery - - -class BaseAlertSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): - """ - Represents a query that is used to locate BaseAlert objects. - """ - VALID_CATEGORIES = ["THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", "CRITICAL"] - VALID_REPUTATIONS = ["KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", "NOT_LISTED", "ADAPTIVE_WHITE_LIST", - "COMMON_WHITE_LIST", "TRUSTED_WHITE_LIST", "COMPANY_BLACK_LIST"] - VALID_ALERT_TYPES = ["CB_ANALYTICS", "VMWARE", "WATCHLIST"] - VALID_WORKFLOW_VALS = ["OPEN", "DISMISSED"] - VALID_FACET_FIELDS = ["ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", - "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", - "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", - "POLICY_APPLIED", "SENSOR_ACTION"] - - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_builder = QueryBuilder() - self._criteria = {} - self._time_filter = {} - self._sortcriteria = {} - self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/workflow/_criteria" - self._count_valid = False - self._total_results = 0 - - def _update_criteria(self, key, newlist): - """ - Updates the criteria being collected for a query. Assumes the specified criteria item is - defined as a list; the list passed in will be set as the value for this criteria item, or - appended to the existing one if there is one. - - :param str key: The key for the criteria item to be set - :param list newlist: List of values to be set for the criteria item - """ - oldlist = self._criteria.get(key, []) - self._criteria[key] = oldlist + newlist - - def set_categories(self, categories): - """ - Restricts the alerts that this query is performed on to the specified categories. - - :param categories list: List of categories to be restricted to. Valid categories are - "THREAT", "MONITORED", "INFO", "MINOR", "SERIOUS", and "CRITICAL." - :return: This instance - """ - if not all((c in BaseAlertSearchQuery.VALID_CATEGORIES) for c in categories): - raise ApiError("One or more invalid category values") - self._update_criteria("category", categories) - return self - - def set_create_time(self, *args, **kwargs): - """ - Restricts the alerts that this query is performed on to the specified - creation time (either specified as a start and end point or as a - range). - - :return: This instance - """ - if kwargs.get("start", None) and kwargs.get("end", None): - if kwargs.get("range", None): - raise ApiError("cannot specify range= in addition to start= and end=") - stime = kwargs["start"] - if not isinstance(stime, str): - stime = stime.isoformat() - etime = kwargs["end"] - if not isinstance(etime, str): - etime = etime.isoformat() - self._time_filter = {"start": stime, "end": etime} - elif kwargs.get("range", None): - if kwargs.get("start", None) or kwargs.get("end", None): - raise ApiError("cannot specify start= or end= in addition to range=") - self._time_filter = {"range": kwargs["range"]} - else: - raise ApiError("must specify either start= and end= or range=") - return self - - def set_device_ids(self, device_ids): - """ - Restricts the alerts that this query is performed on to the specified - device IDs. - - :param device_ids list: list of integer device IDs - :return: This instance - """ - if not all(isinstance(device_id, int) for device_id in device_ids): - raise ApiError("One or more invalid device IDs") - self._update_criteria("device_id", device_ids) - return self - - def set_device_names(self, device_names): - """ - Restricts the alerts that this query is performed on to the specified - device names. - - :param device_names list: list of string device names - :return: This instance - """ - if not all(isinstance(n, str) for n in device_names): - raise ApiError("One or more invalid device names") - self._update_criteria("device_name", device_names) - return self - - def set_device_os(self, device_os): - """ - Restricts the alerts that this query is performed on to the specified - device operating systems. - - :param device_os list: List of string operating systems. Valid values are - "WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", and "OTHER." - :return: This instance - """ - if not all((osval in DeviceSearchQuery.VALID_OS) for osval in device_os): - raise ApiError("One or more invalid operating systems") - self._update_criteria("device_os", device_os) - return self - - def set_device_os_versions(self, device_os_versions): - """ - Restricts the alerts that this query is performed on to the specified - device operating system versions. - - :param device_os_versions list: List of string operating system versions. - :return: This instance - """ - if not all(isinstance(n, str) for n in device_os_versions): - raise ApiError("One or more invalid device OS versions") - self._update_criteria("device_os_version", device_os_versions) - return self - - def set_device_username(self, users): - """ - Restricts the alerts that this query is performed on to the specified - user names. - - :param users list: List of string user names. - :return: This instance - """ - if not all(isinstance(u, str) for u in users): - raise ApiError("One or more invalid user names") - self._update_criteria("device_username", users) - return self - - def set_group_results(self, do_group): - """ - Specifies whether or not to group the results of the query. - - :param do_group boolean: True to group the results, False to not do so. - :return: This instance - """ - self._criteria["group_results"] = True if do_group else False - return self - - def set_alert_ids(self, alert_ids): - """ - Restricts the alerts that this query is performed on to the specified - alert IDs. - - :param alert_ids list: List of string alert IDs. - :return: This instance - """ - if not all(isinstance(v, str) for v in alert_ids): - raise ApiError("One or more invalid alert ID values") - self._update_criteria("id", alert_ids) - return self - - def set_legacy_alert_ids(self, alert_ids): - """ - Restricts the alerts that this query is performed on to the specified - legacy alert IDs. - - :param alert_ids list: List of string legacy alert IDs. - :return: This instance - """ - if not all(isinstance(v, str) for v in alert_ids): - raise ApiError("One or more invalid alert ID values") - self._update_criteria("legacy_alert_id", alert_ids) - return self - - def set_minimum_severity(self, severity): - """ - Restricts the alerts that this query is performed on to the specified - minimum severity level. - - :param severity int: The minimum severity level for alerts. - :return: This instance - """ - self._criteria["minimum_severity"] = severity - return self - - def set_policy_ids(self, policy_ids): - """ - Restricts the alerts that this query is performed on to the specified - policy IDs. - - :param policy_ids list: list of integer policy IDs - :return: This instance - """ - if not all(isinstance(policy_id, int) for policy_id in policy_ids): - raise ApiError("One or more invalid policy IDs") - self._update_criteria("policy_id", policy_ids) - return self - - def set_policy_names(self, policy_names): - """ - Restricts the alerts that this query is performed on to the specified - policy names. - - :param policy_names list: list of string policy names - :return: This instance - """ - if not all(isinstance(n, str) for n in policy_names): - raise ApiError("One or more invalid policy names") - self._update_criteria("policy_name", policy_names) - return self - - def set_process_names(self, process_names): - """ - Restricts the alerts that this query is performed on to the specified - process names. - - :param process_names list: list of string process names - :return: This instance - """ - if not all(isinstance(n, str) for n in process_names): - raise ApiError("One or more invalid process names") - self._update_criteria("process_name", process_names) - return self - - def set_process_sha256(self, shas): - """ - Restricts the alerts that this query is performed on to the specified - process SHA-256 hash values. - - :param shas list: list of string process SHA-256 hash values - :return: This instance - """ - if not all(isinstance(n, str) for n in shas): - raise ApiError("One or more invalid SHA256 values") - self._update_criteria("process_sha256", shas) - return self - - def set_reputations(self, reps): - """ - Restricts the alerts that this query is performed on to the specified - reputation values. - - :param reps list: List of string reputation values. Valid values are - "KNOWN_MALWARE", "SUSPECT_MALWARE", "PUP", "NOT_LISTED", - "ADAPTIVE_WHITE_LIST", "COMMON_WHITE_LIST", - "TRUSTED_WHITE_LIST", and "COMPANY_BLACK_LIST". - :return: This instance - """ - if not all((r in BaseAlertSearchQuery.VALID_REPUTATIONS) for r in reps): - raise ApiError("One or more invalid reputation values") - self._update_criteria("reputation", reps) - return self - - def set_tags(self, tags): - """ - Restricts the alerts that this query is performed on to the specified - tag values. - - :param tags list: list of string tag values - :return: This instance - """ - if not all(isinstance(tag, str) for tag in tags): - raise ApiError("One or more invalid tags") - self._update_criteria("tag", tags) - return self - - def set_target_priorities(self, priorities): - """ - Restricts the alerts that this query is performed on to the specified - target priority values. - - :param priorities list: List of string target priority values. Valid values are - "LOW", "MEDIUM", "HIGH", and "MISSION_CRITICAL". - :return: This instance - """ - if not all((prio in DeviceSearchQuery.VALID_PRIORITIES) for prio in priorities): - raise ApiError("One or more invalid priority values") - self._update_criteria("target_value", priorities) - return self - - def set_threat_ids(self, threats): - """ - Restricts the alerts that this query is performed on to the specified - threat ID values. - - :param threats list: list of string threat ID values - :return: This instance - """ - if not all(isinstance(t, str) for t in threats): - raise ApiError("One or more invalid threat ID values") - self._update_criteria("threat_id", threats) - return self - - def set_types(self, alerttypes): - """ - Restricts the alerts that this query is performed on to the specified - alert type values. - - :param alerttypes list: List of string alert type values. Valid values are - "CB_ANALYTICS", "VMWARE", and "WATCHLIST". - :return: This instance - """ - if not all((t in BaseAlertSearchQuery.VALID_ALERT_TYPES) for t in alerttypes): - raise ApiError("One or more invalid alert type values") - self._update_criteria("type", alerttypes) - return self - - def set_workflows(self, workflow_vals): - """ - Restricts the alerts that this query is performed on to the specified - workflow status values. - - :param workflow_vals list: List of string alert type values. Valid values are - "OPEN" and "DISMISSED". - :return: This instance - """ - if not all((t in BaseAlertSearchQuery.VALID_WORKFLOW_VALS) for t in workflow_vals): - raise ApiError("One or more invalid workflow status values") - self._update_criteria("workflow", workflow_vals) - return self - - def _build_criteria(self): - """ - Builds the criteria object for use in a query. - - :return: The criteria object. - """ - mycrit = self._criteria - if self._time_filter: - mycrit["create_time"] = self._time_filter - return mycrit - - def sort_by(self, key, direction="ASC"): - """Sets the sorting behavior on a query's results. - - Example:: - - >>> cb.select(BaseAlert).sort_by("name") - - :param key: the key in the schema to sort by - :param direction: the sort order, either "ASC" or "DESC" - :rtype: :py:class:`BaseAlertSearchQuery` - """ - if direction not in DeviceSearchQuery.VALID_DIRECTIONS: - raise ApiError("invalid sort direction specified") - self._sortcriteria = {"field": key, "order": direction} - return self - - def _build_request(self, from_row, max_rows, add_sort=True): - """ - Creates the request body for an API call. - - :param int from_row: The row to start the query at. - :param int max_rows: The maximum number of rows to be returned. - :param boolean add_sort: If True(default), the sort criteria will be added as part of the request. - :return: A dict containing the complete request body. - """ - request = {"criteria": self._build_criteria()} - request["query"] = self._query_builder._collapse() - request["rows"] = 100 - if from_row > 0: - request["start"] = from_row - if max_rows >= 0: - request["rows"] = max_rows - if add_sort and self._sortcriteria != {}: - request["sort"] = [self._sortcriteria] - return request - - def _build_url(self, tail_end): - """ - Creates the URL to be used for an API call. - - :param str tail_end: String to be appended to the end of the generated URL. - """ - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) + tail_end - return url - - def _count(self): - """ - Returns the number of results from the run of this query. - - :return: The number of results from the run of this query. - """ - if self._count_valid: - return self._total_results - - url = self._build_url("/_search") - request = self._build_request(0, -1) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - return self._total_results - - def _perform_query(self, from_row=0, max_rows=-1): - """ - Performs the query and returns the results of the query in an iterable fashion. - - :param int from_row: The row to start the query at (default 0). - :param int max_rows: The maximum number of rows to be returned (default -1, meaning "all"). - """ - url = self._build_url("/_search") - current = from_row - numrows = 0 - still_querying = True - while still_querying: - request = self._build_request(current, max_rows) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - results = result.get("results", []) - for item in results: - yield self._doc_class(self._cb, item["id"], item) - current += 1 - numrows += 1 - - if max_rows > 0 and numrows == max_rows: - still_querying = False - break - - from_row = current - if current >= self._total_results: - still_querying = False - break - - def facets(self, fieldlist, max_rows=0): - """ - Return information about the facets for this alert by search, using the defined criteria. - - :param fieldlist list: List of facet field names. Valid names are - "ALERT_TYPE", "CATEGORY", "REPUTATION", "WORKFLOW", "TAG", "POLICY_ID", - "POLICY_NAME", "DEVICE_ID", "DEVICE_NAME", "APPLICATION_HASH", - "APPLICATION_NAME", "STATUS", "RUN_STATE", "POLICY_APPLIED_STATE", - "POLICY_APPLIED", and "SENSOR_ACTION". - :param max_rows int: The maximum number of rows to return. 0 means return all rows. - :return: A list of facet information specified as dicts. - """ - if not all((field in BaseAlertSearchQuery.VALID_FACET_FIELDS) for field in fieldlist): - raise ApiError("One or more invalid term field names") - request = self._build_request(0, -1, False) - request["terms"] = {"fields": fieldlist, "rows": max_rows} - url = self._build_url("/_facet") - resp = self._cb.post_object(url, body=request) - result = resp.json() - return result.get("results", []) - - def _update_status(self, status, remediation, comment): - """ - Updates the status of all alerts matching the given query. - - :param str state: The status to put the alerts into, either "OPEN" or "DISMISSED". - :param remediation str: The remediation state to set for all alerts. - :param comment str: The comment to set for all alerts. - :return: The request ID, which may be used to select a WorkflowStatus object. - """ - request = {"state": status, "criteria": self._build_criteria(), "query": self._query_builder._collapse()} - if remediation is not None: - request["remediation_state"] = remediation - if comment is not None: - request["comment"] = comment - resp = self._cb.post_object(self._bulkupdate_url.format(self._cb.credentials.org_key), body=request) - output = resp.json() - return output["request_id"] - - def update(self, remediation=None, comment=None): - """ - Update all alerts matching the given query. The alerts will be left in an OPEN state after this request. - - :param remediation str: The remediation state to set for all alerts. - :param comment str: The comment to set for all alerts. - :return: The request ID, which may be used to select a WorkflowStatus object. - """ - return self._update_status("OPEN", remediation, comment) - - def dismiss(self, remediation=None, comment=None): - """ - Dismiss all alerts matching the given query. The alerts will be left in a DISMISSED state after this request. - - :param remediation str: The remediation state to set for all alerts. - :param comment str: The comment to set for all alerts. - :return: The request ID, which may be used to select a WorkflowStatus object. - """ - return self._update_status("DISMISSED", remediation, comment) - - -class WatchlistAlertSearchQuery(BaseAlertSearchQuery): - """ - Represents a query that is used to locate WatchlistAlert objects. - """ - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/watchlist/workflow/_criteria" - - def set_watchlist_ids(self, ids): - """ - Restricts the alerts that this query is performed on to the specified - watchlist ID values. - - :param ids list: list of string watchlist ID values - :return: This instance - """ - if not all(isinstance(t, str) for t in ids): - raise ApiError("One or more invalid watchlist IDs") - self._update_criteria("watchlist_id", ids) - return self - - def set_watchlist_names(self, names): - """ - Restricts the alerts that this query is performed on to the specified - watchlist name values. - - :param names list: list of string watchlist name values - :return: This instance - """ - if not all(isinstance(name, str) for name in names): - raise ApiError("One or more invalid watchlist names") - self._update_criteria("watchlist_name", names) - return self - - -class CBAnalyticsAlertSearchQuery(BaseAlertSearchQuery): - """ - Represents a query that is used to locate CBAnalyticsAlert objects. - """ - VALID_THREAT_CATEGORIES = ["UNKNOWN", "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", "RISKY_PROGRAM"] - VALID_LOCATIONS = ["ONSITE", "OFFSITE", "UNKNOWN"] - VALID_KILL_CHAIN_STATUSES = ["RECONNAISSANCE", "WEAPONIZE", "DELIVER_EXPLOIT", "INSTALL_RUN", - "COMMAND_AND_CONTROL", "EXECUTE_GOAL", "BREACH"] - VALID_POLICY_APPLIED = ["APPLIED", "NOT_APPLIED"] - VALID_RUN_STATES = ["DID_NOT_RUN", "RAN", "UNKNOWN"] - VALID_SENSOR_ACTIONS = ["POLICY_NOT_APPLIED", "ALLOW", "ALLOW_AND_LOG", "TERMINATE", "DENY"] - VALID_THREAT_CAUSE_VECTORS = ["EMAIL", "WEB", "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", - "REMOVABLE_MEDIA", "UNKNOWN", "APP_STORE", "THIRD_PARTY"] - - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/cbanalytics/workflow/_criteria" - - def set_blocked_threat_categories(self, categories): - """ - Restricts the alerts that this query is performed on to the specified - threat categories that were blocked. - - :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", - "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". - :return: This instance. - """ - if not all((category in CBAnalyticsAlertSearchQuery.VALID_THREAT_CATEGORIES) - for category in categories): - raise ApiError("One or more invalid threat categories") - self._update_criteria("blocked_threat_category", categories) - return self - - def set_device_locations(self, locations): - """ - Restricts the alerts that this query is performed on to the specified - device locations. - - :param locations list: List of device locations to look for. Valid values are "ONSITE", "OFFSITE", - and "UNKNOWN". - :return: This instance. - """ - if not all((location in CBAnalyticsAlertSearchQuery.VALID_LOCATIONS) - for location in locations): - raise ApiError("One or more invalid device locations") - self._update_criteria("device_location", locations) - return self - - def set_kill_chain_statuses(self, statuses): - """ - Restricts the alerts that this query is performed on to the specified - kill chain statuses. - - :param statuses list: List of kill chain statuses to look for. Valid values are "RECONNAISSANCE", - "WEAPONIZE", "DELIVER_EXPLOIT", "INSTALL_RUN","COMMAND_AND_CONTROL", - "EXECUTE_GOAL", and "BREACH". - :return: This instance. - """ - if not all((status in CBAnalyticsAlertSearchQuery.VALID_KILL_CHAIN_STATUSES) - for status in statuses): - raise ApiError("One or more invalid kill chain status values") - self._update_criteria("kill_chain_status", statuses) - return self - - def set_not_blocked_threat_categories(self, categories): - """ - Restricts the alerts that this query is performed on to the specified - threat categories that were NOT blocked. - - :param categories list: List of threat categories to look for. Valid values are "UNKNOWN", - "NON_MALWARE", "NEW_MALWARE", "KNOWN_MALWARE", and "RISKY_PROGRAM". - :return: This instance. - """ - if not all((category in CBAnalyticsAlertSearchQuery.VALID_THREAT_CATEGORIES) - for category in categories): - raise ApiError("One or more invalid threat categories") - self._update_criteria("not_blocked_threat_category", categories) - return self - - def set_policy_applied(self, applied_statuses): - """ - Restricts the alerts that this query is performed on to the specified - status values showing whether policies were applied. - - :param applied_statuses list: List of status values to look for. Valid values are - "APPLIED" and "NOT_APPLIED". - :return: This instance. - """ - if not all((s in CBAnalyticsAlertSearchQuery.VALID_POLICY_APPLIED) - for s in applied_statuses): - raise ApiError("One or more invalid policy-applied values") - self._update_criteria("policy_applied", applied_statuses) - return self - - def set_reason_code(self, reason): - """ - Restricts the alerts that this query is performed on to the specified - reason codes (enum values). - - :param reason list: List of string reason codes to look for. - :return: This instance. - """ - if not all(isinstance(t, str) for t in reason): - raise ApiError("One or more invalid reason code values") - self._update_criteria("reason_code", reason) - return self - - def set_run_states(self, states): - """ - Restricts the alerts that this query is performed on to the specified run states. - - :param states list: List of run states to look for. Valid values are "DID_NOT_RUN", "RAN", - and "UNKNOWN". - :return: This instance. - """ - if not all((s in CBAnalyticsAlertSearchQuery.VALID_RUN_STATES) - for s in states): - raise ApiError("One or more invalid run states") - self._update_criteria("run_state", states) - return self - - def set_sensor_actions(self, actions): - """ - Restricts the alerts that this query is performed on to the specified sensor actions. - - :param actions list: List of sensor actions to look for. Valid values are "POLICY_NOT_APPLIED", - "ALLOW", "ALLOW_AND_LOG", "TERMINATE", and "DENY". - :return: This instance. - """ - if not all((action in CBAnalyticsAlertSearchQuery.VALID_SENSOR_ACTIONS) - for action in actions): - raise ApiError("One or more invalid sensor actions") - self._update_criteria("sensor_action", actions) - return self - - def set_threat_cause_vectors(self, vectors): - """ - Restricts the alerts that this query is performed on to the specified threat cause vectors. - - :param vectors list: List of threat cause vectors to look for. Valid values are "EMAIL", "WEB", - "GENERIC_SERVER", "GENERIC_CLIENT", "REMOTE_DRIVE", "REMOVABLE_MEDIA", - "UNKNOWN", "APP_STORE", and "THIRD_PARTY". - :return: This instance. - """ - if not all((vector in CBAnalyticsAlertSearchQuery.VALID_THREAT_CAUSE_VECTORS) - for vector in vectors): - raise ApiError("One or more invalid threat cause vectors") - self._update_criteria("threat_cause_vector", vectors) - return self - - -class VMwareAlertSearchQuery(BaseAlertSearchQuery): - """ - Represents a query that is used to locate VMwareAlert objects. - """ - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._bulkupdate_url = "/appservices/v6/orgs/{0}/alerts/vmware/workflow/_criteria" - - def set_group_ids(self, groupids): - """ - Restricts the alerts that this query is performed on to the specified - AppDefense-assigned alarm group IDs. - - :param groupids list: List of (integer) AppDefense-assigned alarm group IDs. - :return: This instance. - """ - if not all(isinstance(groupid, int) for groupid in groupids): - raise ApiError("One or more invalid alarm group IDs") - self._update_criteria("group_id", groupids) - return self diff --git a/src/cbapi/psc/cblr.py b/src/cbapi/psc/cblr.py deleted file mode 100644 index 380450f0..00000000 --- a/src/cbapi/psc/cblr.py +++ /dev/null @@ -1,250 +0,0 @@ -import logging -import threading -from cbapi.six.moves.queue import Queue -from collections import defaultdict -from concurrent.futures import _base - -from cbapi.errors import TimeoutError -from cbapi.live_response_api import CbLRManagerBase, CbLRSessionBase, poll_status - - -OS_LIVE_RESPONSE_ENUM = { - "WINDOWS": 1, - "LINUX": 2, - "MAC": 4 -} - -log = logging.getLogger(__name__) - - -class LiveResponseSession(CbLRSessionBase): - def __init__(self, cblr_manager, session_id, sensor_id, session_data=None): - super(LiveResponseSession, self).__init__(cblr_manager, session_id, sensor_id, session_data=session_data) - from cbapi.psc.defense.models import Device - device_info = self._cb.select(Device, self.sensor_id) - self.os_type = OS_LIVE_RESPONSE_ENUM.get(device_info.deviceType, None) - - -class WorkItem(object): - def __init__(self, fn, sensor_id): - from cbapi.psc.defense.models import Device - self.fn = fn - if isinstance(sensor_id, Device): - self.sensor_id = sensor_id.deviceId - else: - self.sensor_id = int(sensor_id) - - self.future = _base.Future() - - -class CompletionNotification(object): - def __init__(self, sensor_id): - self.sensor_id = sensor_id - - -class WorkerStatus(object): - def __init__(self, sensor_id, status="ready", exception=None): - self.sensor_id = sensor_id - self.status = status - self.exception = exception - - -class JobWorker(threading.Thread): - def __init__(self, cb, sensor_id, result_queue): - super(JobWorker, self).__init__() - self.cb = cb - self.sensor_id = sensor_id - self.job_queue = Queue() - self.lr_session = None - self.result_queue = result_queue - - def run(self): - try: - self.lr_session = self.cb.live_response.request_session(self.sensor_id) - self.result_queue.put(WorkerStatus(self.sensor_id, status="ready")) - - while True: - work_item = self.job_queue.get(block=True) - if not work_item: - self.job_queue.task_done() - return - - self.run_job(work_item) - self.result_queue.put(CompletionNotification(self.sensor_id)) - self.job_queue.task_done() - except Exception as e: - self.result_queue.put(WorkerStatus(self.sensor_id, status="error", exception=e)) - finally: - if self.lr_session: - self.lr_session.close() - self.result_queue.put(WorkerStatus(self.sensor_id, status="exiting")) - - def run_job(self, work_item): - try: - work_item.future.set_result(work_item.fn(self.lr_session)) - except Exception as e: - work_item.future.set_exception(e) - - -class LiveResponseSessionManager(CbLRManagerBase): - cblr_base = "/integrationServices/v3/cblr" - cblr_session_cls = LiveResponseSession - - def submit_job(self, job, sensor): - if self._job_scheduler is None: - # spawn the scheduler thread - self._job_scheduler = LiveResponseJobScheduler(self._cb) - self._job_scheduler.start() - - work_item = WorkItem(job, sensor) - self._job_scheduler.submit_job(work_item) - return work_item.future - - def _get_or_create_session(self, sensor_id): - session_id = self._create_session(sensor_id) - - try: - res = poll_status(self._cb, "{cblr_base}/session/{0}".format(session_id, cblr_base=self.cblr_base), - desired_status="ACTIVE", delay=1, timeout=360) - except Exception: - # "close" the session, otherwise it will stay in a pending state - self._close_session(session_id) - - # the Cb server will return a 404 if we don't establish a session in time, so convert this to a "timeout" - raise TimeoutError(uri="{cblr_base}/session/{0}".format(session_id, cblr_base=self.cblr_base), - message="Could not establish session with sensor {0}".format(sensor_id), - error_code=404) - else: - return session_id, res - - def _close_session(self, session_id): - try: - self._cb.put_object("{cblr_base}/session".format(session_id, cblr_base=self.cblr_base), - {"session_id": session_id, "status": "CLOSE"}) - except Exception: - pass - - def _create_session(self, sensor_id): - response = self._cb.post_object("{cblr_base}/session/{0}".format(sensor_id, cblr_base=self.cblr_base), - {"sensor_id": sensor_id}).json() - session_id = response["id"] - return session_id - - -class LiveResponseJobScheduler(threading.Thread): - daemon = True - - def __init__(self, cb, max_workers=10): - super(LiveResponseJobScheduler, self).__init__() - self._cb = cb - self._job_workers = {} - self._idle_workers = set() - self._unscheduled_jobs = defaultdict(list) - self._max_workers = max_workers - self.schedule_queue = Queue() - - def run(self): - log.debug("Starting Live Response Job Scheduler") - - while True: - log.debug("Waiting for item on Scheduler Queue") - item = self.schedule_queue.get(block=True) - log.debug("Got item: {0}".format(item)) - if isinstance(item, WorkItem): - # new WorkItem available - self._unscheduled_jobs[item.sensor_id].append(item) - elif isinstance(item, CompletionNotification): - # job completed - self._idle_workers.add(item.sensor_id) - elif isinstance(item, WorkerStatus): - if item.status == "error": - log.error("Error encountered by JobWorker[{0}]: {1}".format(item.sensor_id, - item.exception)) - elif item.status == "exiting": - log.debug("JobWorker[{0}] has exited, waiting...".format(item.sensor_id)) - self._job_workers[item.sensor_id].join() - log.debug("JobWorker[{0}] deleted".format(item.sensor_id)) - del self._job_workers[item.sensor_id] - try: - self._idle_workers.remove(item.sensor_id) - except KeyError: - pass - elif item.status == "ready": - log.debug("JobWorker[{0}] now ready to accept jobs, session established".format(item.sensor_id)) - self._idle_workers.add(item.sensor_id) - else: - log.debug("Unknown status from JobWorker[{0}]: {1}".format(item.sensor_id, item.status)) - else: - log.debug("Received unknown item on the scheduler Queue, exiting") - # exiting the scheduler if we get None - # TODO: wait for all worker threads to exit - return - - self._schedule_jobs() - - def _schedule_jobs(self): - log.debug("Entering scheduler") - - # First, see if there are new jobs to schedule on idle workers. - self._schedule_existing_workers() - - # If we have jobs scheduled to run on sensors with no current associated worker, let's spawn new ones. - if set(self._unscheduled_jobs.keys()) - self._idle_workers: - self._cleanup_idle_workers() - self._spawn_new_workers() - self._schedule_existing_workers() - - def _cleanup_idle_workers(self, max=None): - if not max: - max = self._max_workers - - for sensor in list(self._idle_workers)[:max]: - log.debug("asking worker for sensor id {0} to exit".format(sensor)) - self._job_workers[sensor].job_queue.put(None) - - def _schedule_existing_workers(self): - log.debug("There are idle workers for sensor ids {0}".format(self._idle_workers)) - - intersection = self._idle_workers.intersection(set(self._unscheduled_jobs.keys())) - - log.debug("{0} jobs ready to execute in existing execution slots".format(len(intersection))) - - for sensor in intersection: - item = self._unscheduled_jobs[sensor].pop(0) - self._job_workers[sensor].job_queue.put(item) - self._idle_workers.remove(item.sensor_id) - - self._cleanup_unscheduled_jobs() - - def _cleanup_unscheduled_jobs(self): - marked_for_deletion = [] - for k in self._unscheduled_jobs.keys(): - if len(self._unscheduled_jobs[k]) == 0: - marked_for_deletion.append(k) - - for k in marked_for_deletion: - del self._unscheduled_jobs[k] - - def submit_job(self, work_item): - self.schedule_queue.put(work_item) - - def _spawn_new_workers(self): - from cbapi.psc.defense.models import Device - if len(self._job_workers) >= self._max_workers: - return - - schedule_max = self._max_workers - len(self._job_workers) - ''' - sensors = [s for s in self._cb.select(Device) if s.deviceId in self._unscheduled_jobs - and s.deviceId not in self._job_workers - and "AVAILABLE" in s.sensorStates] - ''' - log.debug("spawning new workers to handle unscheduled jobs: {0}".format(self._unscheduled_jobs)) - sensors = [s for s in self._cb.select(Device) if s.deviceId in self._unscheduled_jobs - and s.deviceId not in self._job_workers] - sensors_to_schedule = sensors[:schedule_max] - log.debug("Spawning new workers to handle these sensors: {0}".format(sensors_to_schedule)) - for sensor in sensors_to_schedule: - log.debug("Spawning new JobWorker for sensor id {0}".format(sensor.deviceId)) - self._job_workers[sensor.deviceId] = JobWorker(self._cb, sensor.deviceId, self.schedule_queue) - self._job_workers[sensor.deviceId].start() diff --git a/src/cbapi/psc/defense/__init__.py b/src/cbapi/psc/defense/__init__.py deleted file mode 100644 index bc37a1d5..00000000 --- a/src/cbapi/psc/defense/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Exported public API for the Cb Defense API - -from __future__ import absolute_import - -from .rest_api import CbDefenseAPI -from cbapi.psc.defense.models import Device, Event, Policy diff --git a/src/cbapi/psc/defense/models.py b/src/cbapi/psc/defense/models.py deleted file mode 100644 index 0b02d1b9..00000000 --- a/src/cbapi/psc/defense/models.py +++ /dev/null @@ -1,164 +0,0 @@ -from cbapi.models import MutableBaseModel, CreatableModelMixin, NewBaseModel - -from copy import deepcopy -import logging -import json - -from cbapi.errors import ServerError - -log = logging.getLogger(__name__) - - -class DefenseMutableModel(MutableBaseModel): - _change_object_http_method = "PATCH" - _change_object_key_name = None - - def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=False): - super(DefenseMutableModel, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=force_init, full_doc=full_doc) - if not self._change_object_key_name: - self._change_object_key_name = self.primary_key - - def _parse(self, obj): - if type(obj) == dict and self.info_key in obj: - return obj[self.info_key] - - def _update_object(self): - if self._change_object_http_method != "PATCH": - return self._update_entire_object() - else: - return self._patch_object() - - def _update_entire_object(self): - if self.__class__.primary_key in self._dirty_attributes.keys() or self._model_unique_id is None: - new_object_info = deepcopy(self._info) - try: - if not self._new_object_needs_primary_key: - del(new_object_info[self.__class__.primary_key]) - except Exception: - pass - log.debug("Creating a new {0:s} object".format(self.__class__.__name__)) - ret = self._cb.api_json_request(self.__class__._new_object_http_method, self.urlobject, - data={self.info_key: new_object_info}) - else: - log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) - ret = self._cb.api_json_request(self.__class__._change_object_http_method, - self._build_api_request_uri(), data={self.info_key: self._info}) - - return self._refresh_if_needed(ret) - - def _patch_object(self): - if self.__class__.primary_key in self._dirty_attributes.keys() or self._model_unique_id is None: - log.debug("Creating a new {0:s} object".format(self.__class__.__name__)) - ret = self._cb.api_json_request(self.__class__._new_object_http_method, self.urlobject, - data=self._info) - else: - updates = {} - for k in self._dirty_attributes.keys(): - updates[k] = self._info[k] - log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) - ret = self._cb.api_json_request(self.__class__._change_object_http_method, - self._build_api_request_uri(), data=updates) - - return self._refresh_if_needed(ret) - - def _refresh_if_needed(self, request_ret): - refresh_required = True - - if request_ret.status_code not in range(200, 300): - try: - message = json.loads(request_ret.text)[0] - except Exception: - message = request_ret.text - - raise ServerError(request_ret.status_code, message, - result="Did not update {} record.".format(self.__class__.__name__)) - else: - try: - message = request_ret.json() - log.debug("Received response: %s" % message) - if not isinstance(message, dict): - raise ServerError(request_ret.status_code, message, - result="Unknown error updating {0:s} record.".format(self.__class__.__name__)) - else: - if message.get("success", False): - if isinstance(message.get(self.info_key, None), dict): - self._info = message.get(self.info_key) - self._full_init = True - refresh_required = False - else: - if self._change_object_key_name in message.keys(): - # if all we got back was an ID, try refreshing to get the entire record. - log.debug("Only received an ID back from the server, forcing a refresh") - self._info[self.primary_key] = message[self._change_object_key_name] - refresh_required = True - else: - # "success" is False - raise ServerError(request_ret.status_code, message.get("message", ""), - result="Did not update {0:s} record.".format(self.__class__.__name__)) - except Exception: - pass - - self._dirty_attributes = {} - if refresh_required: - self.refresh() - return self._model_unique_id - - -class Device(DefenseMutableModel): - urlobject = "/integrationServices/v3/device" - primary_key = "deviceId" - info_key = "deviceInfo" - swagger_meta_file = "psc/defense/models/deviceInfo.yaml" - - def __init__(self, cb, model_unique_id, initial_data=None): - super(Device, self).__init__(cb, model_unique_id, initial_data) - - def lr_session(self): - """ - Retrieve a Live Response session object for this Device. - - :return: Live Response session object - :rtype: :py:class:`cbapi.defense.cblr.LiveResponseSession` - :raises ApiError: if there is an error establishing a Live Response session for this Device - - """ - return self._cb._request_lr_session(self._model_unique_id) - - -class Event(NewBaseModel): - urlobject = "/integrationServices/v3/event" - primary_key = "eventId" - info_key = "eventInfo" - - def _parse(self, obj): - if type(obj) == dict and self.info_key in obj: - return obj[self.info_key] - - def __init__(self, cb, model_unique_id, initial_data=None): - super(Event, self).__init__(cb, model_unique_id, initial_data) - - -class Policy(DefenseMutableModel, CreatableModelMixin): - urlobject = "/integrationServices/v3/policy" - info_key = "policyInfo" - swagger_meta_file = "psc/defense/models/policyInfo.yaml" - _change_object_http_method = "PUT" - _change_object_key_name = "policyId" - - @property - def rules(self): - return dict([(r.get("id"), r) for r in self.policy.get("rules", [])]) - - def add_rule(self, new_rule): - self._cb.post_object("{0}/rule".format(self._build_api_request_uri()), {"ruleInfo": new_rule}) - self.refresh() - - def delete_rule(self, rule_id): - self._cb.delete_object("{0}/rule/{1}".format(self._build_api_request_uri(), rule_id)) - self.refresh() - - def replace_rule(self, rule_id, new_rule): - self._cb.put_object("{0}/rule/{1}".format(self._build_api_request_uri(), rule_id), - {"ruleInfo": new_rule}) - self.refresh() diff --git a/src/cbapi/psc/defense/models/deviceInfo.yaml b/src/cbapi/psc/defense/models/deviceInfo.yaml deleted file mode 100644 index 7d7106a7..00000000 --- a/src/cbapi/psc/defense/models/deviceInfo.yaml +++ /dev/null @@ -1,221 +0,0 @@ -type: object -properties: - osVersion: - type: string - activationCode: - type: string - organizationId: - type: integer - format: int64 - deviceId: - type: integer - format: int64 - deviceSessionId: - type: integer - format: int64 - deviceOwnerId: - type: integer - format: int64 - deviceGuid: - type: string - format: uuid - email: - type: string - format: email - assignedToId: - type: integer - format: int64 - assignedToName: - type: string - deviceType: - type: string - x-nullable: true - enum: - - "MAC" - - "WINDOWS" - firstName: - type: string - lastName: - type: string - middleName: - type: string - createTime: - type: integer - format: epoch-ms-date-time - policyId: - type: integer - format: int64 - policyName: - type: string - quarantined: - type: boolean - targetPriorityType: - type: string - x-nullable: true - enum: - - "HIGH" - - "LOW" - - "MEDIUM" - - "MISSION_CRITICAL" - lastVirusActivityTime: - type: integer - format: epoch-ms-date-time - firstVirusActivityTime: - type: integer - format: epoch-ms-date-time - activationCodeExpiryTime: - type: integer - format: epoch-ms-date-time - organizationName: - type: string - sensorVersion: - type: string - registeredTime: - type: integer - format: epoch-ms-date-time - lastContact: - type: integer - format: epoch-ms-date-time - lastReportedTime: - type: integer - format: epoch-ms-date-time - windowsPlatform: - type: string - x-nullable: true - enum: - - "CLIENT_X64" - - "CLIENT_X86" - - "SERVER_X6" - - "SERVER_X86" - vdiBaseDevice: - type: integer - format: int64 - avStatus: - type: array - items: - type: string - x-nullable: true - enum: - - "AV_ACTIVE" - - "AV_BYPASS" - - "AV_DEREGISTERED" - - "AV_NOT_REGISTERED" - - "AV_REGISTERED" - - "FULLY_DISABLED" - - "FULLY_ENABLED" - - "INSTALLED" - - "INSTALLED_SERVER" - - "NOT_INSTALLED" - - "ONACCESS_SCAN_DISABLED" - - "ONDEMAND_SCAN_DISABLED" - - "ONDEMOND_SCAN_DISABLED" - - "PRODUCT_UPDATE_DISABLED" - - "SIGNATURE_UPDATE_DISABLED" - - "UNINSTALLED" - - "UNINSTALLED_SERVER" - deregisteredTime: - type: integer - format: epoch-ms-date-time - sensorStates: - type: array - items: - type: string - x-nullable: true - enum: - - "ACTIVE" - - "CSR_ACTION" - - "DB_CORRUPTION_DETECTED" - - "DRIVER_INIT_ERROR" - - "LOOP_DETECTED" - - "PANICS_DETECTED" - - "REMGR_INIT_ERROR" - - "REPUX_ACTION" - - "SENSOR_MAINTENANCE" - - "SENSOR_RESET_IN_PROGRESS" - - "SENSOR_SHUTDOWN" - - "SENSOR_UNREGISTERED" - - "SENSOR_UPGRADE_IN_PROGRESS" - - "UNSUPPORTED_OS" - - "WATCHDOG" - messages: - type: array - items: - type: object - properties: - message: - type: string - time: - type: integer - format: epoch-ms-date-time - rootedBySensor: - type: boolean - rootedBySensorTime: - type: integer - format: epoch-ms-date-time - lastInternalIpAddress: - type: string - lastExternalIpAddress: - type: string - lastLocation: - type: string - x-nullable: true - enum: - - "OFFSITE" - - "ONSITE" - - "UNKNOWN" - avUpdateServers: - type: array - items: - type: string - passiveMode: - type: boolean - lastResetTime: - type: integer - format: epoch-ms-date-time - lastShutdownTime: - type: integer - format: epoch-ms-date-time - scanStatus: - type: string - scanLastActionTime: - type: integer - format: epoch-ms-date-time - scanLastCompleteTime: - type: integer - format: epoch-ms-date-time - linuxKernelVersion: - type: string - avEngine: - type: string - avLastScanTime: - type: integer - format: epoch-ms-date-time - rootedByAnalytics: - type: boolean - rootedByAnalyticsTime: - type: integer - format: epoch-ms-date-time - testId: - type: integer - avMaster: - type: boolean - uninstalledTime: - type: integer - format: epoch-ms-date-time - name: - type: string - status: - type: string - x-nullable: true - enum: - - "ACTIVE" - - "ALL" - - "BYPASS" - - "BYPASS_ON" - - "DEREGISTERED" - - "ERROR" - - "INACTIVE" - - "PENDING" - - "QUARANTINE" - - "REGISTERED" - - "UNINSTALLED" diff --git a/src/cbapi/psc/defense/models/policyInfo.yaml b/src/cbapi/psc/defense/models/policyInfo.yaml deleted file mode 100644 index b13dc207..00000000 --- a/src/cbapi/psc/defense/models/policyInfo.yaml +++ /dev/null @@ -1,25 +0,0 @@ -type: object -required: -- description -- name -- policy -- priorityLevel -- version -properties: - description: - type: string - id: - type: integer - latestRevision: - type: integer - name: - type: string - policy: - type: object - priorityLevel: - type: string - systemPolicy: - type: boolean - version: - type: integer - diff --git a/src/cbapi/psc/defense/rest_api.py b/src/cbapi/psc/defense/rest_api.py deleted file mode 100644 index 3e719473..00000000 --- a/src/cbapi/psc/defense/rest_api.py +++ /dev/null @@ -1,194 +0,0 @@ -from cbapi.utils import convert_query_params -from cbapi.query import PaginatedQuery - -from cbapi.psc.rest_api import CbPSCBaseAPI -import logging -import time - -log = logging.getLogger(__name__) - - -def convert_to_kv_pairs(q): - k, v = q.split(':', 1) - return k, v - - -class CbDefenseAPI(CbPSCBaseAPI): - """THIS SDK IS DEPRECATED FOR CARBON BLACK CLOUD - - Please see - `Carbon Black Cloud Python SDK on the Developer Network `_ - for details on the replacement Carbon Black Cloud Python SDK. - - The main entry point into the Carbon Black Cloud Endpoint Standard Defense API. - - :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. - Uses the profile named 'default' when not specified. - - Usage:: - - >>> from cbapi import CbDefenseAPI - >>> cb = CbDefenseAPI(profile="production") - """ - - def __init__(self, *args, **kwargs): - super(CbDefenseAPI, self).__init__(*args, **kwargs) - - def _perform_query(self, cls, query_string=None): - return Query(cls, self, query_string) - - def notification_listener(self, interval=60): - """Generator to continually poll the Cloud Endpoint Standard server for notifications (alerts). Note that - this can only be used with a 'SIEM' key generated in the Carbon Black Cloud console. - """ - while True: - for notification in self.get_notifications(): - yield notification - time.sleep(interval) - - def get_notifications(self): - """DEPRECATED: Retrieve queued notifications (alerts) from the Cloud Endpoint Standard server. Note that this can only be - used with a 'SIEM' key generated in the Carbon Black Cloud console. - - :returns: list of dictionary objects representing the notifications, or an empty list if none available. - """ - res = self.get_object("/integrationServices/v3/notification") - return res.get("notifications", []) - - def get_auditlogs(self): - """DEPRECATED: Retrieve queued audit logs from the Carbon Black Cloud Endpoint Standard server. - Note that this can only be used with a 'API' key generated in the CBC console. - :returns: list of dictionary objects representing the audit logs, or an empty list if none available. - """ - res = self.get_object("/integrationServices/v3/auditlogs") - return res.get("notifications", []) - - -class Query(PaginatedQuery): - """Represents a prepared query to the Cloud Endpoint Standard server. - - This object is returned as part of a :py:meth:`CbDefenseAPI.select` - operation on models requested from the Cloud Endpoint Standard server. You should not have to create - this class yourself. - - The query is not executed on the server until it's accessed, either as an iterator (where it will generate values - on demand as they're requested) or as a list (where it will retrieve the entire result set and save to a list). - You can also call the Python built-in ``len()`` on this object to retrieve the total number of items matching - the query. - - Examples:: - - >>> from cbapi.psc.defense import CbDefenseAPI - >>> cb = CbDefenseAPI() - - Notes: - - The slicing operator only supports start and end parameters, but not step. ``[1:-1]`` is legal, but - ``[1:2:-1]`` is not. - - You can chain where clauses together to create AND queries; only objects that match all ``where`` clauses - will be returned. - """ - - def __init__(self, doc_class, cb, query=None): - super(Query, self).__init__(doc_class, cb, None) - if query: - self._query = [query] - else: - self._query = [] - - self._sort_by = None - self._group_by = None - self._batch_size = 100 - - def _clone(self): - nq = self.__class__(self._doc_class, self._cb) - nq._query = self._query[::] - nq._sort_by = self._sort_by - nq._group_by = self._group_by - nq._batch_size = self._batch_size - return nq - - def where(self, q): - """Add a filter to this query. - - :param str q: Query string - :return: Query object - :rtype: :py:class:`Query` - """ - nq = self._clone() - nq._query.append(q) - return nq - - def and_(self, q): - """Add a filter to this query. Equivalent to calling :py:meth:`where` on this object. - - :param str q: Query string - :return: Query object - :rtype: :py:class:`Query` - """ - return self.where(q) - - def prepare_query(self, args): - if self._query: - for qe in self._query: - k, v = convert_to_kv_pairs(qe) - args[k] = v - - return args - - def _count(self): - args = {'limit': 0} - args = self.prepare_query(args) - - query_args = convert_query_params(args) - self._total_results = int(self._cb.get_object(self._doc_class.urlobject, query_parameters=query_args) - .get("totalResults", 0)) - self._count_valid = True - return self._total_results - - def _search(self, start=0, rows=0): - # iterate over total result set, 1000 at a time - args = {} - if start != 0: - args['start'] = start - args['rows'] = self._batch_size - - current = start - numrows = 0 - - args = self.prepare_query(args) - still_querying = True - - while still_querying: - query_args = convert_query_params(args) - result = self._cb.get_object(self._doc_class.urlobject, query_parameters=query_args) - - self._total_results = result.get("totalResults", 0) - self._count_valid = True - - results = result.get('results', []) - - if results is None: - log.debug("Results are None") - if current >= 100000: - log.info("Max result size exceeded. Truncated to 100k.") - break - - for item in results: - yield item - current += 1 - numrows += 1 - if rows and numrows == rows: - still_querying = False - break - - args['start'] = current + 1 # as of 6/2017, the indexing on the Cb Defense backend is still 1-based - - if current >= self._total_results: - break - - if not results: - log.debug("server reported total_results overestimated the number of results for this query by {0}" - .format(self._total_results - current)) - log.debug("resetting total_results for this query to {0}".format(current)) - self._total_results = current - break diff --git a/src/cbapi/psc/devices_query.py b/src/cbapi/psc/devices_query.py index 6ddc431e..5a5e185d 100755 --- a/src/cbapi/psc/devices_query.py +++ b/src/cbapi/psc/devices_query.py @@ -64,7 +64,7 @@ def set_device_ids(self, device_ids): Restricts the devices that this query is performed on to the specified device IDs. - :param ad_group_ids: list of ints + :param device_ids: list of ints :return: This instance """ if not all(isinstance(device_id, int) for device_id in device_ids): diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py index 50805d4b..321dac06 100755 --- a/src/cbapi/psc/models.py +++ b/src/cbapi/psc/models.py @@ -1,8 +1,6 @@ from cbapi.models import MutableBaseModel, UnrefreshableModel from cbapi.errors import ServerError from cbapi.psc.devices_query import DeviceSearchQuery -from cbapi.psc.alerts_query import BaseAlertSearchQuery, WatchlistAlertSearchQuery, \ - CBAnalyticsAlertSearchQuery, VMwareAlertSearchQuery from copy import deepcopy import logging @@ -130,17 +128,6 @@ def _refresh(self): self._last_refresh_time = time.time() return True - def lr_session(self): - """ - Retrieve a Live Response session object for this Device. - - :return: Live Response session object - :rtype: :py:class:`cbapi.defense.cblr.LiveResponseSession` - :raises ApiError: if there is an error establishing a Live Response session for this Device - - """ - return self._cb._request_lr_session(self._model_unique_id) - def background_scan(self, flag): """ Set the background scan option for this device. @@ -192,180 +179,3 @@ def update_sensor_version(self, sensor_version): :param dict sensor_version: New version properties for the sensor. """ return self._cb.device_update_sensor_version([self._model_unique_id], sensor_version) - - -class Workflow(UnrefreshableModel): - swagger_meta_file = "psc/models/workflow.yaml" - - def __init__(self, cb, initial_data=None): - super(Workflow, self).__init__(cb, model_unique_id=None, initial_data=initial_data) - - -class BaseAlert(PSCMutableModel): - urlobject = "/appservices/v6/orgs/{0}/alerts" - urlobject_single = "/appservices/v6/orgs/{0}/alerts/{1}" - primary_key = "id" - swagger_meta_file = "psc/models/base_alert.yaml" - - def __init__(self, cb, model_unique_id, initial_data=None): - super(BaseAlert, self).__init__(cb, model_unique_id, initial_data) - self._workflow = Workflow(cb, initial_data.get("workflow", None) if initial_data else None) - if model_unique_id is not None and initial_data is None: - self._refresh() - - @classmethod - def _query_implementation(cls, cb): - return BaseAlertSearchQuery(cls, cb) - - def _refresh(self): - url = self.urlobject_single.format(self._cb.credentials.org_key, self._model_unique_id) - resp = self._cb.get_object(url) - self._info = resp - self._workflow = Workflow(self._cb, resp.get("workflow", None)) - self._last_refresh_time = time.time() - return True - - @property - def workflow_(self): - return self._workflow - - def _update_workflow_status(self, state, remediation, comment): - """ - Update the workflow status of this alert. - - :param str state: The state to set for this alert, either "OPEN" or "DISMISSED". - :param remediation str: The remediation status to set for the alert. - :param comment str: The comment to set for the alert. - """ - request = {"state": state} - if remediation: - request["remediation_state"] = remediation - if comment: - request["comment"] = comment - url = self.urlobject_single.format(self._cb.credentials.org_key, - self._model_unique_id) + "/workflow" - resp = self._cb.post_object(url, request) - self._workflow = Workflow(self._cb, resp.json()) - self._last_refresh_time = time.time() - - def dismiss(self, remediation=None, comment=None): - """ - Dismiss this alert. - - :param remediation str: The remediation status to set for the alert. - :param comment str: The comment to set for the alert. - """ - self._update_workflow_status("DISMISSED", remediation, comment) - - def update(self, remediation=None, comment=None): - """ - Update this alert. - - :param remediation str: The remediation status to set for the alert. - :param comment str: The comment to set for the alert. - """ - self._update_workflow_status("OPEN", remediation, comment) - - def _update_threat_workflow_status(self, state, remediation, comment): - """ - Update the workflow status of all alerts with the same threat ID, past or future. - - :param str state: The state to set for this alert, either "OPEN" or "DISMISSED". - :param remediation str: The remediation status to set for the alert. - :param comment str: The comment to set for the alert. - """ - request = {"state": state} - if remediation: - request["remediation_state"] = remediation - if comment: - request["comment"] = comment - url = "/appservices/v6/orgs/{0}/threat/{1}/workflow".format(self._cb.credentials.org_key, - self.threat_id) - resp = self._cb.post_object(url, request) - return Workflow(self._cb, resp.json()) - - def dismiss_threat(self, remediation=None, comment=None): - """ - Dismiss alerts for this threat. - - :param remediation str: The remediation status to set for the alert. - :param comment str: The comment to set for the alert. - """ - return self._update_threat_workflow_status("DISMISSED", remediation, comment) - - def update_threat(self, remediation=None, comment=None): - """ - Update alerts for this threat. - - :param remediation str: The remediation status to set for the alert. - :param comment str: The comment to set for the alert. - """ - return self._update_threat_workflow_status("OPEN", remediation, comment) - - -class WatchlistAlert(BaseAlert): - urlobject = "/appservices/v6/orgs/{0}/alerts/watchlist" - - @classmethod - def _query_implementation(cls, cb): - return WatchlistAlertSearchQuery(cls, cb) - - -class CBAnalyticsAlert(BaseAlert): - urlobject = "/appservices/v6/orgs/{0}/alerts/cbanalytics" - - @classmethod - def _query_implementation(cls, cb): - return CBAnalyticsAlertSearchQuery(cls, cb) - - -class VMwareAlert(BaseAlert): - urlobject = "/appservices/v6/orgs/{0}/alerts/vmware" - - @classmethod - def _query_implementation(cls, cb): - return VMwareAlertSearchQuery(cls, cb) - - -class WorkflowStatus(PSCMutableModel): - urlobject_single = "/appservices/v6/orgs/{0}/workflow/status/{1}" - primary_key = "id" - swagger_meta_file = "psc/models/workflow_status.yaml" - - def __init__(self, cb, model_unique_id, initial_data=None): - super(WorkflowStatus, self).__init__(cb, model_unique_id, initial_data) - self._request_id = model_unique_id - self._workflow = None - if model_unique_id is not None: - self._refresh() - - def _refresh(self): - url = self.urlobject_single.format(self._cb.credentials.org_key, self._request_id) - resp = self._cb.get_object(url) - self._info = resp - self._workflow = Workflow(self._cb, resp.get("workflow", None)) - self._last_refresh_time = time.time() - return True - - @property - def id_(self): - return self._request_id - - @property - def workflow_(self): - return self._workflow - - @property - def queued(self): - self._refresh() - return self._info.get("status", "") == "QUEUED" - - @property - def in_progress(self): - self._refresh() - return self._info.get("status", "") == "IN_PROGRESS" - - @property - def finished(self): - self._refresh() - return self._info.get("status", "") == "FINISHED" diff --git a/src/cbapi/psc/models/base_alert.yaml b/src/cbapi/psc/models/base_alert.yaml deleted file mode 100755 index ffc0b4e0..00000000 --- a/src/cbapi/psc/models/base_alert.yaml +++ /dev/null @@ -1,139 +0,0 @@ -type: object -properties: - category: - type: string - description: Alert category - Monitored vs Threat - enum: - - THREAT - - MONITORED - - INFO - - MINOR - - SERIOUS - - CRITICAL - create_time: - type: string - format: date-time - description: Time the alert was created - device_id: - type: integer - format: int64 - description: ID of the device - device_name: - type: string - description: Device name - device_os: - type: string - description: Device OS - enum: - - WINDOWS - - ANDROID - - MAC - - IOS - - LINUX - - OTHER - device_os_version: - type: string - example: Windows 10 x64 - description: Device OS Version - device_username: - type: string - description: Logged on user during the alert. This is filled on a best-effort - approach. If the user is not available it may be populated with the device - owner - first_event_time: - type: string - format: date-time - description: Time of the first event in an alert - group_details: - description: Group details for when alert grouping is on - type: object - properties: - count: - type: integer - format: int64 - description: Number of times the event has occurred - total_devices: - type: integer - format: int64 - description: The number of devices that have seen this alert - id: - type: string - description: Unique ID for this alert - last_event_time: - type: string - format: date-time - description: Time of the last event in an alert - last_update_time: - type: string - format: date-time - description: Time the alert was last updated - legacy_alert_id: - type: string - description: Unique short ID for this alert. This is deprecated and only available - on alerts stored in the old schema. - notes_present: - type: boolean - description: Are notes present for this threatId - org_key: - type: string - example: ABCD1234 - description: Unique identifier for the organization to which the alert belongs - policy_id: - type: integer - format: int64 - description: ID of the policy the device was in at the time of the alert - policy_name: - type: string - description: Name of the policy the device was in at the time of the alert - severity: - type: integer - format: int32 - description: Threat ranking - tags: - type: array - description: Tags for the alert - items: - type: string - target_value: - type: string - description: Device priority as assigned via the policy - enum: - - LOW - - MEDIUM - - HIGH - - MISSION_CRITICAL - threat_id: - type: string - description: ID of the threat to which this alert belongs. Threats are comprised - of a combination of factors that can be repeated across devices. - type: - type: string - description: Type of the alert - enum: - - CB_ANALYTICS - - VMWARE - - WATCHLIST - workflow: - description: User-updatable status of the alert - type: object - properties: - changed_by: - type: string - description: Username of the user who changed the workflow - comment: - type: string - description: Comment when updating the workflow - last_update_time: - type: string - format: date-time - description: When the workflow was last updated - remediation: - type: string - description: Alert remediation code. Indicates the result of the investigation - into the alert - state: - type: string - description: State of the workflow - enum: - - OPEN - - DISMISSED diff --git a/src/cbapi/psc/models/workflow.yaml b/src/cbapi/psc/models/workflow.yaml deleted file mode 100755 index 8807a69f..00000000 --- a/src/cbapi/psc/models/workflow.yaml +++ /dev/null @@ -1,23 +0,0 @@ -type: object -description: Tracking system for alerts as they are triaged and resolved -properties: - changed_by: - type: string - description: Username of the user who changed the workflow - comment: - type: string - description: Comment when updating the workflow - last_update_time: - type: string - format: date-time - description: When the workflow was last updated - remediation: - type: string - description: Alert remediation code. Indicates the result of the investigation - into the alert - state: - type: string - description: State of the workflow - enum: - - OPEN - - DISMISSED diff --git a/src/cbapi/psc/models/workflow_status.yaml b/src/cbapi/psc/models/workflow_status.yaml deleted file mode 100755 index 202e8cb5..00000000 --- a/src/cbapi/psc/models/workflow_status.yaml +++ /dev/null @@ -1,56 +0,0 @@ -type: object -description: Dismiss status response for async calls -properties: - errors: - type: array - description: Errors for dismiss alerts or threats, if no errors it won't be - included in response - items: - type: string - failed_ids: - type: array - description: Failed ids - items: - type: string - id: - type: string - description: Time based id for async job, it's not unique across the orgs - num_hits: - type: integer - format: int64 - description: Total number of alerts to be operated on - num_success: - type: integer - format: int64 - description: Successfully operated number of alerts - status: - type: string - description: Status for the async progress - enum: - - QUEUED - - IN_PROGRESS - - FINISHED - workflow: - description: Requested workflow change - type: object - properties: - changed_by: - type: string - description: Username of the user who changed the workflow - comment: - type: string - description: Comment when updating the workflow - last_update_time: - type: string - format: date-time - description: When the workflow was last updated - remediation: - type: string - description: Alert remediation code. Indicates the result of the investigation - into the alert - state: - type: string - description: State of the workflow - enum: - - OPEN - - DISMISSED diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py index aa12f794..ad7a68da 100755 --- a/src/cbapi/psc/rest_api.py +++ b/src/cbapi/psc/rest_api.py @@ -1,6 +1,5 @@ from cbapi.connection import BaseAPI from cbapi.errors import ApiError, ServerError -from .cblr import LiveResponseSessionManager import logging log = logging.getLogger(__name__) @@ -27,17 +26,6 @@ def _perform_query(self, cls, **kwargs): else: raise ApiError("All PSC models should provide _query_implementation") - # ---- LiveOps - - @property - def live_response(self): - if self._lr_scheduler is None: - self._lr_scheduler = LiveResponseSessionManager(self) - return self._lr_scheduler - - def _request_lr_session(self, sensor_id): - return self.live_response.request_session(sensor_id) - # ---- Device API def _raw_device_action(self, request): diff --git a/src/cbapi/psc/threathunter/__init__.py b/src/cbapi/psc/threathunter/__init__.py deleted file mode 100644 index d1b0e338..00000000 --- a/src/cbapi/psc/threathunter/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# Exported public API for the Cb ThreatHunter API - -from __future__ import absolute_import - -from .rest_api import CbThreatHunterAPI -from cbapi.psc.threathunter.models import ( - Process, Event, Tree, Feed, Report, IOC, IOC_V2, Watchlist, Binary, Downloads -) -from cbapi.psc.threathunter.query import QueryBuilder diff --git a/src/cbapi/psc/threathunter/models.py b/src/cbapi/psc/threathunter/models.py deleted file mode 100644 index 1ac3f31d..00000000 --- a/src/cbapi/psc/threathunter/models.py +++ /dev/null @@ -1,1117 +0,0 @@ -from __future__ import absolute_import -from cbapi.errors import ApiError, InvalidObjectError -from cbapi.models import CreatableModelMixin, MutableBaseModel, UnrefreshableModel -import logging -from cbapi.psc.threathunter.query import Query, AsyncProcessQuery, TreeQuery, FeedQuery, ReportQuery, WatchlistQuery -import validators -import time - -log = logging.getLogger(__name__) - - -class FeedModel(UnrefreshableModel, CreatableModelMixin, MutableBaseModel): - """A common base class for models used by the Feed and Watchlist APIs. - """ - pass - - -class Process(UnrefreshableModel): - """Represents a process retrieved by one of the CbTH endpoints. - """ - default_sort = 'last_update desc' - primary_key = "process_guid" - validation_url = "/api/investigate/v1/orgs/{}/processes/search_validation" - - class Summary(UnrefreshableModel): - """Represents a summary of organization-specific information for - a process. - """ - default_sort = "last_update desc" - primary_key = "process_guid" - urlobject_single = "/api/investigate/v1/orgs/{}/processes/summary" - - def __init__(self, cb, model_unique_id): - url = self.urlobject_single.format(cb.credentials.org_key) - summary = cb.get_object(url, query_parameters={"process_guid": model_unique_id}) - - while summary["incomplete_results"]: - log.debug("summary incomplete, requesting again") - summary = self._cb.get_object( - url, query_parameters={"process_guid": self.process_guid} - ) - - super(Process.Summary, self).__init__(cb, model_unique_id=model_unique_id, - initial_data=summary, force_init=False, - full_doc=True) - - @classmethod - def _query_implementation(cls, cb): - # This will emulate a synchronous process query, for now. - return AsyncProcessQuery(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=True): - super(Process, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=force_init, full_doc=full_doc) - - @property - def summary(self): - """Returns organization-specific information about this process. - """ - return self._cb.select(Process.Summary, self.process_guid) - - def events(self, **kwargs): - """Returns a query for events associated with this process's process GUID. - - :param kwargs: Arguments to filter the event query with. - :return: Returns a Query object with the appropriate search parameters for events - :rtype: :py:class:`cbapi.psc.threathunter.query.Query` - - Example:: - - >>> [print(event) for event in process.events()] - >>> [print(event) for event in process.events(event_type="modload")] - """ - query = self._cb.select(Event).where(process_guid=self.process_guid) - - if kwargs: - query = query.and_(**kwargs) - - return query - - def tree(self): - """Returns a :py:class:`Tree` of children (and possibly siblings) - associated with this process. - - :return: Returns a :py:class:`Tree` object - :rtype: :py:class:`Tree` - - Example: - - >>> tree = process.tree() - """ - data = self._cb.select(Tree).where(process_guid=self.process_guid).all() - return Tree(self._cb, initial_data=data) - - @property - def parents(self): - """Returns a query for parent processes associated with this process. - - :return: Returns a Query object with the appropriate search parameters for parent processes, - or None if the process has no recorded parent - :rtype: :py:class:`cbapi.psc.threathunter.query.AsyncProcessQuery` or None - """ - if "parent_guid" in self._info: - return self._cb.select(Process).where(process_guid=self.parent_guid) - else: - return [] - - @property - def children(self): - """Returns a list of child processes for this process. - - :return: Returns a list of process objects - :rtype: list of :py:class:`Process` - """ - if isinstance(self.summary.children, list): - return [ - Process(self._cb, initial_data=child) - for child in self.summary.children - ] - else: - return [] - - @property - def siblings(self): - """Returns a list of sibling processes for this process. - - :return: Returns a list of process objects - :rtype: list of :py:class:`Process` - """ - return [ - Process(self._cb, initial_data=sibling) - for sibling in self.summary.siblings - ] - - @property - def process_md5(self): - """Returns a string representation of the MD5 hash for this process. - - :return: A string representation of the process's MD5. - :rtype: str - """ - # NOTE: We have to check _info instead of poking the attribute directly - # to avoid the missing attrbute login in NewBaseModel. - if "process_hash" in self._info: - return next((hsh for hsh in self.process_hash if len(hsh) == 32), None) - else: - return None - - @property - def process_sha256(self): - """Returns a string representation of the SHA256 hash for this process. - - :return: A string representation of the process's SHA256. - :rtype: str - """ - if "process_hash" in self._info: - return next((hsh for hsh in self.process_hash if len(hsh) == 64), None) - else: - return None - - @property - def process_pids(self): - """Returns a list of PIDs associated with this process. - - :return: A list of PIDs - :rtype: list of ints - """ - # NOTE(ww): This exists because the API returns the list as "process_pid", - # which is misleading. We just give a slightly clearer name. - return self.process_pid - - -class Event(UnrefreshableModel): - """Events can be queried for via ``CbThreatHunterAPI.select`` - or though an already selected process with ``Process.events()``. - """ - urlobject = '/api/investigate/v2/orgs/{}/events/{}/_search' - validation_url = '/api/investigate/v1/orgs/{}/events/search_validation' - default_sort = 'last_update desc' - primary_key = "process_guid" - - @classmethod - def _query_implementation(cls, cb): - return Query(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=True): - super(Event, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=force_init, full_doc=full_doc) - - -class Tree(UnrefreshableModel): - """The preferred interface for interacting with Tree models - is ``Process.tree()``. - """ - urlobject = '/threathunter/search/v1/orgs/{}/processes/tree' - primary_key = 'process_guid' - - @classmethod - def _query_implementation(cls, cb): - return TreeQuery(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=True): - super(Tree, self).__init__( - cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=force_init, full_doc=full_doc - ) - - @property - def children(self): - """Returns all of the children of the process that this tree is centered around. - - :return: A list of :py:class:`Process` instances - :rtype: list of :py:class:`Process` - """ - return [Process(self._cb, initial_data=child) for child in self.nodes["children"]] - - -class Feed(FeedModel): - """Represents a ThreatHunter feed's metadata. - """ - urlobject = "/threathunter/feedmgr/v2/orgs/{}/feeds" - urlobject_single = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}" - primary_key = "id" - swagger_meta_file = "psc/threathunter/models/feed.yaml" - - @classmethod - def _query_implementation(cls, cb): - return FeedQuery(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None): - item = {} - reports = [] - - if initial_data: - # NOTE(ww): Some endpoints give us the full Feed, others give us just the FeedInfo. - if "feedinfo" in initial_data: - item = initial_data["feedinfo"] - reports = initial_data.get("reports", []) - else: - item = initial_data - elif model_unique_id: - url = self.urlobject_single.format( - cb.credentials.org_key, model_unique_id - ) - resp = cb.get_object(url) - item = resp.get("feedinfo", {}) - reports = resp.get("reports", []) - - feed_id = item.get("id") - - super(Feed, self).__init__(cb, model_unique_id=feed_id, initial_data=item, - force_init=False, full_doc=True) - - self._reports = [Report(cb, initial_data=report, feed_id=feed_id) for report in reports] - - def save(self, public=False): - """Saves this feed on the Enterprise EDR server. - - :param public: Whether to make the feed publicly available - :return: The saved feed - :rtype: :py:class:`Feed` - """ - self.validate() - - body = { - 'feedinfo': self._info, - 'reports': [report._info for report in self._reports], - } - - url = "/threathunter/feedmgr/v2/orgs/{}/feeds".format( - self._cb.credentials.org_key - ) - if public: - url = url + "/public" - - new_info = self._cb.post_object(url, body).json() - self._info.update(new_info) - return self - - def validate(self): - """Validates this feed's state. - - :raise InvalidObjectError: if the feed's state is invalid - """ - super(Feed, self).validate() - - if self.access not in ["public", "private"]: - raise InvalidObjectError("access should be public or private") - - if not validators.url(self.provider_url): - raise InvalidObjectError("provider_url should be a valid URL") - - for report in self._reports: - report.validate() - - def delete(self): - """Deletes this feed from the Enterprise EDR server. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing feed ID") - - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.delete_object(url) - - def update(self, **kwargs): - """Update this feed's metadata with the given arguments. - - >>> feed.update(access="private") - - :param kwargs: The fields to update - :type kwargs: dict(str, str) - :raise InvalidObjectError: if `id` is missing or :py:meth:`validate` fails - :raise ApiError: if an invalid field is specified - """ - if not self.id: - raise InvalidObjectError("missing feed ID") - - for key, value in kwargs.items(): - if key in self._info: - self._info[key] = value - - self.validate() - - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/feedinfo".format( - self._cb.credentials.org_key, - self.id, - ) - new_info = self._cb.put_object(url, self._info).json() - self._info.update(new_info) - - return self - - @property - def reports(self): - """Returns a list of :py:class:`Report` associated with this feed. - - :return: a list of reports - :rtype: list(:py:class:`Report`) - """ - return self._cb.select(Report).where(feed_id=self.id) - - def replace_reports(self, reports): - """Replace this feed's reports with the given reports. - - :param reports: the reports to replace with - :type reports: list(:py:class:`Report`) - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing feed ID") - - rep_dicts = [report._info for report in reports] - body = {"reports": rep_dicts} - - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/reports".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.post_object(url, body) - - def append_reports(self, reports): - """Append the given reports to this feed's current reports. - - :param reports: the reports to append - :type reports: list(:py:class:`Report`) - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing feed ID") - - rep_dicts = [report._info for report in reports] - rep_dicts += [report._info for report in self.reports] - body = {"reports": rep_dicts} - - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/reports".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.post_object(url, body) - - -class Report(FeedModel): - """Represents reports retrieved from a ThreatHunter feed. - """ - urlobject = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/reports" - primary_key = "id" - swagger_meta_file = "psc/threathunter/models/report.yaml" - - @classmethod - def _query_implementation(cls, cb): - return ReportQuery(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None, - feed_id=None, from_watchlist=False): - - super(Report, self).__init__(cb, model_unique_id=initial_data.get("id"), - initial_data=initial_data, - force_init=False, full_doc=True) - - # NOTE(ww): Warn instead of failing since we allow Watchlist reports - # to be created via create(), but we don't actually know that the user - # intends to use them with a watchlist until they call save(). - if not feed_id and not from_watchlist: - log.warning("Report created without feed ID or not from watchlist") - - self._feed_id = feed_id - self._from_watchlist = from_watchlist - - if self.iocs: - self._iocs = IOC(cb, initial_data=self.iocs, report_id=self.id) - if self.iocs_v2: - self._iocs_v2 = [IOC_V2(cb, initial_data=ioc, report_id=self.id) for ioc in self.iocs_v2] - - def save_watchlist(self): - """Saves this report *as a watchlist report*. - - .. NOTE:: - This method **cannot** be used to save a feed report. To - save feed reports, create them with `cb.create` and use - :py:meth:`Feed.replace`. - - :raise InvalidObjectError: if :py:meth:`validate` fails - """ - self.validate() - - # NOTE(ww): Once saved, this object corresponds to a watchlist report. - # As such, we need to tell the model to route calls like update() - # and delete() to the correct (watchlist) endpoints. - self._from_watchlist = True - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports".format( - self._cb.credentials.org_key - ) - new_info = self._cb.post_object(url, self._info).json() - self._info.update(new_info) - return self - - def validate(self): - """Validates this report's state. - - :raise InvalidObjectError: if the report's state is invalid - """ - super(Report, self).validate() - - if self.link and not validators.url(self.link): - raise InvalidObjectError("link should be a valid URL") - - if self.iocs_v2: - [ioc.validate() for ioc in self._iocs_v2] - - def update(self, **kwargs): - """Update this report with the given arguments. - - .. NOTE:: - The report's timestamp is always updated, regardless of whether - passed explicitly. - - >>> report.update(title="My new report title") - - :param kwargs: The fields to update - :type kwargs: dict(str, str) - :return: The updated report - :rtype: :py:class:`Report` - :raises InvalidObjectError: if `id` is missing, or `feed_id` is missing - and this report is a feed report, or :py:meth:`validate` fails - """ - - if not self.id: - raise InvalidObjectError("missing Report ID") - - if self._from_watchlist: - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}".format( - self._cb.credentials.org_key, - self.id - ) - else: - if not self._feed_id: - raise InvalidObjectError("missing Feed ID") - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/reports/{}".format( - self._cb.credentials.org_key, - self._feed_id, - self.id - ) - - for key, value in kwargs.items(): - if key in self._info: - self._info[key] = value - - # NOTE(ww): Updating reports on the watchlist API appears to require - # updated timestamps. - self.timestamp = int(time.time()) - self.validate() - - new_info = self._cb.put_object(url, self._info).json() - self._info.update(new_info) - return self - - def delete(self): - """Deletes this report from the Enterprise EDR server. - - >>> report.delete() - - :raises InvalidObjectError: if `id` is missing, or `feed_id` is missing - and this report is a feed report - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - - if self._from_watchlist: - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}".format( - self._cb.credentials.org_key, - self.id - ) - else: - if not self._feed_id: - raise InvalidObjectError("missing Feed ID") - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/reports/{}".format( - self._cb.credentials.org_key, - self._feed_id, - self.id - ) - - self._cb.delete_object(url) - - @property - def ignored(self): - """Returns the ignore status for this report. - - Only watchlist reports have an ignore status. - - >>> if report.ignored: - ... report.unignore() - - :return: whether or not this report is ignored - :rtype: bool - :raises InvalidObjectError: if `id` is missing or this report is not from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - if not self._from_watchlist: - raise InvalidObjectError("ignore status only applies to watchlist reports") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/ignore".format( - self._cb.credentials.org_key, - self.id - ) - resp = self._cb.get_object(url) - return resp["ignored"] - - def ignore(self): - """Sets the ignore status on this report. - - Only watchlist reports have an ignore status. - - :raises InvalidObjectError: if `id` is missing or this report is not from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - - if not self._from_watchlist: - raise InvalidObjectError("ignoring only applies to watchlist reports") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/ignore".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.put_object(url, None) - - def unignore(self): - """Removes the ignore status on this report. - - Only watchlist reports have an ignore status. - - :raises InvalidObjectError: if `id` is missing or this report is not from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - - if not self._from_watchlist: - raise InvalidObjectError("ignoring only applies to watchlist reports") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/ignore".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.delete_object(url) - - @property - def custom_severity(self): - """Returns the custom severity for this report. - - :return: The custom severity for this report, if it exists - :rtype: :py:class:`ReportSeverity` - :raise InvalidObjectError: if `id` is missing or this report is from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing report ID") - if self._from_watchlist: - raise InvalidObjectError("watchlist reports don't have custom severities") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/severity".format( - self._cb.credentials.org_key, - self.id - ) - resp = self._cb.get_object(url) - return ReportSeverity(self._cb, initial_data=resp) - - @custom_severity.setter - def custom_severity(self, sev_level): - """Sets or removed the custom severity for this report - - :param int sev_level: the new severity, or None to remove the custom severity - :return: The new custom severity, or None if removed - :rtype: :py:class:`ReportSeverity` or None - :raise InvalidObjectError: if `id` is missing or this report is from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing report ID") - if self._from_watchlist: - raise InvalidObjectError("watchlist reports don't have custom severities") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/severity".format( - self._cb.credentials.org_key, - self.id - ) - - if sev_level is None: - self._cb.delete_object(url) - return - - args = { - "report_id": self.id, - "severity": sev_level, - } - - resp = self._cb.put_object(url, args).json() - return ReportSeverity(self._cb, initial_data=resp) - - @property - def iocs_(self): - """Returns a list of :py:class:`IOC_V2` associated with this report. - - >>> for ioc in report.iocs_: - ... print(ioc.values) - - :return: a list of IOCs - :rtype: list(:py:class:`IOC_V2`) - """ - if not self.iocs_v2: - return [] - - # NOTE(ww): This name is underscored because something in the model - # hierarchy is messing up method resolution -- self.iocs and self.iocs_v2 - # are resolving to the attributes rather than the attribute-ified - # methods. - return self._iocs_v2 - - -class IOC(FeedModel): - """Represents a collection of categorized IOCs. - """ - swagger_meta_file = "psc/threathunter/models/iocs.yaml" - - def __init__(self, cb, model_unique_id=None, initial_data=None, report_id=None): - """Creates a new IOC instance. - - :raise ApiError: if `initial_data` is `None` - """ - if not initial_data: - raise ApiError("IOC can only be initialized from initial_data") - - super(IOC, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=False, full_doc=True) - - self._report_id = report_id - - def validate(self): - """Validates this IOC structure's state. - - :raise InvalidObjectError: if the IOC structure's state is invalid - """ - super(IOC, self).validate() - - for md5 in self.md5: - if not validators(md5): - raise InvalidObjectError("invalid MD5 checksum: {}".format(md5)) - for ipv4 in self.ipv4: - if not validators(ipv4): - raise InvalidObjectError("invalid IPv4 address: {}".format(ipv4)) - for ipv6 in self.ipv6: - if not validators(ipv6): - raise InvalidObjectError("invalid IPv6 address: {}".format(ipv6)) - for dns in self.dns: - if not validators(dns): - raise InvalidObjectError("invalid domain: {}".format(dns)) - for query in self.query: - if not self._cb.validate(query["search_query"]): - raise InvalidObjectError("invalid search query: {}".format(query["search_query"])) - - -class IOC_V2(FeedModel): - """Represents a collection of IOCs of a particular type, plus matching criteria and metadata. - """ - primary_key = "id" - swagger_meta_file = "psc/threathunter/models/ioc_v2.yaml" - - def __init__(self, cb, model_unique_id=None, initial_data=None, report_id=None): - """Creates a new IOC_V2 instance. - - :raise ApiError: if `initial_data` is `None` - """ - if not initial_data: - raise ApiError("IOC_V2 can only be initialized from initial_data") - - super(IOC_V2, self).__init__(cb, model_unique_id=initial_data.get(self.primary_key), - initial_data=initial_data, force_init=False, - full_doc=True) - - self._report_id = report_id - - def validate(self): - """Validates this IOC_V2's state. - - :raise InvalidObjectError: if the IOC_V2's state is invalid - """ - super(IOC_V2, self).validate() - - if self.link and not validators.url(self.link): - raise InvalidObjectError("link should be a valid URL") - - @property - def ignored(self): - """Returns whether or not this IOC is ignored - - >>> if ioc.ignored: - ... ioc.unignore() - - :return: the ignore status - :rtype: bool - :raise InvalidObjectError: if this IOC is missing an `id` or is not a watchlist IOC - """ - if not self.id: - raise InvalidObjectError("missing IOC ID") - if not self._report_id: - raise InvalidObjectError("ignore status only applies to watchlist IOCs") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/iocs/{}/ignore".format( - self._cb.credentials.org_key, - self._report_id, - self.id - ) - resp = self._cb.get_object(url) - return resp["ignored"] - - def ignore(self): - """Sets the ignore status on this IOC. - - Only watchlist IOCs have an ignore status. - - :raises InvalidObjectError: if `id` is missing or this IOC is not from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - if not self._report_id: - raise InvalidObjectError("ignoring only applies to watchlist IOCs") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/iocs/{}/ignore".format( - self._cb.credentials.org_key, - self._report_id, - self.id - ) - self._cb.put_object(url, None) - - def unignore(self): - """Removes the ignore status on this IOC. - - Only watchlist IOCs have an ignore status. - - :raises InvalidObjectError: if `id` is missing or this IOC is not from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - if not self._report_id: - raise InvalidObjectError("ignoring only applies to watchlist IOCs") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/iocs/{}/ignore".format( - self._cb.credentials.org_key, - self._report_id, - self.id - ) - self._cb.delete_object(url) - - -class Watchlist(FeedModel): - """Represents a ThreatHunter watchlist. - """ - # NOTE(ww): Not documented. - urlobject = "/threathunter/watchlistmgr/v2/watchlist" - urlobject_single = "/threathunter/watchlistmgr/v2/watchlist/{}" - swagger_meta_file = "psc/threathunter/models/watchlist.yaml" - - @classmethod - def _query_implementation(cls, cb): - return WatchlistQuery(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None): - item = {} - - if initial_data: - item = initial_data - elif model_unique_id: - item = cb.get_object(self.urlobject_single.format(model_unique_id)) - - feed_id = item.get("id") - - super(Watchlist, self).__init__(cb, model_unique_id=feed_id, initial_data=item, - force_init=False, full_doc=True) - - def save(self): - """Saves this watchlist on the Enterprise EDR server. - - :return: The saved watchlist - :rtype: :py:class:`Watchlist` - :raise InvalidObjectError: if :py:meth:`validate` fails - """ - self.validate() - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists".format( - self._cb.credentials.org_key - ) - new_info = self._cb.post_object(url, self._info).json() - self._info.update(new_info) - return self - - def validate(self): - """Validates this watchlist's state. - - :raise InvalidObjectError: if the watchlist's state is invalid - """ - super(Watchlist, self).validate() - - def update(self, **kwargs): - """Updates this watchlist with the given arguments. - - >>> watchlist.update(name="New Name") - - :param kwargs: The fields to update - :type kwargs: dict(str, str) - :raise InvalidObjectError: if `id` is missing or :py:meth:`validate` fails - :raise ApiError: if `report_ids` is given *and* is empty - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - # NOTE(ww): Special case, according to the docs. - if "report_ids" in kwargs and not kwargs["report_ids"]: - raise ApiError("can't update a watchlist to have an empty report list") - - for key, value in kwargs.items(): - if key in self._info: - self._info[key] = value - - self.validate() - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}".format( - self._cb.credentials.org_key, - self.id - ) - new_info = self._cb.put_object(url, self._info).json() - self._info.update(new_info) - - @property - def classifier_(self): - """Returns the classifier key and value, if any, for this watchlist. - - :rtype: tuple(str, str) or None - """ - classifier_dict = self._info.get("classifier") - - if not classifier_dict: - return None - - return (classifier_dict["key"], classifier_dict["value"]) - - def delete(self): - """Deletes this watchlist from the Enterprise EDR server. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.delete_object(url) - - def enable_alerts(self): - """Enable alerts for this watchlist. Alerts are not retroactive. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}/alert".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.put_object(url, None) - - def disable_alerts(self): - """Disable alerts for this watchlist. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}/alert".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.delete_object(url) - - def enable_tags(self): - """Enable tagging for this watchlist. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}/tag".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.put_object(url, None) - - def disable_tags(self): - """Disable tagging for this watchlist. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}/tag".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.delete_object(url) - - @property - def feed(self): - """Returns the feed linked to this watchlist, if there is one. - - :return: the feed linked to this watchlist, if any - :rtype: :py:class:`Feed` or None - """ - if not self.classifier: - return None - if self.classifier["key"] != "feed_id": - log.warning("Unexpected classifier type: {}".format(self.classifier["key"])) - return None - - return self._cb.select(Feed, self.classifier["value"]) - - @property - def reports(self): - """Returns a list of :py:class:`Report` instances associated with this watchlist. - - .. NOTE:: - If this watchlist is a classifier (i.e. feed-linked) watchlist, - `reports` will be empty. To get the reports associated with the linked - feed, use :py:attr:`feed` like: - - >>> for report in watchlist.feed.reports: - ... print(report.title) - - :return: A list of reports - :rtype: list(:py:class:`Report`) - """ - if not self.report_ids: - return [] - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}" - reports_ = [] - for rep_id in self.report_ids: - path = url.format(self._cb.credentials.org_key, rep_id) - resp = self._cb.get_object(path) - reports_.append(Report(self._cb, initial_data=resp, from_watchlist=True)) - - return reports_ - - -class ReportSeverity(FeedModel): - """Represents severity information for a watchlist report. - """ - primary_key = "report_id" - swagger_meta_file = "psc/threathunter/models/report_severity.yaml" - - def __init__(self, cb, initial_data=None): - if not initial_data: - raise ApiError("ReportSeverity can only be initialized from initial_data") - - super(ReportSeverity, self).__init__(cb, model_unique_id=initial_data.get(self.primary_key), - initial_data=initial_data, force_init=False, - full_doc=True) - - -class Binary(UnrefreshableModel): - """Represents a retrievable binary. - """ - primary_key = "sha256" - swagger_meta_file = "psc/threathunter/models/binary.yaml" - urlobject_single = "/ubs/v1/orgs/{}/sha256/{}/metadata" - - class Summary(UnrefreshableModel): - """Represents a summary of organization-specific information - for a retrievable binary. - """ - primary_key = "sha256" - urlobject_single = "/ubs/v1/orgs/{}/sha256/{}/summary/device" - - def __init__(self, cb, model_unique_id): - if not validators.sha256(model_unique_id): - raise ApiError("model_unique_id must be a valid SHA256") - - url = self.urlobject_single.format(cb.credentials.org_key, model_unique_id) - item = cb.get_object(url) - - super(Binary.Summary, self).__init__(cb, model_unique_id=model_unique_id, - initial_data=item, force_init=False, - full_doc=True) - - def __init__(self, cb, model_unique_id): - if not validators.sha256(model_unique_id): - raise ApiError("model_unique_id must be a valid SHA256") - - url = self.urlobject_single.format(cb.credentials.org_key, model_unique_id) - item = cb.get_object(url) - - super(Binary, self).__init__(cb, model_unique_id=model_unique_id, - initial_data=item, force_init=False, - full_doc=True) - - @property - def summary(self): - """Returns organization-specific information about this binary. - """ - return self._cb.select(Binary.Summary, self.sha256) - - @property - def download_url(self, expiration_seconds=3600): - """Returns a URL that can be used to download the file - for this binary. Returns None if no download can be found. - - :param expiration_seconds: How long the download should be valid for - :raise InvalidObjectError: if URL retrieval should be retried - :return: A pre-signed AWS download URL - :rtype: str - """ - downloads = self._cb.select(Downloads, [self.sha256], - expiration_seconds=expiration_seconds) - - if self.sha256 in downloads.not_found: - return None - elif self.sha256 in downloads.error: - raise InvalidObjectError("{} should be retried".format(self.sha256)) - else: - return next((item.url - for item in downloads.found - if self.sha256 == item.sha256), None) - - -class Downloads(UnrefreshableModel): - """Represents download information for a list of process hashes. - """ - urlobject = "/ubs/v1/orgs/{}/file/_download" - - class FoundItem(UnrefreshableModel): - """Represents the download URL and process hash for a successfully - located binary. - """ - primary_key = "sha256" - - def __init__(self, cb, item): - super(Downloads.FoundItem, self).__init__(cb, model_unique_id=item["sha256"], - initial_data=item, force_init=False, - full_doc=True) - - def __init__(self, cb, shas, expiration_seconds=3600): - body = { - "sha256": shas, - "expiration_seconds": expiration_seconds, - } - - url = self.urlobject.format(cb.credentials.org_key) - item = cb.post_object(url, body).json() - - super(Downloads, self).__init__(cb, model_unique_id=None, - initial_data=item, force_init=False, - full_doc=True) - - @property - def found(self): - """Returns a list of :py:class:`Downloads.FoundItem`, one - for each binary found in the binary store. - """ - return [Downloads.FoundItem(self._cb, item) for item in self._info["found"]] diff --git a/src/cbapi/psc/threathunter/models/binary.yaml b/src/cbapi/psc/threathunter/models/binary.yaml deleted file mode 100644 index c19d7b14..00000000 --- a/src/cbapi/psc/threathunter/models/binary.yaml +++ /dev/null @@ -1,79 +0,0 @@ -type: object -required: - - sha256 - - md5 - - file_available - - available_file_size - - file_size - - os_type - - architecture -properties: - sha256: - type: string - description: The SHA-256 hash of the file - md5: - type: string - description: The MD5 hash of the file - file_available: - type: boolean - description: If true, the file is available for download - available_file_size: - type: integer - format: int64 # NOTE(ww): docs say long integer - description: The size of the file available for download - file_size: - type: integer - format: int64 - description: The size of the actual file (represented by the hash) - os_type: - type: string - description: The OS that this file is designed for - architecture: - type: array - items: - type: string - description: The set of architectures that this file was compiled for - lang_id: - type: integer - format: int32 # NOTE(ww): Swagger doesn't have a (u)int16 - description: The Language ID value for the Windows VERSIONINFO resource - charset_id: - type: integer - format: int32 - description: The Character set ID value for the Windows VERSIONINFO resource - internal_name: - type: string - description: The internal name from FileVersionInformation - product_name: - type: string - description: The product name from FileVersionInformation - company_name: - type: string - description: The company name from FileVersionInformation - trademark: - type: string - description: The trademark from FileVersionInformation - file_description: - type: string - description: The file description from FileVersionInformation - file_version: - type: string - description: The file version from FileVersionInformation - comments: - type: string - description: Comments from FileVersionInformation - original_filename: - type: string - description: The original filename from FileVersionInformation - product_description: - type: string - description: The product description from FileVersionInformation - product_version: - type: string - description: The product version from FileVersionInformation - private_build: - type: string - description: The private build from FileVersionInformation - special_build: - type: string - description: The special build from FileVersionInformation diff --git a/src/cbapi/psc/threathunter/models/feed.yaml b/src/cbapi/psc/threathunter/models/feed.yaml deleted file mode 100644 index 27c0e4f4..00000000 --- a/src/cbapi/psc/threathunter/models/feed.yaml +++ /dev/null @@ -1,33 +0,0 @@ -type: object -required: - - name - - owner - - provider_url - - summary - - category - - access -properties: - name: - type: string - description: A human-friendly name for this feed - owner: - type: string - description: The feed owner's connector ID - provider_url: - type: string - description: A URL supplied by the feed's provider - summary: - type: string - description: A human-friendly summary for the feed - category: - type: string - description: The feed's category - source_label: - type: string - description: The feed's source label - access: - type: string - description: The feed's access (public or private) - id: - type: string - description: The feed's unique ID diff --git a/src/cbapi/psc/threathunter/models/ioc_v2.yaml b/src/cbapi/psc/threathunter/models/ioc_v2.yaml deleted file mode 100644 index a65f50cb..00000000 --- a/src/cbapi/psc/threathunter/models/ioc_v2.yaml +++ /dev/null @@ -1,23 +0,0 @@ -type: object -required: - - id - - match_type - - values -properties: - id: - type: string - description: The IOC_V2's unique ID - match_type: - type: string - description: How IOCs in this IOC_V2 are matched - values: - type: array - items: - type: string - description: A list of IOCs - field: - type: string - description: The kind of IOCs contained in this IOC_V2 - link: - type: string - description: A URL for some reference for this IOC_V2 diff --git a/src/cbapi/psc/threathunter/models/iocs.yaml b/src/cbapi/psc/threathunter/models/iocs.yaml deleted file mode 100644 index 274cb7a8..00000000 --- a/src/cbapi/psc/threathunter/models/iocs.yaml +++ /dev/null @@ -1,32 +0,0 @@ -type: object -properties: - md5: - type: array - items: - type: string - description: A list of MD5 checksums - ipv4: - type: array - items: - type: string - description: A list of IPv4 addresses - ipv6: - type: array - items: - type: string - description: A list of IPv6 addresses - dns: - type: array - items: - type: string - description: A list of domain names - query: - type: array - items: - type: object # QueryIOC - properties: - index_type: - type: string - search_query: - type: string - description: A list of dicts, each containing an IOC query diff --git a/src/cbapi/psc/threathunter/models/report.yaml b/src/cbapi/psc/threathunter/models/report.yaml deleted file mode 100644 index 6ef3bad1..00000000 --- a/src/cbapi/psc/threathunter/models/report.yaml +++ /dev/null @@ -1,45 +0,0 @@ -type: object -required: - - id - - timestamp - - title - - description - - severity -properties: - id: - type: string - description: The report's unique ID - timestamp: - type: integer - format: int32 - description: When this report was created - title: - type: string - description: A human-friendly title for this report - description: - type: string - description: A human-friendly description for this report - severity: - type: integer - format: int32 - description: The severity of the IOCs within this report - link: - type: string - description: A URL for some reference for this report - tags: - type: array - items: - type: string - description: A list of tags for this report - iocs: - type: object - # NOTE(ww): Explicitly not documented, since we do almost everything - # through IOC_V2 - iocs_v2: - type: array - items: - type: object - description: A list of IOC_V2 dicts associated with this report - visibility: - type: string - description: The visibility of this report diff --git a/src/cbapi/psc/threathunter/models/report_severity.yaml b/src/cbapi/psc/threathunter/models/report_severity.yaml deleted file mode 100644 index da53e10e..00000000 --- a/src/cbapi/psc/threathunter/models/report_severity.yaml +++ /dev/null @@ -1,12 +0,0 @@ -type: object -required: - - report_id - - severity -properties: - report_id: - type: string - description: The unique ID for the corresponding report - severity: - type: integer - format: int32 - description: The severity level diff --git a/src/cbapi/psc/threathunter/models/watchlist.yaml b/src/cbapi/psc/threathunter/models/watchlist.yaml deleted file mode 100644 index 34990997..00000000 --- a/src/cbapi/psc/threathunter/models/watchlist.yaml +++ /dev/null @@ -1,43 +0,0 @@ -type: object -required: - - name - - description - - create_timestamp - - last_update_timestamp -properties: - name: - type: string - description: A human-friendly name for the watchlist - description: - type: string - description: A short description of the watchlist - id: - type: string - description: The watchlist's unique id - tags_enabled: - type: boolean - description: Whether tags are currently enabled - alerts_enabled: - type: boolean - description: Whether alerts are currently enabled - create_timestamp: - type: integer - format: int32 - description: When this watchlist was created - last_update_timestamp: - type: integer - format: int32 - description: Report IDs associated with this watchlist - report_ids: - type: array - items: - type: string - description: Report IDs associated with this watchlist - classifier: - type: object # ClassifierKeyValue - properties: - key: - type: string - value: - type: string - description: A key, value pair specifying an associated feed diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py deleted file mode 100644 index ae2b1516..00000000 --- a/src/cbapi/psc/threathunter/query.py +++ /dev/null @@ -1,654 +0,0 @@ -from cbapi.query import PaginatedQuery, BaseQuery, SimpleQuery -from cbapi.errors import ApiError, TimeoutError -import time -from solrq import Q -from six import string_types -import logging -import functools - - -log = logging.getLogger(__name__) - - -class QueryBuilder(object): - """ - Provides a flexible interface for building prepared queries for the Carbon Black - Enterprise EDR backend. - - This object can be instantiated directly, or can be managed implicitly - through the :py:meth:`CbThreatHunterAPI.select` API. - - Examples:: - - >>> from cbapi.psc.threathunter import QueryBuilder - >>> # build a query with chaining - >>> query = QueryBuilder().where(process_name="malicious.exe").and_(device_name="suspect") - >>> # start with an initial query, and chain another condition to it - >>> query = QueryBuilder(device_os="WINDOWS").or_(process_username="root") - - """ - def __init__(self, **kwargs): - if kwargs: - self._query = Q(**kwargs) - else: - self._query = None - self._raw_query = None - self._process_guid = None - - def _guard_query_params(func): - """Decorates the query construction methods of *QueryBuilder*, preventing - them from being called with parameters that would result in an intetnally - inconsistent query. - """ - @functools.wraps(func) - def wrap_guard_query_change(self, q, **kwargs): - if self._raw_query is not None and (kwargs or isinstance(q, Q)): - raise ApiError("Cannot modify a raw query with structured parameters") - if self._query is not None and isinstance(q, string_types): - raise ApiError("Cannot modify a structured query with a raw parameter") - return func(self, q, **kwargs) - return wrap_guard_query_change - - @_guard_query_params - def where(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - if self._raw_query is None: - self._raw_query = [] - self._raw_query.append(q) - elif isinstance(q, Q) or kwargs: - if self._query is not None: - raise ApiError("Use .and_() or .or_() for an extant solrq.Q object") - if kwargs: - self._process_guid = self._process_guid or kwargs.get("process_guid") - q = Q(**kwargs) - self._query = q - else: - raise ApiError(".where() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def and_(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - self.where(q) - elif isinstance(q, Q) or kwargs: - if kwargs: - self._process_guid = self._process_guid or kwargs.get("process_guid") - q = Q(**kwargs) - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".and_() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def or_(self, q, **kwargs): - """Adds a disjunctive filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - self._process_guid = self._process_guid or kwargs.get("process_guid") - q = Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query | q - else: - raise ApiError(".or_() only accepts solrq.Q objects") - - return self - - @_guard_query_params - def not_(self, q, **kwargs): - """Adds a negative filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - q = ~ Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".not_() only accepts solrq.Q objects") - - def _collapse(self): - """The query can be represented by either an array of strings - (_raw_query) which is concatenated and passed directly to Solr, or - a solrq.Q object (_query) which is then converted into a string to - pass to Solr. This function will perform the appropriate conversions to - end up with the 'q' string sent into the POST request to the - PSC-R query endpoint.""" - if self._raw_query is not None: - return " ".join(self._raw_query) - elif self._query is not None: - return str(self._query) - else: - return "*:*" # return everything - - -class Query(PaginatedQuery): - """Represents a prepared query to the Carbon Black Enterprise EDR backend. - - This object is returned as part of a :py:meth:`CbThreatHunterPI.select` - operation on models requested from the Enterprise EDR backend. You should not have to create this class yourself. - - The query is not executed on the server until it's accessed, either as an iterator (where it will generate values - on demand as they're requested) or as a list (where it will retrieve the entire result set and save to a list). - You can also call the Python built-in ``len()`` on this object to retrieve the total number of items matching - the query. - - Examples:: - - >>> from cbapi.psc.threathunter import CbThreatHunterAPI,Process - >>> cb = CbThreatHunterAPI() - >>> query = cb.select(Process) - >>> query = query.where(process_name="notepad.exe") - >>> # alternatively: - >>> query = query.where("process_name:notepad.exe") - - Notes: - - The slicing operator only supports start and end parameters, but not step. ``[1:-1]`` is legal, but - ``[1:2:-1]`` is not. - - You can chain where clauses together to create AND queries; only objects that match all ``where`` clauses - will be returned. - """ - - def __init__(self, doc_class, cb): - super(Query, self).__init__(doc_class, cb, None) - - self._query_builder = QueryBuilder() - self._sort_by = None - self._group_by = None - self._batch_size = 100 - self._default_args = {} - - def where(self, q=None, **kwargs): - """Add a filter to this query. - - :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".where() expects a string, a QueryBuilder, a solrq.Q, or kwargs") - - if isinstance(q, QueryBuilder): - self._query_builder = q - else: - self._query_builder.where(q, **kwargs) - return self - - def and_(self, q=None, **kwargs): - """Add a conjunctive filter to this query. - - :param q: Query string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") - - self._query_builder.and_(q, **kwargs) - return self - - def or_(self, q=None, **kwargs): - """Add a disjunctive filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".or_() expects a solrq.Q or kwargs") - - self._query_builder.or_(q, **kwargs) - return self - - def not_(self, q=None, **kwargs): - """Adds a negated filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q and not kwargs: - raise ApiError(".not_() expects a solrq.Q, or kwargs") - - self._query_builder.not_(q, **kwargs) - return self - - def _get_query_parameters(self): - args = self._default_args.copy() - args['query'] = self._query_builder._collapse() - if self._query_builder._process_guid is not None: - args["process_guid"] = self._query_builder._process_guid - args["fields"] = [ - "*", - "parent_hash", - "parent_name", - "process_cmdline", - "backend_timestamp", - "device_external_ip", - "device_group", - "device_internal_ip", - "device_os", - "device_policy", - "process_effective_reputation", - "process_reputation", - "process_start_time", - "ttp" - ] - - return args - - def _count(self): - args = self._get_query_parameters() - - log.debug("args: {}".format(str(args))) - - result = self._cb.post_object( - self._doc_class.urlobject.format( - self._cb.credentials.org_key, - args["process_guid"] - ), body=args - ).json() - - self._total_results = int(result.get('num_available', 0)) - self._count_valid = True - - return self._total_results - - def _validate(self, args): - if not self._doc_class.validation_url: - return - - url = self._doc_class.validation_url.format(self._cb.credentials.org_key) - - if args.get('query', False): - args['q'] = args['query'] - - # v2 search sort key does not work with v1 validation - args.pop('sort', None) - - validated = self._cb.get_object(url, query_parameters=args) - - if not validated.get("valid"): - raise ApiError("Invalid query: {}: {}".format(args, validated["invalid_message"])) - - def _search(self, start=0, rows=0): - # iterate over total result set, 100 at a time - args = self._get_query_parameters() - self._validate(args) - - if start != 0: - args['start'] = start - args['rows'] = self._batch_size - - # args = {"search_params": args} - - current = start - numrows = 0 - - still_querying = True - - while still_querying: - url = self._doc_class.urlobject.format( - self._cb.credentials.org_key, - args["process_guid"] - ) - resp = self._cb.post_object(url, body=args) - result = resp.json() - - self._total_results = result.get("num_available", 0) - self._total_segments = result.get("total_segments", 0) - self._processed_segments = result.get("processed_segments", 0) - self._count_valid = True - - results = result.get('results', []) - - for item in results: - yield item - current += 1 - numrows += 1 - if rows and numrows == rows: - still_querying = False - break - - args['start'] = current + 1 # as of 6/2017, the indexing on the Cb Defense backend is still 1-based - - if current >= self._total_results: - break - if not results: - log.debug("server reported total_results overestimated the number of results for this query by {0}" - .format(self._total_results - current)) - log.debug("resetting total_results for this query to {0}".format(current)) - self._total_results = current - break - - -class AsyncProcessQuery(Query): - """Represents the query logic for an asychronous Process query. - - This class specializes :py:class:`Query` to handle the particulars of - process querying. - """ - def __init__(self, doc_class, cb): - super(AsyncProcessQuery, self).__init__(doc_class, cb) - self._query_token = None - self._timeout = 0 - self._timed_out = False - self._sort = [] - - def sort_by(self, key, direction="ASC"): - """Sets the sorting behavior on a query's results. - - Example:: - - >>> cb.select(Process).where(process_name="cmd.exe").sort_by("device_timestamp") - - :param key: the key in the schema to sort by - :param direction: the sort order, either "ASC" or "DESC" - :rtype: :py:class:`AsyncProcessQuery` - """ - found = False - - for sort_item in self._sort: - if sort_item['field'] == key: - sort_item['order'] = direction - found = True - - if not found: - self._sort.append({'field': key, 'order': direction}) - - self._default_args['sort'] = self._sort - - return self - - def timeout(self, msecs): - """Sets the timeout on a process query. - - Example:: - - >>> cb.select(Process).where(process_name="foo.exe").timeout(5000) - - :param: msecs: the timeout duration, in milliseconds - :return: AsyncProcessQuery object - :rtype: :py:class:`AsyncProcessQuery` - """ - self._timeout = msecs - return self - - def _submit(self): - if self._query_token: - raise ApiError("Query already submitted: token {0}".format(self._query_token)) - - args = self._get_query_parameters() - args['rows'] = 10000 - self._validate(args) - - url = "/api/investigate/v2/orgs/{}/processes/search_jobs".format(self._cb.credentials.org_key) - query_start = self._cb.post_object(url, body=args) - - self._query_token = query_start.json().get("job_id") - - self._timed_out = False - self._submit_time = time.time() * 1000 - - def _still_querying(self): - if not self._query_token: - self._submit() - - status_url = "/api/investigate/v1/orgs/{}/processes/search_jobs/{}".format( - self._cb.credentials.org_key, - self._query_token, - ) - result = self._cb.get_object(status_url) - - searchers_contacted = result.get("contacted", 0) - searchers_completed = result.get("completed", 0) - log.debug("contacted = {}, completed = {}".format(searchers_contacted, searchers_completed)) - if searchers_contacted == 0: - return True - if searchers_completed < searchers_contacted: - if self._timeout != 0 and (time.time() * 1000) - self._submit_time > self._timeout: - self._timed_out = True - return False - return True - - return False - - def _count(self): - if self._count_valid: - return self._total_results - - while self._still_querying(): - time.sleep(.5) - - if self._timed_out: - raise TimeoutError(message="user-specified timeout exceeded while waiting for results") - - result_url = "/api/investigate/v2/orgs/{}/processes/search_jobs/{}/results".format( - self._cb.credentials.org_key, - self._query_token, - ) - result = self._cb.get_object(result_url) - - self._total_results = result.get('num_available', 0) - self._count_valid = True - - return self._total_results - - def _search(self, start=0, rows=0): - if not self._query_token: - self._submit() - - while self._still_querying(): - time.sleep(.5) - - if self._timed_out: - raise TimeoutError(message="user-specified timeout exceeded while waiting for results") - - log.debug("Pulling results, timed_out={}".format(self._timed_out)) - - current = start - rows_fetched = 0 - still_fetching = True - result_url_template = "/api/investigate/v2/orgs/{}/processes/search_jobs/{}/results".format( - self._cb.credentials.org_key, - self._query_token - ) - query_parameters = {} - while still_fetching: - result_url = '{}?start={}&rows={}'.format( - result_url_template, - current, - 10 # Batch gets to reduce API calls - ) - - result = self._cb.get_object(result_url, query_parameters=query_parameters) - - self._total_results = result.get('num_available', 0) - self._count_valid = True - - results = result.get('results', []) - - for item in results: - yield item - current += 1 - rows_fetched += 1 - - if rows and rows_fetched >= rows: - still_fetching = False - break - - if current >= self._total_results: - still_fetching = False - - log.debug("current: {}, total_results: {}".format(current, self._total_results)) - - -class TreeQuery(BaseQuery): - """ Represents the logic for a Tree query. - """ - def __init__(self, doc_class, cb): - super(TreeQuery, self).__init__() - self._doc_class = doc_class - self._cb = cb - self._args = {} - - def where(self, **kwargs): - """Adds a conjunctive filter to this *TreeQuery*. - - Example:: - - >>> cb.select(Tree).where(process_guid="...") - - :param: kwargs: Arguments to invoke the *TreeQuery* with. - :return: this *TreeQuery* - :rtype: :py:class:`TreeQuery` - """ - self._args = dict(self._args, **kwargs) - return self - - def and_(self, **kwargs): - """Adds a conjunctive filter to this *TreeQuery*. - - :param: kwargs: Arguments to invoke the *TreeQuery* with. - :return: this *TreeQuery* - :rtype: :py:class:`TreeQuery` - """ - self.where(**kwargs) - return self - - def or_(self, **kwargs): - """Unsupported. Will raise if called. - - :raise: :py:class:`ApiError` - """ - raise ApiError(".or_() cannot be called on Tree queries") - - def _perform_query(self): - if "process_guid" not in self._args: - raise ApiError("required parameter process_guid missing") - - log.debug("Fetching process tree") - - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) - results = self._cb.get_object(url, query_parameters=self._args) - - while results["incomplete_results"]: - result = self._cb.get_object(url, query_parameters=self._args) - results["nodes"]["children"].extend(result["nodes"]["children"]) - results["incomplete_results"] = result["incomplete_results"] - - return results - - -class FeedQuery(SimpleQuery): - """Represents the logic for a :py:class:`Feed` query. - - >>> cb.select(Feed) - >>> cb.select(Feed, id) - >>> cb.select(Feed).where(include_public=True) - """ - def __init__(self, doc_class, cb): - super(FeedQuery, self).__init__(doc_class, cb) - self._args = {} - - def where(self, **kwargs): - self._args = dict(self._args, **kwargs) - return self - - @property - def results(self): - log.debug("Fetching all feeds") - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) - resp = self._cb.get_object(url, query_parameters=self._args) - results = resp.get("results", []) - return [self._doc_class(self._cb, initial_data=item) for item in results] - - -class ReportQuery(SimpleQuery): - """Represents the logic for a :py:class:`Report` query. - - >>> cb.select(Report).where(feed_id=id) - - .. NOTE:: - Only feed reports can be queried. Watchlist reports - should be interacted with via :py:meth:`Watchlist.reports`. - """ - def __init__(self, doc_class, cb): - super(ReportQuery, self).__init__(doc_class, cb) - self._args = {} - - def where(self, **kwargs): - self._args = dict(self._args, **kwargs) - return self - - @property - def results(self): - if "feed_id" not in self._args: - raise ApiError("required parameter feed_id missing") - - feed_id = self._args["feed_id"] - - log.debug("Fetching all reports") - url = self._doc_class.urlobject.format( - self._cb.credentials.org_key, - feed_id, - ) - resp = self._cb.get_object(url) - results = resp.get("results", []) - return [self._doc_class(self._cb, initial_data=item, feed_id=feed_id) for item in results] - - -class WatchlistQuery(SimpleQuery): - """Represents the logic for a :py:class:`Watchlist` query. - - >>> cb.select(Watchlist) - """ - def __init__(self, doc_class, cb): - super(WatchlistQuery, self).__init__(doc_class, cb) - - @property - def results(self): - log.debug("Fetching all watchlists") - - resp = self._cb.get_object(self._doc_class.urlobject) - results = resp.get("results", []) - return [self._doc_class(self._cb, initial_data=item) for item in results] diff --git a/src/cbapi/psc/threathunter/rest_api.py b/src/cbapi/psc/threathunter/rest_api.py deleted file mode 100644 index b163d6ba..00000000 --- a/src/cbapi/psc/threathunter/rest_api.py +++ /dev/null @@ -1,115 +0,0 @@ -from cbapi.psc.threathunter.query import Query -from cbapi.psc.rest_api import CbPSCBaseAPI -from cbapi.psc.threathunter.models import ReportSeverity -from cbapi.errors import CredentialError -import logging - -log = logging.getLogger(__name__) - - -class CbThreatHunterAPI(CbPSCBaseAPI): - """The main entry point into the Carbon Black Cloud Enterprise EDR API. - - :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. - Uses the profile named 'default' when not specified. - - Usage:: - - >>> from cbapi.psc.threathunter import CbThreatHunterAPI - >>> cb = CbThreatHunterAPI(profile="production") - """ - def __init__(self, *args, **kwargs): - super(CbThreatHunterAPI, self).__init__(*args, **kwargs) - - if not self.credentials.get("org_key", None): - raise CredentialError("No organization key specified") - - def _perform_query(self, cls, **kwargs): - if hasattr(cls, "_query_implementation"): - return cls._query_implementation(self) - else: - return Query(cls, self, **kwargs) - - def create(self, cls, data=None): - """Creates a new model. - - >>> feed = cb.create(Feed, feed_data) - - :param cls: The model being created - :param data: The data to pre-populate the model with - :type data: dict(str, object) - :return: an instance of `cls` - """ - return cls(self, initial_data=data) - - def validate_query(self, query): - """Validates the given IOC query. - - >>> cb.validate_query("process_name:chrome.exe") # True - - :param str query: the query to validate - :return: whether or not the query is valid - :rtype: bool - """ - args = {"q": query} - url = "/threathunter/search/v1/orgs/{}/processes/search_validation".format( - self.credentials.org_key - ) - resp = self.get_object(url, query_parameters=args) - - return resp.get("valid", False) - - def convert_query(self, query): - """Converts a legacy Carbon Black EDR query to an Enterprise EDR query. - - :param str query: the query to convert - :return: the converted query - :rtype: str - """ - args = {"query": query} - resp = self.post_object("/threathunter/feedmgr/v2/query/translate", args).json() - - return resp.get("query") - - @property - def custom_severities(self): - """Returns a list of active :py:class:`ReportSeverity` instances - - :rtype: list[:py:class:`ReportSeverity`] - """ - # TODO(ww): There's probably a better place to put this. - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/severity".format( - self.credentials.org_key - ) - resp = self.get_object(url) - items = resp.get("results", []) - return [self.create(ReportSeverity, item) for item in items] - - def queries(self): - """Retrieves a list of queries, active or complete, known by - the Enterprise EDR server. - - :return: a list of query ids - :rtype: list(str) - """ - url = "/threathunter/search/v1/orgs/{}/processes/search_jobs".format( - self.credentials.org_key - ) - ids = self.get_object(url) - return ids.get("query_ids", []) - - def limits(self): - """Returns a dictionary containing API limiting information. - - Example: - - >>> cb.limits() - {u'status_code': 200, u'time_bounds': {u'upper': 1545335070095, u'lower': 1542779216139}} - - :return: a dict of limiting information - :rtype: dict(str, str) - """ - url = "/threathunter/search/v1/orgs/{}/processes/limits".format( - self.credentials.org_key - ) - return self.get_object(url) diff --git a/test/cbapi/psc/test_alertsv6_api.py b/test/cbapi/psc/test_alertsv6_api.py deleted file mode 100755 index c787bba5..00000000 --- a/test/cbapi/psc/test_alertsv6_api.py +++ /dev/null @@ -1,535 +0,0 @@ -import pytest -from cbapi.errors import ApiError -from cbapi.psc.models import BaseAlert, CBAnalyticsAlert, VMwareAlert, WatchlistAlert, WorkflowStatus -from cbapi.psc.rest_api import CbPSCBaseAPI -from test.cbtest import StubResponse, patch_cbapi - - -def test_query_basealert_with_all_bells_and_whistles(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/_search" - assert body == {"query": "Blort", - "criteria": {"category": ["SERIOUS", "CRITICAL"], "device_id": [6023], "device_name": ["HAL"], - "device_os": ["LINUX"], "device_os_version": ["0.1.2"], - "device_username": ["JRN"], "group_results": True, "id": ["S0L0"], - "legacy_alert_id": ["S0L0_1"], "minimum_severity": 6, "policy_id": [8675309], - "policy_name": ["Strict"], "process_name": ["IEXPLORE.EXE"], - "process_sha256": ["0123456789ABCDEF0123456789ABCDEF"], - "reputation": ["SUSPECT_MALWARE"], "tag": ["Frood"], "target_value": ["HIGH"], - "threat_id": ["B0RG"], "type": ["WATCHLIST"], "workflow": ["OPEN"]}, - "sort": [{"field": "name", "order": "DESC"}]} - _was_called = True - return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", - "workflow": {"state": "OPEN"}}], "num_found": 1}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.select(BaseAlert).where("Blort").set_categories(["SERIOUS", "CRITICAL"]).set_device_ids([6023]) \ - .set_device_names(["HAL"]).set_device_os(["LINUX"]).set_device_os_versions(["0.1.2"]) \ - .set_device_username(["JRN"]).set_group_results(True).set_alert_ids(["S0L0"]) \ - .set_legacy_alert_ids(["S0L0_1"]).set_minimum_severity(6).set_policy_ids([8675309]) \ - .set_policy_names(["Strict"]).set_process_names(["IEXPLORE.EXE"]) \ - .set_process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).set_reputations(["SUSPECT_MALWARE"]) \ - .set_tags(["Frood"]).set_target_priorities(["HIGH"]).set_threat_ids(["B0RG"]).set_types(["WATCHLIST"]) \ - .set_workflows(["OPEN"]).sort_by("name", "DESC") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_basealert_with_create_time_as_start_end(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/_search" - assert body == {"query": "Blort", - "criteria": {"create_time": {"start": "2019-09-30T12:34:56", "end": "2019-10-01T12:00:12"}}} - _was_called = True - return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", - "workflow": {"state": "OPEN"}}], "num_found": 1}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.select(BaseAlert).where("Blort").set_create_time(start="2019-09-30T12:34:56", - end="2019-10-01T12:00:12") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_basealert_with_create_time_as_range(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/_search" - assert body == {"query": "Blort", "criteria": {"create_time": {"range": "-3w"}}} - _was_called = True - return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", - "workflow": {"state": "OPEN"}}], "num_found": 1}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.select(BaseAlert).where("Blort").set_create_time(range="-3w") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_basealert_facets(monkeypatch): - _was_called = False - - def _run_facet_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/_facet" - assert body["query"] == "Blort" - t = body["criteria"] - assert t["workflow"] == ["OPEN"] - t = body["terms"] - assert t["rows"] == 0 - assert t["fields"] == ["REPUTATION", "STATUS"] - _was_called = True - return StubResponse({"results": [{"field": {}, - "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, - {"field": {}, - "values": [{"id": "status", "name": "statusX", "total": 9}]}]}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_facet_query) - query = api.select(BaseAlert).where("Blort").set_workflows(["OPEN"]) - f = query.facets(["REPUTATION", "STATUS"]) - assert _was_called - assert f == [{"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, - {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]}] - - -def test_query_basealert_invalid_create_time_combinations(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(BaseAlert).set_create_time() - with pytest.raises(ApiError): - api.select(BaseAlert).set_create_time(start="2019-09-30T12:34:56", - end="2019-10-01T12:00:12", range="-3w") - with pytest.raises(ApiError): - api.select(BaseAlert).set_create_time(start="2019-09-30T12:34:56", range="-3w") - with pytest.raises(ApiError): - api.select(BaseAlert).set_create_time(end="2019-10-01T12:00:12", range="-3w") - - -def test_query_basealert_invalid_criteria_values(): - tests = [ - {"method": "set_categories", "arg": ["DOUBLE_DARE"]}, - {"method": "set_device_ids", "arg": ["Bogus"]}, - {"method": "set_device_names", "arg": [42]}, - {"method": "set_device_os", "arg": ["TI994A"]}, - {"method": "set_device_os_versions", "arg": [8808]}, - {"method": "set_device_username", "arg": [-1]}, - {"method": "set_alert_ids", "arg": [9001]}, - {"method": "set_legacy_alert_ids", "arg": [9001]}, - {"method": "set_policy_ids", "arg": ["Bogus"]}, - {"method": "set_policy_names", "arg": [323]}, - {"method": "set_process_names", "arg": [7071]}, - {"method": "set_process_sha256", "arg": [123456789]}, - {"method": "set_reputations", "arg": ["MICROSOFT_FUDWARE"]}, - {"method": "set_tags", "arg": [-1]}, - {"method": "set_target_priorities", "arg": ["DOGWASH"]}, - {"method": "set_threat_ids", "arg": [4096]}, - {"method": "set_types", "arg": ["ERBOSOFT"]}, - {"method": "set_workflows", "arg": ["IN_LIMBO"]}, - ] - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - query = api.select(BaseAlert) - for t in tests: - meth = getattr(query, t["method"], None) - with pytest.raises(ApiError): - meth(t["arg"]) - - -def test_query_cbanalyticsalert_with_all_bells_and_whistles(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/_search" - assert body == {"query": "Blort", - "criteria": {"category": ["SERIOUS", "CRITICAL"], "device_id": [6023], "device_name": ["HAL"], - "device_os": ["LINUX"], "device_os_version": ["0.1.2"], - "device_username": ["JRN"], "group_results": True, "id": ["S0L0"], - "legacy_alert_id": ["S0L0_1"], "minimum_severity": 6, "policy_id": [8675309], - "policy_name": ["Strict"], "process_name": ["IEXPLORE.EXE"], - "process_sha256": ["0123456789ABCDEF0123456789ABCDEF"], - "reputation": ["SUSPECT_MALWARE"], "tag": ["Frood"], "target_value": ["HIGH"], - "threat_id": ["B0RG"], "type": ["WATCHLIST"], "workflow": ["OPEN"], - "blocked_threat_category": ["RISKY_PROGRAM"], "device_location": ["ONSITE"], - "kill_chain_status": ["EXECUTE_GOAL"], - "not_blocked_threat_category": ["NEW_MALWARE"], "policy_applied": ["APPLIED"], - "reason_code": ["ATTACK_VECTOR"], "run_state": ["RAN"], "sensor_action": ["DENY"], - "threat_cause_vector": ["WEB"]}, "sort": [{"field": "name", "order": "DESC"}]} - _was_called = True - return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", - "workflow": {"state": "OPEN"}}], "num_found": 1}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.select(CBAnalyticsAlert).where("Blort").set_categories(["SERIOUS", "CRITICAL"]) \ - .set_device_ids([6023]).set_device_names(["HAL"]).set_device_os(["LINUX"]).set_device_os_versions(["0.1.2"]) \ - .set_device_username(["JRN"]).set_group_results(True).set_alert_ids(["S0L0"]).set_legacy_alert_ids(["S0L0_1"]) \ - .set_minimum_severity(6).set_policy_ids([8675309]).set_policy_names(["Strict"]) \ - .set_process_names(["IEXPLORE.EXE"]).set_process_sha256(["0123456789ABCDEF0123456789ABCDEF"]) \ - .set_reputations(["SUSPECT_MALWARE"]).set_tags(["Frood"]).set_target_priorities(["HIGH"]) \ - .set_threat_ids(["B0RG"]).set_types(["WATCHLIST"]).set_workflows(["OPEN"]) \ - .set_blocked_threat_categories(["RISKY_PROGRAM"]).set_device_locations(["ONSITE"]) \ - .set_kill_chain_statuses(["EXECUTE_GOAL"]).set_not_blocked_threat_categories(["NEW_MALWARE"]) \ - .set_policy_applied(["APPLIED"]).set_reason_code(["ATTACK_VECTOR"]).set_run_states(["RAN"]) \ - .set_sensor_actions(["DENY"]).set_threat_cause_vectors(["WEB"]).sort_by("name", "DESC") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_cbanalyticsalert_facets(monkeypatch): - _was_called = False - - def _run_facet_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/_facet" - assert body == {"query": "Blort", "criteria": {"workflow": ["OPEN"]}, - "terms": {"rows": 0, "fields": ["REPUTATION", "STATUS"]}} - _was_called = True - return StubResponse({"results": [{"field": {}, - "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, - {"field": {}, - "values": [{"id": "status", "name": "statusX", "total": 9}]}]}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_facet_query) - query = api.select(CBAnalyticsAlert).where("Blort").set_workflows(["OPEN"]) - f = query.facets(["REPUTATION", "STATUS"]) - assert _was_called - assert f == [{"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, - {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]}] - - -def test_query_cbanalyticsalert_invalid_criteria_values(): - tests = [ - {"method": "set_blocked_threat_categories", "arg": ["MINOR"]}, - {"method": "set_device_locations", "arg": ["NARNIA"]}, - {"method": "set_kill_chain_statuses", "arg": ["SPAWN_COPIES"]}, - {"method": "set_not_blocked_threat_categories", "arg": ["MINOR"]}, - {"method": "set_policy_applied", "arg": ["MAYBE"]}, - {"method": "set_reason_code", "arg": [55]}, - {"method": "set_run_states", "arg": ["MIGHT_HAVE"]}, - {"method": "set_sensor_actions", "arg": ["FLIP_A_COIN"]}, - {"method": "set_threat_cause_vectors", "arg": ["NETWORK"]} - ] - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - query = api.select(CBAnalyticsAlert) - for t in tests: - meth = getattr(query, t["method"], None) - with pytest.raises(ApiError): - meth(t["arg"]) - - -def test_query_vmwarealert_with_all_bells_and_whistles(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/vmware/_search" - assert body == {"query": "Blort", - "criteria": {"category": ["SERIOUS", "CRITICAL"], "device_id": [6023], "device_name": ["HAL"], - "device_os": ["LINUX"], "device_os_version": ["0.1.2"], - "device_username": ["JRN"], "group_results": True, "id": ["S0L0"], - "legacy_alert_id": ["S0L0_1"], "minimum_severity": 6, "policy_id": [8675309], - "policy_name": ["Strict"], "process_name": ["IEXPLORE.EXE"], - "process_sha256": ["0123456789ABCDEF0123456789ABCDEF"], - "reputation": ["SUSPECT_MALWARE"], "tag": ["Frood"], "target_value": ["HIGH"], - "threat_id": ["B0RG"], "type": ["WATCHLIST"], "workflow": ["OPEN"], - "group_id": [14]}, "sort": [{"field": "name", "order": "DESC"}]} - _was_called = True - return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", - "workflow": {"state": "OPEN"}}], "num_found": 1}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.select(VMwareAlert).where("Blort").set_categories(["SERIOUS", "CRITICAL"]).set_device_ids([6023]) \ - .set_device_names(["HAL"]).set_device_os(["LINUX"]).set_device_os_versions(["0.1.2"]) \ - .set_device_username(["JRN"]).set_group_results(True).set_alert_ids(["S0L0"]) \ - .set_legacy_alert_ids(["S0L0_1"]).set_minimum_severity(6).set_policy_ids([8675309]) \ - .set_policy_names(["Strict"]).set_process_names(["IEXPLORE.EXE"]) \ - .set_process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).set_reputations(["SUSPECT_MALWARE"]) \ - .set_tags(["Frood"]).set_target_priorities(["HIGH"]).set_threat_ids(["B0RG"]).set_types(["WATCHLIST"]) \ - .set_workflows(["OPEN"]).set_group_ids([14]).sort_by("name", "DESC") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_vmwarealert_facets(monkeypatch): - _was_called = False - - def _run_facet_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/vmware/_facet" - assert body == {"query": "Blort", "criteria": {"workflow": ["OPEN"]}, - "terms": {"rows": 0, "fields": ["REPUTATION", "STATUS"]}} - _was_called = True - return StubResponse({"results": [{"field": {}, - "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, - {"field": {}, - "values": [{"id": "status", "name": "statusX", "total": 9}]}]}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_facet_query) - query = api.select(VMwareAlert).where("Blort").set_workflows(["OPEN"]) - f = query.facets(["REPUTATION", "STATUS"]) - assert _was_called - assert f == [{"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, - {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]}] - - -def test_query_vmwarealert_invalid_group_ids(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(VMwareAlert).set_group_ids(["Bogus"]) - - -def test_query_watchlistalert_with_all_bells_and_whistles(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/_search" - assert body == {"query": "Blort", - "criteria": {"category": ["SERIOUS", "CRITICAL"], "device_id": [6023], "device_name": ["HAL"], - "device_os": ["LINUX"], "device_os_version": ["0.1.2"], - "device_username": ["JRN"], "group_results": True, "id": ["S0L0"], - "legacy_alert_id": ["S0L0_1"], "minimum_severity": 6, "policy_id": [8675309], - "policy_name": ["Strict"], "process_name": ["IEXPLORE.EXE"], - "process_sha256": ["0123456789ABCDEF0123456789ABCDEF"], - "reputation": ["SUSPECT_MALWARE"], "tag": ["Frood"], "target_value": ["HIGH"], - "threat_id": ["B0RG"], "type": ["WATCHLIST"], "workflow": ["OPEN"], - "watchlist_id": ["100"], "watchlist_name": ["Gandalf"]}, - "sort": [{"field": "name", "order": "DESC"}]} - _was_called = True - return StubResponse({"results": [{"id": "S0L0", "org_key": "Z100", "threat_id": "B0RG", - "workflow": {"state": "OPEN"}}], "num_found": 1}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.select(WatchlistAlert).where("Blort").set_categories(["SERIOUS", "CRITICAL"]).set_device_ids([6023]) \ - .set_device_names(["HAL"]).set_device_os(["LINUX"]).set_device_os_versions(["0.1.2"]) \ - .set_device_username(["JRN"]).set_group_results(True).set_alert_ids(["S0L0"]) \ - .set_legacy_alert_ids(["S0L0_1"]).set_minimum_severity(6).set_policy_ids([8675309]) \ - .set_policy_names(["Strict"]).set_process_names(["IEXPLORE.EXE"]) \ - .set_process_sha256(["0123456789ABCDEF0123456789ABCDEF"]).set_reputations(["SUSPECT_MALWARE"]) \ - .set_tags(["Frood"]).set_target_priorities(["HIGH"]).set_threat_ids(["B0RG"]).set_types(["WATCHLIST"]) \ - .set_workflows(["OPEN"]).set_watchlist_ids(["100"]).set_watchlist_names(["Gandalf"]).sort_by("name", "DESC") - a = query.one() - assert _was_called - assert a.id == "S0L0" - assert a.org_key == "Z100" - assert a.threat_id == "B0RG" - assert a.workflow_.state == "OPEN" - - -def test_query_watchlistalert_facets(monkeypatch): - _was_called = False - - def _run_facet_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/_facet" - assert body == {"query": "Blort", "criteria": {"workflow": ["OPEN"]}, - "terms": {"rows": 0, "fields": ["REPUTATION", "STATUS"]}} - _was_called = True - return StubResponse({"results": [{"field": {}, - "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, - {"field": {}, - "values": [{"id": "status", "name": "statusX", "total": 9}]}]}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_facet_query) - query = api.select(WatchlistAlert).where("Blort").set_workflows(["OPEN"]) - f = query.facets(["REPUTATION", "STATUS"]) - assert _was_called - assert f == [{"field": {}, "values": [{"id": "reputation", "name": "reputationX", "total": 4}]}, - {"field": {}, "values": [{"id": "status", "name": "statusX", "total": 9}]}] - - -def test_query_watchlistalert_invalid_criteria_values(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(WatchlistAlert).set_watchlist_ids([888]) - with pytest.raises(ApiError): - api.select(WatchlistAlert).set_watchlist_names([69]) - - -def test_alerts_bulk_dismiss(monkeypatch): - _was_called = False - - def _do_dismiss(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/workflow/_criteria" - assert body == {"query": "Blort", "state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir", - "criteria": {"device_name": ["HAL9000"]}} - _was_called = True - return StubResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_do_dismiss) - q = api.select(BaseAlert).where("Blort").set_device_names(["HAL9000"]) - reqid = q.dismiss("Fixed", "Yessir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_undismiss(monkeypatch): - _was_called = False - - def _do_update(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/workflow/_criteria" - assert body == {"query": "Blort", "state": "OPEN", "remediation_state": "Fixed", "comment": "NoSir", - "criteria": {"device_name": ["HAL9000"]}} - _was_called = True - return StubResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_do_update) - q = api.select(BaseAlert).where("Blort").set_device_names(["HAL9000"]) - reqid = q.update("Fixed", "NoSir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_dismiss_watchlist(monkeypatch): - _was_called = False - - def _do_dismiss(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/watchlist/workflow/_criteria" - assert body == {"query": "Blort", "state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir", - "criteria": {"device_name": ["HAL9000"]}} - _was_called = True - return StubResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_do_dismiss) - q = api.select(WatchlistAlert).where("Blort").set_device_names(["HAL9000"]) - reqid = q.dismiss("Fixed", "Yessir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_dismiss_cbanalytics(monkeypatch): - _was_called = False - - def _do_dismiss(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/cbanalytics/workflow/_criteria" - assert body == {"query": "Blort", "state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir", - "criteria": {"device_name": ["HAL9000"]}} - _was_called = True - return StubResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_do_dismiss) - q = api.select(CBAnalyticsAlert).where("Blort").set_device_names(["HAL9000"]) - reqid = q.dismiss("Fixed", "Yessir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_dismiss_vmware(monkeypatch): - _was_called = False - - def _do_dismiss(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/vmware/workflow/_criteria" - assert body == {"query": "Blort", "state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir", - "criteria": {"device_name": ["HAL9000"]}} - _was_called = True - return StubResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_do_dismiss) - q = api.select(VMwareAlert).where("Blort").set_device_names(["HAL9000"]) - reqid = q.dismiss("Fixed", "Yessir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_dismiss_threat(monkeypatch): - _was_called = False - - def _do_dismiss(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/threat/workflow/_criteria" - assert body == {"threat_id": ["B0RG", "F3R3NG1"], "state": "DISMISSED", "remediation_state": "Fixed", - "comment": "Yessir"} - _was_called = True - return StubResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_do_dismiss) - reqid = api.bulk_threat_dismiss(["B0RG", "F3R3NG1"], "Fixed", "Yessir") - assert _was_called - assert reqid == "497ABX" - - -def test_alerts_bulk_undismiss_threat(monkeypatch): - _was_called = False - - def _do_update(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/threat/workflow/_criteria" - assert body == {"threat_id": ["B0RG", "F3R3NG1"], "state": "OPEN", "remediation_state": "Fixed", - "comment": "NoSir"} - _was_called = True - return StubResponse({"request_id": "497ABX"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_do_update) - reqid = api.bulk_threat_update(["B0RG", "F3R3NG1"], "Fixed", "NoSir") - assert _was_called - assert reqid == "497ABX" - - -def test_load_workflow(monkeypatch): - _was_called = False - - def _get_workflow(url, parms=None, default=None): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/workflow/status/497ABX" - _was_called = True - return {"errors": [], "failed_ids": [], "id": "497ABX", "num_hits": 0, "num_success": 0, "status": "QUEUED", - "workflow": {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}} - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_workflow) - workflow = api.select(WorkflowStatus, "497ABX") - assert _was_called - assert workflow.id_ == "497ABX" diff --git a/test/cbapi/psc/test_models.py b/test/cbapi/psc/test_models.py index 4b86dede..c9158fa7 100755 --- a/test/cbapi/psc/test_models.py +++ b/test/cbapi/psc/test_models.py @@ -1,5 +1,5 @@ import pytest -from cbapi.psc.models import Device, BaseAlert, WorkflowStatus +from cbapi.psc.models import Device from cbapi.psc.rest_api import CbPSCBaseAPI from test.cbtest import StubResponse, patch_cbapi @@ -177,137 +177,3 @@ def _update_sensor_version(url, body, **kwargs): dev = Device(api, 6023, {"id": 6023}) dev.update_sensor_version({"RHEL": "2.3.4.5"}) assert _was_called - - -def test_BaseAlert_dismiss(monkeypatch): - _was_called = False - - def _do_dismiss(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/ESD14U2C/workflow" - assert body == {"state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir"} - _was_called = True - return StubResponse({"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_do_dismiss) - alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "workflow": {"state": "OPEN"}}) - alert.dismiss("Fixed", "Yessir") - assert _was_called - assert alert.workflow_.changed_by == "Robocop" - assert alert.workflow_.state == "DISMISSED" - assert alert.workflow_.remediation == "Fixed" - assert alert.workflow_.comment == "Yessir" - assert alert.workflow_.last_update_time == "2019-10-31T16:03:13.951Z" - - -def test_BaseAlert_undismiss(monkeypatch): - _was_called = False - - def _do_update(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/alerts/ESD14U2C/workflow" - assert body == {"state": "OPEN", "remediation_state": "Fixed", "comment": "NoSir"} - _was_called = True - return StubResponse({"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_do_update) - alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "workflow": {"state": "DISMISS"}}) - alert.update("Fixed", "NoSir") - assert _was_called - assert alert.workflow_.changed_by == "Robocop" - assert alert.workflow_.state == "OPEN" - assert alert.workflow_.remediation == "Fixed" - assert alert.workflow_.comment == "NoSir" - assert alert.workflow_.last_update_time == "2019-10-31T16:03:13.951Z" - - -def test_BaseAlert_dismiss_threat(monkeypatch): - _was_called = False - - def _do_dismiss(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/threat/B0RG/workflow" - assert body == {"state": "DISMISSED", "remediation_state": "Fixed", "comment": "Yessir"} - _was_called = True - return StubResponse({"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_do_dismiss) - alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "threat_id": "B0RG", "workflow": {"state": "OPEN"}}) - wf = alert.dismiss_threat("Fixed", "Yessir") - assert _was_called - assert wf.changed_by == "Robocop" - assert wf.state == "DISMISSED" - assert wf.remediation == "Fixed" - assert wf.comment == "Yessir" - assert wf.last_update_time == "2019-10-31T16:03:13.951Z" - - -def test_BaseAlert_undismiss_threat(monkeypatch): - _was_called = False - - def _do_update(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/threat/B0RG/workflow" - assert body == {"state": "OPEN", "remediation_state": "Fixed", "comment": "NoSir"} - _was_called = True - return StubResponse({"state": "OPEN", "remediation": "Fixed", "comment": "NoSir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_do_update) - alert = BaseAlert(api, "ESD14U2C", {"id": "ESD14U2C", "threat_id": "B0RG", "workflow": {"state": "OPEN"}}) - wf = alert.update_threat("Fixed", "NoSir") - assert _was_called - assert wf.changed_by == "Robocop" - assert wf.state == "OPEN" - assert wf.remediation == "Fixed" - assert wf.comment == "NoSir" - assert wf.last_update_time == "2019-10-31T16:03:13.951Z" - - -def test_WorkflowStatus(monkeypatch): - _times_called = 0 - - def _get_workflow(url, parms=None, default=None): - nonlocal _times_called - assert url == "/appservices/v6/orgs/Z100/workflow/status/W00K13" - if _times_called >= 0 and _times_called <= 3: - _stat = "QUEUED" - elif _times_called >= 4 and _times_called <= 6: - _stat = "IN_PROGRESS" - elif _times_called >= 7 and _times_called <= 9: - _stat = "FINISHED" - else: - pytest.fail("_get_workflow called too many times") - _times_called = _times_called + 1 - return {"errors": [], "failed_ids": [], "id": "W00K13", "num_hits": 0, "num_success": 0, "status": _stat, - "workflow": {"state": "DISMISSED", "remediation": "Fixed", "comment": "Yessir", - "changed_by": "Robocop", "last_update_time": "2019-10-31T16:03:13.951Z"}} - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_workflow) - wfstat = WorkflowStatus(api, "W00K13") - assert wfstat.workflow_.changed_by == "Robocop" - assert wfstat.workflow_.state == "DISMISSED" - assert wfstat.workflow_.remediation == "Fixed" - assert wfstat.workflow_.comment == "Yessir" - assert wfstat.workflow_.last_update_time == "2019-10-31T16:03:13.951Z" - assert _times_called == 1 - assert wfstat.queued - assert not wfstat.in_progress - assert not wfstat.finished - assert _times_called == 4 - assert not wfstat.queued - assert wfstat.in_progress - assert not wfstat.finished - assert _times_called == 7 - assert not wfstat.queued - assert not wfstat.in_progress - assert wfstat.finished - assert _times_called == 10 diff --git a/tests/test_defense_policy.py b/tests/test_defense_policy.py deleted file mode 100644 index 48653764..00000000 --- a/tests/test_defense_policy.py +++ /dev/null @@ -1,53 +0,0 @@ -import sys -import time - -import glob -import json -import os -import unittest - -from cbapi.psc.defense import * - - -sys.path.append(os.path.dirname(__file__)) -import requests_cache - -@unittest.skip("temporarily disabled") -def test_policy(rulefiles): - requests_cache.uninstall_cache() - defense_api = CbDefenseAPI(profile="test") - - - default_policies = [policy for policy in defense_api.select(Policy) if policy.name == "default"] - new_policy = defense_api.create(Policy) - new_policy.policy = default_policies[0].policy - new_policy.name = "cbapi-python-test-%d" % time.time() - new_policy.priorityLevel = "LOW" - new_policy.description = "Test policy" - new_policy.version = 2 - new_policy.save() - - for t in rulefiles: - try: - test_rule(new_policy, t) - print("Added rule %s" % t) - except Exception as e: - print("Exception adding rule %s: %s" % (t, e)) - - new_policy.delete() - - -@unittest.skip("temporarily disabled") -def test_rule(new_policy, fn): - new_rule = json.load(open(fn, "r")) - new_policy.add_rule(new_rule) - - -if __name__ == '__main__': - rulefiles = glob.glob(os.path.join(os.path.dirname(__file__), "data", "defense", "policy_rules", "*.json")) - print(rulefiles) - - test_policy(rulefiles) - - - unittest.main() From 223815cc77e280593f34ed49d209b46b9b48a601 Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Thu, 25 Jul 2024 15:59:34 +0300 Subject: [PATCH 195/197] more cleanup --- examples/livequery/manage_run.py | 149 ------- examples/livequery/run_device_summary.py | 88 ---- examples/livequery/run_facets.py | 106 ----- examples/livequery/run_search.py | 73 ---- src/cbapi/__init__.py | 2 - src/cbapi/example_helpers.py | 31 -- src/cbapi/psc/__init__.py | 6 - src/cbapi/psc/base_query.py | 271 ------------- src/cbapi/psc/devices_query.py | 361 ----------------- src/cbapi/psc/livequery/__init__.py | 7 - src/cbapi/psc/livequery/models.py | 294 -------------- .../psc/livequery/models/device_summary.yaml | 52 --- src/cbapi/psc/livequery/models/facet.yaml | 22 - src/cbapi/psc/livequery/models/result.yaml | 41 -- src/cbapi/psc/livequery/models/run.yaml | 86 ---- src/cbapi/psc/livequery/query.py | 375 ----------------- src/cbapi/psc/livequery/rest_api.py | 36 -- src/cbapi/psc/models.py | 181 --------- src/cbapi/psc/models/device.yaml | 310 -------------- src/cbapi/psc/rest_api.py | 191 --------- test/cbapi/__init__.py | 0 test/cbapi/psc/__init__.py | 0 test/cbapi/psc/livequery/__init__.py | 0 test/cbapi/psc/livequery/test_models.py | 234 ----------- test/cbapi/psc/livequery/test_rest_api.py | 161 -------- test/cbapi/psc/test_devicev6_api.py | 383 ------------------ test/cbapi/psc/test_models.py | 179 -------- 27 files changed, 3639 deletions(-) delete mode 100644 examples/livequery/manage_run.py delete mode 100755 examples/livequery/run_device_summary.py delete mode 100755 examples/livequery/run_facets.py delete mode 100644 examples/livequery/run_search.py delete mode 100644 src/cbapi/psc/__init__.py delete mode 100755 src/cbapi/psc/base_query.py delete mode 100755 src/cbapi/psc/devices_query.py delete mode 100644 src/cbapi/psc/livequery/__init__.py delete mode 100644 src/cbapi/psc/livequery/models.py delete mode 100755 src/cbapi/psc/livequery/models/device_summary.yaml delete mode 100755 src/cbapi/psc/livequery/models/facet.yaml delete mode 100644 src/cbapi/psc/livequery/models/result.yaml delete mode 100644 src/cbapi/psc/livequery/models/run.yaml delete mode 100644 src/cbapi/psc/livequery/query.py delete mode 100644 src/cbapi/psc/livequery/rest_api.py delete mode 100755 src/cbapi/psc/models.py delete mode 100755 src/cbapi/psc/models/device.yaml delete mode 100755 src/cbapi/psc/rest_api.py delete mode 100755 test/cbapi/__init__.py delete mode 100755 test/cbapi/psc/__init__.py delete mode 100755 test/cbapi/psc/livequery/__init__.py delete mode 100755 test/cbapi/psc/livequery/test_models.py delete mode 100755 test/cbapi/psc/livequery/test_rest_api.py delete mode 100755 test/cbapi/psc/test_devicev6_api.py delete mode 100755 test/cbapi/psc/test_models.py diff --git a/examples/livequery/manage_run.py b/examples/livequery/manage_run.py deleted file mode 100644 index 46e67552..00000000 --- a/examples/livequery/manage_run.py +++ /dev/null @@ -1,149 +0,0 @@ -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_livequery_object -from cbapi.psc.livequery.models import Run - - -def create_run(cb, args): - query = cb.query(args.sql) - - if args.device_ids: - query.device_ids(args.device_ids) - if args.device_types: - query.device_types(args.device_types) - if args.policy_ids: - query.policy_ids(args.policy_ids) - if args.notify: - query.notify_on_finish() - if args.name: - query.name(args.name) - - run = query.submit() - print(run) - - -def run_status(cb, args): - run = cb.select(Run, args.id) - print(run) - - -def run_stop(cb, args): - run = cb.select(Run, args.id) - if run.stop(): - print("Run {} has been stopped.".format(run.id)) - print(run) - else: - print("Unable to stop run {}".format(run.id)) - - -def run_delete(cb, args): - run = cb.select(Run, args.id) - if run.delete(): - print("Run {} has been deleted.".format(run.id)) - else: - print("Unable to delete run {}".format(run.id)) - - -def run_history(cb, args): - results = cb.query_history(args.query) - if args.sort_by: - dir = "DESC" if args.descending_results else "ASC" - results.sort_by(args.sort_by, direction=dir) - for result in results: - print(result) - - -def main(): - parser = build_cli_parser("Create and manage LiveQuery runs") - commands = parser.add_subparsers(help="Commands", dest="command_name") - - create_command = commands.add_parser("create", help="Create a new LiveQuery run") - create_command.add_argument( - "-s", "--sql", type=str, required=True, help="The query to run" - ) - create_command.add_argument( - "-n", - "--notify", - action="store_true", - help="Notify by email when the run finishes", - ) - create_command.add_argument( - "-N", "--name", type=str, required=False, help="The name of the run" - ) - create_command.add_argument( - "--device_ids", - nargs="+", - type=int, - required=False, - help="Device IDs to filter on", - ) - create_command.add_argument( - "--device_types", - nargs="+", - type=str, - required=False, - help="Device types to filter on", - ) - create_command.add_argument( - "--policy_ids", - nargs="+", - type=str, - required=False, - help="Policy IDs to filter on", - ) - - status_command = commands.add_parser( - "status", help="Retrieve information about a run" - ) - status_command.add_argument( - "-i", "--id", type=str, required=True, help="The run ID" - ) - - stop_command = commands.add_parser( - "stop", help="Stops/cancels a current run" - ) - stop_command.add_argument( - "-i", "--id", type=str, required=True, help="The run ID" - ) - - delete_command = commands.add_parser( - "delete", help="Permanently delete a run" - ) - delete_command.add_argument( - "-i", "--id", type=str, required=True, help="The run ID" - ) - - history_command = commands.add_parser( - "history", help="List history of all runs" - ) - history_command.add_argument( - "-q", "--query", type=str, required=False, help="Query string to use" - ) - history_command.add_argument( - "-S", "--sort_by", type=str, help="sort by this field", required=False - ) - history_command.add_argument( - "-D", - "--descending_results", - help="return results in descending order", - action="store_true", - required=False - ) - - args = parser.parse_args() - cb = get_cb_livequery_object(args) - - if args.command_name == "create": - return create_run(cb, args) - elif args.command_name == "status": - return run_status(cb, args) - elif args.command_name == "stop": - return run_stop(cb, args) - elif args.command_name == "delete": - return run_delete(cb, args) - elif args.command_name == "history": - return run_history(cb, args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/livequery/run_device_summary.py b/examples/livequery/run_device_summary.py deleted file mode 100755 index 33ee8931..00000000 --- a/examples/livequery/run_device_summary.py +++ /dev/null @@ -1,88 +0,0 @@ -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_livequery_object -from cbapi.psc.livequery.models import Result - - -def main(): - parser = build_cli_parser("Search the device summaries of a LiveQuery run") - parser.add_argument("-i", "--id", type=str, required=True, help="Run ID") - parser.add_argument("-q", "--query", type=str, required=False, help="Search query") - - parser.add_argument( - "--device_ids", - nargs="+", - type=int, - required=False, - help="Device IDs to filter on", - ) - parser.add_argument( - "--device_names", - nargs="+", - type=int, - required=False, - help="Device names to filter on", - ) - parser.add_argument( - "--policy_ids", - nargs="+", - type=int, - required=False, - help="Policy IDs to filter on", - ) - parser.add_argument( - "--policy_names", - nargs="+", - type=int, - required=False, - help="Policy names to filter on", - ) - parser.add_argument( - "--statuses", - nargs="+", - type=str, - required=False, - help="Statuses to filter on", - ) - parser.add_argument( - "-S", "--sort_by", type=str, help="sort by this field", required=False - ) - parser.add_argument( - "-D", - "--descending_results", - help="return results in descending order", - action="store_true", - ) - - args = parser.parse_args() - cb = get_cb_livequery_object(args) - - results = cb.select(Result).run_id(args.id) - result = results.first() - if result is None: - print("ERROR: No results.") - return 1 - - summaries = result.query_device_summaries() - if args.query: - summaries.where(args.query) - if args.device_ids: - summaries.criteria(device_id=args.device_ids) - if args.device_names: - summaries.criteria(device_name=args.device_names) - if args.policy_ids: - summaries.criteria(policy_id=args.policy_ids) - if args.policy_names: - summaries.criteria(policy_name=args.policy_names) - if args.statuses: - summaries.criteria(status=args.statuses) - if args.sort_by: - dir = "DESC" if args.descending_results else "ASC" - summaries.sort_by(args.sort_by, direction=dir) - - for summary in summaries: - print(summary) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/livequery/run_facets.py b/examples/livequery/run_facets.py deleted file mode 100755 index 14bfc6c4..00000000 --- a/examples/livequery/run_facets.py +++ /dev/null @@ -1,106 +0,0 @@ -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_livequery_object -from cbapi.psc.livequery.models import Result - - -def main(): - parser = build_cli_parser("Search the facets of a LiveQuery run") - parser.add_argument("-i", "--id", type=str, required=True, help="Run ID") - parser.add_argument( - "--result", - action="store_true", - help="Run facet query on results" - ) - parser.add_argument( - "--device_summary", - action="store_true", - help="Run facet query on device summaries" - ) - parser.add_argument( - "-f", - "--fields", - nargs="+", - type=str, - required=False, - help="Fields to be displayed in results", - ) - - parser.add_argument("-q", "--query", type=str, required=False, help="Search query") - parser.add_argument( - "--device_ids", - nargs="+", - type=int, - required=False, - help="Device IDs to filter on", - ) - parser.add_argument( - "--device_names", - nargs="+", - type=int, - required=False, - help="Device names to filter on", - ) - parser.add_argument( - "--policy_ids", - nargs="+", - type=int, - required=False, - help="Policy IDs to filter on", - ) - parser.add_argument( - "--policy_names", - nargs="+", - type=int, - required=False, - help="Policy names to filter on", - ) - parser.add_argument( - "--statuses", - nargs="+", - type=str, - required=False, - help="Statuses to filter on", - ) - - args = parser.parse_args() - if not (args.result or args.device_summary): - print("ERROR: One of --result or --device_summary must be specified") - return 1 - if args.result and args.device_summary: - print("ERROR: --result and --device_summary cannot both be specified") - return 1 - - cb = get_cb_livequery_object(args) - - results = cb.select(Result).run_id(args.id) - result = results.first() - if result is None: - print("ERROR: No results.") - return 1 - - if args.result: - facets = result.query_result_facets() - elif args.device_summary: - facets = result.query_device_summary_facets() - if args.fields: - facets.facet_field(args.fields) - if args.query: - facets.where(args.query) - if args.device_ids: - facets.criteria(device_id=args.device_ids) - if args.device_names: - facets.criteria(device_name=args.device_names) - if args.policy_ids: - facets.criteria(policy_id=args.policy_ids) - if args.policy_names: - facets.criteria(policy_name=args.policy_names) - if args.statuses: - facets.criteria(status=args.statuses) - - for facet in facets: - print(facet) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/livequery/run_search.py b/examples/livequery/run_search.py deleted file mode 100644 index 66b2e6a7..00000000 --- a/examples/livequery/run_search.py +++ /dev/null @@ -1,73 +0,0 @@ -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_livequery_object -from cbapi.psc.livequery.models import Result - - -def main(): - parser = build_cli_parser("Search the results of a LiveQuery run") - parser.add_argument("-i", "--id", type=str, required=True, help="Run ID") - parser.add_argument("-q", "--query", type=str, required=False, help="Search query") - parser.add_argument( - "-F", "--fields_only", action="store_true", help="Show only fields" - ) - parser.add_argument( - "--device_ids", - nargs="+", - type=int, - required=False, - help="Device IDs to filter on", - ) - parser.add_argument( - "--device_types", - nargs="+", - type=str, - required=False, - help="Device types to filter on", - ) - parser.add_argument( - "--statuses", - nargs="+", - type=str, - required=False, - help="Statuses to filter on", - ) - parser.add_argument( - "-S", "--sort_by", type=str, help="sort by this field", required=False - ) - parser.add_argument( - "-D", - "--descending_results", - help="return results in descending order", - action="store_true", - ) - - args = parser.parse_args() - cb = get_cb_livequery_object(args) - - results = cb.select(Result).run_id(args.id) - if args.query: - results = results.where(args.query) - - if args.device_ids: - results.criteria(device_id=args.device_ids) - if args.device_types: - results.criteria(device_type=args.device_types) - if args.statuses: - results.criteria(status=args.statuses) - - if args.sort_by: - direction = "ASC" - if args.descending_results: - direction = "DESC" - results.sort_by(args.sort_by, direction=direction) - - for result in results: - if args.fields_only: - print(result.fields_) - else: - print(result) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index 163499f0..d90ed52d 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -11,5 +11,3 @@ # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI from cbapi.protection.rest_api import CbEnterpriseProtectionAPI, CbProtectionAPI -from cbapi.psc import CbPSCBaseAPI -from cbapi.psc.livequery import CbLiveQueryAPI diff --git a/src/cbapi/example_helpers.py b/src/cbapi/example_helpers.py index e0ee2731..798b94cf 100644 --- a/src/cbapi/example_helpers.py +++ b/src/cbapi/example_helpers.py @@ -15,8 +15,6 @@ import hashlib from cbapi.protection import CbEnterpriseProtectionAPI -from cbapi.psc import CbPSCBaseAPI -from cbapi.psc.livequery import CbLiveQueryAPI from cbapi.response import CbEnterpriseResponseAPI log = logging.getLogger(__name__) @@ -78,35 +76,6 @@ def get_cb_protection_object(args): return cb -def get_cb_psc_object(args): - if args.verbose: - logging.basicConfig() - logging.getLogger("cbapi").setLevel(logging.DEBUG) - logging.getLogger("__main__").setLevel(logging.DEBUG) - - if args.cburl and args.apitoken: - cb = CbPSCBaseAPI(url=args.cburl, token=args.apitoken, ssl_verify=(not args.no_ssl_verify)) - else: - cb = CbPSCBaseAPI(profile=args.profile) - - return cb - - -def get_cb_livequery_object(args): - if args.verbose: - logging.basicConfig() - logging.getLogger("cbapi").setLevel(logging.DEBUG) - logging.getLogger("__main__").setLevel(logging.DEBUG) - - if args.cburl and args.apitoken and args.orgkey: - cb = CbLiveQueryAPI(url=args.cburl, token=args.apitoken, org_key=args.orgkey, - ssl_verify=(not args.no_ssl_verify)) - else: - cb = CbLiveQueryAPI(profile=args.profile) - - return cb - - def get_object_by_name_or_id(cb, cls, name_field="name", id=None, name=None, force_init=True): clsname = cls.__name__ try: diff --git a/src/cbapi/psc/__init__.py b/src/cbapi/psc/__init__.py deleted file mode 100644 index 239b1eb6..00000000 --- a/src/cbapi/psc/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Exported public API for the Cb PSC API - -from __future__ import absolute_import - -from .rest_api import CbPSCBaseAPI -from .models import Device diff --git a/src/cbapi/psc/base_query.py b/src/cbapi/psc/base_query.py deleted file mode 100755 index 7812b3d8..00000000 --- a/src/cbapi/psc/base_query.py +++ /dev/null @@ -1,271 +0,0 @@ -from cbapi.errors import ApiError, MoreThanOneResultError -import functools -from six import string_types -from solrq import Q - - -class QueryBuilder(object): - """ - Provides a flexible interface for building prepared queries for the CB - PSC backend. - - This object can be instantiated directly, or can be managed implicitly - through the :py:meth:`select` API. - """ - - def __init__(self, **kwargs): - if kwargs: - self._query = Q(**kwargs) - else: - self._query = None - self._raw_query = None - - def _guard_query_params(func): - """Decorates the query construction methods of *QueryBuilder*, preventing - them from being called with parameters that would result in an internally - inconsistent query. - """ - - @functools.wraps(func) - def wrap_guard_query_change(self, q, **kwargs): - if self._raw_query is not None and (kwargs or isinstance(q, Q)): - raise ApiError("Cannot modify a raw query with structured parameters") - if self._query is not None and isinstance(q, string_types): - raise ApiError("Cannot modify a structured query with a raw parameter") - return func(self, q, **kwargs) - - return wrap_guard_query_change - - @_guard_query_params - def where(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - if self._raw_query is None: - self._raw_query = [] - self._raw_query.append(q) - elif isinstance(q, Q) or kwargs: - if self._query is not None: - raise ApiError("Use .and_() or .or_() for an extant solrq.Q object") - if kwargs: - q = Q(**kwargs) - self._query = q - else: - raise ApiError(".where() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def and_(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - self.where(q) - elif isinstance(q, Q) or kwargs: - if kwargs: - q = Q(**kwargs) - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".and_() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def or_(self, q, **kwargs): - """Adds a disjunctive filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - q = Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query | q - else: - raise ApiError(".or_() only accepts solrq.Q objects") - - return self - - @_guard_query_params - def not_(self, q, **kwargs): - """Adds a negative filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - q = ~Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".not_() only accepts solrq.Q objects") - - def _collapse(self): - """The query can be represented by either an array of strings - (_raw_query) which is concatenated and passed directly to Solr, or - a solrq.Q object (_query) which is then converted into a string to - pass to Solr. This function will perform the appropriate conversions to - end up with the 'q' string sent into the POST request to the - PSC-R query endpoint.""" - if self._raw_query is not None: - return " ".join(self._raw_query) - elif self._query is not None: - return str(self._query) - else: - return None # return everything - - -class PSCQueryBase: - """ - Represents the base of all LiveQuery query classes. - """ - - def __init__(self, doc_class, cb): - self._doc_class = doc_class - self._cb = cb - self._count_valid = False - - -class QueryBuilderSupportMixin: - """ - A mixin that supplies wrapper methods to access the _query_builder. - """ - def where(self, q=None, **kwargs): - """Add a filter to this query. - - :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q: - return self - if isinstance(q, QueryBuilder): - self._query_builder = q - else: - self._query_builder.where(q, **kwargs) - return self - - def and_(self, q=None, **kwargs): - """Add a conjunctive filter to this query. - - :param q: Query string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") - - self._query_builder.and_(q, **kwargs) - return self - - def or_(self, q=None, **kwargs): - """Add a disjunctive filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".or_() expects a solrq.Q or kwargs") - - self._query_builder.or_(q, **kwargs) - return self - - def not_(self, q=None, **kwargs): - """Adds a negated filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q and not kwargs: - raise ApiError(".not_() expects a solrq.Q, or kwargs") - - self._query_builder.not_(q, **kwargs) - return self - - -class IterableQueryMixin: - """ - A mix-in to provide iterability to a query. - """ - def all(self): - """ - Returns all the items of a query as a list. - - :return: List of query items - """ - return self._perform_query() - - def first(self): - """ - Returns the first item that would be returned as the result of a query. - - :return: First query item - """ - allres = list(self) - res = allres[:1] - if not len(res): - return None - return res[0] - - def one(self): - """ - Returns the only item that would be returned by a query. - - :return: Sole query return item - :raises MoreThanOneResultError: If the query returns zero items, or more than one item - """ - allres = list(self) - res = allres[:2] - if len(res) == 0: - raise MoreThanOneResultError( - message="0 results for query {0:s}".format(self._query) - ) - if len(res) > 1: - raise MoreThanOneResultError( - message="{0:d} results found for query {1:s}".format( - len(self), self._query - ) - ) - return res[0] - - def __len__(self): - return self._count() - - def __getitem__(self, item): - return None - - def __iter__(self): - return self._perform_query() diff --git a/src/cbapi/psc/devices_query.py b/src/cbapi/psc/devices_query.py deleted file mode 100755 index 5a5e185d..00000000 --- a/src/cbapi/psc/devices_query.py +++ /dev/null @@ -1,361 +0,0 @@ -from cbapi.errors import ApiError -from .base_query import PSCQueryBase, QueryBuilder, QueryBuilderSupportMixin, IterableQueryMixin - - -class DeviceSearchQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): - """ - Represents a query that is used to locate Device objects. - """ - VALID_OS = ["WINDOWS", "ANDROID", "MAC", "IOS", "LINUX", "OTHER"] - VALID_STATUSES = ["PENDING", "REGISTERED", "UNINSTALLED", "DEREGISTERED", - "ACTIVE", "INACTIVE", "ERROR", "ALL", "BYPASS_ON", - "BYPASS", "QUARANTINE", "SENSOR_OUTOFDATE", - "DELETED", "LIVE"] - VALID_PRIORITIES = ["LOW", "MEDIUM", "HIGH", "MISSION_CRITICAL"] - VALID_DIRECTIONS = ["ASC", "DESC"] - - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_builder = QueryBuilder() - self._criteria = {} - self._time_filter = {} - self._exclusions = {} - self._sortcriteria = {} - - def _update_criteria(self, key, newlist): - """ - Updates the criteria being collected for a query. Assumes the specified criteria item is - defined as a list; the list passed in will be set as the value for this criteria item, or - appended to the existing one if there is one. - - :param str key: The key for the criteria item to be set - :param list newlist: List of values to be set for the criteria item - """ - oldlist = self._criteria.get(key, []) - self._criteria[key] = oldlist + newlist - - def _update_exclusions(self, key, newlist): - """ - Updates the exclusion criteria being collected for a query. Assumes the specified criteria item is - defined as a list; the list passed in will be set as the value for this criteria item, or - appended to the existing one if there is one. - - :param str key: The key for the criteria item to be set - :param list newlist: List of values to be set for the criteria item - """ - oldlist = self._exclusions.get(key, []) - self._exclusions[key] = oldlist + newlist - - def set_ad_group_ids(self, ad_group_ids): - """ - Restricts the devices that this query is performed on to the specified - AD group IDs. - - :param ad_group_ids: list of ints - :return: This instance - """ - if not all(isinstance(ad_group_id, int) for ad_group_id in ad_group_ids): - raise ApiError("One or more invalid AD group IDs") - self._update_criteria("ad_group_id", ad_group_ids) - return self - - def set_device_ids(self, device_ids): - """ - Restricts the devices that this query is performed on to the specified - device IDs. - - :param device_ids: list of ints - :return: This instance - """ - if not all(isinstance(device_id, int) for device_id in device_ids): - raise ApiError("One or more invalid device IDs") - self._update_criteria("id", device_ids) - return self - - def set_last_contact_time(self, *args, **kwargs): - """ - Restricts the devices that this query is performed on to the specified - last contact time (either specified as a start and end point or as a - range). - - :return: This instance - """ - if kwargs.get("start", None) and kwargs.get("end", None): - if kwargs.get("range", None): - raise ApiError("cannot specify range= in addition to start= and end=") - stime = kwargs["start"] - if not isinstance(stime, str): - stime = stime.isoformat() - etime = kwargs["end"] - if not isinstance(etime, str): - etime = etime.isoformat() - self._time_filter = {"start": stime, "end": etime} - elif kwargs.get("range", None): - if kwargs.get("start", None) or kwargs.get("end", None): - raise ApiError("cannot specify start= or end= in addition to range=") - self._time_filter = {"range": kwargs["range"]} - else: - raise ApiError("must specify either start= and end= or range=") - return self - - def set_os(self, operating_systems): - """ - Restricts the devices that this query is performed on to the specified - operating systems. - - :param operating_systems: list of operating systems - :return: This instance - """ - if not all((osval in DeviceSearchQuery.VALID_OS) for osval in operating_systems): - raise ApiError("One or more invalid operating systems") - self._update_criteria("os", operating_systems) - return self - - def set_policy_ids(self, policy_ids): - """ - Restricts the devices that this query is performed on to the specified - policy IDs. - - :param policy_ids: list of ints - :return: This instance - """ - if not all(isinstance(policy_id, int) for policy_id in policy_ids): - raise ApiError("One or more invalid policy IDs") - self._update_criteria("policy_id", policy_ids) - return self - - def set_status(self, statuses): - """ - Restricts the devices that this query is performed on to the specified - status values. - - :param statuses: list of strings - :return: This instance - """ - if not all((stat in DeviceSearchQuery.VALID_STATUSES) for stat in statuses): - raise ApiError("One or more invalid status values") - self._update_criteria("status", statuses) - return self - - def set_target_priorities(self, target_priorities): - """ - Restricts the devices that this query is performed on to the specified - target priority values. - - :param target_priorities: list of strings - :return: This instance - """ - if not all((prio in DeviceSearchQuery.VALID_PRIORITIES) for prio in target_priorities): - raise ApiError("One or more invalid target priority values") - self._update_criteria("target_priority", target_priorities) - return self - - def set_exclude_sensor_versions(self, sensor_versions): - """ - Restricts the devices that this query is performed on to exclude specified - sensor versions. - - :param sensor_versions: List of sensor versions to exclude - :return: This instance - """ - if not all(isinstance(v, str) for v in sensor_versions): - raise ApiError("One or more invalid sensor versions") - self._update_exclusions("sensor_version", sensor_versions) - return self - - def sort_by(self, key, direction="ASC"): - """Sets the sorting behavior on a query's results. - - Example:: - - >>> cb.select(Device).sort_by("name") - - :param key: the key in the schema to sort by - :param direction: the sort order, either "ASC" or "DESC" - :rtype: :py:class:`DeviceSearchQuery` - """ - if direction not in DeviceSearchQuery.VALID_DIRECTIONS: - raise ApiError("invalid sort direction specified") - self._sortcriteria = {"field": key, "order": direction} - return self - - def _build_request(self, from_row, max_rows): - """ - Creates the request body for an API call. - - :param int from_row: The row to start the query at. - :param int max_rows: The maximum number of rows to be returned. - :return: A dict containing the complete request body. - """ - mycrit = self._criteria - if self._time_filter: - mycrit["last_contact_time"] = self._time_filter - request = {"criteria": mycrit, "exclusions": self._exclusions} - request["query"] = self._query_builder._collapse() - if from_row > 0: - request["start"] = from_row - if max_rows >= 0: - request["rows"] = max_rows - if self._sortcriteria != {}: - request["sort"] = [self._sortcriteria] - return request - - def _build_url(self, tail_end): - """ - Creates the URL to be used for an API call. - - :param str tail_end: String to be appended to the end of the generated URL. - """ - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) + tail_end - return url - - def _count(self): - """ - Returns the number of results from the run of this query. - - :return: The number of results from the run of this query. - """ - if self._count_valid: - return self._total_results - - url = self._build_url("/_search") - request = self._build_request(0, -1) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - return self._total_results - - def _perform_query(self, from_row=0, max_rows=-1): - """ - Performs the query and returns the results of the query in an iterable fashion. - - :param int from_row: The row to start the query at (default 0). - :param int max_rows: The maximum number of rows to be returned (default -1, meaning "all"). - """ - url = self._build_url("/_search") - current = from_row - numrows = 0 - still_querying = True - while still_querying: - request = self._build_request(current, max_rows) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - results = result.get("results", []) - for item in results: - yield self._doc_class(self._cb, item["id"], item) - current += 1 - numrows += 1 - - if max_rows > 0 and numrows == max_rows: - still_querying = False - break - - from_row = current - if current >= self._total_results: - still_querying = False - break - - def download(self): - """ - Uses the query parameters that have been set to download all - device listings in CSV format. - - Example:: - - >>> cb.select(Device).set_status(["ALL"]).download() - - :return: The CSV raw data as returned from the server. - """ - tmp = self._criteria.get("status", []) - if not tmp: - raise ApiError("at least one status must be specified to download") - query_params = {"status": ",".join(tmp)} - tmp = self._criteria.get("ad_group_id", []) - if tmp: - query_params["ad_group_id"] = ",".join([str(t) for t in tmp]) - tmp = self._criteria.get("policy_id", []) - if tmp: - query_params["policy_id"] = ",".join([str(t) for t in tmp]) - tmp = self._criteria.get("target_priority", []) - if tmp: - query_params["target_priority"] = ",".join(tmp) - tmp = self._query_builder._collapse() - if tmp: - query_params["query_string"] = tmp - if self._sortcriteria: - query_params["sort_field"] = self._sortcriteria["field"] - query_params["sort_order"] = self._sortcriteria["order"] - url = self._build_url("/_search/download") - # AGRB 10/3/2019 - Header is TEMPORARY until bug is fixed in API. Remove when fix deployed. - return self._cb.get_raw_data(url, query_params, headers={"Content-Type": "application/json"}) - - def _bulk_device_action(self, action_type, options=None): - """ - Perform a bulk action on all devices matching the current search criteria. - - :param str action_type: The action type to be performed. - :param dict options: Options for the bulk device action. Default None. - """ - request = {"action_type": action_type, "search": self._build_request(0, -1)} - if options: - request["options"] = options - return self._cb._raw_device_action(request) - - def background_scan(self, scan): - """ - Set the background scan option for the specified devices. - - :param boolean scan: True to turn background scan on, False to turn it off. - """ - return self._bulk_device_action("BACKGROUND_SCAN", self._cb._action_toggle(scan)) - - def bypass(self, enable): - """ - Set the bypass option for the specified devices. - - :param boolean enable: True to enable bypass, False to disable it. - """ - return self._bulk_device_action("BYPASS", self._cb._action_toggle(enable)) - - def delete_sensor(self): - """ - Delete the specified sensor devices. - """ - return self._bulk_device_action("DELETE_SENSOR") - - def uninstall_sensor(self): - """ - Uninstall the specified sensor devices. - """ - return self._bulk_device_action("UNINSTALL_SENSOR") - - def quarantine(self, enable): - """ - Set the quarantine option for the specified devices. - - :param boolean enable: True to enable quarantine, False to disable it. - """ - return self._bulk_device_action("QUARANTINE", self._cb._action_toggle(enable)) - - def update_policy(self, policy_id): - """ - Set the current policy for the specified devices. - - :param int policy_id: ID of the policy to set for the devices. - """ - return self._bulk_device_action("UPDATE_POLICY", {"policy_id": policy_id}) - - def update_sensor_version(self, sensor_version): - """ - Update the sensor version for the specified devices. - - :param dict sensor_version: New version properties for the sensor. - """ - return self._bulk_device_action("UPDATE_SENSOR_VERSION", {"sensor_version": sensor_version}) diff --git a/src/cbapi/psc/livequery/__init__.py b/src/cbapi/psc/livequery/__init__.py deleted file mode 100644 index d48b5b1d..00000000 --- a/src/cbapi/psc/livequery/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# Exported public API for the Cb LiveQuery API - -from __future__ import absolute_import - -from .rest_api import CbLiveQueryAPI -from cbapi.psc.livequery.models import Run, RunHistory, Result, DeviceSummary -from cbapi.psc.livequery.models import ResultFacet, DeviceSummaryFacet diff --git a/src/cbapi/psc/livequery/models.py b/src/cbapi/psc/livequery/models.py deleted file mode 100644 index 38514034..00000000 --- a/src/cbapi/psc/livequery/models.py +++ /dev/null @@ -1,294 +0,0 @@ -from __future__ import absolute_import -from cbapi.models import UnrefreshableModel, NewBaseModel -from cbapi.errors import ApiError, ServerError -from .query import RunQuery, RunHistoryQuery, ResultQuery, FacetQuery -import logging -import time - -log = logging.getLogger(__name__) - - -class Run(NewBaseModel): - """ - Represents a LiveQuery run. - - Example:: - - >>> run = cb.select(Run, run_id) - >>> print(run.name, run.sql, run.create_time) - >>> print(run.status, run.match_count) - >>> run.refresh() - """ - primary_key = "id" - swagger_meta_file = "psc/livequery/models/run.yaml" - urlobject = "/livequery/v1/orgs/{}/runs" - urlobject_single = "/livequery/v1/orgs/{}/runs/{}" - _is_deleted = False - - def __init__(self, cb, model_unique_id=None, initial_data=None): - if initial_data is not None: - item = initial_data - elif model_unique_id is not None: - url = self.urlobject_single.format(cb.credentials.org_key, model_unique_id) - item = cb.get_object(url) - - model_unique_id = item.get("id") - - super(Run, self).__init__( - cb, - model_unique_id=model_unique_id, - initial_data=item, - force_init=False, - full_doc=True, - ) - - @classmethod - def _query_implementation(cls, cb): - return RunQuery(cls, cb) - - def _refresh(self): - if self._is_deleted: - raise ApiError("cannot refresh a deleted query") - url = self.urlobject_single.format(self._cb.credentials.org_key, self.id) - resp = self._cb.get_object(url) - self._info = resp - self._last_refresh_time = time.time() - return True - - def stop(self): - if self._is_deleted: - raise ApiError("cannot stop a deleted query") - url = self.urlobject_single.format(self._cb.credentials.org_key, self.id) + "/status" - result = self._cb.put_object(url, {'status': 'CANCELLED'}) - if (result.status_code == 200): - try: - self._info = result.json() - self._last_refresh_time = time.time() - return True - except Exception: - raise ServerError(result.status_code, "Cannot parse response as JSON: {0:s}".format(result.content)) - return False - - def delete(self): - if self._is_deleted: - return True # already deleted - url = self.urlobject_single.format(self._cb.credentials.org_key, self.id) - result = self._cb.delete_object(url) - if result.status_code == 200: - self._is_deleted = True - return True - return False - - -class RunHistory(Run): - """ - Represents a historical LiveQuery ``Run``. - """ - urlobject_history = "/livequery/v1/orgs/{}/runs/_search" - - def __init__(self, cb, initial_data=None): - item = initial_data - model_unique_id = item.get("id") - super(Run, self).__init__(cb, - model_unique_id, initial_data=item, - force_init=False, full_doc=True) - - @classmethod - def _query_implementation(cls, cb): - return RunHistoryQuery(cls, cb) - - -class Result(UnrefreshableModel): - """ - Represents a single result from a LiveQuery ``Run``. - """ - primary_key = "id" - swagger_meta_file = "psc/livequery/models/result.yaml" - urlobject = "/livequery/v1/orgs/{}/runs/{}/results/_search" - - class Device(UnrefreshableModel): - """ - Represents device information for a result. - """ - primary_key = "id" - - def __init__(self, cb, initial_data): - super(Result.Device, self).__init__( - cb, - model_unique_id=initial_data["id"], - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - - class Fields(UnrefreshableModel): - """ - Represents the fields of a result. - """ - def __init__(self, cb, initial_data): - super(Result.Fields, self).__init__( - cb, - model_unique_id=None, - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - - class Metrics(UnrefreshableModel): - """ - Represents the metrics for a result. - """ - def __init__(self, cb, initial_data): - super(Result.Metrics, self).__init__( - cb, - model_unique_id=None, - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - - @classmethod - def _query_implementation(cls, cb): - return ResultQuery(cls, cb) - - def __init__(self, cb, initial_data): - super(Result, self).__init__( - cb, - model_unique_id=initial_data["id"], - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - self._run_id = initial_data["id"] - self._device = Result.Device(cb, initial_data=initial_data["device"]) - self._fields = Result.Fields(cb, initial_data=initial_data["fields"]) - if "metrics" in initial_data: - self._metrics = Result.Metrics(cb, initial_data=initial_data["metrics"]) - else: - self._metrics = Result.Metrics(cb, initial_data=None) - - @property - def device_(self): - """ - Returns the reified ``Result.Device`` for this result. - """ - return self._device - - @property - def fields_(self): - """ - Returns the reified ``Result.Fields`` for this result. - """ - return self._fields - - @property - def metrics_(self): - """ - Returns the reified ``Result.Metrics`` for this result. - """ - return self._metrics - - def query_device_summaries(self): - return self._cb.select(DeviceSummary).run_id(self._run_id) - - def query_result_facets(self): - return self._cb.select(ResultFacet).run_id(self._run_id) - - def query_device_summary_facets(self): - return self._cb.select(DeviceSummaryFacet).run_id(self._run_id) - - -class DeviceSummary(UnrefreshableModel): - """ - Represents the summary of results from a single device during a single LiveQuery ``Run``. - """ - primary_key = "id" - swagger_meta_file = "psc/livequery/models/device_summary.yaml" - urlobject = "/livequery/v1/orgs/{}/runs/{}/results/device_summaries/_search" - - class Metrics(UnrefreshableModel): - """ - Represents the metrics for a result. - """ - def __init__(self, cb, initial_data): - super(DeviceSummary.Metrics, self).__init__( - cb, - model_unique_id=None, - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - - @classmethod - def _query_implementation(cls, cb): - return ResultQuery(cls, cb) - - def __init__(self, cb, initial_data): - super(DeviceSummary, self).__init__( - cb, - model_unique_id=initial_data["id"], - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - self._metrics = DeviceSummary.Metrics(cb, initial_data=initial_data["metrics"]) - - @property - def metrics_(self): - """ - Returns the reified ``DeviceSummary.Metrics`` for this result. - """ - return self._metrics - - -class ResultFacet(UnrefreshableModel): - """ - Represents the summary of results for a single field in a LiveQuery ``Run``. - """ - primary_key = "field" - swagger_meta_file = "psc/livequery/models/facet.yaml" - urlobject = "/livequery/v1/orgs/{}/runs/{}/results/_facet" - - class Values(UnrefreshableModel): - """ - Represents the values associated with a field. - """ - def __init__(self, cb, initial_data): - super(ResultFacet.Values, self).__init__( - cb, - model_unique_id=None, - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - - @classmethod - def _query_implementation(cls, cb): - return FacetQuery(cls, cb) - - def __init__(self, cb, initial_data): - super(ResultFacet, self).__init__( - cb, - model_unique_id=None, - initial_data=initial_data, - force_init=False, - full_doc=True - ) - self._values = ResultFacet.Values(cb, initial_data=initial_data["values"]) - - @property - def values_(self): - """ - Returns the reified ``ResultFacet.Values`` for this result. - """ - return self._values - - -class DeviceSummaryFacet(ResultFacet): - """ - Represents the summary of results for a single device summary in a LiveQuery ``Run``. - """ - urlobject = "/livequery/v1/orgs/{}/runs/{}/results/device_summaries/_facet" - - def __init__(self, cb, initial_data): - super(DeviceSummaryFacet, self).__init__(cb, initial_data) diff --git a/src/cbapi/psc/livequery/models/device_summary.yaml b/src/cbapi/psc/livequery/models/device_summary.yaml deleted file mode 100755 index a4b32c88..00000000 --- a/src/cbapi/psc/livequery/models/device_summary.yaml +++ /dev/null @@ -1,52 +0,0 @@ -type: object -required: [] # TODO(ww): Find out which result fields are required -properties: - id: - type: string - description: The result's unique ID - total_results: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Number of results returned for this particular device - device_id: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The device's unique ID - device_name: - type: string - description: The device's name - time_received: - type: string - description: The time at which this result was received # NOTE(ww): This is a guess - format: date-time - status: - type: string - description: The result's status - device_message: - type: string - description: Placeholder # TODO(ww): Needs description - os: - type: string - description: The device's operating system - policy_id: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The device's policy ID - policy_name: - type: string - description: The device's policy name - metrics: - type: array - description: Metrics associated with the device - items: - type: object - description: Individual metrics entries - properties: - key: - type: string - description: The name of the particular metric - value: - type: number - format: float - description: The value of the particular metric - \ No newline at end of file diff --git a/src/cbapi/psc/livequery/models/facet.yaml b/src/cbapi/psc/livequery/models/facet.yaml deleted file mode 100755 index b20151b3..00000000 --- a/src/cbapi/psc/livequery/models/facet.yaml +++ /dev/null @@ -1,22 +0,0 @@ -type: object -required: [] # TODO(ww): Find out which result fields are required -properties: - field: - type: string - description: The name of the field being summarized - values: - type: array - items: - type: object - properties: - total: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The total number of times this value appears in the query output - id: - type: string - description: The ID of the value being enumerated - name: - type: string - description: The name of the value being enumerated - \ No newline at end of file diff --git a/src/cbapi/psc/livequery/models/result.yaml b/src/cbapi/psc/livequery/models/result.yaml deleted file mode 100644 index ff1d3bc8..00000000 --- a/src/cbapi/psc/livequery/models/result.yaml +++ /dev/null @@ -1,41 +0,0 @@ -type: object -required: [] # TODO(ww): Find out which result fields are required -properties: - id: - type: string - description: The result's unique ID - device: - type: object - description: The device associated with the result - properties: - id: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The device's unique ID - name: - type: string - description: The device's name - policy_id: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The device's policy ID - policy_name: - type: string - description: The device's policy name - status: - type: string - description: The result's status - time_received: - type: string - description: The time at which this result was received # NOTE(ww): This is a guess - device_message: - type: string - description: Placeholder # TODO(ww): Needs description - fields: - type: object - description: The fields returned by the backing osquery query - metrics: # TODO(ww): Document each field in metrics? - type: object - description: Metrics associated with the result's host - - diff --git a/src/cbapi/psc/livequery/models/run.yaml b/src/cbapi/psc/livequery/models/run.yaml deleted file mode 100644 index 089407be..00000000 --- a/src/cbapi/psc/livequery/models/run.yaml +++ /dev/null @@ -1,86 +0,0 @@ -type: object -required: # TODO(ww): Find out which run fields are required - - id -properties: - template_id: - type: string # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - org_key: - type: string - description: The organization key for this run - name: - type: string - description: The name of the LiveQuery run - id: - type: string - description: The run's unique ID - sql: - type: string - description: The LiveQuery query - created_by: - type: string - description: Placeholder # TODO(ww): Needs description - create_time: - type: string - description: When this run was created - status_update_time: - type: string - description: When the status of this run was last updated - timeout_time: - type: string - description: Placeholder # TODO(ww): Needs description - cancellation_time: - type: string - description: Placeholder # TODO(ww): Needs description - cancelled_by: - type: string # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - archive_time: - type: string - description: Placeholder # TODO(ww): Needs description - archived_by: - type: string # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - notify_on_finish: - type: boolean - description: Whether or not to send an email on query completion - active_org_devices: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The number of devices active in the organization - status: - type: string - description: The run status - device_filter: - type: object - description: Any device filter rules associated with the run - schedule: - type: string # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - last_result_time: - type: string - description: When the most recent result for this run was reported - total_results: - type: integer - format: int32 - description: Placeholder # TODO(ww): Needs description - match_count: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - no_match_count: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - error_count: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - not_supported_count: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - cancelled_count: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description diff --git a/src/cbapi/psc/livequery/query.py b/src/cbapi/psc/livequery/query.py deleted file mode 100644 index f4a5a05e..00000000 --- a/src/cbapi/psc/livequery/query.py +++ /dev/null @@ -1,375 +0,0 @@ -from cbapi.errors import ApiError -from cbapi.psc.base_query import QueryBuilder, PSCQueryBase -from cbapi.psc.base_query import QueryBuilderSupportMixin, IterableQueryMixin -import logging -from six import string_types - -log = logging.getLogger(__name__) - - -class RunQuery(PSCQueryBase): - """ - Represents a query that either creates or retrieves the - status of a LiveQuery run. - """ - - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_token = None - self._query_body = {"device_filter": {}} - self._device_filter = self._query_body["device_filter"] - - def device_ids(self, device_ids): - """ - Restricts the devices that this LiveQuery run is performed on - to the given IDs. - - :param device_ids: list of ints - :return: This instance - """ - if not all(isinstance(device_id, int) for device_id in device_ids): - raise ApiError("One or more invalid device IDs") - self._device_filter["device_ids"] = device_ids - return self - - def device_types(self, device_types): - """ - Restricts the devices that this LiveQuery run is performed on - to the given device types. - - :param device_types: list of strs - :return: This instance - """ - if not all(isinstance(device_type, str) for device_type in device_types): - raise ApiError("One or more invalid device types") - self._device_filter["device_types"] = device_types - return self - - def policy_ids(self, policy_ids): - """ - Restricts this LiveQuery run to the given policy IDs. - - :param policy_ids: list of ints - :return: This instance - """ - if not all(isinstance(policy_id, int) for policy_id in policy_ids): - raise ApiError("One or more invalid policy IDs") - self._device_filter["policy_ids"] = policy_ids - return self - - def where(self, sql): - """ - Sets this LiveQuery run's underlying SQL. - - :param sql: The SQL to execute - :return: This instance - """ - self._query_body["sql"] = sql - return self - - def name(self, name): - """ - Sets this LiveQuery run's name. If no name is explicitly set, - the run is named after its SQL. - - :param name: The run name - :return: This instance - """ - self._query_body["name"] = name - return self - - def notify_on_finish(self): - """ - Sets the notify-on-finish flag on this LiveQuery run. - - :return: This instance - """ - self._query_body["notify_on_finish"] = True - return self - - def submit(self): - """ - Submits this LiveQuery run. - - :return: A new ``Run`` instance containing the run's status - """ - if self._query_token is not None: - raise ApiError( - "Query already submitted: token {0}".format(self._query_token) - ) - - if "sql" not in self._query_body: - raise ApiError("Missing LiveQuery SQL") - - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) - resp = self._cb.post_object(url, body=self._query_body) - - return self._doc_class(self._cb, initial_data=resp.json()) - - -class RunHistoryQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): - """ - Represents a query that retrieves historic LiveQuery runs. - """ - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_builder = QueryBuilder() - self._sort = {} - - def sort_by(self, key, direction="ASC"): - """Sets the sorting behavior on a query's results. - - Example:: - - >>> cb.select(Result).run_id(my_run).where(username="foobar").sort_by("uid") - - :param key: the key in the schema to sort by - :param direction: the sort order, either "ASC" or "DESC" - :rtype: :py:class:`ResultQuery` - """ - self._sort.update({"field": key, "order": direction}) - return self - - def _build_request(self, start, rows): - request = {"start": start} - - if self._query_builder: - request["query"] = self._query_builder._collapse() - if rows != 0: - request["rows"] = rows - if self._sort: - request["sort"] = [self._sort] - - return request - - def _count(self): - if self._count_valid: - return self._total_results - - url = self._doc_class.urlobject_history.format( - self._cb.credentials.org_key - ) - request = self._build_request(start=0, rows=0) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - return self._total_results - - def _perform_query(self, start=0, rows=0): - url = self._doc_class.urlobject_history.format( - self._cb.credentials.org_key - ) - current = start - numrows = 0 - still_querying = True - while still_querying: - request = self._build_request(start, rows) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - results = result.get("results", []) - for item in results: - yield self._doc_class(self._cb, item) - current += 1 - numrows += 1 - - if rows and numrows == rows: - still_querying = False - break - - start = current - if current >= self._total_results: - still_querying = False - break - - -class ResultQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): - """ - Represents a query that retrieves results from a LiveQuery run. - """ - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_builder = QueryBuilder() - self._criteria = {} - self._sort = {} - self._batch_size = 100 - self._run_id = None - - def criteria(self, **kwargs): - """Sets the filter criteria on a query's results. - - Example:: - - >>> cb.select(Result).run_id(my_run).criteria(device_id=[123, 456]) - - """ - self._criteria.update(kwargs) - return self - - def sort_by(self, key, direction="ASC"): - """Sets the sorting behavior on a query's results. - - Example:: - - >>> cb.select(Result).run_id(my_run).where(username="foobar").sort_by("uid") - - :param key: the key in the schema to sort by - :param direction: the sort order, either "ASC" or "DESC" - :rtype: :py:class:`ResultQuery` - """ - self._sort.update({"field": key, "order": direction}) - return self - - def run_id(self, run_id): - """Sets the run ID to query results for. - - Example:: - - >>> cb.select(Result).run_id(my_run) - """ - self._run_id = run_id - return self - - def _build_request(self, start, rows): - request = {"start": start, "query": self._query_builder._collapse()} - - if rows != 0: - request["rows"] = rows - if self._criteria: - request["criteria"] = self._criteria - if self._sort: - request["sort"] = [self._sort] - - return request - - def _count(self): - if self._count_valid: - return self._total_results - - if self._run_id is None: - raise ApiError("Can't retrieve count without a run ID") - - url = self._doc_class.urlobject.format( - self._cb.credentials.org_key, self._run_id - ) - request = self._build_request(start=0, rows=0) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - return self._total_results - - def _perform_query(self, start=0, rows=0): - if self._run_id is None: - raise ApiError("Can't retrieve results without a run ID") - - url = self._doc_class.urlobject.format( - self._cb.credentials.org_key, self._run_id - ) - current = start - numrows = 0 - still_querying = True - while still_querying: - request = self._build_request(start, rows) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - results = result.get("results", []) - for item in results: - yield self._doc_class(self._cb, item) - current += 1 - numrows += 1 - - if rows and numrows == rows: - still_querying = False - break - - start = current - if current >= self._total_results: - still_querying = False - break - - -class FacetQuery(PSCQueryBase, QueryBuilderSupportMixin, IterableQueryMixin): - """ - Represents a query that receives facet information from a LiveQuery run. - """ - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_builder = QueryBuilder() - self._facet_fields = [] - self._criteria = {} - self._run_id = None - - def facet_field(self, field): - """Sets the facet fields to be received by this query. - - Example:: - - >>> cb.select(ResultFacet).run_id(my_run).facet_field(["device.policy_name", "device.os"]) - - :param field: Field(s) to be received, either single string or list of strings - :return: Query object - :rtype: :py:class:`Query` - """ - if isinstance(field, string_types): - self._facet_fields.append(field) - else: - for name in field: - self._facet_fields.append(name) - return self - - def criteria(self, **kwargs): - """Sets the filter criteria on a query's results. - - Example:: - - >>> cb.select(ResultFacet).run_id(my_run).criteria(device_id=[123, 456]) - - """ - self._criteria.update(kwargs) - return self - - def run_id(self, run_id): - """Sets the run ID to query results for. - - Example:: - - >>> cb.select(ResultFacet).run_id(my_run) - """ - self._run_id = run_id - return self - - def _build_request(self, rows): - terms = {"fields": self._facet_fields} - if rows != 0: - terms["rows"] = rows - request = {"query": self._query_builder._collapse(), "terms": terms} - if self._criteria: - request["criteria"] = self._criteria - return request - - def _perform_query(self, rows=0): - if self._run_id is None: - raise ApiError("Can't retrieve results without a run ID") - - url = self._doc_class.urlobject.format( - self._cb.credentials.org_key, self._run_id - ) - request = self._build_request(rows) - resp = self._cb.post_object(url, body=request) - result = resp.json() - results = result.get("terms", []) - for item in results: - yield self._doc_class(self._cb, item) diff --git a/src/cbapi/psc/livequery/rest_api.py b/src/cbapi/psc/livequery/rest_api.py deleted file mode 100644 index 824677f0..00000000 --- a/src/cbapi/psc/livequery/rest_api.py +++ /dev/null @@ -1,36 +0,0 @@ -from cbapi.psc.livequery.models import Run, RunHistory -from cbapi.psc.rest_api import CbPSCBaseAPI -from cbapi.errors import CredentialError, ApiError -import logging - -log = logging.getLogger(__name__) - - -class CbLiveQueryAPI(CbPSCBaseAPI): - """The main entry point into the Carbon Black Cloud LiveQuery API. - - :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. - Uses the profile named 'default' when not specified. - - Usage:: - - >>> from cbapi.psc.livequery import CbLiveQueryAPI - >>> cb = CbLiveQueryAPI(profile="production") - """ - def __init__(self, *args, **kwargs): - super(CbLiveQueryAPI, self).__init__(*args, **kwargs) - - if not self.credentials.get("org_key", None): - raise CredentialError("No organization key specified") - - def _perform_query(self, cls, **kwargs): - if hasattr(cls, "_query_implementation"): - return cls._query_implementation(self) - else: - raise ApiError("All LiveQuery models should provide _query_implementation") - - def query(self, sql): - return self.select(Run).where(sql=sql) - - def query_history(self, query=None): - return self.select(RunHistory).where(query) diff --git a/src/cbapi/psc/models.py b/src/cbapi/psc/models.py deleted file mode 100755 index 321dac06..00000000 --- a/src/cbapi/psc/models.py +++ /dev/null @@ -1,181 +0,0 @@ -from cbapi.models import MutableBaseModel, UnrefreshableModel -from cbapi.errors import ServerError -from cbapi.psc.devices_query import DeviceSearchQuery - -from copy import deepcopy -import logging -import json -import time - -log = logging.getLogger(__name__) - - -class PSCMutableModel(MutableBaseModel): - _change_object_http_method = "PATCH" - _change_object_key_name = None - - def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=False): - super(PSCMutableModel, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=force_init, full_doc=full_doc) - if not self._change_object_key_name: - self._change_object_key_name = self.primary_key - - def _parse(self, obj): - if type(obj) == dict and self.info_key in obj: - return obj[self.info_key] - - def _update_object(self): - if self._change_object_http_method != "PATCH": - return self._update_entire_object() - else: - return self._patch_object() - - def _update_entire_object(self): - if self.__class__.primary_key in self._dirty_attributes.keys() or self._model_unique_id is None: - new_object_info = deepcopy(self._info) - try: - if not self._new_object_needs_primary_key: - del(new_object_info[self.__class__.primary_key]) - except Exception: - pass - log.debug("Creating a new {0:s} object".format(self.__class__.__name__)) - ret = self._cb.api_json_request(self.__class__._new_object_http_method, self.urlobject, - data={self.info_key: new_object_info}) - else: - log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) - ret = self._cb.api_json_request(self.__class__._change_object_http_method, - self._build_api_request_uri(), data={self.info_key: self._info}) - - return self._refresh_if_needed(ret) - - def _patch_object(self): - if self.__class__.primary_key in self._dirty_attributes.keys() or self._model_unique_id is None: - log.debug("Creating a new {0:s} object".format(self.__class__.__name__)) - ret = self._cb.api_json_request(self.__class__._new_object_http_method, self.urlobject, - data=self._info) - else: - updates = {} - for k in self._dirty_attributes.keys(): - updates[k] = self._info[k] - log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) - ret = self._cb.api_json_request(self.__class__._change_object_http_method, - self._build_api_request_uri(), data=updates) - - return self._refresh_if_needed(ret) - - def _refresh_if_needed(self, request_ret): - refresh_required = True - - if request_ret.status_code not in range(200, 300): - try: - message = json.loads(request_ret.text)[0] - except Exception: - message = request_ret.text - - raise ServerError(request_ret.status_code, message, - result="Did not update {} record.".format(self.__class__.__name__)) - else: - try: - message = request_ret.json() - log.debug("Received response: %s" % message) - if not isinstance(message, dict): - raise ServerError(request_ret.status_code, message, - result="Unknown error updating {0:s} record.".format(self.__class__.__name__)) - else: - if message.get("success", False): - if isinstance(message.get(self.info_key, None), dict): - self._info = message.get(self.info_key) - self._full_init = True - refresh_required = False - else: - if self._change_object_key_name in message.keys(): - # if all we got back was an ID, try refreshing to get the entire record. - log.debug("Only received an ID back from the server, forcing a refresh") - self._info[self.primary_key] = message[self._change_object_key_name] - refresh_required = True - else: - # "success" is False - raise ServerError(request_ret.status_code, message.get("message", ""), - result="Did not update {0:s} record.".format(self.__class__.__name__)) - except Exception: - pass - - self._dirty_attributes = {} - if refresh_required: - self.refresh() - return self._model_unique_id - - -class Device(PSCMutableModel): - urlobject = "/appservices/v6/orgs/{0}/devices" - urlobject_single = "/appservices/v6/orgs/{0}/devices/{1}" - primary_key = "id" - swagger_meta_file = "psc/models/device.yaml" - - def __init__(self, cb, model_unique_id, initial_data=None): - super(Device, self).__init__(cb, model_unique_id, initial_data) - if model_unique_id is not None and initial_data is None: - self._refresh() - - @classmethod - def _query_implementation(cls, cb): - return DeviceSearchQuery(cls, cb) - - def _refresh(self): - url = self.urlobject_single.format(self._cb.credentials.org_key, self._model_unique_id) - resp = self._cb.get_object(url) - self._info = resp - self._last_refresh_time = time.time() - return True - - def background_scan(self, flag): - """ - Set the background scan option for this device. - - :param boolean flag: True to turn background scan on, False to turn it off. - """ - return self._cb.device_background_scan([self._model_unique_id], flag) - - def bypass(self, flag): - """ - Set the bypass option for this device. - - :param boolean flag: True to enable bypass, False to disable it. - """ - return self._cb.device_bypass([self._model_unique_id], flag) - - def delete_sensor(self): - """ - Delete this sensor device. - """ - return self._cb.device_delete_sensor([self._model_unique_id]) - - def uninstall_sensor(self): - """ - Uninstall this sensor device. - """ - return self._cb.device_uninstall_sensor([self._model_unique_id]) - - def quarantine(self, flag): - """ - Set the quarantine option for this device. - - :param boolean flag: True to enable quarantine, False to disable it. - """ - return self._cb.device_quarantine([self._model_unique_id], flag) - - def update_policy(self, policy_id): - """ - Set the current policy for this device. - - :param int policy_id: ID of the policy to set for the devices. - """ - return self._cb.device_update_policy([self._model_unique_id], policy_id) - - def update_sensor_version(self, sensor_version): - """ - Update the sensor version for this device. - - :param dict sensor_version: New version properties for the sensor. - """ - return self._cb.device_update_sensor_version([self._model_unique_id], sensor_version) diff --git a/src/cbapi/psc/models/device.yaml b/src/cbapi/psc/models/device.yaml deleted file mode 100755 index f03c3a3d..00000000 --- a/src/cbapi/psc/models/device.yaml +++ /dev/null @@ -1,310 +0,0 @@ -type: object -properties: - activation_code: - type: string - description: Device activation code - activation_code_expiry_time: - type: string - description: When the expiration code expires and cannot be used to register a device - ad_group_id: - type: integer - format: int64 - description: Device's AD group - av_ave_version: - type: string - description: AVE version (part of AV Version) - av_engine: - type: string - example: '4.3.0.203-ave.8.3.42.106:avpack.8.4.2.36:vdf.8.12.142.100' - description: Current AV version - av_last_scan_time: - type: string - description: Last AV scan time - av_master: - type: boolean - description: Whether the device is an AV Master (?) - av_pack_version: - type: string - description: Pack version (part of AV Version) - av_product_version: - type: string - description: AV Product version (part of AV Version) - av_status: - type: array - description: AV Statuses - items: - type: string - enum: - - AV_NOT_REGISTERED - - AV_REGISTERED - - AV_DEREGISTERED - - AV_ACTIVE - - AV_BYPASS - - NOT_INSTALLED - - INSTALLED - - UNINSTALLED - - INSTALLED_SERVER - - UNINSTALLED_SERVER - - FULLY_ENABLED - - FULLY_DISABLED - - SIGNATURE_UPDATE_DISABLED - - ONACCESS_SCAN_DISABLED - - ONDEMOND_SCAN_DISABLED - - ONDEMAND_SCAN_DISABLED - - PRODUCT_UPDATE_DISABLED - av_update_servers: - type: array - description: Device's AV servers - items: - type: string - av_vdf_version: - type: string - description: VDF version (part of AV Version) - current_sensor_policy_name: - type: string - description: Current MSM policy name - deregistered_time: - type: string - format: date-time - description: When the device was deregistered with the PSC backend - device_id: - type: integer - format: int64 - description: ID of the device - device_meta_data_item_list: - type: array - description: MSM Device metadata - items: - type: object - properties: - key_name: - type: string - key_value: - type: string - position: - type: integer - format: int32 - device_owner_id: - type: integer - format: int64 - description: ID of the user who owns the device - email: - type: string - description: Email of the user who owns the device - encoded_activation_code: - type: string - description: Encoded device activation code - first_name: - type: string - description: First name of the user who owns the device - id: - type: integer - format: int64 - description: ID of the device - last_contact_time: - type: string - format: date-time - description: Time the device last checked into the PSC backend - last_device_policy_changed_time: - type: string - format: date-time - description: Last time the device's policy was changed - last_device_policy_requested_time: - type: string - format: date-time - description: Last time the device requested policy updates - last_external_ip_address: - type: string - description: Device's external IP - last_internal_ip_address: - type: string - description: Device's internal IP - last_location: - type: string - description: Location of the device (on-/off-premises) - enum: - - UNKNOWN - - ONSITE - - OFFSITE - last_name: - type: string - description: Last name of the user who owns the device - last_policy_updated_time: - type: string - format: date-time - description: Last time the device was MSM processed - last_reported_time: - type: string - format: date-time - description: Time when device last reported an event to PSC backend - last_reset_time: - type: string - format: date-time - description: When the sensor was last reset - last_shutdown_time: - type: string - format: date-time - description: When the device last shut down - linux_kernel_version: - type: string - description: Linux kernel version - login_user_name: - type: string - description: Last acive logged in username - mac_address: - type: string - description: Device's hardware MAC address - middle_name: - type: string - description: Middle name of the user who owns the device - name: - type: string - description: Device Hostname - organization_id: - type: integer - format: int64 - example: 1000 - description: Org ID to which the device belongs - organization_name: - type: string - description: Name of the org that owns this device - os: - type: string - example: WINDOWS - description: Device type - enum: - - WINDOWS - - ANDROID - - MAC - - IOS - - LINUX - - OTHER - os_version: - type: string - example: 'Windows 7 x86 SP: 1' - description: Version of the OS - passive_mode: - type: boolean - description: Whether the device is in passive mode (bypass?) - policy_id: - type: integer - format: int64 - description: ID of the policy this device is using - policy_name: - type: string - description: Name of the policy this device is using - policy_override: - type: boolean - description: Manually assigned policy (overrides mass sensor management) - quarantined: - type: boolean - description: Whether the device is quarantined - registered_time: - type: string - format: date-time - description: When the device was registered with the PSC backend - scan_last_action_time: - type: string - format: date-time - description: When the background scan was last active - scan_last_complete_time: - type: string - format: date-time - description: When the background scan was last completed - scan_status: - type: string - description: Background scan status - enum: - - NEVER_RUN - - STOPPED - - IN_PROGRESS - - COMPLETED - sensor_out_of_date: - type: boolean - description: Whether the device is out of date - sensor_states: - type: array - description: Active sensor states - items: - type: string - enum: - - ACTIVE - - PANICS_DETECTED - - LOOP_DETECTED - - DB_CORRUPTION_DETECTED - - CSR_ACTION - - REPUX_ACTION - - DRIVER_INIT_ERROR - - REMGR_INIT_ERROR - - UNSUPPORTED_OS - - SENSOR_UPGRADE_IN_PROGRESS - - SENSOR_UNREGISTERED - - WATCHDOG - - SENSOR_RESET_IN_PROGRESS - - DRIVER_INIT_REBOOT_REQUIRED - - SENSOR_SHUTDOWN - - SENSOR_MAINTENANCE - - DEBUG_MODE_ENABLED - - AUTO_UPDATE_DISABLED - - SELF_PROTECT_DISABLED - - VDI_MODE_ENABLED - - POC_MODE_ENABLED - - SECURITY_CENTER_OPTLN_DISABLED - - LIVE_RESPONSE_RUNNING - - LIVE_RESPONSE_NOT_RUNNING - - LIVE_RESPONSE_KILLED - - LIVE_RESPONSE_NOT_KILLED - - LIVE_RESPONSE_ENABLED - - LIVE_RESPONSE_DISABLED - sensor_version: - type: string - example: 3.4.0.0 - description: Version of the PSC sensor - status: - type: string - description: Device status - enum: - - PENDING - - REGISTERED - - UNINSTALLED - - DEREGISTERED - - ACTIVE - - INACTIVE - - ERROR - - ALL - - BYPASS_ON - - BYPASS - - QUARANTINE - - SENSOR_OUTOFDATE - - DELETED - - LIVE - target_priority_type: - type: string - example: MISSION_CRITICAL - description: Priority of the device - enum: - - LOW - - MEDIUM - - HIGH - - MISSION_CRITICAL - uninstall_code: - type: string - description: Code to enter to uninstall this device - vdi_base_device: - type: integer - format: int64 - description: VDI Base device - virtual_machine: - type: boolean - description: Whether this device is a Virtual Machine (VMware AppDefense integration - virtualization_provider: - type: string - description: VM Virtualization Provider - windows_platform: - type: string - description: 'Type of windows platform (client/server, x86/x64)' - enum: - - CLIENT_X86 - - CLIENT_X64 - - SERVER_X86 - - SERVER_X64 diff --git a/src/cbapi/psc/rest_api.py b/src/cbapi/psc/rest_api.py deleted file mode 100755 index ad7a68da..00000000 --- a/src/cbapi/psc/rest_api.py +++ /dev/null @@ -1,191 +0,0 @@ -from cbapi.connection import BaseAPI -from cbapi.errors import ApiError, ServerError -import logging - -log = logging.getLogger(__name__) - - -class CbPSCBaseAPI(BaseAPI): - """The main entry point into the Cb PSC API. - - :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. - Uses the profile named 'default' when not specified. - - Usage:: - - >>> from cbapi import CbPSCBaseAPI - >>> cb = CbPSCBaseAPI(profile="production") - """ - def __init__(self, *args, **kwargs): - super(CbPSCBaseAPI, self).__init__(product_name="psc", *args, **kwargs) - self._lr_scheduler = None - - def _perform_query(self, cls, **kwargs): - if hasattr(cls, "_query_implementation"): - return cls._query_implementation(self) - else: - raise ApiError("All PSC models should provide _query_implementation") - - # ---- Device API - - def _raw_device_action(self, request): - """ - Invokes the API method for a device action. - - :param dict request: The request body to be passed as JSON to the API method. - :return: The parsed JSON output from the request. - :raises ServerError: If the API method returns an HTTP error code. - """ - url = "/appservices/v6/orgs/{0}/device_actions".format(self.credentials.org_key) - resp = self.post_object(url, body=request) - if resp.status_code == 200: - return resp.json() - elif resp.status_code == 204: - return None - else: - raise ServerError(error_code=resp.status_code, message="Device action error: {0}".format(resp.content)) - - def _device_action(self, device_ids, action_type, options=None): - """ - Executes a device action on multiple device IDs. - - :param list device_ids: The list of device IDs to execute the action on. - :param str action_type: The action type to be performed. - :param dict options: Options for the bulk device action. Default None. - """ - request = {"action_type": action_type, "device_id": device_ids} - if options: - request["options"] = options - return self._raw_device_action(request) - - def _action_toggle(self, flag): - """ - Converts a boolean flag value into a "toggle" option. - - :param boolean flag: The value to be converted. - :return: A dict containing the appropriate "toggle" element. - """ - if flag: - return {"toggle": "ON"} - else: - return {"toggle": "OFF"} - - def device_background_scan(self, device_ids, scan): - """ - Set the background scan option for the specified devices. - - :param list device_ids: List of IDs of devices to be set. - :param boolean scan: True to turn background scan on, False to turn it off. - """ - return self._device_action(device_ids, "BACKGROUND_SCAN", self._action_toggle(scan)) - - def device_bypass(self, device_ids, enable): - """ - Set the bypass option for the specified devices. - - :param list device_ids: List of IDs of devices to be set. - :param boolean enable: True to enable bypass, False to disable it. - """ - return self._device_action(device_ids, "BYPASS", self._action_toggle(enable)) - - def device_delete_sensor(self, device_ids): - """ - Delete the specified sensor devices. - - :param list device_ids: List of IDs of devices to be deleted. - """ - return self._device_action(device_ids, "DELETE_SENSOR") - - def device_uninstall_sensor(self, device_ids): - """ - Uninstall the specified sensor devices. - - :param list device_ids: List of IDs of devices to be uninstalled. - """ - return self._device_action(device_ids, "UNINSTALL_SENSOR") - - def device_quarantine(self, device_ids, enable): - """ - Set the quarantine option for the specified devices. - - :param list device_ids: List of IDs of devices to be set. - :param boolean enable: True to enable quarantine, False to disable it. - """ - return self._device_action(device_ids, "QUARANTINE", self._action_toggle(enable)) - - def device_update_policy(self, device_ids, policy_id): - """ - Set the current policy for the specified devices. - - :param list device_ids: List of IDs of devices to be changed. - :param int policy_id: ID of the policy to set for the devices. - """ - return self._device_action(device_ids, "UPDATE_POLICY", {"policy_id": policy_id}) - - def device_update_sensor_version(self, device_ids, sensor_version): - """ - Update the sensor version for the specified devices. - - :param list device_ids: List of IDs of devices to be changed. - :param dict sensor_version: New version properties for the sensor. - """ - return self._device_action(device_ids, "UPDATE_SENSOR_VERSION", {"sensor_version": sensor_version}) - - # ---- Alerts API - - def alert_search_suggestions(self, query): - """ - Returns suggestions for keys and field values that can be used in a search. - - :param query str: A search query to use. - :return: A list of search suggestions expressed as dict objects. - """ - query_params = {"suggest.q": query} - url = "/appservices/v6/orgs/{0}/alerts/search_suggestions".format(self.credentials.org_key) - output = self.get_object(url, query_params) - return output["suggestions"] - - def _bulk_threat_update_status(self, threat_ids, status, remediation, comment): - """ - Update the status of alerts associated with multiple threat IDs, past and future. - - :param list threat_ids: List of string threat IDs. - :param str status: The status to set for all alerts, either "OPEN" or "DISMISSED". - :param str remediation: The remediation state to set for all alerts. - :param str comment: The comment to set for all alerts. - """ - if not all(isinstance(t, str) for t in threat_ids): - raise ApiError("One or more invalid threat ID values") - request = {"state": status, "threat_id": threat_ids} - if remediation is not None: - request["remediation_state"] = remediation - if comment is not None: - request["comment"] = comment - url = "/appservices/v6/orgs/{0}/threat/workflow/_criteria".format(self.credentials.org_key) - resp = self.post_object(url, body=request) - output = resp.json() - return output["request_id"] - - def bulk_threat_update(self, threat_ids, remediation=None, comment=None): - """ - Update the alert status of alerts associated with multiple threat IDs. - The alerts will be left in an OPEN state after this request. - - :param threat_ids list: List of string threat IDs. - :param remediation str: The remediation state to set for all alerts. - :param comment str: The comment to set for all alerts. - :return: The request ID, which may be used to select a WorkflowStatus object. - """ - return self._bulk_threat_update_status(threat_ids, "OPEN", remediation, comment) - - def bulk_threat_dismiss(self, threat_ids, remediation=None, comment=None): - """ - Dismiss the alerts associated with multiple threat IDs. - The alerts will be left in a DISMISSED state after this request. - - :param threat_ids list: List of string threat IDs. - :param remediation str: The remediation state to set for all alerts. - :param comment str: The comment to set for all alerts. - :return: The request ID, which may be used to select a WorkflowStatus object. - """ - return self._bulk_threat_update_status(threat_ids, "DISMISSED", remediation, comment) diff --git a/test/cbapi/__init__.py b/test/cbapi/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/test/cbapi/psc/__init__.py b/test/cbapi/psc/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/test/cbapi/psc/livequery/__init__.py b/test/cbapi/psc/livequery/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/test/cbapi/psc/livequery/test_models.py b/test/cbapi/psc/livequery/test_models.py deleted file mode 100755 index cc227c2e..00000000 --- a/test/cbapi/psc/livequery/test_models.py +++ /dev/null @@ -1,234 +0,0 @@ -import pytest -from cbapi.psc.livequery.rest_api import CbLiveQueryAPI -from cbapi.psc.livequery.models import Run, Result -from cbapi.psc.livequery.query import ResultQuery, FacetQuery -from cbapi.errors import ApiError -from test.cbtest import StubResponse, patch_cbapi - - -def test_run_refresh(monkeypatch): - _was_called = False - - def _get_run(url, parms=None, default=None): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" - _was_called = True - return {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "COMPLETE"} - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_run) - run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "ACTIVE"}) - rc = run.refresh() - assert _was_called - assert rc - assert run.org_key == "Z100" - assert run.name == "FoobieBletch" - assert run.id == "abcdefg" - assert run.status == "COMPLETE" - - -def test_run_stop(monkeypatch): - _was_called = False - - def _execute_stop(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/status" - assert body == {"status": "CANCELLED"} - _was_called = True - return StubResponse({"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "CANCELLED"}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, PUT=_execute_stop) - run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "ACTIVE"}) - rc = run.stop() - assert _was_called - assert rc - assert run.org_key == "Z100" - assert run.name == "FoobieBletch" - assert run.id == "abcdefg" - assert run.status == "CANCELLED" - - -def test_run_stop_failed(monkeypatch): - _was_called = False - - def _execute_stop(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/status" - assert body == {"status": "CANCELLED"} - _was_called = True - return StubResponse({"error_message": "The query is not presently running."}, 409) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, PUT=_execute_stop) - run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "CANCELLED"}) - rc = run.stop() - assert _was_called - assert not rc - - -def test_run_delete(monkeypatch): - _was_called = False - - def _execute_delete(url): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" - if _was_called: - pytest.fail("_execute_delete should not be called twice!") - _was_called = True - return StubResponse(None) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, DELETE=_execute_delete) - run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "ACTIVE"}) - rc = run.delete() - assert _was_called - assert rc - assert run._is_deleted - # Now ensure that certain operations that don't make sense on a deleted object raise ApiError - with pytest.raises(ApiError): - run.refresh() - with pytest.raises(ApiError): - run.stop() - # And make sure that deleting a deleted object returns True immediately - rc = run.delete() - assert rc - - -def test_run_delete_failed(monkeypatch): - _was_called = False - - def _execute_delete(url): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" - _was_called = True - return StubResponse(None, 403) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, DELETE=_execute_delete) - run = Run(api, "abcdefg", {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg", "status": "ACTIVE"}) - rc = run.delete() - assert _was_called - assert not rc - assert not run._is_deleted - - -def test_result_device_summaries(monkeypatch): - _was_called = False - - def _run_summaries(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/results/device_summaries/_search" - assert body == {"query": "foo", "criteria": {"device_name": ["AxCx", "A7X"]}, - "sort": [{"field": "device_name", "order": "ASC"}], "start": 0} - _was_called = True - return StubResponse({"org_key": "Z100", "num_found": 2, - "results": [{"id": "ghijklm", "total_results": 2, "device_id": 314159, - "metrics": [{"key": "aaa", "value": 0.0}, {"key": "bbb", "value": 0.0}]}, - {"id": "mnopqrs", "total_results": 3, "device_id": 271828, - "metrics": [{"key": "aaa", "value": 0.0}, {"key": "bbb", "value": 0.0}]}]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_summaries) - result = Result(api, {"id": "abcdefg", "device": {"id": "abcdefg"}, "fields": {}, "metrics": {}}) - query = result.query_device_summaries().where("foo").criteria(device_name=["AxCx", "A7X"]).sort_by("device_name") - assert isinstance(query, ResultQuery) - count = 0 - for item in query.all(): - if item.id == "ghijklm": - assert item.total_results == 2 - assert item.device_id == 314159 - elif item.id == "mnopqrs": - assert item.total_results == 3 - assert item.device_id == 271828 - else: - pytest.fail("Invalid object with ID %s seen" % item.id) - count = count + 1 - assert _was_called - assert count == 2 - - -def test_result_query_result_facets(monkeypatch): - _was_called = False - - def _run_facets(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/results/_facet" - assert body == {"query": "xyzzy", "criteria": {"device_name": ["AxCx", "A7X"]}, - "terms": {"fields": ["alpha", "bravo", "charlie"]}} - _was_called = True - return StubResponse({"terms": [{"field": "alpha", "values": [{"total": 1, "id": "alpha1", "name": "alpha1"}, - {"total": 2, "id": "alpha2", "name": "alpha2"}]}, - {"field": "bravo", "values": [{"total": 1, "id": "bravo1", "name": "bravo1"}, - {"total": 2, "id": "bravo2", "name": "bravo2"}]}, - {"field": "charlie", "values": [{"total": 1, "id": "charlie1", - "name": "charlie1"}, - {"total": 2, "id": "charlie2", - "name": "charlie2"}]}]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_facets) - result = Result(api, {"id": "abcdefg", "device": {"id": "abcdefg"}, "fields": {}, "metrics": {}}) - query = result.query_result_facets().where("xyzzy").facet_field("alpha").facet_field(["bravo", "charlie"]) \ - .criteria(device_name=["AxCx", "A7X"]) - assert isinstance(query, FacetQuery) - count = 0 - for item in query.all(): - vals = item.values - if item.field == "alpha": - assert vals[0]["id"] == "alpha1" - assert vals[1]["id"] == "alpha2" - elif item.field == "bravo": - assert vals[0]["id"] == "bravo1" - assert vals[1]["id"] == "bravo2" - elif item.field == "charlie": - assert vals[0]["id"] == "charlie1" - assert vals[1]["id"] == "charlie2" - else: - pytest.fail("Unknown field name %s seen" % item.field) - count = count + 1 - assert _was_called - assert count == 3 - - -def test_result_query_device_summary_facets(monkeypatch): - _was_called = False - - def _run_facets(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/results/device_summaries/_facet" - assert body == {"query": "xyzzy", "criteria": {"device_name": ["AxCx", "A7X"]}, - "terms": {"fields": ["alpha", "bravo", "charlie"]}} - _was_called = True - return StubResponse({"terms": [{"field": "alpha", "values": [{"total": 1, "id": "alpha1", "name": "alpha1"}, - {"total": 2, "id": "alpha2", "name": "alpha2"}]}, - {"field": "bravo", "values": [{"total": 1, "id": "bravo1", "name": "bravo1"}, - {"total": 2, "id": "bravo2", "name": "bravo2"}]}, - {"field": "charlie", "values": [{"total": 1, "id": "charlie1", - "name": "charlie1"}, - {"total": 2, "id": "charlie2", - "name": "charlie2"}]}]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_facets) - result = Result(api, {"id": "abcdefg", "device": {"id": "abcdefg"}, "fields": {}, "metrics": {}}) - query = result.query_device_summary_facets().where("xyzzy").facet_field("alpha") \ - .facet_field(["bravo", "charlie"]).criteria(device_name=["AxCx", "A7X"]) - assert isinstance(query, FacetQuery) - count = 0 - for item in query.all(): - vals = item.values - if item.field == "alpha": - assert vals[0]["id"] == "alpha1" - assert vals[1]["id"] == "alpha2" - elif item.field == "bravo": - assert vals[0]["id"] == "bravo1" - assert vals[1]["id"] == "bravo2" - elif item.field == "charlie": - assert vals[0]["id"] == "charlie1" - assert vals[1]["id"] == "charlie2" - else: - pytest.fail("Unknown field name %s seen" % item.field) - count = count + 1 - assert _was_called - assert count == 3 diff --git a/test/cbapi/psc/livequery/test_rest_api.py b/test/cbapi/psc/livequery/test_rest_api.py deleted file mode 100755 index b9d38a17..00000000 --- a/test/cbapi/psc/livequery/test_rest_api.py +++ /dev/null @@ -1,161 +0,0 @@ -import pytest -from cbapi.psc.livequery.rest_api import CbLiveQueryAPI -from cbapi.psc.livequery.models import Run -from cbapi.psc.livequery.query import RunQuery, RunHistoryQuery -from cbapi.errors import ApiError, CredentialError -from test.cbtest import StubResponse, patch_cbapi - - -def test_no_org_key(): - with pytest.raises(CredentialError): - CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", ssl_verify=True) # note: no org_key - - -def test_simple_get(monkeypatch): - _was_called = False - - def _get_run(url, parms=None, default=None): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" - _was_called = True - return {"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"} - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_run) - run = api.select(Run, "abcdefg") - assert _was_called - assert run.org_key == "Z100" - assert run.name == "FoobieBletch" - assert run.id == "abcdefg" - - -def test_query(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs" - assert body == {"sql": "select * from whatever;", "device_filter": {}} - _was_called = True - return StubResponse({"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.query("select * from whatever;") - assert isinstance(query, RunQuery) - run = query.submit() - assert _was_called - assert run.org_key == "Z100" - assert run.name == "FoobieBletch" - assert run.id == "abcdefg" - - -def test_query_with_everything(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs" - assert body == {"sql": "select * from whatever;", "name": "AmyWasHere", "notify_on_finish": True, - "device_filter": {"device_ids": [1, 2, 3], "device_types": ["Alpha", "Bravo", "Charlie"], - "policy_ids": [16, 27, 38]}} - _was_called = True - return StubResponse({"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.query("select * from whatever;").device_ids([1, 2, 3]).device_types(["Alpha", "Bravo", "Charlie"]) \ - .policy_ids([16, 27, 38]).name("AmyWasHere").notify_on_finish() - assert isinstance(query, RunQuery) - run = query.submit() - assert _was_called - assert run.org_key == "Z100" - assert run.name == "FoobieBletch" - assert run.id == "abcdefg" - - -def test_query_device_ids_broken(): - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - query = api.query("select * from whatever;") - with pytest.raises(ApiError): - query = query.device_ids(["Bogus"]) - - -def test_query_device_types_broken(): - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - query = api.query("select * from whatever;") - with pytest.raises(ApiError): - query = query.device_types([420]) - - -def test_query_policy_ids_broken(): - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - query = api.query("select * from whatever;") - with pytest.raises(ApiError): - query = query.policy_ids(["Bogus"]) - - -def test_query_history(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/_search" - assert body == {"query": "xyzzy", "start": 0} - _was_called = True - return StubResponse({"org_key": "Z100", "num_found": 3, - "results": [{"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}, - {"org_key": "Z100", "name": "Aoxomoxoa", "id": "cdefghi"}, - {"org_key": "Z100", "name": "Read_Me", "id": "efghijk"}]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.query_history("xyzzy") - assert isinstance(query, RunHistoryQuery) - count = 0 - for item in query.all(): - assert item.org_key == "Z100" - if item.id == "abcdefg": - assert item.name == "FoobieBletch" - elif item.id == "cdefghi": - assert item.name == "Aoxomoxoa" - elif item.id == "efghijk": - assert item.name == "Read_Me" - else: - pytest.fail("Unknown item ID: %s" % item.id) - count = count + 1 - assert _was_called - assert count == 3 - - -def test_query_history_with_everything(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/_search" - assert body == {"query": "xyzzy", "sort": [{"field": "id", "order": "ASC"}], "start": 0} - _was_called = True - return StubResponse({"org_key": "Z100", "num_found": 3, - "results": [{"org_key": "Z100", "name": "FoobieBletch", "id": "abcdefg"}, - {"org_key": "Z100", "name": "Aoxomoxoa", "id": "cdefghi"}, - {"org_key": "Z100", "name": "Read_Me", "id": "efghijk"}]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.query_history("xyzzy").sort_by("id") - assert isinstance(query, RunHistoryQuery) - count = 0 - for item in query.all(): - assert item.org_key == "Z100" - if item.id == "abcdefg": - assert item.name == "FoobieBletch" - elif item.id == "cdefghi": - assert item.name == "Aoxomoxoa" - elif item.id == "efghijk": - assert item.name == "Read_Me" - else: - pytest.fail("Unknown item ID: %s" % item.id) - count = count + 1 - assert _was_called - assert count == 3 diff --git a/test/cbapi/psc/test_devicev6_api.py b/test/cbapi/psc/test_devicev6_api.py deleted file mode 100755 index a73570e7..00000000 --- a/test/cbapi/psc/test_devicev6_api.py +++ /dev/null @@ -1,383 +0,0 @@ -import pytest -from cbapi.errors import ApiError -from cbapi.psc.models import Device -from cbapi.psc.rest_api import CbPSCBaseAPI -from test.cbtest import StubResponse, patch_cbapi - - -def test_get_device(monkeypatch): - _was_called = False - - def _get_device(url): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/devices/6023" - _was_called = True - return {"device_id": 6023, "organization_name": "thistestworks"} - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_device) - rc = api.select(Device, 6023) - assert _was_called - assert isinstance(rc, Device) - assert rc.device_id == 6023 - assert rc.organization_name == "thistestworks" - - -def test_device_background_scan(monkeypatch): - _was_called = False - - def _call_background_scan(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "BACKGROUND_SCAN", "device_id": [6023], "options": {"toggle": "ON"}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_call_background_scan) - api.device_background_scan([6023], True) - assert _was_called - - -def test_device_bypass(monkeypatch): - _was_called = False - - def _call_bypass(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "BYPASS", "device_id": [6023], "options": {"toggle": "OFF"}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_call_bypass) - api.device_bypass([6023], False) - assert _was_called - - -def test_device_delete_sensor(monkeypatch): - _was_called = False - - def _call_delete_sensor(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "DELETE_SENSOR", "device_id": [6023]} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_call_delete_sensor) - api.device_delete_sensor([6023]) - assert _was_called - - -def test_device_uninstall_sensor(monkeypatch): - _was_called = False - - def _call_uninstall_sensor(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "UNINSTALL_SENSOR", "device_id": [6023]} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_call_uninstall_sensor) - api.device_uninstall_sensor([6023]) - assert _was_called - - -def test_device_quarantine(monkeypatch): - _was_called = False - - def _call_quarantine(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "QUARANTINE", "device_id": [6023], "options": {"toggle": "ON"}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_call_quarantine) - api.device_quarantine([6023], True) - assert _was_called - - -def test_device_update_policy(monkeypatch): - _was_called = False - - def _call_update_policy(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "UPDATE_POLICY", "device_id": [6023], "options": {"policy_id": 8675309}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_call_update_policy) - api.device_update_policy([6023], 8675309) - assert _was_called - - -def test_device_update_sensor_version(monkeypatch): - _was_called = False - - def _call_update_sensor_version(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "UPDATE_SENSOR_VERSION", "device_id": [6023], - "options": {"sensor_version": {"RHEL": "2.3.4.5"}}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_call_update_sensor_version) - api.device_update_sensor_version([6023], {"RHEL": "2.3.4.5"}) - assert _was_called - - -def test_query_device_with_all_bells_and_whistles(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/devices/_search" - assert body == {"query": "foobar", - "criteria": {"ad_group_id": [14, 25], "os": ["LINUX"], "policy_id": [8675309], - "status": ["ALL"], "target_priority": ["HIGH"]}, - "exclusions": {"sensor_version": ["0.1"]}, - "sort": [{"field": "name", "order": "DESC"}]} - _was_called = True - return StubResponse({"results": [{"id": 6023, "organization_name": "thistestworks"}], - "num_found": 1}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.select(Device).where("foobar").set_ad_group_ids([14, 25]).set_os(["LINUX"]) \ - .set_policy_ids([8675309]).set_status(["ALL"]).set_target_priorities(["HIGH"]) \ - .set_exclude_sensor_versions(["0.1"]).sort_by("name", "DESC") - d = query.one() - assert _was_called - assert d.id == 6023 - assert d.organization_name == "thistestworks" - - -def test_query_device_with_last_contact_time_as_start_end(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/devices/_search" - assert body == {"query": "foobar", - "criteria": {"last_contact_time": {"start": "2019-09-30T12:34:56", - "end": "2019-10-01T12:00:12"}}, "exclusions": {}} - _was_called = True - return StubResponse({"results": [{"id": 6023, "organization_name": "thistestworks"}], - "num_found": 1}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.select(Device).where("foobar") \ - .set_last_contact_time(start="2019-09-30T12:34:56", end="2019-10-01T12:00:12") - d = query.one() - assert _was_called - assert d.id == 6023 - assert d.organization_name == "thistestworks" - - -def test_query_device_with_last_contact_time_as_range(monkeypatch): - _was_called = False - - def _run_query(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/devices/_search" - assert body == {"query": "foobar", "criteria": {"last_contact_time": {"range": "-3w"}}, "exclusions": {}} - _was_called = True - return StubResponse({"results": [{"id": 6023, "organization_name": "thistestworks"}], - "num_found": 1}) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_run_query) - query = api.select(Device).where("foobar").set_last_contact_time(range="-3w") - d = query.one() - assert _was_called - assert d.id == 6023 - assert d.organization_name == "thistestworks" - - -def test_query_device_invalid_last_contact_time_combinations(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).set_last_contact_time() - with pytest.raises(ApiError): - api.select(Device).set_last_contact_time(start="2019-09-30T12:34:56", end="2019-10-01T12:00:12", - range="-3w") - with pytest.raises(ApiError): - api.select(Device).set_last_contact_time(start="2019-09-30T12:34:56", range="-3w") - with pytest.raises(ApiError): - api.select(Device).set_last_contact_time(end="2019-10-01T12:00:12", range="-3w") - - -def test_query_device_invalid_criteria_values(): - tests = [ - {"method": "set_ad_group_ids", "arg": ["Bogus"]}, - {"method": "set_policy_ids", "arg": ["Bogus"]}, - {"method": "set_os", "arg": ["COMMODORE_64"]}, - {"method": "set_status", "arg": ["Bogus"]}, - {"method": "set_target_priorities", "arg": ["Bogus"]}, - {"method": "set_exclude_sensor_versions", "arg": [12703]} - ] - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - query = api.select(Device) - for t in tests: - meth = getattr(query, t["method"], None) - with pytest.raises(ApiError): - meth(t["arg"]) - - -def test_query_device_invalid_sort_direction(): - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - with pytest.raises(ApiError): - api.select(Device).sort_by("policy_name", "BOGUS") - - -def test_query_device_download(monkeypatch): - _was_called = False - - def _run_download(url, query_params, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/devices/_search/download" - assert query_params == {"status": "ALL", "ad_group_id": "14,25", "policy_id": "8675309", - "target_priority": "HIGH", "query_string": "foobar", "sort_field": "name", - "sort_order": "DESC"} - _was_called = True - return "123456789,123456789,123456789" - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, RAW_GET=_run_download) - rc = api.select(Device).where("foobar").set_ad_group_ids([14, 25]).set_policy_ids([8675309]) \ - .set_status(["ALL"]).set_target_priorities(["HIGH"]).sort_by("name", "DESC").download() - assert _was_called - assert rc == "123456789,123456789,123456789" - - -def test_query_device_do_background_scan(monkeypatch): - _was_called = False - - def _background_scan(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "BACKGROUND_SCAN", - "search": {"query": "foobar", "criteria": {}, "exclusions": {}}, "options": {"toggle": "ON"}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_background_scan) - api.select(Device).where("foobar").background_scan(True) - assert _was_called - - -def test_query_device_do_bypass(monkeypatch): - _was_called = False - - def _bypass(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "BYPASS", - "search": {"query": "foobar", "criteria": {}, "exclusions": {}}, "options": {"toggle": "OFF"}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_bypass) - api.select(Device).where("foobar").bypass(False) - assert _was_called - - -def test_query_device_do_delete_sensor(monkeypatch): - _was_called = False - - def _delete_sensor(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "DELETE_SENSOR", - "search": {"query": "foobar", "criteria": {}, "exclusions": {}}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_delete_sensor) - api.select(Device).where("foobar").delete_sensor() - assert _was_called - - -def test_query_device_do_uninstall_sensor(monkeypatch): - _was_called = False - - def _uninstall_sensor(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "UNINSTALL_SENSOR", - "search": {"query": "foobar", "criteria": {}, "exclusions": {}}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_uninstall_sensor) - api.select(Device).where("foobar").uninstall_sensor() - assert _was_called - - -def test_query_device_do_quarantine(monkeypatch): - _was_called = False - - def _quarantine(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "QUARANTINE", - "search": {"query": "foobar", "criteria": {}, "exclusions": {}}, "options": {"toggle": "ON"}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_quarantine) - api.select(Device).where("foobar").quarantine(True) - assert _was_called - - -def test_query_device_do_update_policy(monkeypatch): - _was_called = False - - def _update_policy(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "UPDATE_POLICY", - "search": {"query": "foobar", "criteria": {}, "exclusions": {}}, - "options": {"policy_id": 8675309}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_update_policy) - api.select(Device).where("foobar").update_policy(8675309) - assert _was_called - - -def test_query_device_do_update_sensor_version(monkeypatch): - _was_called = False - - def _update_sensor_version(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "UPDATE_SENSOR_VERSION", - "search": {"query": "foobar", "criteria": {}, "exclusions": {}}, - "options": {"sensor_version": {"RHEL": "2.3.4.5"}}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, POST=_update_sensor_version) - api.select(Device).where("foobar").update_sensor_version({"RHEL": "2.3.4.5"}) - assert _was_called diff --git a/test/cbapi/psc/test_models.py b/test/cbapi/psc/test_models.py deleted file mode 100755 index c9158fa7..00000000 --- a/test/cbapi/psc/test_models.py +++ /dev/null @@ -1,179 +0,0 @@ -import pytest -from cbapi.psc.models import Device -from cbapi.psc.rest_api import CbPSCBaseAPI -from test.cbtest import StubResponse, patch_cbapi - - -class StubScheduler: - def __init__(self, expected_id): - self.expected_id = expected_id - self.was_called = False - - def request_session(self, sensor_id): - assert sensor_id == self.expected_id - self.was_called = True - return {"itworks": True} - - -def test_Device_lr_session(monkeypatch): - - def _get_session(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/devices/6023" - return {"id": 6023} - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - sked = StubScheduler(6023) - api._lr_scheduler = sked - patch_cbapi(monkeypatch, api, GET=_get_session) - dev = Device(api, 6023, {"id": 6023}) - sess = dev.lr_session() - assert sess["itworks"] - assert sked.was_called - - -def test_Device_background_scan(monkeypatch): - _was_called = False - - def _get_device(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/devices/6023" - return {"id": 6023} - - def _background_scan(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "BACKGROUND_SCAN", "device_id": [6023], "options": {"toggle": "ON"}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_device, POST=_background_scan) - dev = Device(api, 6023, {"id": 6023}) - dev.background_scan(True) - assert _was_called - - -def test_Device_bypass(monkeypatch): - _was_called = False - - def _get_device(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/devices/6023" - return {"id": 6023} - - def _bypass(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "BYPASS", "device_id": [6023], "options": {"toggle": "OFF"}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_device, POST=_bypass) - dev = Device(api, 6023, {"id": 6023}) - dev.bypass(False) - assert _was_called - - -def test_Device_delete_sensor(monkeypatch): - _was_called = False - - def _get_device(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/devices/6023" - return {"id": 6023} - - def _delete_sensor(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "DELETE_SENSOR", "device_id": [6023]} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_device, POST=_delete_sensor) - dev = Device(api, 6023, {"id": 6023}) - dev.delete_sensor() - assert _was_called - - -def test_Device_uninstall_sensor(monkeypatch): - _was_called = False - - def _get_device(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/devices/6023" - return {"id": 6023} - - def _uninstall_sensor(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "UNINSTALL_SENSOR", "device_id": [6023]} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_device, POST=_uninstall_sensor) - dev = Device(api, 6023, {"id": 6023}) - dev.uninstall_sensor() - assert _was_called - - -def test_Device_quarantine(monkeypatch): - _was_called = False - - def _get_device(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/devices/6023" - return {"id": 6023} - - def _quarantine(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "QUARANTINE", "device_id": [6023], "options": {"toggle": "ON"}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_device, POST=_quarantine) - dev = Device(api, 6023, {"id": 6023}) - dev.quarantine(True) - assert _was_called - - -def test_Device_update_policy(monkeypatch): - _was_called = False - - def _get_device(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/devices/6023" - return {"id": 6023} - - def _update_policy(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "UPDATE_POLICY", "device_id": [6023], "options": {"policy_id": 8675309}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_device, POST=_update_policy) - dev = Device(api, 6023, {"id": 6023}) - dev.update_policy(8675309) - assert _was_called - - -def test_Device_update_sensor_version(monkeypatch): - _was_called = False - - def _get_device(url, parms=None, default=None): - assert url == "/appservices/v6/orgs/Z100/devices/6023" - return {"id": 6023} - - def _update_sensor_version(url, body, **kwargs): - nonlocal _was_called - assert url == "/appservices/v6/orgs/Z100/device_actions" - assert body == {"action_type": "UPDATE_SENSOR_VERSION", "device_id": [6023], - "options": {"sensor_version": {"RHEL": "2.3.4.5"}}} - _was_called = True - return StubResponse(None, 204) - - api = CbPSCBaseAPI(url="https://example.com", token="ABCD/1234", org_key="Z100", ssl_verify=True) - patch_cbapi(monkeypatch, api, GET=_get_device, POST=_update_sensor_version) - dev = Device(api, 6023, {"id": 6023}) - dev.update_sensor_version({"RHEL": "2.3.4.5"}) - assert _was_called From 707dcf1bf57cd6127a224525723a3fd52814965e Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Thu, 25 Jul 2024 16:02:21 +0300 Subject: [PATCH 196/197] one more --- src/cbapi/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cbapi/auth.py b/src/cbapi/auth.py index 60cce5a8..bfc6c163 100644 --- a/src/cbapi/auth.py +++ b/src/cbapi/auth.py @@ -78,7 +78,7 @@ def get_credentials(self, profile=None): class FileCredentialStore(object): def __init__(self, product_name, **kwargs): - if product_name not in ("response", "protection", "psc"): + if product_name not in ("response", "protection"): raise CredentialError("Product name {0:s} not valid".format(product_name)) self.credential_search_path = [ From 9dea00137db30096d8ca154264b982901aecf946 Mon Sep 17 00:00:00 2001 From: Emanuela Mitreva Date: Fri, 26 Jul 2024 10:21:21 +0300 Subject: [PATCH 197/197] Bump version --- README.md | 2 +- docs/changelog.rst | 7 +++++++ docs/conf.py | 2 +- setup.py | 2 +- src/cbapi/__init__.py | 2 +- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 26625acc..a736bf24 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.7.10** +**Latest Version: 2.0.0** _**Notice**:_ * The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI diff --git a/docs/changelog.rst b/docs/changelog.rst index 37ba3b0c..ac13cca6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,13 @@ Updates `Carbon Black Cloud Python SDK on the Developer Network `_ for details. +CbAPI 2.0.0 - Release July 29, 2024 +------------------------------------ + +Breaking Changes + * Removing psc functionalities. + + CbAPI 1.7.10 - Release Feb 1, 2023 ------------------------------------ diff --git a/docs/conf.py b/docs/conf.py index 52251114..33cecda7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # The short X.Y version. version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.7.10' +release = u'2.0.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 50703cc8..ef4e12d5 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires.extend(['packaging']) setup( name='cbapi', - version='1.7.10', + version='2.0.0', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py index d90ed52d..aea6d6ff 100644 --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' __copyright__ = 'Copyright 2018-2022 VMware Carbon Black' -__version__ = '1.7.10' +__version__ = '2.0.0' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI
KeyValue