diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 00000000..9a53bbab --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: bug +assignees: kebringer-cb, talfano-cb + +--- + +**I am seeing this behaviour on: (please complete the following information):** + - OS: [e.g. iOS] + - Carbon Black Product: [e.g. CB Protection, CB Defense] + - Python Version: [e.g. 2.7] + +**Describe the bug** +A clear and concise description of what the bug is. + +**Steps to Reproduce** +Steps to reproduce the behavior (Provide a log message if relevant): +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots or code to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..c82af637 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,49 @@ +## Pull request checklist + +Please check if your PR fulfills the following requirements: + +- [ ] Docs have been reviewed and added / updated if needed (for bug fixes / features) +- [ ] Tests have been added that prove the fix is effective or that the feature works. +- [ ] New and existing tests pass locally with the changes. +- [ ] Code follows the style guidelines of this project (PEP8, clean code). +- [ ] Linter has passed locally and any fixes were made for failures. +- [ ] A self-review of the code has been done. + +## Pull request type + + + +Please check the type of change your PR introduces: +- [ ] Bugfix +- [ ] Feature +- [ ] Code style update (formatting, renaming) +- [ ] Refactoring (no functional changes, no api changes) +- [ ] Build related changes +- [ ] Documentation content changes (not tied to bugs/features) +- [ ] Other (please describe): + + +## What is the ticket or issue number? + + +- Ticket Number: N/A + +- Issue Number: N/A + +## Pull Request Description + + +## Does this introduce a breaking change? + +- [ ] Yes +- [ ] No + + + +## How Has This Been Tested? + + + +## Other information: + + diff --git a/.gitignore b/.gitignore index db1e3353..7397522a 100644 --- a/.gitignore +++ b/.gitignore @@ -60,14 +60,17 @@ target/ #Ipython Notebook .ipynb_checkpoints - -.idea/ *.ipynb .DS_Store +# IntelliJ IDEA +/.idea +*.iml + # Eclipse/PyDev /.project /.pydevproject +/.settings # Credential files .carbonblack/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..ac48e9db --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,30 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + builder: dirhtml +# fail_on_warning: true + +# If using Sphinx, optionally build your docs in additional formats, such as PDF +formats: + - pdf + - epub + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt + - method: setuptools + path: . diff --git a/LICENSE b/LICENSE index 2b532005..89ad6804 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2016-2019 Carbon Black +Copyright (c) 2016-2021 Carbon Black Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.md b/README.md old mode 100755 new mode 100644 index c7297dd3..a736bf24 --- a/README.md +++ b/README.md @@ -1,23 +1,26 @@ # Python bindings for Carbon Black REST API -**Latest Version: 1.5.4** +**Latest Version: 2.0.0** -These are the new Python bindings for the Carbon Black Enterprise Response and Enterprise Protection REST APIs. +_**Notice**:_ +* The Carbon Black Cloud portion of CBAPI has moved to https://github.com/carbonblack/carbon-black-cloud-sdk-python. Any future development and bug fixes for Carbon Black Cloud APIs will be made there. Carbon Black EDR and App Control will remain supported at CBAPI +* Carbon Black EDR (Endpoint Detection and Response) is the new name for the product formerly called CB Response. +* Carbon Black App Control is the new name for the product formerly called CB Protection. + +These are the Python bindings for the Carbon Black EDR and App Control REST APIs. To learn more about the REST APIs, visit the Carbon Black Developer Network Website at https://developer.carbonblack.com. -Please visit https://cbapi.readthedocs.io for detailed documentation on this API. Additionally, we have a slideshow -available at https://developer.carbonblack.com/2016/07/presentation-on-the-new-carbon-black-python-api/ that provides -an overview of the concepts that underly this API binding. +Please visit https://cbapi.readthedocs.io for detailed documentation on this API. ## Support -If you have questions on the Carbon Black API or these API Bindings, please contact us at dev-support@carbonblack.com. -Also review the documentation and guides available on the -[Carbon Black Developer Network website](https://developer.carbonblack.com) +1. View all API and integration offerings on the [Developer Network](https://developer.carbonblack.com/) along with reference documentation, video tutorials, and how-to guides. +2. Use the [Developer Community Forum](https://community.carbonblack.com/t5/Developer-Relations/bd-p/developer-relations) to discuss issues and get answers from other API developers in the Carbon Black Community. +3. Report bugs and change requests to [Carbon Black Support](https://www.vmware.com/support/services.html). ## Requirements -The new cbapi is designed to work on Python 2.6.6 and above (including 3.x). If you're just starting out, +The cbapi package is designed to work on Python 2.6.6 and above (including 3.x). If you're just starting out, we recommend using the latest version of Python 3.6.x or above. All requirements are installed as part of `pip install`. @@ -26,11 +29,10 @@ The legacy cbapi (`cbapi.CbApi`) and legacy bit9api (`cbapi.bit9Api`) are still ## Backwards Compatibility Backwards compatibility with old scripts is maintained through the `cbapi.legacy` module. Old scripts that import -`cbapi.CbApi` directly will continue to work. Once cbapi 2.0.0 is released, the old `CbApi` will be deprecated and -removed entirely no earlier than January 2017. +`cbapi.CbApi` directly will continue to work. -New scripts should use the `cbapi.CbResponseAPI` (for CB Response) and -`cbapi.CbProtectionAPI` (for CB Protection / former Bit9) API entry points. +New scripts should use the `cbapi.CbResponseAPI` (for EDR (CB Response)) and +`cbapi.CbProtectionAPI` (for App Control (CB Protection)) API entry points. ## Getting Started @@ -47,10 +49,10 @@ Clone this repository, cd into `cbapi-python` then run setup.py with the `develo ### Sample Code -There are several examples in the `examples` directory for both Carbon Black Enterprise Response and Protection. We -will be adding more samples over time. For a quick start, see the following code snippets: +There are several examples in the `examples` directory for both EDR and App Control. +For a quick start, see the following code snippets: -**Carbon Black Enterprise Response** +**Carbon Black EDR** from cbapi.response.models import Process, Binary, Sensor, Feed, Watchlist, Investigation from cbapi.response.rest_api import CbEnterpriseResponseAPI @@ -76,7 +78,7 @@ will be adding more samples over time. For a quick start, see the following code s.save() -**Carbon Black Enterprise Protection** +**Carbon Black App Control** from cbapi.protection.models import * from cbapi.protection.rest_api import CbEnterpriseProtectionAPI @@ -98,9 +100,8 @@ will be adding more samples over time. For a quick start, see the following code In order to perform any queries via the API, you will need to get the API token for your CB user. See the documentation on the Developer Network website on how to acquire the API token for -[CB Response](http://developer.carbonblack.com/reference/enterprise-response/authentication/), -[CB Protection](http://developer.carbonblack.com/reference/enterprise-protection/authentication/), or -[CB Defense](http://developer.carbonblack.com/reference/cb-defense/authentication/). +[CB Response](http://developer.carbonblack.com/reference/enterprise-response/authentication/) or +[CB Protection](http://developer.carbonblack.com/reference/enterprise-protection/authentication/). Once you acquire your API token, place it in one of the default credentials file locations: @@ -110,15 +111,14 @@ Once you acquire your API token, place it in one of the default credentials file For distinction between credentials of different Carbon Black products, use the following naming convention for your credentials files: -* ``credentials.psc`` for CB Defense, CB ThreatHunter, and CB LiveOps -* ``credentials.response`` for CB Response -* ``credentials.protection`` for CB Protection +* ``credentials.response`` for EDR (CB Response) +* ``credentials.protection`` for App Control (CB Protection) -For example, if you use a PSC product, you should have created a credentials file in one of these locations: +For example, if you use a Carbon Black Cloud product, you should have created a credentials file in one of these locations: -* ``/etc/carbonblack/credentials.psc`` -* ``~/.carbonblack/credentials.psc`` -* ``/current_working_directory/.carbonblack/credentials.psc`` +* ``/etc/carbonblack/credentials.response`` +* ``~/.carbonblack/credentials.response`` +* ``/current_working_directory/.carbonblack/credentials.response`` Credentials found in a later path will overwrite earlier ones. @@ -142,17 +142,17 @@ by key-value pairs providing the necessary credential information:: The possible options for each credential profile are: -* **url**: The base URL of the CB server. This should include the protocol (https) and the hostname, and nothing else. +* **url**: The base URL of the Carbon Black server. This should include the protocol (https) and the hostname, and nothing else. * **token**: The API token for the user ID. More than one credential profile can be specified for a given server, with different tokens for each. * **ssl_verify**: True or False; controls whether the SSL/TLS certificate presented by the server is validated against the local trusted CA store. -* **org_key**: The organization key. This is required to access the PSC, and can be found in the console. The format is ``123ABC45``. +* **org_key**: The organization key. This is required to access the Carbon Black Cloud, and can be found in the console. The format is ``123ABC45``. * **proxy**: A proxy specification that will be used when connecting to the CB server. The format is: ``http://myusername:mypassword@proxy.company.com:8001/`` where the hostname of the proxy is ``proxy.company.com``, port 8001, and using username/password ``myusername`` and ``mypassword`` respectively. * **ignore_system_proxy**: If you have a system-wide proxy specified, setting this to True will force cbapi to bypass the proxy and directly connect to the CB server. -Future versions of cbapi will also provide the ability to "pin" the TLS certificate so as to provide certificate +Future versions of cbapi may provide the ability to "pin" the TLS certificate so as to provide certificate verification on self-signed or internal CA signed certificates. diff --git a/bin/cbapi-defense b/bin/cbapi-defense deleted file mode 100644 index 8e6c2443..00000000 --- a/bin/cbapi-defense +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python - -import argparse -import contextlib - -from cbapi.six import iteritems -from cbapi.six.moves import input -import os -import sys -import cbapi.six as six -if six.PY3: - from io import StringIO as StringIO -else: - from cStringIO import StringIO - -from cbapi.six.moves.configparser import RawConfigParser - - -@contextlib.contextmanager -def temp_umask(umask): - oldmask = os.umask(umask) - try: - yield - finally: - os.umask(oldmask) - - -def configure(opts): - credential_path = os.path.join(os.path.expanduser("~"), ".carbonblack") - credential_file = os.path.join(credential_path, "credentials.defense") - - print("Welcome to the CbAPI.") - if os.path.exists(credential_file): - print("An existing credential file exists at {0}.".format(credential_file)) - resp = input("Do you want to continue and overwrite the existing configuration? [Y/N] ") - if resp.strip().upper() != "Y": - print("Exiting.") - return 1 - - if not os.path.exists(credential_path): - os.makedirs(credential_path, 0o700) - - url = input("URL to the Cb Defense API server (do not include '/integrationServices') [https://hostname]: ") - - ssl_verify = True - - connector_id = input("Connector ID: ") - token = input("API key: ") - - config = RawConfigParser() - config.readfp(StringIO('[default]')) - config.set("default", "url", url) - config.set("default", "token", "{0}/{1}".format(token, connector_id)) - config.set("default", "ssl_verify", ssl_verify) - with temp_umask(0): - with os.fdopen(os.open(credential_file, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0o600), 'w') as fp: - os.chmod(credential_file, 0o600) - config.write(fp) - print("Successfully wrote credentials to {0}.".format(credential_file)) - - -command_map = { - "configure": { - "extra_args": {}, - "help": "Configure CbAPI", - "method": configure - } -} - - -def main(args): - parser = argparse.ArgumentParser() - commands = parser.add_subparsers(dest="command_name", help="CbAPI subcommand") - - for cmd_name, cmd_config in iteritems(command_map): - cmd_parser = commands.add_parser(cmd_name, help=cmd_config.get("help", None)) - for cmd_arg_name, cmd_arg_config in iteritems(cmd_config.get("extra_args", {})): - cmd_parser.add_argument(cmd_arg_name, **cmd_arg_config) - - opts = parser.parse_args(args) - command = command_map.get(opts.command_name) - if not command: - parser.print_usage() - return - - command_method = command.get("method", None) - if command_method: - return command_method(opts) - else: - parser.print_usage() - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/bin/cbapi-protection b/bin/cbapi-protection index e8773543..d45ccd3a 100644 --- a/bin/cbapi-protection +++ b/bin/cbapi-protection @@ -55,7 +55,7 @@ def configure(opts): token = input("API token: ") config = RawConfigParser() - config.readfp(StringIO('[default]')) + config.read_file(StringIO('[default]')) config.set("default", "url", url) config.set("default", "token", token) config.set("default", "ssl_verify", ssl_verify) diff --git a/bin/cbapi-psc b/bin/cbapi-psc deleted file mode 100644 index a52b461e..00000000 --- a/bin/cbapi-psc +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python - -import argparse -import contextlib - -from cbapi.six import iteritems -from cbapi.six.moves import input -import os -import sys -import cbapi.six as six -if six.PY3: - from io import StringIO as StringIO -else: - from cStringIO import StringIO - -from cbapi.six.moves.configparser import RawConfigParser - - -@contextlib.contextmanager -def temp_umask(umask): - oldmask = os.umask(umask) - try: - yield - finally: - os.umask(oldmask) - - -def configure(opts): - credential_path = os.path.join(os.path.expanduser("~"), ".carbonblack") - credential_file = os.path.join(credential_path, "credentials.psc") - - print("Welcome to the CbAPI.") - if os.path.exists(credential_file): - print("An existing credential file exists at {0}.".format(credential_file)) - resp = input("Do you want to continue and overwrite the existing configuration? [Y/N] ") - if resp.strip().upper() != "Y": - print("Exiting.") - return 1 - - if not os.path.exists(credential_path): - os.makedirs(credential_path, 0o700) - - url = input("URL to the Cb PSC API server (do not include '/integrationServices') [https://hostname]: ") - - ssl_verify = True - - connector_id = input("Connector ID: ") - token = input("API key: ") - - org_key = input("Org Key: ") - - config = RawConfigParser() - config.readfp(StringIO('[default]')) - config.set("default", "url", url) - config.set("default", "token", "{0}/{1}".format(token, connector_id)) - config.set("default", "org_key", org_key) - config.set("default", "ssl_verify", ssl_verify) - with temp_umask(0): - with os.fdopen(os.open(credential_file, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0o600), 'w') as fp: - os.chmod(credential_file, 0o600) - config.write(fp) - print("Successfully wrote credentials to {0}.".format(credential_file)) - - -command_map = { - "configure": { - "extra_args": {}, - "help": "Configure CbAPI", - "method": configure - } -} - - -def main(args): - parser = argparse.ArgumentParser() - commands = parser.add_subparsers(dest="command_name", help="CbAPI subcommand") - - for cmd_name, cmd_config in iteritems(command_map): - cmd_parser = commands.add_parser(cmd_name, help=cmd_config.get("help", None)) - for cmd_arg_name, cmd_arg_config in iteritems(cmd_config.get("extra_args", {})): - cmd_parser.add_argument(cmd_arg_name, **cmd_arg_config) - - opts = parser.parse_args(args) - command = command_map.get(opts.command_name) - if not command: - parser.print_usage() - return - - command_method = command.get("method", None) - if command_method: - return command_method(opts) - else: - parser.print_usage() - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/bin/cbapi-response b/bin/cbapi-response index 84b45fa0..b04c4f44 100644 --- a/bin/cbapi-response +++ b/bin/cbapi-response @@ -70,7 +70,7 @@ def configure(opts): return 1 config = RawConfigParser() - config.readfp(StringIO('[default]')) + config.read_file(StringIO('[default]')) config.set("default", "url", url) config.set("default", "token", token) config.set("default", "ssl_verify", ssl_verify) diff --git a/docs/changelog.rst b/docs/changelog.rst index bd3b10dd..ac13cca6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,5 +1,219 @@ CbAPI Changelog =============== +.. top-of-changelog (DO NOT REMOVE THIS COMMENT) + +Documentation - Release Feb 14, 2024 +------------------------------------ + +Updates + * Removed references to and documentation about Carbon Black Cloud. CBAPI (this SDK) is not maintained for Carbon Black Cloud. + + Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. Please see + `Carbon Black Cloud Python SDK on the Developer Network `_ + for details. + +CbAPI 2.0.0 - Release July 29, 2024 +------------------------------------ + +Breaking Changes + * Removing psc functionalities. + + +CbAPI 1.7.10 - Release Feb 1, 2023 +------------------------------------ + +Bug Fixes + * Update CbAPI to use packaging instead of distutils for python3.7+ + +CbAPI 1.7.9 - Release Sept 29, 2022 +------------------------------------ + +Bug Fixes + * Adjust Live Response Worker creation for EDR sensors to optimize for sensor specific jobs + +CbAPI 1.7.8 - Release Jun 17, 2022 +------------------------------------ + +Bug Fixes + * Vendor the `attrdict` module because of `ImportError` for Python3.10 + +CbAPI 1.7.7 - Release Jan 28, 2022 +------------------------------------ + +Bug Fixes + * Changed the sort order for EDR sensor searches from 'last_checkin_time' (default when none provided explicitly) to 'hostname' to make the sort stable as sensors checkin during paging + + +CbAPI 1.7.6 - Release Dec 20, 2021 +------------------------------------ + +Bug Fixes + * Removed the requirement for an admin token to connect + * Added sensor paginated query + +General + * Updated version of lxml library + + +CbAPI 1.7.5 - Released June 16, 2021 +------------------------------------ + +Updates + +* General + * Allow the CbAPI to accept a pre-configured Session object to be used for access, to get around unusual configuration requirements. + +CbAPI 1.7.4 - Released April 7, 2021 +------------------------------------ + +Updates + +* General + * Fix example code in the documentation for Facets +* EDR (CB Response) + * Add missing fields for SensorGroup class and fix example script to properly create SensorGroup + * Fix example script sensor_group_operations.py to list groups without ipaddresses + * Fix alert.save() + * Allow blocked processes to be accessed through the Process (processblocks) + + +CbAPI 1.7.3 - Released January 15, 2021 +------------------------------------ + +Updates + +* General + * Fix resource warnings regarding unclosed file object + * Notice added to readme for Carbon Black Cloud features moving to Carbon Black Cloud SDK repo +* Carbon Black Cloud + * Increase default rows of alerts to 100 + * Add get_auditlogs function to API object +* CB Threathunter + * Fix typo in process query + * Bump lxml from 4.4.1 to 4.6.2 for Threat Intelligence example +* EDR (CB Response) + * Add Sensor Builds + * Alert.set_ignored() and AlertQuery.set_ignored(): + * Added a docstring to specify what happens with this method + * Modified the payload keys based on manual testing + * Alert.change_status() and AlertQuery.change_status(): + * Added a status check to ensure it's a valid status + + +CbAPI 1.7.2 - Released July 22, 2020 +------------------------------------ + +Updates + +* General + * Allow passing in proxy configuration as direct parameters during class instantiation of base API. + + +CbAPI 1.7.1 - Released July 22, 2020 +------------------------------------ + +Updates + +* General + * Documentation updates to indicate changed product names +* Carbon Black Cloud + * Process Search v2 rows defaults to 10k to match UI behavior +* EDR (CB Response) + * Add support for fetching alert by ID + + +CbAPI 1.7.0 - Released July 14, 2020 +------------------------------------ + +Updates + +* General + * Updates to pool defaults in base API. + * Changes to exception handling to better discriminate ConnectionErrors and queries with invalid syntax. + * Various minor bug fixes throughout. +* Carbon Black Cloud + * Bug fixes to query implementation. + * Live Response: Account for sensor queue depth when submitting jobs. +* CB Defense + * Added examples for Dell BIOS verification. +* CB ThreatHunter + * Bug fixes to query implementation. + * Update process and event searches to v2. + * examples/create_feed: Make report optional during feed creation + * examples/process_exporter: Add headers to CSV file writer + * examples/threat_intelligence: Simplify report validation, add severity conversion to percent + +CbAPI 1.6.2 - Released April 08, 2020 +------------------------------------- + +Updates + +* CB Response + * Changes to align with limits placed on the sensor update function in CB Response 7.1.0. Release notes are available on User Exchange, the ID is `CB 28683 `_. + +CbAPI 1.6.1 - Released January 13, 2020 +--------------------------------------- + +Updates + +* CB Response + * Fix Alert.save() to use alert v1 API +* Carbon Black Cloud + * Fix Live Response flow to use integrationServices/v3/device to prevent need for multiple API keys +* CB ThreatHunter + * Update example for ThreatHunter Query + +CbAPI 1.6.0 - Released December 3, 2019 +--------------------------------------- + +Updates + +* New Carbon Black Cloud API Support + * Support for Devices v6: + * List and search for devices + * Export device information to CSV + * Device control actions: quarantine, bypass, background scan, deregister/delete, update + * Support for Alerts v6: + * Search for and retrieve alerts + * Update alert status (dismiss alerts) + +Examples + +* Devices v6: + * psc/device_control.py + * psc/download_device_list.py + * psc/list_devices.py +* Alerts v6: + * psc/alert_search_suggestions.py + * psc/bulk_update_alerts.py + * psc/bulk_update_cbanalytics_alerts.py + * psc/bulk_update_threat_alerts.py + * psc/bulk_update_vmware_alerts.py + * psc/bulk_update_watchlist_alerts.py + * psc/list_alert_facets.py + * psc/list_alerts.py + * psc/list_cbanalytics_alert_facets.py + * psc/list_cbanalytics_alerts.py + * psc/list_vmware_alert_facets.py + * psc/list_vmware_alerts.py + * psc/list_watchlist_alert_facets.py + * psc/list_watchlist_alerts.py + +CbAPI 1.5.6 - Released November 19, 2019 +---------------------------------------- + +Updates + +* General + * Name change to Carbon Black Cloud from PSC. + +CbAPI 1.5.5 - Released November 12, 2019 +---------------------------------------- + +Updates + +* CB ThreatHunter + * Fix List object that was not callable. CbAPI 1.5.4 - Released October 24, 2019 ---------------------------------------- @@ -197,7 +411,7 @@ CB Response: You can also use the special segment "0" to retrieve process events across all segments. * Fix ``cmdline_filters`` in the ``IngressFilter`` model object. -CB Protection: +App Control (CB Protection): * Tamper Protection can now be set and cleared in the ``Computer`` model object. @@ -217,7 +431,7 @@ Security fix: Bug fixes: * Add rule filename parameter to CB Defense ``policy_operations.py`` script's ``add-rule`` command. -* Add support for ``tamperProtectionActive`` attribute to CB Protection's ``Computer`` object. +* Add support for ``tamperProtectionActive`` attribute to App Control's (CB Protection) ``Computer`` object. * Work around CB Response issue- the ``/api/v1/sensor`` route incorrectly returns an HTTP 500 if no sensors match the provided query. CbAPI now catches this exception and will instead return an empty set back to the caller. @@ -240,7 +454,7 @@ Security changes: * Add the ability to "pin" a specific server certificate to a credential profile. - * You can now force TLS certificate verification on self-signed, on-premise installations of CB Response or Protection + * You can now force TLS certificate verification on self-signed, on-premise installations of EDR (CB Response) or App Control (Protection) through the ``ssl_cert_file`` option in the credential profile. * To "pin" a server certificate, save the PEM-formatted server certificate to a file, and put the full path to that PEM file in the ``ssl_cert_file`` option of that server's credential profile. diff --git a/docs/concepts.rst b/docs/concepts.rst index b8ef6855..639fdd60 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -5,7 +5,7 @@ There are a few critical concepts that will make understanding and using the cba explained below, and also covered in a slide deck presented at the Carbon Black regional User Exchanges in 2016. You can see the slide deck `here `_. -At a high level, the cbapi tries to represent data in CB Response or CB Protection as Python objects. If you've worked +At a high level, the cbapi tries to represent data in EDR (CB Response) or App Control (CB Protection) as Python objects. If you've worked with SQL Object-relational Mapping (ORM) frameworks before, then this structure may seem familiar -- cbapi was designed to operate much like an ORM such as SQLAlchemy or Ruby's ActiveRecord. If you haven't worked with one of these libraries, don't worry! The concepts will become clear after a little practice. @@ -14,9 +14,9 @@ Model Objects ------------- Everything in cbapi is represented in terms of "Model Objects". A Model Object in cbapi represents a single instance -of a specific type of data in CB Response or Protection. For example, a process document from CB Response (as seen +of a specific type of data in EEDR (CB Response) or App Control (CB Protection). For example, a process document from EDR (as seen on an Analyze Process page in the Web UI) is represented as a :py:mod:`cbapi.response.models.Process` Model Object. -Similarly, a file instance in CB Protection is represented as a :py:mod:`cbapi.protection.models.FileInstance` +Similarly, a file instance in App Control (CB Protection) is represented as a :py:mod:`cbapi.protection.models.FileInstance` Model Object. Once you have an instance of a Model Object, you can access all of the data contained within as Python properties. @@ -27,7 +27,7 @@ in the ``cmdline`` property), you would write the code:: This would automatically retrieve the ``cmdline`` attribute of the process and print it out to your screen. -The data in CB Response and Protection may change rapidly, and so a comprehensive list of valid properties is difficult +The data in EDR (CB Response) or App Control (CB Protection) may change rapidly, and so a comprehensive list of valid properties is difficult to keep up-to-date. Therefore, if you are curious what properties are available on a specific Model Object, you can print that Model Object to the screen. It will dump all of the available properties and their current values. For example:: @@ -56,15 +56,15 @@ method to retrieve the property, and return a default value if the property does In summary, Model Objects contain all the data associated with a specific type of API call. In this example, the :py:mod:`cbapi.response.models.Binary` Model Object reflects all the data available via the -``/api/v1/binary`` API route on a CB Response server. +``/api/v1/binary`` API route on an EDR (CB Response) server. Joining Model Objects --------------------- Many times, there are relationships between different Model Objects. To make navigating these relationships easy, -cbapi provides special properties to "join" Model Objects together. For example, a :py:mod:`cbapi.response.models.Process` -Model Object can reference the :py:mod:`cbapi.response.models.Sensor` or :py:mod:`cbapi.response.models.Binary` -associated with this Process. +cbapi provides special properties to "join" Model Objects together. For example, a +:py:mod:`cbapi.response.models.Process` Model Object can reference the :py:mod:`cbapi.response.models.Sensor` or +:py:mod:`cbapi.response.models.Binary` associated with this Process. In this case, special "join" properties are provided for you. When you use one of these properties, cbapi will automatically retrieve the associated Model Object, if necessary. @@ -90,13 +90,13 @@ Queries Now that we've covered how to get data out of a specific Model Object, we now need to learn how to obtain Model Objects in the first place! To do this, we have to create and execute a Query. cbapi Queries use the same query -syntax accepted by CB Response or Protection's APIs, and add a few little helpful features along the way. +syntax accepted by EDR (CB Response) or App Control (CB Protection) APIs, and add a few little helpful features along the way. To create a query in cbapi, use the ``.select()`` method on the CbResponseAPI or CbProtectionAPI object. Pass the Model Object type as a parameter to the ``.select()`` call and optionally add filtering criteria with ``.where()`` clauses. -Let's start with a simple query for CB Response:: +Let's start with a simple query for EDR (CB Response):: >>> from cbapi.response import * >>> cb = CbResponseAPI() @@ -152,15 +152,15 @@ will throw a :py:mod:`MoreThanOneResultError` exception if there are zero or mor second method is ``.first()``, which will return the first result from the result set, or None if there are no results. Every time you access a Query object, it will perform a REST API query to the Carbon Black server. For large result -sets, the results are retrieved in batches- by default, 100 results per API request on CB Response and 1,000 results -per API request on CB Protection. The search queries themselves are not cached, but the resulting Model Objects are. +sets, the results are retrieved in batches- by default, 100 results per API request on EDR (CB Response) and 1,000 results +per API request on App Control (CB Protection). The search queries themselves are not cached, but the resulting Model Objects are. Retrieving Objects by ID ------------------------ Every Model Object (and in fact any object addressable via the REST API) has a unique ID associated with it. If you already have a unique ID for a given Model Object, for example, a Process GUID for CB Response, or a Computer ID -for CB Protection, you can ask cbapi to give you the associated Model Object for that ID by passing that ID to the +for App Control (CB Protection), you can ask cbapi to give you the associated Model Object for that ID by passing that ID to the ``.select()`` call. For example:: >>> binary = cb.select(Binary, "CA4FAFFA957C71C006B59E29DFE3EB8B") @@ -178,16 +178,16 @@ object and if it does not exist, cbapi will throw a :py:mod:`ObjectNotFoundError Creating New Objects -------------------- -The CB Response and Protection REST APIs provide the ability to insert new data under certain circumstances. For -example, the CB Response REST API allows you to insert a new banned hash into its database. Model Objects that +The EDR (CB Response) and App Control (CB Protection) REST APIs provide the ability to insert new data under certain circumstances. For +example, the EDR REST API allows you to insert a new banned hash into its database. Model Objects that represent these data types can be "created" in cbapi by using the ``create()`` method:: >>> bh = cb.create(BannedHash) If you attempt to create a Model Object that cannot be created, you will receive a :py:mod:`ApiError` exception. -Once a Model Object is created, it's blank (it has no data). You will need to set the required properties and then call the -``.save()`` method:: +Once a Model Object is created, it's blank (it has no data). You will need to set the required properties and then call +the ``.save()`` method:: >>> bh = cb.create(BannedHash) >>> bh.text = "Banned from API" @@ -199,7 +199,7 @@ exception with a list of the properties that are required and not currently set. Once the ``.save()`` method is called, the appropriate REST API call is made to create the object. The Model Object is then updated to the current state returned by the API, which may include additional data properties initialized -by CB Response or Protection. +by EDR (CB Response) or App Control (CB Protection). Modifying Existing Objects -------------------------- @@ -207,9 +207,10 @@ Modifying Existing Objects The same ``.save()`` method can be used to modify existing Model Objects if the REST API provides that capability. If you attempt to modify a Model Object that cannot be changed, you will receive a :py:mod:`ApiError` exception. -For example, if you want to change the "jgarman" user's password to "cbisawesome":: +For example, if you want to change the "jgarman" user's password from the "cb" to "cbisawesome":: >>> user = cb.select(User, "jgarman") + >>> user.current_password = "cb" >>> user.password = "cbisawesome" >>> user.save() diff --git a/docs/conf.py b/docs/conf.py index e7d7ba46..33cecda7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,18 +12,18 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys -import os +# import sys (imports not needed) +# import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -44,14 +44,14 @@ source_suffix = '.rst' # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'cbapi' -copyright = u'2016-2019, Carbon Black Developer Network' +copyright = u'2016-2021, VMware Carbon Black' author = u'Carbon Black Developer Network' # The version info for the project you're documenting, acts as replacement for @@ -59,22 +59,22 @@ # built documents. # # The short X.Y version. -version = u'1.5' +version = u'1.7' # The full version, including alpha/beta/rc tags. -release = u'1.5.4' +release = u'2.0.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -83,27 +83,27 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'tango' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True @@ -124,14 +124,14 @@ } # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. # " v documentation" by default. -#html_title = u'cbapi v0.9.1' +# html_title = u'cbapi v0.9.1' # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. @@ -140,7 +140,7 @@ # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -150,64 +150,64 @@ # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not None, a 'Last updated on:' timestamp is inserted at every page # bottom, using the given strftime format. # The empty string is equivalent to '%b %d, %Y'. -#html_last_updated_fmt = None +# html_last_updated_fmt = None # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # 'ja' uses this config value. # 'zh' user can custom change `jieba` dictionary path. -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. htmlhelp_basename = 'CarbonBlackAPI-PythonBindingsdoc' @@ -222,23 +222,23 @@ # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- @@ -251,7 +251,7 @@ ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -266,16 +266,16 @@ ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False latex_elements = { # Additional stuff for the LaTeX preamble. @@ -291,11 +291,12 @@ # Latex figure (float) alignment # 'figure_align': 'htbp', 'preamble': "".join(( - '\DeclareUnicodeCharacter{25A0}{=}', # Solid box + '\\DeclareUnicodeCharacter{25A0}{=}', # Solid box )), } autoclass_content = 'both' + def setup(app): - app.add_stylesheet('css/custom.css') + app.add_css_file('css/custom.css') diff --git a/docs/defense-api.rst b/docs/defense-api.rst index cbf012ab..1c983a39 100644 --- a/docs/defense-api.rst +++ b/docs/defense-api.rst @@ -1,33 +1,12 @@ .. _defense_api: -CB Defense API -============== +Cloud Endpoint Standard API - DEPRECATED +======================================== -This page documents the public interfaces exposed by cbapi when communicating with a CB Defense server. +Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. -Main Interface --------------- +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. -To use cbapi with Carbon Black Defense, you will be using the CBDefenseAPI. -The CBDefenseAPI object then exposes two main methods to select data on the Carbon Black server: - -.. autoclass:: cbapi.psc.defense.rest_api.CbDefenseAPI - :members: - :inherited-members: - - .. :automethod:: select - .. :automethod:: create - -Queries -------- - -.. autoclass:: cbapi.psc.defense.rest_api.Query - :members: - - -Models ------- - -.. automodule:: cbapi.psc.defense.models - :members: - :undoc-members: +CBAPI is not maintained for Carbon Black Cloud. diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 1742a151..3f0d7a0e 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -4,28 +4,29 @@ Getting Started First, let's make sure that your API authentication tokens have been imported into cbapi. Once that's done, then read on for the key concepts that will explain how to interact with Carbon Black APIs via cbapi. -Feel free to follow along with this document or watch the `Development Environment Setup video `_ +Feel free to follow along with this document or watch the +`Development Environment Setup video `_ on the Developer Network website. API Authentication ------------------ -CB Response and CB Protection use a per-user API secret token to authenticate requests via the API. The API token +EDR (CB Response) and App Control (CB Protection) use a per-user API secret token to authenticate requests via the API. The API token confers the same permissions and authorization as the user it is associated with, so protect the API token with the same care as a password. To learn how to obtain the API token for a user, see the Developer Network website: there you will find instructions -for obtaining an API token for `CB Response `_ -and `CB Protection `_. +for obtaining an API token for `EDR (CB Response) `_ +and `App Control (CB Protection) `_. Once you have the API token, cbapi helps keep your credentials secret by enforcing the use of a credential file. To encourage sharing of scripts across the community while at the same time protecting the security of our customers, cbapi strongly discourages embedding credentials in individual scripts. Instead, you can place credentials for several -CB Response or CB Protection servers inside the API credential file and select which "profile" you would like to use +EDR (CB Response) or App Control (CB Protection) servers inside the API credential file and select which "profile" you would like to use at runtime. -To create the initial credential file, a simple-to-use script is provided. Just run the ``cbapi-response``, -``cbapi-protection``, or ``cbapi-psc`` script with the ``configure`` argument. On Mac OS X and Linux:: +To create the initial credential file, a simple-to-use script is provided. Just run the ``cbapi-response`` or +``cbapi-protection`` script with the ``configure`` argument. On Mac OS X and Linux:: $ cbapi-response configure @@ -36,8 +37,6 @@ Alternatively, if you're using Windows (change ``c:\python27`` if Python is inst This configuration script will walk you through entering your API credentials and will save them to your current user's credential file location, which is located in the ``.carbonblack`` directory in your user's home directory. -If using cbapi-psc, you will also be asked to provide an org key. An org key is required to access the PSC, and can be found in the console under Settings -> API Keys. - Your First Query ---------------- diff --git a/docs/index.rst b/docs/index.rst index 3196c1d0..7e1fdf1b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,4 +1,4 @@ -.. cbapi documentation master file, created by +.. CBAPI documentation master file, created by sphinx-quickstart on Thu Apr 28 09:52:29 2016. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. @@ -8,13 +8,12 @@ cbapi: Carbon Black API for Python Release v\ |release|. -cbapi provides a straightforward interface to the Carbon Black products: CB Protection, Response, and Defense. -This library provides a Pythonic layer to access the raw power of the REST APIs of all CB products, making it trivial -to do the easy stuff and handling all of the "sharp corners" behind the scenes for you. Take a look:: +CBAPI provides a straightforward interface to the VMware Carbon Black products Carbon Black EDR and Carbon Black App Control. +This library provides a Pythonic layer to access the raw power of the REST APIs of these Carbon Black products, making it easier to query data from on-premise APIs, combine data from multiple API calls, manage all API credentials in one place, and manipulate data as Python objects. Take a look:: >>> from cbapi.response import CbResponseAPI, Process, Binary, Sensor >>> # - >>> # Create our CbAPI object + >>> # Create our EDR API object >>> # >>> c = CbResponseAPI() >>> # @@ -39,12 +38,12 @@ to do the easy stuff and handling all of the "sharp corners" behind the scenes f ... s.network_isolation_enabled = True ... s.save() -If you're more a CB Protection fellow, then you're in luck as well:: +If you're a Carbon Black App Control customer (formerly CB Protection), you may use:: >>> from cbapi.protection.models import FileInstance >>> from cbapi.protection import CbProtectionAPI >>> # - >>> # Create our CB Protection API object + >>> # Create our App Control API object >>> # >>> p = CbProtectionAPI() >>> # @@ -62,68 +61,49 @@ If you're more a CB Protection fellow, then you're in luck as well:: >>> fi.computer.policyId = 3 >>> fi.computer.save() -As of version 1.2, cbapi now provides support for CB Defense too! - - >>> from cbapi.psc.defense import * - >>> # - >>> # Create our CB Defense API object - >>> # - >>> p = CbDefenseAPI() - >>> # - >>> # Select any devices that have the hostname WIN-IA9NQ1GN8OI and an internal IP address of 192.168.215.150 - >>> # - >>> devices = c.select(Device).where('hostNameExact:WIN-IA9NQ1GN8OI').and_("ipAddress:192.168.215.150").first() - >>> # - >>> # Change those devices' policy into the Windows_Restrictive_Workstation policy. - >>> # - >>> for dev in devices: - >>> dev.policyName = "Restrictive_Windows_Workstation" - >>> dev.save() - Major Features -------------- - **Enhanced Live Response API** - The new cbapi now provides a robust interface to the CB Response Live Response capability. + The new CBAPI now provides a robust interface to the Carbon Black EDR Live Response capability. Easily create Live Response sessions, initiate commands on remote hosts, and pull down data as necessary to make your Incident Response process much more efficient and automated. -- **Consistent API for CB Response, Protection and Defense platforms** - We now support CB Response, Protection and Defense users in the same API layer. Even better, - the object model is the same for both; if you know one API you can easily transition to the other. cbapi - hides all the differences between the three REST APIs behind a single, consistent Python-like interface. +- **Consistent API across VMware Carbon Black platforms** + CBAPI supports Carbon Black EDR and Carbon Black App Control customers from a single API layer. Even better, + the object model is the same for all three, and if you know one API, you can easily transition to another. CBAPI + manages the differences among the two REST APIs behind a single, consistent Python-like interface. - **Enhanced Performance** - cbapi now provides a built in caching layer to reduce the query load on the Carbon Black server. This is especially - useful when taking advantage of cbapi's new "joining" features. You can transparently access, for example, the - binary associated with a given process in CB Response. Since many processes may be associated + CBAPI now provides a built in caching layer to reduce the query load on the Carbon Black server. This is especially + useful when taking advantage of CBAPI's new "joining" features. You can transparently access, for example, the + binary associated with a given process in Carbon Black EDR. Since many processes may be associated with the same binary, it does not make sense to repeatedly request the same binary information from the server - over and over again. Therefore cbapi now caches this information to avoid unnecessary requests. + over and over again. Therefore CBAPI now caches this information to avoid unnecessary requests. - **Reduce Complexity** - cbapi now provides a friendly - dare I say "fun" - interface to the data. This greatly improves developer + CBAPI provides a friendly interface for accessing Carbon Black data. This greatly improves developer productivity and lowers the bar to entry. - **Python 3 and Python 2 compatible** - Use all the new features and modules available in Python 3 with cbapi. This module is compatible with Python + Use all the new features and modules available in Python 3 with CBAPI. This module is compatible with Python versions 2.6.6 and above, 2.7.x, 3.4.x, and 3.5.x. - **Better support for multiple CB servers** - cbapi now introduces the concept of Credential Profiles; named collections of URL, API keys, and optional proxy - configuration for connecting to any number of CB Protection, Defense, or Response servers. + CBAPI introduces the concept of Credential Profiles; named collections of URL, API keys, and optional proxy + configuration for connecting to any number of Carbon Black EDR or Carbon Black App Control servers. API Credentials --------------- -The new cbapi as of version 0.9.0 enforces the use of credential files. +CBAPI version 0.9.0 enforces the use of credential files. In order to perform any queries via the API, you will need to get the API token for your CB user. See the documentation on the Developer Network website on how to acquire the API token for -`CB Response `_, -`CB Protection `_, or -`CB Defense `_. +`Carbon Black EDR (CB Response) `_, or +`Carbon Black App Control (CB Protection) `_. Once you acquire your API token, place it in one of the default credentials file locations: @@ -131,17 +111,18 @@ Once you acquire your API token, place it in one of the default credentials file * ``~/.carbonblack/`` * ``/current_working_directory/.carbonblack/`` -For distinction between credentials of different Carbon Black products, use the following naming convention for your credentials files: +For distinction between credentials of different Carbon Black products, use the following naming convention for your +credentials files: -* ``credentials.psc`` for CB Defense, CB ThreatHunter, and CB LiveOps -* ``credentials.response`` for CB Response -* ``credentials.protection`` for CB Protection +* ``credentials.response`` for Carbon Black EDR (CB Response) +* ``credentials.protection`` for Carbon Black App Control (CB Protection) -For example, if you use a PSC product, you should have created a credentials file in one of these locations: +For example, if you use Carbon Black EDR, you should have created a credentials file in one of these +locations: -* ``/etc/carbonblack/credentials.psc`` -* ``~/.carbonblack/credentials.psc`` -* ``/current_working_directory/.carbonblack/credentials.psc`` +* ``/etc/carbonblack/credentials.response`` +* ``~/.carbonblack/credentials.response`` +* ``/current_working_directory/.carbonblack/credentials.response`` Credentials found in a later path will overwrite earlier ones. @@ -165,59 +146,62 @@ by key-value pairs providing the necessary credential information:: The possible options for each credential profile are: -* **url**: The base URL of the CB server. This should include the protocol (https) and the hostname, and nothing else. +* **url**: The base URL of the Carbon Black server. This should include the protocol (https) and the hostname, and nothing else. * **token**: The API token for the user ID. More than one credential profile can be specified for a given server, with different tokens for each. * **ssl_verify**: True or False; controls whether the SSL/TLS certificate presented by the server is validated against the local trusted CA store. -* **org_key**: The organization key. This is required to access the PSC, and can be found in the console. The format is ``123ABC45``. -* **proxy**: A proxy specification that will be used when connecting to the CB server. The format is: +* **proxy**: A proxy specification that will be used when connecting to the Carbon Black server. The format is: ``http://myusername:mypassword@proxy.company.com:8001/`` where the hostname of the proxy is ``proxy.company.com``, port 8001, and using username/password ``myusername`` and ``mypassword`` respectively. -* **ignore_system_proxy**: If you have a system-wide proxy specified, setting this to True will force cbapi to bypass - the proxy and directly connect to the CB server. +* **ignore_system_proxy**: If you have a system-wide proxy specified, setting this to True will force CBAPI to bypass + the proxy and directly connect to the Carbon Black server. -Future versions of cbapi will also provide the ability to "pin" the TLS certificate so as to provide certificate +Future versions of CBAPI will also provide the ability to "pin" the TLS certificate so as to provide certificate verification on self-signed or internal CA signed certificates. Environment Variable Support -The latest cbapi for python supports specifying API credentials in the following three environment variables: +The latest CBAPI for Python supports specifying API credentials in the following three environment variables: -`CBAPI_TOKEN` the envar for holding the CbR/CbP api token or the ConnectorId/APIKEY combination for CB Defense/PSC. +`CBAPI_TOKEN` the envar for holding the EDR (CbR) or App Control (CbP) api token. -The `CBAPI_URL` envar holds the FQDN of the target, a CbR , CBD, or CbD/PSC server specified just as they are in the +The `CBAPI_URL` envar holds the FQDN of the target, an EDR (CbR) server specified just as they are in the configuration file format specified above. -The optional `CBAPI_SSL_VERIFY` envar can be used to control SSL validation(True/False or 0/1), which will default to ON when -not explicitly set by the user. +The optional `CBAPI_SSL_VERIFY` envar can be used to control SSL validation(True/False or 0/1), which will default to +ON when not explicitly set by the user. + +For environments where complex outbound network filters and proxy configurations are used (eg. anything other than +an unauthenticated or basic password authenticated proxy) a prepared `requests.Session` object may be supplied as a +`proxy_session` parameter. This session will then be used for all communication with the API. Construction of such a +`Session` is beyond the scope of this document, consult your local network/security administrators for assistance. Backwards & Forwards Compatibility ---------------------------------- -The previous versions (0.8.x and earlier) of cbapi and bit9Api are now deprecated and will no longer receive updates. -However, existing scripts will work without change as cbapi includes both in its legacy package. -The legacy package is imported by default and placed in the top level cbapi namespace when the cbapi module +The previous versions (0.8.x and earlier) of CBAPI and bit9Api are now deprecated and will no longer receive updates. +However, existing scripts will work without change as CBAPI includes both in its legacy package. +The legacy package is imported by default and placed in the top level CBAPI namespace when the CBAPI module is imported on a Python 2.x interpreter. Therefore, scripts that expect to import cbapi.CbApi will continue to work exactly as they had previously. Since the old API was not compatible with Python 3, the legacy package is not importable in Python 3.x and therefore legacy scripts cannot run under Python 3. -Once cbapi 1.0.0 is released, the old :py:mod:`cbapi.legacy.CbApi` will be deprecated and removed entirely no earlier +Once CBAPI 1.0.0 is released, the old :py:mod:`cbapi.legacy.CbApi` will be deprecated and removed entirely no earlier than January 2017. New scripts should use the :py:mod:`cbapi.response.rest_api.CbResponseAPI` -(for CB Response), :py:mod:`cbapi.protection.rest_api.CbProtectionAPI` -(for CB Protection), or :py:mod:`cbapi.defense.rest_api.CbDefenseAPI` API entry points. +(for Carbon Black EDR (CB Response)), :py:mod:`cbapi.protection.rest_api.CbProtectionAPI` +(for Carbon Black App Control (CB Protection)), or :py:mod:`cbapi.defense.rest_api.CbDefenseAPI` API entry points. -The API is frozen as of version 1.0; afterward, any changes in the 1.x version branch +The API is frozen as of version 1.0; any changes in the 1.x version branch will be additions/bug fixes only. Breaking changes to the API will increment the major version number (2.x). User Guide ---------- -Let's get started with cbapi. Once you've mastered the concepts here, then you can always hop over to the API -Documentation (below) for detailed information on the objects and methods exposed by cbapi. +Get started with CBAPI here. For detailed information on the objects and methods exposed by CBAPI, see the full API Documentation below. .. toctree:: :maxdepth: 2 @@ -235,19 +219,15 @@ Documentation (below) for detailed information on the objects and methods expose API Documentation ----------------- -Once you've taken a look at the User Guide, read through some of the -`examples on GitHub `_, -and maybe even written some code of your own, the API documentation can help you get the most out of cbapi by -documenting all of the methods available to you. +Once you have read the User Guide, you can view `examples on GitHub `_ +or try writing code of your own. You can use the full API documentation below to see all the methods available in CBAPI +and unlock the full functionality of the SDK. .. toctree:: :maxdepth: 2 response-api protection-api - defense-api - threathunter-api - livequery-api exceptions Indices and tables diff --git a/docs/installation.rst b/docs/installation.rst index 41e79b93..2168e8dc 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1,12 +1,11 @@ Installation ============ -Before installing cbapi, make sure that you have access to a working CB Response or CB Protection server. The server -can be either on-premise or in the cloud. CB Response clusters are also supported. Once you have access to a working +Before installing cbapi, make sure that you have access to a working EDR (CB Response) or App Control (CB Protection) server. The server +can be either on-premise or in the cloud. EDR (CB Response) clusters are also supported. Once you have access to a working can use the standard Python packaging tools to install cbapi on your local machine. -Feel free to follow along with this document or watch the `Development Environment Setup video `_ -on the Developer Network website. +Documentation is also available on the `Developer Network `. If you already have Python installed, you can skip right down to "Using Pip". @@ -40,8 +39,7 @@ https://www.python.org/ftp/python/3.6.4/python-3.6.4-amd64.exe. :alt: Windows installation options showing "Add python.exe to path" :align: right -Ensure that the "Add Python to PATH" option is -checked. +Ensure that the "Add Python to PATH" option is checked. If for some reason you do not have pip installed, follow the instructions at this `handy guide `_. @@ -76,4 +74,3 @@ Once you have a copy of the source, you can install it in "development" mode int This will link the version of cbapi-python you checked out into your Python site-packages directory. Any changes you make to the checked out version of cbapi will be reflected in your local Python installation. This is a good choice if you are thinking of changing or developing on cbapi itself. - diff --git a/docs/live-response.rst b/docs/live-response.rst index 7c657561..69c87fb7 100644 --- a/docs/live-response.rst +++ b/docs/live-response.rst @@ -1,7 +1,7 @@ CbAPI and Live Response ======================= -Working with the CB Live Response REST API directly can be difficult. Thankfully, just like the rest of Carbon +Working with the Live Response REST API directly can be difficult. Thankfully, just like the rest of Carbon Black's REST APIs, cbapi provides Pythonic APIs to make working with the Live Response API much easier. In addition to easy-to-use APIs to call into Live Response, cbapi also provides a "job-based" interface that allows @@ -30,7 +30,7 @@ Since the Live Response API is synchronous, the script will not continue until e established and the file contents are retrieved, or an exception occurs (in this case, either a timeout error or an error reading the file). -As seen in the example above, the ``.lr_session()`` method is context-aware. CB Response has a limited number of +As seen in the example above, the ``.lr_session()`` method is context-aware. EDR (CB Response) has a limited number of concurrent Live Response session slots (by default, only ten). By wrapping the ``.lr_session()`` call within a ``with`` context, the session is automatically closed at the end of the block and frees that slot for another concurrent Live Response session in another script or user context. @@ -92,4 +92,3 @@ back from the endpoint, and submit the ``.run()`` method to the Live Response Jo Your script resumes execution immediately after the call to ``.submit_job()``. The job(s) that you've submitted will be executed in a set of background threads managed by cbapi. - diff --git a/docs/livequery-api.rst b/docs/livequery-api.rst index d80bb872..8224bbc5 100644 --- a/docs/livequery-api.rst +++ b/docs/livequery-api.rst @@ -1,49 +1,12 @@ .. _livequery_api: -CB LiveQuery API -=================== +CB LiveQuery API - DEPRECATED +============================= -This page documents the public interfaces exposed by cbapi when communicating with -Carbon Black LiveQuery devices. +Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. -Main Interface --------------- - -To use cbapi with Carbon Black LiveQuery, you use CbLiveQueryAPI objects. - -The LiveQuery API is used in two stages: run submission and result retrieval. - -.. autoclass:: cbapi.psc.livequery.rest_api.CbLiveQueryAPI - :members: - :inherited-members: - -Queries -------- - -The LiveQuery API uses QueryBuilder instances to construct structured -or unstructured (i.e., raw string) queries. You can either construct these -instances manually, or allow ``CbLiveQueryAPI.select()`` to do it for you: - -.. autoclass:: cbapi.psc.livequery.query.QueryBuilder - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.livequery.query.RunQuery - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.livequery.models.ResultQuery - :members: - :inherited-members: - -Models ------- - -.. autoclass:: cbapi.psc.livequery.models.Run - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.livequery.models.Result - :members: - :inherited-members: +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. +CBAPI is not maintained for Carbon Black Cloud. diff --git a/docs/livequery-examples.rst b/docs/livequery-examples.rst index 9dff3f5b..ac32dea4 100644 --- a/docs/livequery-examples.rst +++ b/docs/livequery-examples.rst @@ -1,110 +1,10 @@ -CB LiveQuery API Examples -======================== +CB LiveQuery API Examples - DEPRECATED +====================================== -Let's cover a few example functions that our LiveQuery Python bindings enable. To begin, we need to import the -relevant libraries:: +Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. - >>> import sys - >>> from cbapi.psc.livequery import CbLiveQueryAPI - >>> from cbapi.psc.livequery.models import Run, Result +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. - -Now that we've imported the necessary libraries, we can perform some queries on our endpoints. - -Create a Query Run ----------------------------------- - -Let's create a Query Run. First, we specify which profile to use for authentication from our credentials.psc file and create the LiveQuery object. - - >>> profile = "default' - >>> cb = CbLiveQueryAPI(profile=profile) - -Now, we specify the SQL query that we want to run, name of the run, device IDs, and device types. - - - >>> sql = 'select * from logged_in_users;' - >>> name_of_run = 'Selecting all logged in users' - >>> device_ids = '1234567' - >>> device_types = 'WINDOWS' - -Now, we create a query and add these values to it. - - >>> query = cb.query(sql) - >>> query.name(name_of_run) - >>> query.device_ids(device_ids) - >>> query.device_types(device_types) - -Finally, we submit the query and print the results. - - >>> run = query.submit() - >>> print(run) - -This query should return all logged in Windows endpoints with a ``device_id`` of ``1234567``. - -The same query can be executed with the example script `manage_run.py `_. :: - - python manage_run.py --profile default create --sql 'select * from logged_in_users;' --name 'Selecting all logged in users' --device_ids '1234567' --device_types 'WINDOWS' - -Other possible arguments to ``manage_run.py`` include ``--notify`` and ``--policy_ids``. - -Get Query Run Status ---------------------- - -Now that we've created a Query Run, let's check the status. If we haven't already authenticated with a credentials profile, we begin by specifying which profile to authenticate with. - - >>> profile = 'default' - >>> cb = CbLiveQueryAPI(profile=profile) - -Next, we select the run with the unique run ID. - - >>> run_id = 'a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz' - >>> run = cb.select(Run, run_id) - >>> print(run) - -This can also be accomplished with the example script `manage_run.py `_:: - - python manage_run.py --profile default --id a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz - -In addition, you can specify which order you want results returned. To change from the default ascending order, use the flag ``-d`` or ``--descending_results``:: - - python manage_run.py --profile default --id a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz --descending_results - -Get Query Run Results ---------------------- - -Let's view the results of a run. If we haven't already authenticated, we must start with that. - - >>> profile = 'default' - >>> cb = CbLiveQueryAPI(profile=profile) - -To view the results of a run, we must specify the run ID. - - >>> run_id = 'a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz' - >>> results = cb.select(Result).run_id(run_id) - -Finally, we print the results. - - >>> for result in results: - ... print(result) - -Results can be narrowed down with the following criteria:: - - device_ids - status - -Examples of using these criteria are below:: - - >>> device_id = '1234567' - >>> results.criteria(device_id=device_id) - >>> status = 'matched' - >>> results.criteria(status=status) - -Finally, we print the results. - - >>> for result in results: - ... print(result) - - -You can also retrieve run results with the example script `run_search.py `_:: - - python run_search.py --profile default --id a4oh4fqtmrr8uxrdj6mm0mbjsyhdhhvz --device_ids '1234567' --status 'matched' +CBAPI is not maintained for Carbon Black Cloud. diff --git a/docs/protection-api.rst b/docs/protection-api.rst index d5631e3c..1799bb5a 100644 --- a/docs/protection-api.rst +++ b/docs/protection-api.rst @@ -1,15 +1,12 @@ .. _protection_api: -CB Protection API -================= - -This page documents the public interfaces exposed by cbapi when communicating with a Carbon Black Enterprise -Protection server. +Carbon Black App Control (CB Protection) API +=========================================== Main Interface -------------- -To use cbapi with Carbon Black Protection, you will be using the CbProtectionAPI. +To use cbapi with Carbon Black App Control (CB Protection), you will be using the CbProtectionAPI. The CbProtectionAPI object then exposes two main methods to select data on the Carbon Black server: .. autoclass:: cbapi.protection.rest_api.CbProtectionAPI diff --git a/docs/psc-api.rst b/docs/psc-api.rst new file mode 100755 index 00000000..712ad928 --- /dev/null +++ b/docs/psc-api.rst @@ -0,0 +1,12 @@ +.. _psc_api: + +VMware Carbon Black Cloud API - DEPRECATED +========================================== + +Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. + +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. + +CBAPI is not maintained for Carbon Black Cloud. diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..a662ecf9 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,7 @@ +# Defining the exact version will make sure things don't break +sphinx==6.2.1 +sphinx-copybutton==0.4.0 +sphinx-rtd-theme==1.2.2 +sphinxcontrib-apidoc +sphinx-copybutton==0.4.0 +pygments diff --git a/docs/response-api.rst b/docs/response-api.rst index 441ee260..24761824 100644 --- a/docs/response-api.rst +++ b/docs/response-api.rst @@ -1,15 +1,12 @@ .. _response_api: -CB Response API +EDR (CB Response) API =============== -This page documents the public interfaces exposed by cbapi when communicating with a Carbon Black Enterprise -Response server. - Main Interface -------------- -To use cbapi with Carbon Black Response, you will be using the CbResponseAPI. +To use cbapi with Carbon Black EDR (Response), you will be using the CbResponseAPI. The CbResponseAPI object then exposes two main methods to access data on the Carbon Black server: ``select`` and ``create``. .. autoclass:: cbapi.response.rest_api.CbResponseAPI @@ -77,6 +74,3 @@ Process Operations .. automethod:: cbapi.live_response_api.CbLRSessionBase.kill_process .. automethod:: cbapi.live_response_api.CbLRSessionBase.create_process .. automethod:: cbapi.live_response_api.CbLRSessionBase.list_processes - - - diff --git a/docs/response-examples.rst b/docs/response-examples.rst index 6f39a207..6cbd0da1 100644 --- a/docs/response-examples.rst +++ b/docs/response-examples.rst @@ -1,9 +1,5 @@ -CB Response API Examples -======================== - -Now that we've covered the basics, let's step through a few examples using the CB Response API. In these examples, -we will assume the following boilerplate code to enable logging and establish a connection to the "default" -CB Response server in our credential file:: +EDR (CB Response) API Examples +============================== >>> import logging >>> root = logging.getLogger() @@ -15,10 +11,10 @@ CB Response server in our credential file:: With that boilerplate out of the way, let's take a look at a few examples. -Download a Binary from CB Response ----------------------------------- +Download a Binary from EDR (CB Response) +---------------------------------------- -Let's grab a binary that CB Response has collected from one of the endpoints. This can be useful if you want to +Let's grab a binary that EDR (CB Response) has collected from one of the endpoints. This can be useful if you want to send this binary for further automated analysis or pull it down for manual reverse engineering. You can see a full example with command line options in the examples directory: ``binary_download.py``. @@ -58,13 +54,13 @@ Now let's take this binary and add a Banning rule for it. To do this, we create Received response: {u'result': u'success'} HTTP GET /api/v1/banning/blacklist/7FB55F5A62E78AF9B58D08AAEEAEF848 took 0.039s (response 200) -Note that if the hash is already banned in CB Response, then you will receive a `ServerError` exception with the message that +Note that if the hash is already banned in EDR (CB Response), then you will receive a `ServerError` exception with the message that the banned hash already exists. Isolate a Sensor ---------------- -Switching gears, let's take a Sensor and quarantine it from the network. The CB Response network isolation +Switching gears, let's take a Sensor and quarantine it from the network. The EDR (CB Response) network isolation functionality allows administrators to isolate endpoints that may be actively involved in an incident, while preserving access to perform Live Response on that endpoint and collect further endpoint telemetry. @@ -82,7 +78,7 @@ This will select the first sensor that matches the hostname ``HOSTNAME``. Now we ... True -The ``.isolate()`` method will keep polling the CB Response server until the sensor has confirmed that it is now +The ``.isolate()`` method will keep polling the EDR (CB Response) server until the sensor has confirmed that it is now isolated from the network. If the sensor is offline or otherwise unreachable, this call could never return. Therefore, there is also a ``timeout=`` keyword parameter that can be used to set an optional timeout that, if reached, will throw a ``TimeoutError`` exception. The ``.isolate()`` function returns True when the sensor is successfully @@ -104,7 +100,7 @@ you can optionally specify a timeout using the ``timeout=`` keyword parameter. Querying Processes and Events ----------------------------- -Now, let's do some queries into the CB Response database. The true power of CB Response is its continuous recording +Now, let's do some queries into the EDR (CB Response) database. The true power of EDR (CB Response) is its continuous recording and powerful query language that allows you to go back in time and track the root cause of any security incident on your endpoints. Let's start with a simple query to find instances of a specific behavioral IOC, where our attacker used the built-in Windows tool ``net.exe`` to mount an internal network share. We will iterate over all uses @@ -175,42 +171,42 @@ New Filters: Group By, Time Restrictions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In the query above, there is an extra ``.group_by()`` method. This method is new in cbapi 1.1.0 and is part of five -new query filters available when communicating with a CB Response 6.1 server. These filters are accessible via methods +new query filters available when communicating with a EDR (CB Response) 6.1 server. These filters are accessible via methods on the ``Process`` Query object. These new methods are: * ``.group_by()`` - Group the result set by a field in the response. Typically you will want to group by ``id``, which will ensure that the result set only has one result per *process* rather than one result per *event segment*. For - more information on processes, process segments, and how segments are stored in CB Response 6.0, see the - `Process API Changes for CB Response 6.0 `_ + more information on processes, process segments, and how segments are stored in EDR (CB Response) 6.0, see the + `Process API Changes for EDR (CB Response) 6.0 `_ page on the Developer Network website. * ``.min_last_update()`` - Only return processes that have events after a given date/time stamp (relative to the individual sensor's clock) * ``.max_last_update()`` - Only return processes that have events before a given date/time stamp (relative to the individual sensor's clock) * ``.min_last_server_update()`` - Only return processes that have events after a given date/time stamp (relative to the - CB Response server's clock) + EDR (CB Response) server's clock) * ``.max_last_server_update()`` - Only return processes that have events before a given date/time stamp (relative to the - CB Response server's clock) + EDR (CB Response) server's clock) -CB Response 6.1 uses a new way of recording process events that greatly increases the speed and scale of collection, +EDR (CB Response) 6.1 uses a new way of recording process events that greatly increases the speed and scale of collection, allowing you to store and search data for more endpoints on the same hardware. Details on the new database format -can be found on the Developer Network website at the `Process API Changes for CB Response 6.0 +can be found on the Developer Network website at the `Process API Changes for EDR (CB Response) 6.0 `_ page. The ``Process`` Model Object traditionally referred to a single "segment" of events in the CB Response database. In -CB Response versions prior to 6.0, a single segment will include up to 10,000 individual endpoint events, enough to +EDR (CB Response) versions prior to 6.0, a single segment will include up to 10,000 individual endpoint events, enough to handle over 95% of the typical event activity for a given process. Therefore, even though a ``Process`` Model Object technically refers to a single *segment* in a process, since most processes had less than 10,000 events and therefore were only comprised of a single segment, this distinction wasn't necessary. However, now that processes are split across many segments, a better way of handling this is necessary. Therefore, -CB Response 6.0 introduces the new ``.group_by()`` method. +EDR (CB Response) 6.0 introduces the new ``.group_by()`` method. More on Filters ~~~~~~~~~~~~~~~ Querying for a process will return *all* segments that match. For example, if you search for ``process_name:cmd.exe``, -the result set will include *all* segments of *all* ``cmd.exe`` processes. Therefore, CB Response 6.1 introduced +the result set will include *all* segments of *all* ``cmd.exe`` processes. Therefore, EDR (CB Response) 6.1 introduced the ability to "group" result sets by a field in the result. Typically you will want to group by the internal process id (the ``id`` field), and this is what we did in the query above. Grouping by the ``id`` field will ensure that only one result is returned per *process* rather than per *segment*. @@ -230,7 +226,7 @@ Let's take a look at an example:: 00000001-0000-0e48-01d2-c2a397f4cfe0 1495463176570 00000001-0000-0e48-01d2-c2a397f4cfe0 1495463243492 -Notice that the "same" process ID is returned seven times, but with seven different segment IDs. CB Response will +Notice that the "same" process ID is returned seven times, but with seven different segment IDs. EDR (CB Response) will return *every* process event segment that matches a given query, in this case, any event segment that contains the process command name ``cmd.exe``. @@ -246,9 +242,9 @@ the command name ``cmd.exe``. Just add the ``.group_by("id")`` filter to your qu Feed and Watchlist Maintenance ------------------------------ -The cbapi provides several helper functions to assist in creating watchlists and +The cbapi provides several helper functions to assist in creating watchlists and feeds. -Watchlists are simply saved Queries that are automatically run on the CB Response server on a periodic basis. Results +Watchlists are simply saved Queries that are automatically run on the EDR (CB Response) server on a periodic basis. Results of the watchlist are tagged in the database and optionally trigger alerts. Therefore, a cbapi Query can easily be converted into a watchlist through the Query ``.create_watchlist()`` function:: @@ -295,9 +291,10 @@ The cbapi provides helper functions to manage alerts and threat reports in bulk. the ThreatReport and Alert Model Objects provide a few bulk operations to help manage large numbers of Threat Reports and Alerts, respectively. -To mark a large number of Threat Reports as false positives, create a query that matches the Reports you're interested in. -For example, if every Report from the Feed named "SOC" that contains the word "FUZZYWOMBAT" in the report title should be -considered a false positive (and no longer trigger Alerts), you can write the following code to do so:: +To mark a large number of Threat Reports as false positives, create a query that matches the Reports you're +interested in. For example, if every Report from the Feed named "SOC" that contains the word "FUZZYWOMBAT" in the +report title should be considered a false positive (and no longer trigger Alerts), you can write the following code +to do so:: >>> feed = c.select(Feed).where("name:SOC").one() >>> report_query = feed.reports.where("title:FUZZYWOMBAT") @@ -359,7 +356,7 @@ Those few lines of Python above are jam-packed with functionality. Now for each contextual information on the source host, the group that host is part of, and details about the signing status of the binary that was executed. The magic is performed behind the scenes when we use the ``.binary`` and ``.sensor`` properties on the Process Model Object. Just like our previous example, cbapi's caching layer ensures that we do not overload -the CB Response server with duplicate requests for the same data. In this example, multiple redundant requests for sensor, +the EDR (CB Response) server with duplicate requests for the same data. In this example, multiple redundant requests for sensor, sensor group, and binary data are all eliminated by cbapi's cache. Facets @@ -371,7 +368,7 @@ pulls the most common process names for our sample host:: >>> def print_facet_histogram(facets): ... for entry in facets: - ... print("%15s: %5s%% %s" % (entry["name"][:15], entry["ratio"], u"\u25A0"*(int(entry["percent"])/2))) + ... print("%15s: %5s%% %s" % (entry["name"][:15], entry["ratio"], u"\u25A0"*int((entry["percent"])/2))) ... >>> facet_query = cb.select(Process).where("hostname:WIN-IA9NQ1GN8OI").and_("username:bit9rad") @@ -395,7 +392,7 @@ Administrative Tasks In addition to querying data, you can also perform various administrative tasks using cbapi. -Let's create a user on our CB Response server:: +Let's create a user on our EDR (CB Response) server:: >>> user = cb.create(User) >>> user.username = "jgarman" @@ -429,4 +426,3 @@ How about moving a sensor to a new Sensor Group:: Sending HTTP PUT /api/v1/sensor/3 with {"boot_id": "2", "build_id": 2, "build_version_string": "005.002.000.60922", ... HTTP PUT /api/v1/sensor/3 took 0.087s (response 204) HTTP GET /api/v1/sensor/3 took 0.030s (response 200) - diff --git a/docs/threathunter-api.rst b/docs/threathunter-api.rst index 01ff3eeb..82d58bc7 100644 --- a/docs/threathunter-api.rst +++ b/docs/threathunter-api.rst @@ -1,95 +1,12 @@ .. _threathunter_api: -CB ThreatHunter API -=================== +VMware Carbon Black Cloud Enterprise EDR API - DEPRECATED +========================================================= -This page documents the public interfaces exposed by cbapi when communicating with a -Carbon Black PSC ThreatHunter server. +Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. -Main Interface --------------- +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. -To use cbapi with Carbon Black ThreatHunter, you use CbThreatHunterAPI objects. -These objects expose two main methods to access data on the -ThreatHunter server: ``select`` and ``create``. - -.. autoclass:: cbapi.psc.threathunter.rest_api.CbThreatHunterAPI - :members: - :inherited-members: - -Queries -------- - -The ThreatHunter API uses QueryBuilder instances to construct structured -or unstructured (i.e., raw string) queries. You can either construct these -instances manually, or allow ``CbThreatHunterAPI.select()`` to do it for you: - -.. autoclass:: cbapi.psc.threathunter.query.QueryBuilder - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.query.Query - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.AsyncProcessQuery - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.query.FeedQuery - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.query.ReportQuery - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.query.WatchlistQuery - :members: - :inherited-members: - -Models ------- - -.. autoclass:: cbapi.psc.threathunter.models.Process - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Event - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Tree - :members: - -.. autoclass:: cbapi.psc.threathunter.models.Feed - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Report - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.IOC - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.IOC_V2 - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Watchlist - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.ReportSeverity - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Binary - :members: - :inherited-members: - -.. autoclass:: cbapi.psc.threathunter.models.Downloads - :members: - :inherited-members: +CBAPI is not maintained for Carbon Black Cloud. diff --git a/examples/README.md b/examples/README.md new file mode 100755 index 00000000..bd8e5b14 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,24 @@ +# Running the Example Scripts + +To run the example scripts, first set up the CBAPI with either the `pip install cbapi` or `python setup.py develop` +commands as detailed in the top-level `README.md` file. You may also set your `PYTHONPATH` environment variable to +point to the `{cbapi}/src` directory, where `{cbapi}` refers to the top-level directory where you have cloned +the CBAPI repository. + +You should also have an API key and have set up a `credentials` file as detailed in the "API Token" section of the +top-level `README.md` file. + +Once you have done so, you should be able to run any example script with the command: + + python scriptname.py [arguments] + +Executing any script with the `--help` argument should give you a detailed message about the arguments that can +be supplied to the script when it is executed. + +### Note: Users of Carbon Black Cloud must transition to the Carbon Black Cloud Python SDK. + +Please see +`Carbon Black Cloud Python SDK on the Developer Network `_ +for details. + +CBAPI is not maintained for Carbon Black Cloud. \ No newline at end of file diff --git a/examples/defense/cblr/examplejob.py b/examples/defense/cblr/examplejob.py deleted file mode 100755 index b5bf0eec..00000000 --- a/examples/defense/cblr/examplejob.py +++ /dev/null @@ -1,10 +0,0 @@ -class GetFileJob(object): - def __init__(self, file_name): - self._file_name = file_name - - def run(self, session): - return session.get_file(self._file_name) - - -def getjob(): - return GetFileJob("c:\\test.txt") diff --git a/examples/defense/cblr/jobrunner.py b/examples/defense/cblr/jobrunner.py deleted file mode 100755 index e9da853b..00000000 --- a/examples/defense/cblr/jobrunner.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python - -from cbapi.defense import * -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object -from concurrent.futures import as_completed -import sys -from datetime import datetime, timedelta - - -def main(): - parser = build_cli_parser() - parser.add_argument("--job", action="store", default="examplejob", required=True) - - args = parser.parse_args() - - cb = get_cb_defense_object(args) - - sensor_query = cb.select(Device) - - # Retrieve the list of sensors that are online - # calculate based on sensors that have checked in during the last five minutes - now = datetime.utcnow() - delta = timedelta(minutes=5) - - online_sensors = [] - offline_sensors = [] - for sensor in sensor_query: - if now - sensor.lastContact < delta: - online_sensors.append(sensor) - else: - offline_sensors.append(sensor) - - print("The following sensors are offline and will not be queried:") - for sensor in offline_sensors: - print(" {0}: {1}".format(sensor.deviceId, sensor.name)) - - print("The following sensors are online and WILL be queried:") - for sensor in online_sensors: - print(" {0}: {1}".format(sensor.deviceId, sensor.name)) - - # import our job object from the jobfile - job = __import__(args.job) - jobobject = job.getjob() - - completed_sensors = [] - futures = {} - - # collect 'future' objects for all jobs - for sensor in online_sensors: - f = cb.live_response.submit_job(jobobject.run, sensor) - futures[f] = sensor.deviceId - - # iterate over all the futures - for f in as_completed(futures.keys(), timeout=100): - if f.exception() is None: - print("Sensor {0} had result:".format(futures[f])) - print(f.result()) - completed_sensors.append(futures[f]) - else: - print("Sensor {0} had error:".format(futures[f])) - print(f.exception()) - - - still_to_do = set([s.deviceId for s in online_sensors]) - set(completed_sensors) - print("The following sensors were attempted but not completed or errored out:") - for sensor in still_to_do: - print(" {0}".format(still_to_do)) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/examples/defense/cblr_cli.py b/examples/defense/cblr_cli.py deleted file mode 100644 index e2207e39..00000000 --- a/examples/defense/cblr_cli.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -import sys - -import logging - -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object, CblrCli -from cbapi.psc.defense import Device - -log = logging.getLogger(__name__) - - -def connect_callback(cb, line): - try: - sensor_id = int(line) - except ValueError: - sensor_id = None - - if not sensor_id: - q = cb.select(Device).where("hostNameExact:{0}".format(line)) - sensor = q.first() - else: - sensor = cb.select(Device, sensor_id) - - return sensor - - -def main(): - parser = build_cli_parser("Cb Defense Live Response CLI") - parser.add_argument("--log", help="Log activity to a file", default='') - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.log: - file_handler = logging.FileHandler(args.log) - file_handler.setLevel(logging.DEBUG) - log.addHandler(file_handler) - - cli = CblrCli(cb, connect_callback) - cli.cmdloop() - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/defense/list_devices.py b/examples/defense/list_devices.py deleted file mode 100644 index 2777c4a4..00000000 --- a/examples/defense/list_devices.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object -from cbapi.psc.defense import Device - - -def main(): - parser = build_cli_parser("List devices") - device_options = parser.add_mutually_exclusive_group(required=False) - device_options.add_argument("-i", "--id", type=int, help="Device ID of sensor") - device_options.add_argument("-n", "--hostname", help="Hostname") - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.id: - devices = [cb.select(Device, args.id)] - elif args.hostname: - devices = list(cb.select(Device).where("hostNameExact:{0}".format(args.hostname))) - else: - devices = list(cb.select(Device)) - - print("{0:9} {1:40}{2:18}{3}".format("ID", "Hostname", "IP Address", "Last Checkin Time")) - for device in devices: - print("{0:9} {1:40s}{2:18s}{3}".format(device.deviceId, device.name or "None", device.lastInternalIpAddress or "Unknown", device.lastContact)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/defense/list_events.py b/examples/defense/list_events.py deleted file mode 100644 index e6056b8f..00000000 --- a/examples/defense/list_events.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python -# Example of using cbapi to get event data -# usage: -# python list_events.py --hostname --start --end - - -# Notes on this script: -# - script can only pull up to 2 weeks of events at one time ( this is an API limitation) -# - if no data exists between the start and end range, the script will pull no data. -# - script can be run with no arguments, it will return events from all endpoints for the past 2 weeks - -import sys -import re -from datetime import datetime -from cbapi.psc.defense.models import Event -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object - - -# Function to format epoch time -def convert_time(epoch_time): - converted_time = datetime.fromtimestamp(int(epoch_time / 1000.0)).strftime(' %b %d %Y %H:%M:%S') - return converted_time - - -# Function to strip HTML from a string. -def strip_html(string): - p = re.compile(r'<.*?>') - return p.sub('', string) - - -def main(): - parser = build_cli_parser("List Events for a device") - event_options = parser.add_mutually_exclusive_group(required=False) - event_date_options = parser.add_argument_group("Date Range Arguments") - event_date_options.add_argument("--start", help="start time") - event_date_options.add_argument("--end", help="end time") - event_options.add_argument("-n", "--hostname", help="Hostname") - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.hostname: - events = list(cb.select(Event).where("hostNameExact:{0}".format(args.hostname))) - elif args.start and args.end: - # flipped the start and end arguments around so script can be called with the start date being the earliest date. - # it's just easier on the eyes for most folks. - - events = list(cb.select(Event).where("startTime:{0}".format(args.end))) and ( - cb.select(Event).where("endTime:{0}".format(args.start))) - else: - events = list(cb.select(Event)) - - for event in events: - # convert event and create times - event_time = str(convert_time(event.createTime)) - create_time = str(convert_time(event.eventTime)) - - # stripping HTML tags out of the long description - long_description = strip_html(event.longDescription) - - # format and print out the event time, Event ID, Creation time, Event type and Description - print("{0:^25}{1:^25}{2:^32}{3}".format("Event Time", "Event ID", "Create Time", "Event Type")) - print("{0} | {1} | {2} | {3}".format(event_time, event.eventId, create_time, event.eventType)) - print("{0:50}".format(" ")) - print("{0} {1}".format("Description: ", long_description)) - print("{0:50}".format("------------------------------------")) - print("{0:50}".format(" ")) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/defense/list_events_with_cmdline_csv.py b/examples/defense/list_events_with_cmdline_csv.py deleted file mode 100644 index e282bd00..00000000 --- a/examples/defense/list_events_with_cmdline_csv.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Example of using cbapi to get event data -# usage: -# python list_events.py --hostname --start --end - - -# Notes on this script: -# - based on https://github.com/carbonblack/cbapi-python/blob/master/examples/defense/list_events.py with 2 primary changes -# 1. this script outputs the command line of the main process process -# 2. this script places a '|' delimiter between fields so it can be read into a spreadsheet -# - can only pull up to 2 weeks of events at one time ( this is an API limitation) -# - if no data exists between the start and end range, the script will pull no data. -# - script can be run with no arguments, it will return events from all endpoints for the past 2 weeks - -import sys -import re -import unicodedata -from datetime import datetime -from cbapi.psc.defense.models import Event -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object - - -# Function to format epoch time -def convert_time(epoch_time): - converted_time = datetime.fromtimestamp(int(epoch_time / 1000.0)).strftime(' %b %d %Y %H:%M:%S') - return converted_time - - -# Function to strip HTML from a string. -def strip_html(string): - p = re.compile(r'<.*?>') - return p.sub('', string) - - -def main(): - parser = build_cli_parser("List Events for a device") - event_options = parser.add_mutually_exclusive_group(required=False) - event_date_options = parser.add_argument_group("Date Range Arguments") - event_date_options.add_argument("--start", help="start time") - event_date_options.add_argument("--end", help="end time") - event_options.add_argument("-n", "--hostname", help="Hostname") - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.hostname: - events = list(cb.select(Event).where("hostNameExact:{0}".format(args.hostname))) - elif args.start and args.end: - # flipped the start and end arguments around so script can be called with the start date being the earliest date. - # it's just easier on the eyes for most folks. - - events = list(cb.select(Event).where("startTime:{0}".format(args.end))) and ( - cb.select(Event).where("endTime:{0}".format(args.start))) - else: - events = list(cb.select(Event)) - - # print the column headers - print("Event Time|Event ID|Create Time|Event Type|Description|Command Line") - - for event in events: - # convert event and create times - event_time = str(convert_time(event.createTime)) - create_time = str(convert_time(event.eventTime)) - - # stripping HTML tags out of the long description - long_description = unicodedata.normalize('NFD',strip_html(event.longDescription)) - - if event.processDetails: - # stripping out the command line arguments from the processDetails field - processDetails = str(event.processDetails) - start_cmdline = processDetails.find("u'commandLine'") - end_cmdline = processDetails.find(", u'parentName'") - commandline = processDetails[start_cmdline + 18: end_cmdline -1] - print("{0}|{1}|{2}|{3}|{4}|{5}".format(event_time, event.eventId, create_time, event.eventType, long_description, commandline)) - else: - print("{0}|{1}|{2}|{3}|{4}".format(event_time, event.eventId, create_time, event.eventType, long_description)) - # format and print out the event time, Event ID, Creation time, Event type and Description - - - - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/defense/move_device.py b/examples/defense/move_device.py deleted file mode 100644 index f28289bb..00000000 --- a/examples/defense/move_device.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object -from cbapi.psc.defense import Device - - -def main(): - parser = build_cli_parser("Move a device into a new security policy") - device_options = parser.add_mutually_exclusive_group(required=True) - device_options.add_argument("-i", "--id", type=int, help="Device ID of sensor to move") - device_options.add_argument("-n", "--hostname", help="Hostname to move") - - policy_options = parser.add_mutually_exclusive_group(required=True) - policy_options.add_argument("--policyid", type=int, help="Policy ID") - policy_options.add_argument("--policyname", help="Policy name") - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.id: - devices = [cb.select(Device, args.id)] - else: - devices = list(cb.select(Device).where("hostNameExact:{0}".format(args.hostname))) - - for device in devices: - if args.policyid: - destpolicy = int(args.policyid) - device.policyId = int(args.policyid) - else: - destpolicy = args.policyname - device.policyName = args.policyname - - device.save() - print("Moved device id {0} (hostname {1}) into policy {2}".format(device.deviceId, device.name, destpolicy)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/defense/notifications.py b/examples/defense/notifications.py deleted file mode 100644 index e414b05f..00000000 --- a/examples/defense/notifications.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python - -import sys -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object -import json - - -def main(): - parser = build_cli_parser("Listen to real-time notifications") - parser.add_argument("-s", type=int, help="# of seconds to sleep between polls", default=30) - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - while True: - for notification in cb.notification_listener(args.s): - print(json.dumps(notification, indent=2)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/defense/policy_operations.py b/examples/defense/policy_operations.py deleted file mode 100644 index 00656cc1..00000000 --- a/examples/defense/policy_operations.py +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env python -# - -import sys - -import json -import logging - -from cbapi.errors import ServerError -from cbapi.example_helpers import build_cli_parser, get_cb_defense_object -from cbapi.psc.defense import Policy - -log = logging.getLogger(__name__) - - -def get_policy_by_name_or_id(cb, policy_id=None, name=None, return_all_if_none=False): - policies = [] - - try: - if policy_id: - if isinstance(policy_id, list): - attempted_to_find = "IDs of {0}".format(", ".join([str(pid) for pid in policy_id])) - policies = [p for p in cb.select(Policy) if p.id in policy_id] - else: - attempted_to_find = "ID of {0}".format(policy_id) - policies = [cb.select(Policy, policy_id, force_init=True)] - elif name: - if isinstance(name, list): - attempted_to_find = "names of {0}".format(", ".join(name)) - policies = [p for p in cb.select(Policy) if p.name in name] - else: - attempted_to_find = "name {0}".format(name) - policies = [p for p in cb.select(Policy) if p.name == name] - elif return_all_if_none: - attempted_to_find = "all policies" - policies = list(cb.select(Policy)) - except Exception as e: - print("Could not find policy with {0}: {1}".format(attempted_to_find, str(e))) - - return policies - - -def list_policies(cb, parser, args): - for p in cb.select(Policy): - print(u"Policy id {0}: {1} {2}".format(p.id, p.name, "({0})".format(p.description) if p.description else "")) - print("Rules:") - for r in p.rules.values(): - print(" {0}: {1} when {2} {3} is {4}".format(r.get('id'), r.get("action"), - r.get("application", {}).get("type"), - r.get("application", {}).get("value"), r.get("operation"))) - - -def import_policy(cb, parser, args): - p = cb.create(Policy) - - p.policy = json.load(open(args.policyfile, "r")) - p.description = args.description - p.name = args.name - p.priorityLevel = args.prioritylevel - p.version = 2 - - try: - p.save() - except ServerError as se: - print("Could not add policy: {0}".format(str(se))) - except Exception as e: - print("Could not add policy: {0}".format(str(e))) - else: - print("Added policy. New policy ID is {0}".format(p.id)) - - -def delete_policy(cb, parser, args): - policies = get_policy_by_name_or_id(cb, args.id, args.name) - if len(policies) == 0: - return - - num_matching_policies = len(policies) - if num_matching_policies > 1 and not args.force: - print("{0:d} policies match and --force not specified. No action taken.".format(num_matching_policies)) - return - - for p in policies: - try: - p.delete() - except Exception as e: - print("Could not delete policy: {0}".format(str(e))) - else: - print("Deleted policy id {0} with name {1}".format(p.id, p.name)) - - -def export_policy(cb, parser, args): - policies = get_policy_by_name_or_id(cb, args.id, args.name, return_all_if_none=True) - - for p in policies: - json.dump(p.policy, open("policy-{0}.json".format(p.id), "w"), indent=2) - print("Wrote policy {0} {1} to file policy-{0}.json".format(p.id, p.name)) - - -def add_rule(cb, parser, args): - policies = get_policy_by_name_or_id(cb, args.id, args.name) - - num_matching_policies = len(policies) - if num_matching_policies < 1: - print("No policies match. No action taken.".format(num_matching_policies)) - - for policy in policies: - policy.add_rule(json.load(open(args.rulefile, "r"))) - print("Added rule from {0} to policy {1}.".format(args.rulefile, policy.name)) - - -def del_rule(cb, parser, args): - policies = get_policy_by_name_or_id(cb, args.id, args.name) - - num_matching_policies = len(policies) - if num_matching_policies != 1: - print("{0:d} policies match. No action taken.".format(num_matching_policies)) - - policy = policies[0] - policy.delete_rule(args.ruleid) - - print("Removed rule id {0} from policy {1}.".format(args.ruleid, policy.name)) - - -def replace_rule(cb, parser, args): - policies = get_policy_by_name_or_id(cb, args.id, args.name) - - num_matching_policies = len(policies) - if num_matching_policies != 1: - print("{0:d} policies match. No action taken.".format(num_matching_policies)) - - policy = policies[0] - policy.replace_rule(args.ruleid, json.load(open(args.rulefile, "r"))) - - print("Replaced rule id {0} from policy {1} with rule from file {2}.".format(args.ruleid, policy.name, - args.rulefile)) - -def main(): - parser = build_cli_parser("Policy operations") - commands = parser.add_subparsers(help="Policy commands", dest="command_name") - - list_command = commands.add_parser("list", help="List all configured policies") - - import_policy_command = commands.add_parser("import", help="Import policy from JSON file") - import_policy_command.add_argument("-N", "--name", help="Name of new policy", required=True) - import_policy_command.add_argument("-d", "--description", help="Description of new policy", required=True) - import_policy_command.add_argument("-p", "--prioritylevel", help="Priority level (HIGH, MEDIUM, LOW)", - default="LOW") - import_policy_command.add_argument("-f", "--policyfile", help="Filename containing the JSON policy description", - required=True) - - export_policy_command = commands.add_parser("export", help="Export policy to JSON file") - export_policy_specifier = export_policy_command.add_mutually_exclusive_group(required=False) - export_policy_specifier.add_argument("-i", "--id", type=int, help="ID of policy") - export_policy_specifier.add_argument("-N", "--name", help="Name of policy") - - del_command = commands.add_parser("delete", help="Delete policies") - del_policy_specifier = del_command.add_mutually_exclusive_group(required=True) - del_policy_specifier.add_argument("-i", "--id", type=int, help="ID of policy to delete") - del_policy_specifier.add_argument("-N", "--name", help="Name of policy to delete. Specify --force to delete" - " multiple policies that have the same name") - del_command.add_argument("--force", help="If NAME matches multiple policies, delete all matching policies", - action="store_true", default=False) - - add_rule_command = commands.add_parser("add-rule", help="Add rule to existing policy from JSON rule file") - add_rule_specifier = add_rule_command.add_mutually_exclusive_group(required=True) - add_rule_specifier.add_argument("-i", "--id", type=int, help="ID of policy (can specify multiple)", - action="append", metavar="POLICYID") - add_rule_specifier.add_argument("-N", "--name", help="Name of policy (can specify multiple)", - action="append", metavar="POLICYNAME") - add_rule_command.add_argument("-f", "--rulefile", help="Filename containing the JSON rule", required=True) - - del_rule_command = commands.add_parser("del-rule", help="Delete rule from existing policy") - del_rule_specifier = del_rule_command.add_mutually_exclusive_group(required=True) - del_rule_specifier.add_argument("-i", "--id", type=int, help="ID of policy") - del_rule_specifier.add_argument("-N", "--name", help="Name of policy") - del_rule_command.add_argument("-r", "--ruleid", type=int, help="ID of rule", required=True) - - replace_rule_command = commands.add_parser("replace-rule", help="Replace existing rule with a new one") - replace_rule_specifier = replace_rule_command.add_mutually_exclusive_group(required=True) - replace_rule_specifier.add_argument("-i", "--id", type=int, help="ID of policy") - replace_rule_specifier.add_argument("-N", "--name", help="Name of policy") - replace_rule_command.add_argument("-r", "--ruleid", type=int, help="ID of rule", required=True) - replace_rule_command.add_argument("-f", "--rulefile", help="Filename containing the JSON rule", required=True) - - args = parser.parse_args() - cb = get_cb_defense_object(args) - - if args.command_name == "list": - return list_policies(cb, parser, args) - elif args.command_name == "import": - return import_policy(cb, parser, args) - elif args.command_name == "export": - return export_policy(cb, parser, args) - elif args.command_name == "delete": - return delete_policy(cb, parser, args) - elif args.command_name == "add-rule": - return add_rule(cb, parser, args) - elif args.command_name == "del-rule": - return del_rule(cb, parser, args) - elif args.command_name == "replace-rule": - return replace_rule(cb, parser, args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/livequery/manage_run.py b/examples/livequery/manage_run.py deleted file mode 100644 index 493f4097..00000000 --- a/examples/livequery/manage_run.py +++ /dev/null @@ -1,149 +0,0 @@ -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_livequery_object -from cbapi.psc.livequery.models import Run - - -def create_run(cb, args): - query = cb.query(args.sql) - - if args.device_ids: - query.device_ids(args.device_ids) - if args.device_types: - query.device_types(args.device_types) - if args.policy_ids: - query.policy_ids(args.policy_ids) - if args.notify: - query.notify_on_finish() - if args.name: - query.name(args.name) - - run = query.submit() - print(run) - - -def run_status(cb, args): - run = cb.select(Run, args.id) - print(run) - - -def run_stop(cb, args): - run = cb.select(Run, args.id) - if run.stop(): - print("Run {} has been stopped.".format(run.id)) - print(run) - else: - print("Unable to stop run {}".format(run.id)) - - -def run_delete(cb, args): - run = cb.select(Run, args.id) - if run.delete(): - print("Run {} has been deleted.".format(run.id)) - else: - print("Unable to delete run {}".format(run.id)) - - -def run_history(cb, args): - results = cb.query_history(args.query) - if args.sort_by: - dir = "DESC" if args.descending_results else "ASC" - results.sort_by(args.sort_by, direction=dir) - for result in results: - print(result) - - - -def main(): - parser = build_cli_parser("Create and manage LiveQuery runs") - commands = parser.add_subparsers(help="Commands", dest="command_name") - - create_command = commands.add_parser("create", help="Create a new LiveQuery run") - create_command.add_argument( - "-s", "--sql", type=str, required=True, help="The query to run" - ) - create_command.add_argument( - "-n", - "--notify", - action="store_true", - help="Notify by email when the run finishes", - ) - create_command.add_argument( - "-N", "--name", type=str, required=False, help="The name of the run" - ) - create_command.add_argument( - "--device_ids", - nargs="+", - type=int, - required=False, - help="Device IDs to filter on", - ) - create_command.add_argument( - "--device_types", - nargs="+", - type=str, - required=False, - help="Device types to filter on", - ) - create_command.add_argument( - "--policy_ids", - nargs="+", - type=str, - required=False, - help="Policy IDs to filter on", - ) - - status_command = commands.add_parser( - "status", help="Retrieve information about a run" - ) - status_command.add_argument( - "-i", "--id", type=str, required=True, help="The run ID" - ) - - stop_command = commands.add_parser( - "stop", help="Stops/cancels a current run" - ) - stop_command.add_argument( - "-i", "--id", type=str, required=True, help="The run ID" - ) - - delete_command = commands.add_parser( - "delete", help="Permanently delete a run" - ) - delete_command.add_argument( - "-i", "--id", type=str, required=True, help="The run ID" - ) - - history_command = commands.add_parser( - "history", help="List history of all runs" - ) - history_command.add_argument( - "-q", "--query", type=str, required=False, help="Query string to use" - ) - history_command.add_argument( - "-S", "--sort_by", type=str, help="sort by this field", required=False - ) - history_command.add_argument( - "-D", - "--descending_results", - help="return results in descending order", - action="store_true", - required=False - ) - - args = parser.parse_args() - cb = get_cb_livequery_object(args) - - if args.command_name == "create": - return create_run(cb, args) - elif args.command_name == "status": - return run_status(cb, args) - elif args.command_name == "stop": - return run_stop(cb, args) - elif args.command_name == "delete": - return run_delete(cb, args) - elif args.command_name == "history": - return run_history(cb, args) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/livequery/run_device_summary.py b/examples/livequery/run_device_summary.py deleted file mode 100755 index 12bd1a69..00000000 --- a/examples/livequery/run_device_summary.py +++ /dev/null @@ -1,88 +0,0 @@ -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_livequery_object -from cbapi.psc.livequery.models import Result - - -def main(): - parser = build_cli_parser("Search the device summaries of a LiveQuery run") - parser.add_argument("-i", "--id", type=str, required=True, help="Run ID") - parser.add_argument("-q", "--query", type=str, required=False, help="Search query") - - parser.add_argument( - "--device_ids", - nargs="+", - type=int, - required=False, - help="Device IDs to filter on", - ) - parser.add_argument( - "--device_names", - nargs="+", - type=int, - required=False, - help="Device names to filter on", - ) - parser.add_argument( - "--policy_ids", - nargs="+", - type=int, - required=False, - help="Policy IDs to filter on", - ) - parser.add_argument( - "--policy_names", - nargs="+", - type=int, - required=False, - help="Policy names to filter on", - ) - parser.add_argument( - "--statuses", - nargs="+", - type=str, - required=False, - help="Statuses to filter on", - ) - parser.add_argument( - "-S", "--sort_by", type=str, help="sort by this field", required=False - ) - parser.add_argument( - "-D", - "--descending_results", - help="return results in descending order", - action="store_true", - ) - - args = parser.parse_args() - cb = get_cb_livequery_object(args) - - results = cb.select(Result).run_id(args.id) - result = results.first() - if result is None: - print("ERROR: No results.") - return 1 - - summaries = result.query_device_summaries() - if args.query: - summaries.where(args.query) - if args.device_ids: - summaries.criteria(device_id=args.device_ids) - if args.device_names: - summaries.criteria(device_name=args.device_names) - if args.policy_ids: - summaries.criteria(policy_id=args.policy_ids) - if args.policy_names: - summaries.criteria(policy_name=args.policy_names) - if args.statuses: - summaries.criteria(status=args.statuses) - if args.sort_by: - dir = "DESC" if args.descending_results else "ASC" - summaries.sort_by(args.sort_by, direction=dir) - - for summary in summaries: - print(summary) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/livequery/run_facets.py b/examples/livequery/run_facets.py deleted file mode 100755 index 8cabb8a6..00000000 --- a/examples/livequery/run_facets.py +++ /dev/null @@ -1,107 +0,0 @@ -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_livequery_object -from cbapi.psc.livequery.models import Result - - -def main(): - parser = build_cli_parser("Search the facets of a LiveQuery run") - parser.add_argument("-i", "--id", type=str, required=True, help="Run ID") - parser.add_argument( - "--result", - action="store_true", - help="Run facet query on results" - ) - parser.add_argument( - "--device_summary", - action="store_true", - help="Run facet query on device summaries" - ) - parser.add_argument( - "-f", - "--fields", - nargs="+", - type=str, - required=False, - help="Fields to be displayed in results", - ) - - parser.add_argument("-q", "--query", type=str, required=False, help="Search query") - parser.add_argument( - "--device_ids", - nargs="+", - type=int, - required=False, - help="Device IDs to filter on", - ) - parser.add_argument( - "--device_names", - nargs="+", - type=int, - required=False, - help="Device names to filter on", - ) - parser.add_argument( - "--policy_ids", - nargs="+", - type=int, - required=False, - help="Policy IDs to filter on", - ) - parser.add_argument( - "--policy_names", - nargs="+", - type=int, - required=False, - help="Policy names to filter on", - ) - parser.add_argument( - "--statuses", - nargs="+", - type=str, - required=False, - help="Statuses to filter on", - ) - - args = parser.parse_args() - if not (args.result or args.device_summary): - print("ERROR: One of --result or --device_summary must be specified") - return 1 - if args.result and args.device_summary: - print("ERROR: --result and --device_summary cannot both be specified") - return 1 - - cb = get_cb_livequery_object(args) - - results = cb.select(Result).run_id(args.id) - result = results.first() - if result is None: - print("ERROR: No results.") - return 1 - - if args.result: - facets = result.query_result_facets() - elif args.device_summary: - facets = result.query_device_summary_facets() - if args.fields: - facets.facet_field(args.fields) - if args.query: - facets.where(args.query) - if args.device_ids: - facets.criteria(device_id=args.device_ids) - if args.device_names: - facets.criteria(device_name=args.device_names) - if args.policy_ids: - facets.criteria(policy_id=args.policy_ids) - if args.policy_names: - facets.criteria(policy_name=args.policy_names) - if args.statuses: - facets.criteria(status=args.statuses) - - for facet in facets: - print(facet) - - -if __name__ == "__main__": - sys.exit(main()) - diff --git a/examples/livequery/run_search.py b/examples/livequery/run_search.py deleted file mode 100644 index 66b2e6a7..00000000 --- a/examples/livequery/run_search.py +++ /dev/null @@ -1,73 +0,0 @@ -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_livequery_object -from cbapi.psc.livequery.models import Result - - -def main(): - parser = build_cli_parser("Search the results of a LiveQuery run") - parser.add_argument("-i", "--id", type=str, required=True, help="Run ID") - parser.add_argument("-q", "--query", type=str, required=False, help="Search query") - parser.add_argument( - "-F", "--fields_only", action="store_true", help="Show only fields" - ) - parser.add_argument( - "--device_ids", - nargs="+", - type=int, - required=False, - help="Device IDs to filter on", - ) - parser.add_argument( - "--device_types", - nargs="+", - type=str, - required=False, - help="Device types to filter on", - ) - parser.add_argument( - "--statuses", - nargs="+", - type=str, - required=False, - help="Statuses to filter on", - ) - parser.add_argument( - "-S", "--sort_by", type=str, help="sort by this field", required=False - ) - parser.add_argument( - "-D", - "--descending_results", - help="return results in descending order", - action="store_true", - ) - - args = parser.parse_args() - cb = get_cb_livequery_object(args) - - results = cb.select(Result).run_id(args.id) - if args.query: - results = results.where(args.query) - - if args.device_ids: - results.criteria(device_id=args.device_ids) - if args.device_types: - results.criteria(device_type=args.device_types) - if args.statuses: - results.criteria(status=args.statuses) - - if args.sort_by: - direction = "ASC" - if args.descending_results: - direction = "DESC" - results.sort_by(args.sort_by, direction=direction) - - for result in results: - if args.fields_only: - print(result.fields_) - else: - print(result) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/protection/empty_analysis_connector.py b/examples/protection/empty_analysis_connector.py index 882efa53..f4162a1e 100644 --- a/examples/protection/empty_analysis_connector.py +++ b/examples/protection/empty_analysis_connector.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from cbapi.protection import Connector, PendingAnalysis, Notification +from cbapi.protection import Connector, Notification from cbapi.example_helpers import get_cb_protection_object, build_cli_parser import logging import time @@ -48,4 +48,3 @@ def process_request(api, i): for i in vt_connector.pendingAnalyses: process_request(api, i) time.sleep(10) - diff --git a/examples/protection/policy_return.py b/examples/protection/policy_return.py index dabb79aa..3c0e13b5 100644 --- a/examples/protection/policy_return.py +++ b/examples/protection/policy_return.py @@ -2,6 +2,7 @@ from cbapi.protection.models import Computer, Policy import sys + def main(): parser = build_cli_parser("Revert computers in policy to previous policy") parser.add_argument("--policy", "-p", help="Policy name or ID", required=True) @@ -20,5 +21,6 @@ def main(): computer.save() print("%s is now in %s" % (computer.name, computer.policyName)) + if __name__ == '__main__': sys.exit(main()) diff --git a/examples/protection/remove_duplicates.py b/examples/protection/remove_duplicates.py index 06d68e5a..369b10f2 100644 --- a/examples/protection/remove_duplicates.py +++ b/examples/protection/remove_duplicates.py @@ -29,5 +29,6 @@ def main(): print("deleting from server...") p.select(Computer, computer_id["id"]).delete() + if __name__ == '__main__': sys.exit(main()) diff --git a/examples/protection/virus_total_connector.py b/examples/protection/virus_total_connector.py index 15d48d3a..1c229068 100644 --- a/examples/protection/virus_total_connector.py +++ b/examples/protection/virus_total_connector.py @@ -38,7 +38,7 @@ def __init__(self, api, vt_token=None, connector_name='VirusTotal', allow_upload raise TypeError("Missing required VT authentication token.") self.vt_token = vt_token self.vt_url = 'https://www.virustotal.com/vtapi/v2' - self.polling_frequency = 30 # seconds + self.polling_frequency = 30 # seconds self.malicious_threshold = malicious_threshold self.potential_threshold = potential_threshold @@ -83,7 +83,8 @@ def process_response(self, binary, scanResults): self.schedule_check(binary, binary.fileHash) else: binary.analysisStatus = PendingAnalysis.StatusCancelled - log.info("%s: VirusTotal has no information and we aren't allowed to upload it. Cancelling the analysis request." % binary.fileHash) + log.info("%s: VirusTotal has no information and we aren't allowed to upload it. " + "Cancelling the analysis request." % binary.fileHash) binary.save() def report_result(self, binary, scanResults): @@ -143,9 +144,7 @@ def upload_to_vt(self, binary): r = requests.post(self.vt_url + "/file/scan", files=files, params={'apikey': self.vt_token}) if r.status_code != 200: vt_upload_error = True - else: - scanId = r.json()['scan_id'] - except: + except Exception: log.exception("Could not send file %s to VirusTotal" % (binary.fileHash,)) vt_upload_error = True @@ -157,13 +156,14 @@ def upload_to_vt(self, binary): binary.save() else: - log.info("%s: VirusTotal has no information on this hash. Waiting for agent to upload it." % binary.fileHash) + log.info("%s: VirusTotal has no information on this hash. Waiting for agent to upload it." + % binary.fileHash) def schedule_check(self, binary, scanId): next_check = datetime.datetime.now() + datetime.timedelta(0, 3600) self.awaiting_results[binary.fileHash] = {'scanId': scanId, 'nextCheck': next_check} - log.info("%s: Waiting for analysis to complete. Will check back after %s." % (binary.fileHash, - next_check.strftime("%Y-%m-%d %H:%M:%S"))) + log.info("%s: Waiting for analysis to complete. Will check back after %s." + % (binary.fileHash, next_check.strftime("%Y-%m-%d %H:%M:%S"))) def process_request(self, binary): if binary.fileHash in self.awaiting_results: @@ -218,7 +218,7 @@ def main(): log.info("Configuration:") for k, v in iteritems(config): - log.info(" %-20s: %s" % (k,v)) + log.info(" %-20s: %s" % (k, v)) api = get_cb_protection_object(args) @@ -235,4 +235,3 @@ def main(): if __name__ == '__main__': sys.exit(main()) - diff --git a/examples/response/alert_search.py b/examples/response/alert_search.py index 3d19a659..2248dfab 100755 --- a/examples/response/alert_search.py +++ b/examples/response/alert_search.py @@ -27,4 +27,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/examples/response/binary_export.py b/examples/response/binary_export.py index 0d5e0837..24a8eb3d 100755 --- a/examples/response/binary_export.py +++ b/examples/response/binary_export.py @@ -19,5 +19,6 @@ def main(): return 0 + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/response/binary_search.py b/examples/response/binary_search.py index ed374658..218df892 100755 --- a/examples/response/binary_search.py +++ b/examples/response/binary_search.py @@ -13,13 +13,14 @@ def main(): cb = get_cb_response_object(args) binary_query = cb.select(Binary).where(args.query) - # for each result + # for each result for binary in binary_query: print(binary.md5sum) print("-" * 80) print("%-20s : %s" % ('Size (bytes)', binary.size)) print("%-20s : %s" % ('Signature Status', binary.signed)) - print("%-20s : %s" % ('Publisher', binary.digsig_publisher) if binary.signed == True else "%-20s : %s" % ('Publisher', 'n/a')) + print("%-20s : %s" % ('Publisher', binary.digsig_publisher) if binary.signed + else "%-20s : %s" % ('Publisher', 'n/a')) print("%-20s : %s" % ('Product Version', binary.product_version)) print("%-20s : %s" % ('File Version', binary.file_version)) print("%-20s : %s" % ('64-bit (x64)', binary.is_64bit)) @@ -30,5 +31,6 @@ def main(): print('\n') + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/response/cmd_exe_filemods.py b/examples/response/cmd_exe_filemods.py index 6c9439ef..3e3afac9 100755 --- a/examples/response/cmd_exe_filemods.py +++ b/examples/response/cmd_exe_filemods.py @@ -31,5 +31,6 @@ def main(): print("%s,%s,%s,%s,%s,%s,%s,%s,%s" % (str(fm.timestamp), proc.hostname, proc.username, proc.path, fm.path, fm.type, fm.md5, signed, product_name)) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/response/download_from_lr.py b/examples/response/download_from_lr.py index 533b9f31..f9ebb125 100755 --- a/examples/response/download_from_lr.py +++ b/examples/response/download_from_lr.py @@ -2,9 +2,8 @@ # import sys -from cbapi.response import * +from cbapi.response import Sensor from cbapi.example_helpers import build_cli_parser, get_cb_response_object -from cbapi.errors import ServerError import logging log = logging.getLogger(__name__) @@ -12,7 +11,7 @@ def download_file(sensor, path, output_filename=None): with sensor.lr_session() as session: - + # get basename of the file, if output_filename is None if output_filename is None: if session.os_type == 1: # Windows uses backslashes @@ -25,7 +24,7 @@ def download_file(sensor, path, output_filename=None): print("Successfully wrote {0} to local file {1}.".format(path, output_filename)) return 0 - + def main(): parser = build_cli_parser("Download binary from endpoint through Live Response") @@ -41,7 +40,8 @@ def main(): if sensor.status == "Online": return download_file(sensor, args.path, args.output) - print("No sensors for hostname {0} are online, exiting".format(hostname)) + print("No sensors for hostname {0} are online, exiting".format(args.hostname)) + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/response/dump_all_binaries.py b/examples/response/dump_all_binaries.py index 224f90c7..db34ff59 100755 --- a/examples/response/dump_all_binaries.py +++ b/examples/response/dump_all_binaries.py @@ -21,7 +21,7 @@ def get_path_for_md5(d, basepath=''): def create_directory(pathname): try: os.makedirs(os.path.dirname(pathname)) - except: + except Exception: pass @@ -35,21 +35,21 @@ def already_exists(self, pathname, item): filesize = os.path.getsize(pathname) if filesize == item.copied_size: return True - except: + except Exception: pass return False def run(self): - l = logging.getLogger(__name__) - l.setLevel(logging.INFO) + log = logging.getLogger(__name__) + log.setLevel(logging.INFO) while True: item = worker_queue.get() pathname = get_path_for_md5(item.md5sum, self.basepath) if self.already_exists(pathname, item): - l.info('already have %s' % item.md5sum) + log.info('already have %s' % item.md5sum) else: create_directory(pathname) write_progress = 0 @@ -58,15 +58,15 @@ def run(self): write_progress += 1 json.dump(item.original_document, open(pathname + '.json', 'w')) write_progress += 1 - except: + except Exception: pass if not write_progress: - l.error(u'Could not grab {0:s}'.format(item.md5sum)) + log.error(u'Could not grab {0:s}'.format(item.md5sum)) elif write_progress == 1: - l.info(u'Grabbed {0:s} binary'.format(item.md5sum)) + log.info(u'Grabbed {0:s} binary'.format(item.md5sum)) elif write_progress == 2: - l.info(u'Grabbed {0:s} binary & process document'.format(item.md5sum)) + log.info(u'Grabbed {0:s} binary & process document'.format(item.md5sum)) worker_queue.task_done() @@ -112,4 +112,3 @@ def main(): if __name__ == '__main__': sys.exit(main()) - diff --git a/examples/response/enumerate_usb_devices.py b/examples/response/enumerate_usb_devices.py index e60f9c48..f024e950 100755 --- a/examples/response/enumerate_usb_devices.py +++ b/examples/response/enumerate_usb_devices.py @@ -13,7 +13,7 @@ def main(): args = parser.parse_args() cb = get_cb_response_object(args) - query_string = r'regmod:registry\machine\system\currentcontrolset\control\deviceclasses\{53f56307-b6bf-11d0-94f2-00a0c91efb8b}\*' + query_string = r'regmod:registry\machine\system\currentcontrolset\control\deviceclasses\{53f56307-b6bf-11d0-94f2-00a0c91efb8b}\*' # noqa: E501 if args.start_time: query_string += ' start:{0:s}'.format(args.start_time) @@ -24,7 +24,7 @@ def main(): if len(pieces) < 2: print("WARN:::: {0}".format(str(pieces))) else: - device_info = pieces[1] #.split('{53f56307-b6bf-11d0-94f2-00a0c91efb8b}')[0] + device_info = pieces[1] # .split('{53f56307-b6bf-11d0-94f2-00a0c91efb8b}')[0] print(device_info) diff --git a/examples/response/event/get_reg_autoruns.py b/examples/response/event/get_reg_autoruns.py index 402d6f76..14d5f532 100644 --- a/examples/response/event/get_reg_autoruns.py +++ b/examples/response/event/get_reg_autoruns.py @@ -30,8 +30,7 @@ import sys import time - -autoruns_regex = re.compile("|".join("""\\registry\\machine\\system\\currentcontrolset\\control\\session manager\\bootexecute(.*) +autoruns = """\\registry\\machine\\system\\currentcontrolset\\control\\session manager\\bootexecute(.*) \\registry\\machine\\system\\currentcontrolset\\services(.*) \\registry\\machine\\system\\currentcontrolset\\services(.*) \\registry\\machine\\software\\microsoft\\windows\\currentversion\\runservicesonce(.*) @@ -53,8 +52,9 @@ \\registry\\user\\(.*)\\software\\microsoft\\windows\\currentversion\\run(.*) \\registry\\user\\(.*)\\software\\microsoft\\windows\\currentversion\\runonce(.*) \\registry\\user\\(.*)\\software\\microsoft\\windows\\currentversion\\policies\\explorer\\run(.*) -\\registry\\user\\(.*)\\software\\microsoft\\windows nt\\currentversion\\windows\\load(.*)""".replace("\\", "\\\\").split("\n"))) +\\registry\\user\\(.*)\\software\\microsoft\\windows nt\\currentversion\\windows\\load(.*)""" +autoruns_regex = re.compile("|".join(autoruns.replace("\\", "\\\\").split("\n"))) class GetRegistryValue(object): @@ -85,7 +85,7 @@ def process_callback(cb, event_type, event_data): def print_result(registry_job): try: timestamp, sensor_id, registry_key, registry_value = registry_job.result() - except: + except Exception: print("Error encountered when pulling registry key: {0}".format(registry_job.exception())) else: print("Got result for sensor ID {0} registry key {1}: value is {2}".format(sensor_id, registry_key, @@ -118,7 +118,5 @@ def main(): print(error["exception"]) - if __name__ == "__main__": sys.exit(main()) - diff --git a/examples/response/event/watchlist_automation.py b/examples/response/event/watchlist_automation.py index 175c9a63..f15d661c 100644 --- a/examples/response/event/watchlist_automation.py +++ b/examples/response/event/watchlist_automation.py @@ -75,9 +75,11 @@ def perform_liveresponse(lr_session): db.row_factory = sqlite3.Row cur = db.cursor() cur.execute( - "SELECT url, title, datetime(last_visit_time / 1000000 + (strftime('%s', '1601-01-01')), 'unixepoch') as last_visit_time FROM urls ORDER BY last_visit_time DESC LIMIT 10") + "SELECT url, title, " + "datetime(last_visit_time / 1000000 + (strftime('%s', '1601-01-01')), 'unixepoch') " + "as last_visit_time FROM urls ORDER BY last_visit_time DESC LIMIT 10") urls = [dict(u) for u in cur.fetchall()] - except: + except Exception: pass else: results[user] = urls @@ -90,7 +92,7 @@ def perform_liveresponse(lr_session): def print_result(lr_job): try: sensor_id, running_services, urls = lr_job.result() - except: + except Exception: print("Error encountered when pulling Live Response data: {0}".format(lr_job.exception())) else: print("Running services for sensor ID {0}:".format(sensor_id)) @@ -140,7 +142,8 @@ def process_callback(cb, event_type, event_data): def main(): - parser = build_cli_parser("Event-driven example to BAN hashes, ISOLATE sensors, or LOCK (both ban & isolate) based on watchlist hits") + parser = build_cli_parser("Event-driven example to BAN hashes, ISOLATE sensors, or LOCK " + "(both ban & isolate) based on watchlist hits") args = parser.parse_args() cb = get_cb_response_object(args) @@ -164,4 +167,3 @@ def main(): if __name__ == "__main__": sys.exit(main()) - diff --git a/examples/response/event_export.py b/examples/response/event_export.py index d718f520..b0fcc38c 100755 --- a/examples/response/event_export.py +++ b/examples/response/event_export.py @@ -8,7 +8,8 @@ from cbapi.example_helpers import build_cli_parser, get_cb_response_object import csv from cbapi.six import PY3 -from cbapi.response.models import CbChildProcEvent, CbFileModEvent, CbNetConnEvent, CbRegModEvent, CbModLoadEvent, CbCrossProcEvent +from cbapi.response.models import CbChildProcEvent, CbFileModEvent, CbNetConnEvent, CbRegModEvent, \ + CbModLoadEvent, CbCrossProcEvent # UnicodeWriter class from http://python3porting.com/problems.html diff --git a/examples/response/feed_operations.py b/examples/response/feed_operations.py index d135c466..2fe30fd9 100755 --- a/examples/response/feed_operations.py +++ b/examples/response/feed_operations.py @@ -2,7 +2,7 @@ # import sys -from cbapi.response.models import Feed, FeedAction +from cbapi.response.models import Feed from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id from cbapi.errors import ServerError import logging @@ -145,7 +145,7 @@ def main(): parser = build_cli_parser() commands = parser.add_subparsers(help="Feed commands", dest="command_name") - list_command = commands.add_parser("list", help="List all configured feeds") + commands.add_parser("list", help="List all configured feeds") list_actions_command = commands.add_parser("list-actions", help="List actions associated with a feed") list_actions_specifier = list_actions_command.add_mutually_exclusive_group(required=True) @@ -195,7 +195,7 @@ def main(): elif args.command_name == "delete": return delete_feed(cb, parser, args) elif args.command_name in ("disable", "enable"): - return toggle_feed(cb, args.feedname, enable=args.command_name=="enable") + return toggle_feed(cb, args.feedname, enable=(args.command_name == "enable")) if __name__ == "__main__": diff --git a/examples/response/new_binaries_after_date.py b/examples/response/new_binaries_after_date.py index fc405e90..597b05dc 100755 --- a/examples/response/new_binaries_after_date.py +++ b/examples/response/new_binaries_after_date.py @@ -27,7 +27,7 @@ def main(): # parser = build_cli_parser("System Check After Specified Date") parser.add_argument("-d", "--date-to-query", action="store", dest="date", - help="New since DATE, format YYYY-MM-DD") + help="New since DATE, format YYYY-MM-DD") parser.add_argument("-f", "--output-file", action="store", dest="output_file", help="output file in csv format") @@ -122,11 +122,10 @@ def main(): binary.host_count, binary_timestamp, number_of_times_executed)) - except: - print binary + except Exception: + print(binary) pbar.finish() + if __name__ == "__main__": sys.exit(main()) - - diff --git a/examples/response/new_binaries_with_netconns.py b/examples/response/new_binaries_with_netconns.py index d81de5f0..402637ba 100755 --- a/examples/response/new_binaries_with_netconns.py +++ b/examples/response/new_binaries_with_netconns.py @@ -29,7 +29,7 @@ def main(): # parser = build_cli_parser("New Binaries with Netconns") parser.add_argument("-d", "--date-to-query", action="store", dest="date", - help="New since DATE, format YYYY-MM-DD") + help="New since DATE, format YYYY-MM-DD") parser.add_argument("-f", "--output-file", action="store", dest="output_file", help="output file in csv format") @@ -111,10 +111,11 @@ def main(): binary.server_added_timestamp, binary.host_count, binary_timestamp)) - except: - print binary + except Exception: + print(binary) pbar.finish() + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/response/partition_operations.py b/examples/response/partition_operations.py index c19628e1..bd896907 100755 --- a/examples/response/partition_operations.py +++ b/examples/response/partition_operations.py @@ -1,9 +1,11 @@ import sys -from distutils.version import LooseVersion +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from cbapi.response.models import StoragePartition -from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id -from cbapi.errors import ServerError +from cbapi.example_helpers import build_cli_parser, get_cb_response_object import logging log = logging.getLogger(__name__) @@ -58,9 +60,9 @@ def main(): parser = build_cli_parser() commands = parser.add_subparsers(help="Storage Partition commands", dest="command_name") - list_command = commands.add_parser("list", help="List all storage partitions") + commands.add_parser("list", help="List all storage partitions") - create_command = commands.add_parser("create", help="Create new active writer partition") + commands.add_parser("create", help="Create new active writer partition") del_command = commands.add_parser("delete", help="Delete partition") del_command.add_argument("-N", "--name", help="Name of partition to delete.", required=True) @@ -74,7 +76,7 @@ def main(): args = parser.parse_args() cb = get_cb_response_object(args) - if cb.cb_server_version < LooseVersion("6.1.0"): + if cb.cb_server_version < parse("6.1.0"): parser.error("This script can only work with server versions >= 6.1.0; {0} is running {1}" .format(cb.url, cb.cb_server_version)) return 1 diff --git a/examples/response/process_netconn_rate.py b/examples/response/process_netconn_rate.py index 866ec571..572114c4 100755 --- a/examples/response/process_netconn_rate.py +++ b/examples/response/process_netconn_rate.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -#The MIT License (MIT) +# The MIT License (MIT) ## # Copyright (c) 2015 Bit9 + Carbon Black # diff --git a/examples/response/s3-watchlist-ban.py b/examples/response/s3-watchlist-ban.py index 4457e644..e9286d69 100755 --- a/examples/response/s3-watchlist-ban.py +++ b/examples/response/s3-watchlist-ban.py @@ -1,7 +1,5 @@ import boto3 import json -import pprint -import traceback import time from cbapi.response.models import BannedHash from cbapi.example_helpers import build_cli_parser, get_cb_response_object @@ -47,7 +45,6 @@ def process_events(data): print(e.message) - def save_progress(processed_list): # # Save our progress in a log file @@ -55,6 +52,7 @@ def save_progress(processed_list): with open('script_progress.log', 'wb') as hfile: hfile.write(json.dumps(list(processed_list))) + def listen_mode(bucket): print("[+]: Listen Mode") @@ -74,7 +72,7 @@ def listen_mode(bucket): for obj in bucket.objects.all(): key = obj.key if key not in current_list: - print "[+]: New File: {}".format(key) + print("[+]: New File: {}".format(key)) # # We have not processed this file. # @@ -142,7 +140,7 @@ def listen_mode(bucket): with open('script_progress.log', 'rb') as hfile: for item in json.loads(hfile.read()): processed_list.add(item) - except: + except Exception: print("[?]: No previous progress file found: script_progress.log") processed_list = set() @@ -170,4 +168,3 @@ def listen_mode(bucket): print("[+]: saving progess to file script_progress.log") save_progress(processed_list) - diff --git a/examples/response/sensor_export.py b/examples/response/sensor_export.py index 383b37c9..0a5077ea 100755 --- a/examples/response/sensor_export.py +++ b/examples/response/sensor_export.py @@ -34,7 +34,8 @@ def main(): parser = build_cli_parser(description="Export CbR Sensors from your environment as CSV") parser.add_argument("--output", "-o", dest="exportfile", help="The file to export to", required=True) parser.add_argument("--fields", "-f", dest="exportfields", help="The fields to export", - default="id,hostname,group_id,network_interfaces,os_environment_display_string,build_version_string,network_isolation_enabled,last_checkin_time", + default="id,hostname,group_id,network_interfaces,os_environment_display_string," + "build_version_string,network_isolation_enabled,last_checkin_time", required=False) parser.add_argument("--query", "-q", dest="query", help="optional query to filter exported sensors", required=False) args = parser.parse_args() diff --git a/examples/response/sensor_group_operations.py b/examples/response/sensor_group_operations.py index b11e8200..bedf9cae 100755 --- a/examples/response/sensor_group_operations.py +++ b/examples/response/sensor_group_operations.py @@ -26,7 +26,9 @@ def list_sensors(cb, parser, args): print(" {0:40}{1:18}{2}".format("Hostname", "IP Address", "Last Checkin Time")) for sensor in group.sensors: ipaddrs = [iface.ipaddr for iface in sensor.network_interfaces if iface.ipaddr not in ("127.0.0.1", "0.0.0.0")] - print(" {0:40}{1:18}{2}".format(sensor.hostname, ipaddrs[0], sensor.last_checkin_time)) + print(" {0:40}{1:18}{2}".format(sensor.hostname, + ipaddrs[0] if len(ipaddrs) else '', + sensor.last_checkin_time)) def add_sensor_group(cb, parser, args): @@ -48,6 +50,8 @@ def add_sensor_group(cb, parser, args): g.name = args.new_group_name g.site = site + g.sensorbackend_server = args.sensorbackend_server + g.sensor_version_windows = g.sensor_version_linux = g.sensor_version_osx = 'Manual' try: g.save() @@ -76,8 +80,8 @@ def delete_sensor_group(cb, parser, args): num_matching_sensor_groups = len(groups) if num_matching_sensor_groups > 1 and not args.force: - print("{0:d} sensor groups match {1:s} and --force not specified. No action taken.".format(num_matching_sensor_groups, - attempted_to_find)) + print("{0:d} sensor groups match {1:s} and --force not specified. No action taken." + .format(num_matching_sensor_groups, attempted_to_find)) return for g in groups: @@ -93,7 +97,7 @@ def main(): parser = build_cli_parser() commands = parser.add_subparsers(help="Sensor Group commands", dest="command_name") - list_command = commands.add_parser("list", help="List all configured sensor groups") + commands.add_parser("list", help="List all configured sensor groups") add_command = commands.add_parser("add", help="Add new sensor group") add_command.add_argument("-n", "--name", action="store", help="Sensor group name", required=True, @@ -101,13 +105,17 @@ def main(): site_group = add_command.add_mutually_exclusive_group(required=False) site_group.add_argument("-s", "--site", action="store", help="Site name", dest="site_name") site_group.add_argument("-i", "--site-id", action="store", help="Site ID", dest="site_id") + add_command.add_argument("-b", "--sensorbackend-server", action="store", help="Sensor backend server", required=True, + dest="sensorbackend_server") del_command = commands.add_parser("delete", help="Delete sensor groups") del_sensor_group_specifier = del_command.add_mutually_exclusive_group(required=True) del_sensor_group_specifier.add_argument("-i", "--id", type=int, help="ID of sensor group to delete") - del_sensor_group_specifier.add_argument("-n", "--name", help="Name of sensor group to delete. Specify --force to delete" - " multiple sensor groups that have the same name", dest="groupname") - del_command.add_argument("--force", help="If NAME matches multiple sensor groups, delete all matching sensor groups", + del_sensor_group_specifier.add_argument("-n", "--name", + help="Name of sensor group to delete. Specify --force to delete" + " multiple sensor groups that have the same name", dest="groupname") + del_command.add_argument("--force", + help="If NAME matches multiple sensor groups, delete all matching sensor groups", action="store_true", default=False) list_sensors_command = commands.add_parser("list-sensors", help="List all sensors in a sensor group") diff --git a/examples/response/sensor_operations.py b/examples/response/sensor_operations.py index d48f53ae..55a11111 100755 --- a/examples/response/sensor_operations.py +++ b/examples/response/sensor_operations.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -#ZE 2018 AD +# ZE 2018 AD import sys from cbapi.response.models import Alert @@ -11,9 +11,10 @@ ''' This is a utility designed to use watchlists to perform operations on affected sensors: -supported operations: memory dump, isolation and process termination. +supported operations: memory dump, isolation and process termination. ''' + def sensor_operations(cb, watchlists, operation, dryrun=False): print("Trying to {0} based on watchlists: {1}".format(operation, watchlists)) where_clause = " or ".join(("watchlist_name:" + wl for wl in watchlists.split(","))) @@ -34,7 +35,7 @@ def sensor_operations(cb, watchlists, operation, dryrun=False): lr.close() else: print("DRYRUN: would have {0} sensor {1}".format(sensor, operation)) - except Exception as e: + except Exception: print(traceback.format_exc(0)) print("Sensor operations finished") diff --git a/examples/response/tf.py b/examples/response/tf.py index 7b5f1a05..e7b0b565 100755 --- a/examples/response/tf.py +++ b/examples/response/tf.py @@ -4,7 +4,7 @@ import sys from cbapi.response.models import Process from cbapi.example_helpers import build_cli_parser, get_cb_response_object -from cbapi.errors import ServerError, ObjectNotFoundError +from cbapi.errors import ObjectNotFoundError import logging log = logging.getLogger(__name__) @@ -68,13 +68,13 @@ def process_hit(cb, parent_proc_name, proc_name, value, ratio, child_behavior): def main(): parser = build_cli_parser("Term Frequency Analysis") parser.add_argument("-p", "--percentage", action="store", default="2", dest="percentless", - help="Max Percentage of Term Frequency e.g., 2 ") + help="Max Percentage of Term Frequency e.g., 2 ") process_selection = parser.add_mutually_exclusive_group(required=True) process_selection.add_argument("-t", "--term", action="store", default=None, dest="procname", - help="Comma separated list of parent processes to get term frequency") + help="Comma separated list of parent processes to get term frequency") process_selection.add_argument("-f", "--termfile", action="store", default=None, dest="procnamefile", - help="Text file new line separated list of parent processes to get term frequency") + help="Text file new line separated list of parent processes to get term frequency") output_selection = parser.add_mutually_exclusive_group(required=False) output_selection.add_argument("--count", action="store_true", diff --git a/examples/response/user_operations.py b/examples/response/user_operations.py index e2dedccb..0231da3f 100755 --- a/examples/response/user_operations.py +++ b/examples/response/user_operations.py @@ -3,7 +3,7 @@ import sys from cbapi.response.models import User, Team, SensorGroup -from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id +from cbapi.example_helpers import build_cli_parser, get_cb_response_object from cbapi.errors import ServerError import logging import getpass @@ -118,8 +118,8 @@ def main(): parser = build_cli_parser() commands = parser.add_subparsers(help="User commands", dest="command_name") - list_command = commands.add_parser("list", help="List all configured users") - list_teams_command = commands.add_parser("list-teams", help="List all configured user teams") + commands.add_parser("list", help="List all configured users") + commands.add_parser("list-teams", help="List all configured user teams") add_command = commands.add_parser("add", help="Add new user") add_command.add_argument("-u", "--username", help="New user's username", required=True) diff --git a/examples/response/walk_children.py b/examples/response/walk_children.py index 54d9771a..a59263bb 100755 --- a/examples/response/walk_children.py +++ b/examples/response/walk_children.py @@ -30,7 +30,7 @@ def main(): if args.process: try: procs = [c.select(Process, args.process, max_children=args.children, force_init=True)] - except ObjectNotFoundError as e: + except ObjectNotFoundError: print("Could not find process {0:s}".format(args.procss)) return 1 except ApiError as e: @@ -53,7 +53,7 @@ def main(): duration = str(root_proc.end - root_proc.start) print("Process {0:s} on {1:s} executed by {2:s}:".format(root_proc.cmdline, root_proc.hostname, - root_proc.username)) + root_proc.username)) print("started at {0} ({1})".format(str(root_proc.start), duration)) print("Cb Response console link: {0}".format(root_proc.webui_link)) root_proc.walk_children(visitor) diff --git a/examples/response/watchlist_exporter.py b/examples/response/watchlist_exporter.py index f7ca3675..728ed015 100755 --- a/examples/response/watchlist_exporter.py +++ b/examples/response/watchlist_exporter.py @@ -4,8 +4,7 @@ import sys import cbapi.six as six from cbapi.response.models import Watchlist -from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id -from cbapi.errors import ServerError +from cbapi.example_helpers import build_cli_parser, get_cb_response_object import logging from datetime import datetime import json @@ -16,7 +15,7 @@ if six.PY3: confirm_input = input else: - confirm_input = raw_input + confirm_input = raw_input # noqa: F821 def confirm(watch_list): @@ -51,13 +50,18 @@ def export_watchlists(cb, args): if not confirm(watchlist.name): continue + watchlist_actions = cb.get_object('/api/v1/watchlist/{0}/action_type'.format(watchlist.id)) + exported_watchlists.append( { + "Created": str(watchlist.date_added), "Name": watchlist.name, "URL": watchlist.search_query, "Type": watchlist.index_type, "SearchString": watchlist.query, - "Description": "Please fill in if you intend to share this." + "Description": watchlist.description, + "Actions": watchlist_actions, + "Enabled": watchlist.enabled } ) diff --git a/examples/response/watchlist_importer.py b/examples/response/watchlist_importer.py index 6fa7e75a..e5cd5514 100755 --- a/examples/response/watchlist_importer.py +++ b/examples/response/watchlist_importer.py @@ -4,10 +4,8 @@ import sys import cbapi.six as six from cbapi.response.models import Watchlist -from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id -from cbapi.errors import ServerError +from cbapi.example_helpers import build_cli_parser, get_cb_response_object import logging -from datetime import datetime import json log = logging.getLogger(__name__) @@ -16,7 +14,7 @@ if six.PY3: confirm_input = input else: - confirm_input = raw_input + confirm_input = raw_input # noqa: F821 def confirm(watch_list): diff --git a/examples/response/watchlist_operations.py b/examples/response/watchlist_operations.py index 655e2647..b6e69989 100755 --- a/examples/response/watchlist_operations.py +++ b/examples/response/watchlist_operations.py @@ -50,7 +50,7 @@ def delete_watchlist(cb, parser, args): num_matching_watchlists = len(watchlists) if num_matching_watchlists > 1 and not args.force: print("{0:d} watchlists match {1:s} and --force not specified. No action taken.".format(num_matching_watchlists, - attempted_to_find)) + attempted_to_find)) return for f in watchlists: @@ -78,7 +78,7 @@ def main(): parser = build_cli_parser() commands = parser.add_subparsers(help="Watchlist commands", dest="command_name") - list_command = commands.add_parser("list", help="List all configured watchlists") + commands.add_parser("list", help="List all configured watchlists") list_actions_command = commands.add_parser("list-actions", help="List actions associated with a watchlist") list_actions_specifier = list_actions_command.add_mutually_exclusive_group(required=True) diff --git a/examples/threathunter/create_feed.py b/examples/threathunter/create_feed.py deleted file mode 100644 index 2c143e80..00000000 --- a/examples/threathunter/create_feed.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python - -import sys -import time - -from cbapi.example_helpers import read_iocs, build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Feed - - -def main(): - parser = build_cli_parser("Create a CbTH feed and report from a stream of IOCs") - - # Feed metadata arguments. - parser.add_argument("--name", type=str, help="Feed name", required=True) - parser.add_argument("--owner", type=str, help="Feed owner", required=True) - parser.add_argument("--url", type=str, help="Feed provider url", required=True) - parser.add_argument("--summary", type=str, help="Feed summary", required=True) - parser.add_argument("--category", type=str, help="Feed category", required=True) - parser.add_argument("--source_label", type=str, help="Feed source label", required=True) - parser.add_argument("--access", type=str, help="Feed access scope", default="private") - - # Report metadata arguments. - parser.add_argument("--rep_timestamp", type=int, help="Report timestamp", default=int(time.time())) - parser.add_argument("--rep_title", type=str, help="Report title", required=True) - parser.add_argument("--rep_desc", type=str, help="Report description", required=True) - parser.add_argument("--rep_severity", type=int, help="Report severity", default=1) - parser.add_argument("--rep_link", type=str, help="Report link") - parser.add_argument("--rep_tags", type=str, help="Report tags, comma separated") - parser.add_argument("--rep_visibility", type=str, help="Report visibility") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - feed_info = { - "name": args.name, - "owner": args.owner, - "provider_url": args.url, - "summary": args.summary, - "category": args.category, - "access": args.access, - } - - rep_tags = [] - if args.rep_tags: - rep_tags = args.rep_tags.split(",") - - report = { - "timestamp": args.rep_timestamp, - "title": args.rep_title, - "description": args.rep_desc, - "severity": args.rep_severity, - "link": args.rep_link, - "tags": rep_tags, - "iocs_v2": [], # NOTE(ww): The feed server will convert IOCs to v2s for us. - } - - report_id, iocs = read_iocs(cb) - - report["id"] = report_id - report["iocs"] = iocs - - feed = { - "feedinfo": feed_info, - "reports": [report] - } - - feed = cb.create(Feed, feed) - feed.save() - - print(feed) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/threathunter/events.py b/examples/threathunter/events.py deleted file mode 100644 index d0fe23d0..00000000 --- a/examples/threathunter/events.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Event - - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-p", type=str, help="process guid", default=None) - parser.add_argument("-n", type=int, help="only output N events", default=None) - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if not args.p: - print("Error: Missing Process GUID to search for events with") - sys.exit(1) - - events = cb.select(Event).where(process_guid=args.p) - - if args.n: - events = events[0:args.n] - - for event in events: - print("Event type: {}".format(event.event_type)) - print("\tEvent GUID: {}".format(event.event_guid)) - print("\tEvent Timestamp: {}".format(event.event_timestamp)) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/threathunter/events_exporter.py b/examples/threathunter/events_exporter.py deleted file mode 100644 index f9fe8d59..00000000 --- a/examples/threathunter/events_exporter.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Event -import json -import csv - - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-p", type=str, help="process guid", default=None) - parser.add_argument("-s",type=bool, help="silent mode",default=False) - parser.add_argument("-n", type=int, help="only output N events", default=None) - parser.add_argument("-f", type=str, help="output file name",default=None) - parser.add_argument("-of", type=str,help="output file format: csv or json",default="json") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if not args.p: - print("Error: Missing Process GUID to search for events with") - sys.exit(1) - - events = cb.select(Event).where(process_guid=args.p) - - if args.n: - events = events[0:args.n] - - if not args.s: - for event in events: - print("Event type: {}".format(event.event_type)) - print("\tEvent GUID: {}".format(event.event_guid)) - print("\tEvent Timestamp: {}".format(event.event_timestamp)) - - if args.f is not None: - if args.of == "json": - with open(args.f, 'w') as outfile: - for event in events: - json.dump(event.original_document, outfile) - else: - with open(args.f, 'w') as outfile: - csvwriter = csv.writer(outfile) - for event in events: - csvwriter.writerows(event) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/threathunter/feed_operations.py b/examples/threathunter/feed_operations.py deleted file mode 100644 index 9ab4469c..00000000 --- a/examples/threathunter/feed_operations.py +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python -# - -import sys -from cbapi.psc.threathunter.models import Feed, Report -from cbapi.example_helpers import eprint, build_cli_parser, get_cb_threathunter_object -import logging -import json - -log = logging.getLogger(__name__) - - -def get_feed(cb, feed_id=None, feed_name=None): - if feed_id: - return cb.select(Feed, feed_id) - elif feed_name: - feeds = [feed for feed in cb.select(Feed) if feed.name == feed_name] - - if not feeds: - eprint("No feeds named '{}'".format(feed_name)) - sys.exit(1) - elif len(feeds) > 1: - eprint("More than one feed named '{}'".format(feed_name)) - sys.exit(1) - - return feeds[0] - else: - raise ValueError("expected either feed_id or feed_name") - - -def get_report(feed, report_id=None, report_name=None): - if report_id: - reports = [report for report in feed.reports if report.id == report_id] - - if not reports: - eprint("No reports with ID '{}'".format(report_id)) - sys.exit(1) - elif len(reports) > 1: - eprint("More than one report with ID '{}'".format(report_id)) - sys.exit(1) - elif report_name: - reports = [report for report in feed.reports if report.title == report_name] - - if not reports: - eprint("No reports named '{}'".format(report_name)) - sys.exit(1) - elif len(reports) > 1: - eprint("More than one report named '{}'".format(report_name)) - sys.exit(1) - else: - raise ValueError("expected either report_id or report_name") - - return reports[0] - - -def list_feeds(cb, parser, args): - if args.iocs and not args.reports: - eprint("--iocs specified without --reports") - sys.exit(1) - - feeds = cb.select(Feed).where(include_public=args.public) - - for feed in feeds: - print(feed) - if args.reports: - for report in feed.reports: - print(report) - if args.iocs: - for ioc in report.iocs_: - print(ioc) - - -def list_iocs(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - - for report in feed.reports: - for ioc in report.iocs_: - print(ioc) - - -def export_feed(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - - exported = {} - - exported["feedinfo"] = feed._info - exported["reports"] = [report._info for report in feed.reports] - print(json.dumps(exported)) - - -def import_feed(cb, parser, args): - imported = json.loads(sys.stdin.read()) - - if args.feedname: - imported["feedinfo"]["name"] = args.feedname - - feed = cb.create(Feed, imported) - feed.save(public=args.public) - - -def delete_feed(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - feed.delete() - - -def export_report(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - report = get_report(feed, report_id=args.reportid, report_name=args.reportname) - - print(json.dumps(report._info)) - - -def import_report(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - - imp_dict = json.loads(sys.stdin.read()) - - reports = feed.reports - existing_report = next( - (report for report in reports if imp_dict["id"] == report.id), None - ) - - if existing_report: - eprint("Report already exists; use replace-report.") - sys.exit(1) - else: - imp_report = cb.create(Report, imp_dict) - feed.append_reports([imp_report]) - - -def delete_report(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - report = get_report(feed, report_id=args.reportid, report_name=args.reportname) - report.delete() - - -def replace_report(cb, parser, args): - feed = get_feed(cb, feed_id=args.id, feed_name=args.feedname) - - imported = json.loads(sys.stdin.read()) - - reports = feed.reports - existing_report = next( - (report for report in reports if imported["id"] == report.id), None - ) - - if existing_report: - existing_report.update(**imported) - else: - eprint("No existing report to replace") - sys.exit(1) - - -def main(): - parser = build_cli_parser() - commands = parser.add_subparsers(help="Feed commands", dest="command_name") - - list_command = commands.add_parser("list", help="List all configured feeds") - list_command.add_argument( - "-P", - "--public", - help="Include public feeds", - action="store_true", - default=False, - ) - list_command.add_argument( - "-r", - "--reports", - help="Include reports for each feed", - action="store_true", - default=False, - ) - list_command.add_argument( - "-i", - "--iocs", - help="Include IOCs for each feed's reports", - action="store_true", - default=False, - ) - - list_iocs_command = commands.add_parser( - "list-iocs", help="List all IOCs for a feed" - ) - specifier = list_iocs_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - - export_command = commands.add_parser( - "export", help="Export a feed into an importable format" - ) - specifier = export_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - - import_command = commands.add_parser( - "import", help="Import a previously exported feed" - ) - import_command.add_argument( - "-f", "--feedname", type=str, help="Renames the imported feed" - ) - import_command.add_argument( - "-P", "--public", help="Make the feed public", action="store_true" - ) - - del_command = commands.add_parser("delete", help="Delete feed") - specifier = del_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - - export_report_command = commands.add_parser( - "export-report", help="Export a feed's report into an importable format" - ) - specifier = export_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - specifier = export_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-I", "--reportid", type=str, help="Report ID") - specifier.add_argument("-r", "--reportname", type=str, help="Report Name") - - import_report_command = commands.add_parser( - "import-report", help="Import a previously exported report" - ) - specifier = import_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - - delete_report_command = commands.add_parser( - "delete-report", help="Delete a report from a feed" - ) - specifier = delete_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - specifier = delete_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-I", "--reportid", type=str, help="Report ID") - specifier.add_argument("-r", "--reportname", type=str, help="Report Name") - - replace_report_command = commands.add_parser( - "replace-report", help="Replace a feed's report" - ) - specifier = replace_report_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--id", type=str, help="Feed ID") - specifier.add_argument("-f", "--feedname", type=str, help="Feed Name") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if args.command_name == "list": - return list_feeds(cb, parser, args) - elif args.command_name == "list-iocs": - return list_iocs(cb, parser, args) - elif args.command_name == "export": - return export_feed(cb, parser, args) - elif args.command_name == "import": - return import_feed(cb, parser, args) - elif args.command_name == "delete": - return delete_feed(cb, parser, args) - elif args.command_name == "export-report": - return export_report(cb, parser, args) - elif args.command_name == "import-report": - return import_report(cb, parser, args) - elif args.command_name == "delete-report": - return delete_report(cb, parser, args) - elif args.command_name == "replace-report": - return replace_report(cb, parser, args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/threathunter/import_response_feeds.py b/examples/threathunter/import_response_feeds.py deleted file mode 100644 index 3bd48279..00000000 --- a/examples/threathunter/import_response_feeds.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -# -import sys -from cbapi.psc.threathunter import CbThreatHunterAPI -from cbapi.psc.threathunter.models import Feed as FeedTH -from cbapi.response.models import Feed, ThreatReport -from cbapi.example_helpers import build_cli_parser, get_cb_response_object, get_object_by_name_or_id -from cbapi.errors import ServerError -from urllib.parse import unquote -import logging - -log = logging.getLogger(__name__) - -""" -Lists the feeds in CB Response -""" -def list_feeds(cb, parser, args): - for f in cb.select(Feed): - for fieldname in ["id", "category", "display_name", "enabled", "provider_url", "summary", "tech_data", - "feed_url", "use_proxy", "validate_server_cert"]: - print("%-20s: %s" % (fieldname, getattr(f, fieldname, ""))) - - if f.username: - for fieldname in ["username", "password"]: - print("%-20s: %s" % (fieldname, getattr(f, fieldname, ""))) - - if f.ssl_client_crt: - for fieldname in ["ssl_client_crt", "ssl_client_key"]: - print("%-20s: %s" % (fieldname, getattr(f, fieldname, ""))) - - print("\n") - -""" -Lists the reports in a feed from CB Response -:param: id - The ID of a feed -""" -def list_reports(cb, parser, args): - feed = cb.select(Feed, args.id, force_init=True) - for report in feed.reports: - print (report) - print("\n") - -""" -Converts and copies a feed from CB Response to CB Threat Hunter -:param: id - The ID of a feed from CB Response - -Requires a credentials profile for both CB Response and CB Threat Hunter - Ensure that your credentials for CB Threat Hunter have permissions to the Feed Manager APIs -""" -def convert_feed(cb, cb_th, parser, args): - th_feed = { "feedinfo": {}, "reports": []} - # Fetches the CB Response feed - feed = cb.select(Feed, args.id, force_init=True) - - th_feed["feedinfo"]["name"] = feed.name - th_feed["feedinfo"]["provider_url"] = feed.provider_url - th_feed["feedinfo"]["summary"] = feed.summary - th_feed["feedinfo"]["category"] = feed.category - th_feed["feedinfo"]["access"] = "private" - - # Temporary values until refresh - th_feed["feedinfo"]["owner"] = "org_key" - th_feed["feedinfo"]["id"] = "id" - - # Iterates the reports in the CB Response feed - for report in feed.reports: - th_report = {} - th_report["id"] = report.id - th_report["timestamp"] = report.timestamp - th_report["title"] = report.title - th_report["severity"] = (report.score % 10) + 1 - if hasattr(report, "description"): - th_report["description"] = report.description - else: - th_report["description"] = "" - if hasattr(report, "link"): - th_report["link"] = report.link - th_report["iocs"] = {} - if report.iocs: - if "md5" in report.iocs: - th_report["iocs"]["md5"] = report.iocs["md5"] - if "ipv4" in report.iocs: - th_report["iocs"]["ipv4"] = report.iocs["ipv4"] - if "ipv6" in report.iocs: - th_report["iocs"]["ipv6"] = report.iocs["ipv6"] - if "dns" in report.iocs: - th_report["iocs"]["dns"] = report.iocs["dns"] - - if "query" in report.iocs: - th_report["iocs"]["query"] = [] - for query in report.iocs.get("query", []): - try: - search = query.get('search_query', "") - if "q=" in search: - params = search.split('&') - for p in params: - if "q=" in p: - search = unquote(p[2:]) - # Converts the CB Response query to CB Threat Hunter - th_query = cb_th.convert_query(search) - if th_query: - query["search_query"] = th_query - th_report["iocs"]["query"].append(query) - except ServerError as e: - print ('Invalid query {}'.format(query.get('search_query', ""))) - - th_feed["reports"].append(th_report) - - # Pushes the new feed to CB Threat Hunter - new_feed = cb_th.create(FeedTH, th_feed) - new_feed.save() - print ("{}\n".format(new_feed)) - -def main(): - parser = build_cli_parser() - parser.add_argument("-thp", "--threatprofile", help="Threat Hunter profile", default="default") - commands = parser.add_subparsers(help="Feed commands", dest="command_name") - - list_command = commands.add_parser("list", help="List all configured feeds") - - list_reports_command = commands.add_parser("list-reports", help="List all configured reports for a feed") - list_reports_command.add_argument("-i", "--id", type=str, help="Feed ID") - - convert_feed_command = commands.add_parser("convert", help="Convert feed from CB Response to CB Threat Hunter") - convert_feed_command.add_argument("-i", "--id", type=str, help="Feed ID") - - args = parser.parse_args() - cb = get_cb_response_object(args) - cb_th = CbThreatHunterAPI(profile=args.threatprofile) - - if args.command_name == "list": - return list_feeds(cb, parser, args) - if args.command_name == "list-reports": - return list_reports(cb, parser, args) - if args.command_name == "convert": - return convert_feed(cb, cb_th, parser, args) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/threathunter/modify_feed.py b/examples/threathunter/modify_feed.py deleted file mode 100644 index 8f543b8a..00000000 --- a/examples/threathunter/modify_feed.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_feed_object - - -def main(): - parser = build_cli_parser("Modify a CbTH feed") - parser.add_argument("--id", type=str, help="Feed ID", default=None) - parser.add_argument("--name", type=str, help="Feed name", default=None) - parser.add_argument("--owner", type=str, help="Feed owner", default=None) - parser.add_argument("--url", type=str, help="Feed provider url", default="https://example.com") - parser.add_argument("--summary", type=str, help="Feed summary", default=None) - parser.add_argument("--category", type=str, help="Feed category", default="Partner") - parser.add_argument("--access", type=str, help="Feed access scope", default="private") - - args = parser.parse_args() - cb = get_cb_threathunter_feed_object(args) - - feed = cb.feed(args.id) - - print("Before modification:") - print("=" * 80) - print(feed) - print("=" * 80) - - metadata = {} - if args.name: - metadata["name"] = args.name - if args.owner: - metadata["owner"] = args.owner - if args.url: - metadata["provider_url"] = args.url - if args.summary: - metadata["summary"] = args.summary - if args.category: - metadata["category"] = args.category - if args.access: - metadata["access"] = args.access - - feed.feedinfo.update(**metadata) - - print("After modification:") - print("=" * 80) - print(feed) - print("=" * 80) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/threathunter/process_exporter.py b/examples/threathunter/process_exporter.py deleted file mode 100644 index 562c83c8..00000000 --- a/examples/threathunter/process_exporter.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Process -import json -import csv - - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-p", type=str, help="process guid", default=None) - parser.add_argument("-q",type=str,help="query string",default=None) - parser.add_argument("-s",type=bool, help="silent mode",default=False) - parser.add_argument("-n", type=int, help="only output N events", default=None) - parser.add_argument("-f", type=str, help="output file name",default=None) - parser.add_argument("-of", type=str,help="output file format: csv or json",default="json") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if not args.p and not args.q: - print("Error: Missing Process GUID to search for events with") - sys.exit(1) - - if args.q: - processes = cb.select(Process).where(args.q) - else: - processes = cb.select(Process).where(process_guid=args.p) - - if args.n: - processes = [ p for p in processes[0:args.n]] - - if not args.s: - for process in processes: - print("Process: {}".format(process.process_name)) - print("\tPIDs: {}".format(process.process_pids)) - print("\tSHA256: {}".format(process.process_sha256)) - print("\tGUID: {}".format(process.process_guid)) - - if args.f is not None: - if args.of == "json": - with open(args.f, 'w') as outfile: - for p in processes: - json.dump(p.original_document, outfile) - else: - with open(args.f, 'w') as outfile: - csvwriter = csv.writer(outfile) - for p in processes: - csvwriter.writerow(p.original_document) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/threathunter/process_query.py b/examples/threathunter/process_query.py deleted file mode 100644 index d1c2845a..00000000 --- a/examples/threathunter/process_query.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Process, Binary -from cbapi.errors import ObjectNotFoundError - - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-q", type=str, help="process query", default="process_name:notepad.exe") - parser.add_argument("-n", type=int, help="only output N processes", default=None) - parser.add_argument("-b", action="store_true", help="show binary information", default=False) - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - processes = cb.select(Process).where(args.q) - - if args.n: - processes = processes[0:args.n] - - for process in processes: - print("Process: {}".format(process.process_name)) - print("\tPIDs: {}".format(process.process_pids)) - print("\tSHA256: {}".format(process.process_sha256)) - print("\tGUID: {}".format(process.process_guid)) - - if args.b: - try: - binary = cb.select(Binary, process.process_sha256) - print(binary) - print(binary.summary) - except ObjectNotFoundError: - print("No binary found for process with hash: {}".format(process.process_sha256)) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/threathunter/process_tree.py b/examples/threathunter/process_tree.py deleted file mode 100644 index c6e1cb3d..00000000 --- a/examples/threathunter/process_tree.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Process - - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-p", type=str, help="process guid", default=None) - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if not args.p: - print("Error: Missing Process GUID to query the process tree with") - sys.exit(1) - - tree = cb.select(Process).where(process_guid=args.p)[0].tree() - for idx, child in enumerate(tree.children): - print("Child #{}".format(idx)) - print("\tName: {}".format(child.process_name)) - print("\tGUID: {}".format(child.process_guid)) - print("\tNumber of children: {}".format(len(child.children))) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/threathunter/process_tree_exporter.py b/examples/threathunter/process_tree_exporter.py deleted file mode 100644 index 63321b0a..00000000 --- a/examples/threathunter/process_tree_exporter.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Process -import csv -import json - -def main(): - parser = build_cli_parser("Query processes") - parser.add_argument("-p", type=str, help="process guid", default=None) - parser.add_argument("-f", type=str, help="output file name", default=None) - parser.add_argument("-of", type=str, help="output file format: csv or json", default="json") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if not args.p: - print("Error: Missing Process GUID to query the process tree with") - sys.exit(1) - - tree = cb.select(Process).where(process_guid=args.p)[0].tree() - - for idx, child in enumerate(tree.children): - print("Child #{}".format(idx)) - print("\tName: {}".format(child.process_name)) - print("\tNumber of children: {}".format(len(child.children))) - - if args.f is not None: - if args.of == "json": - with open(args.f, 'w') as outfile: - for idx, child in enumerate(tree.children): - json.dump(child.original_document, outfile) - else: - with open(args.f, 'w') as outfile: - csvwriter = csv.writer(outfile) - for idx,child in enumerate(tree.children): - csvwriter.writerow(child.original_document) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/threathunter/search.py b/examples/threathunter/search.py deleted file mode 100644 index 067c045b..00000000 --- a/examples/threathunter/search.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python - -import sys - -from cbapi.example_helpers import build_cli_parser, get_cb_threathunter_object -from cbapi.psc.threathunter import Process, Event, Tree -from solrq import Range, Value - - -def main(): - parser = build_cli_parser("Search processes") - parser.add_argument("-q", type=str, help="process query", default="process_name:notepad.exe") - parser.add_argument("-f", help="show full objects", action="store_true", default=False) - parser.add_argument("-n", type=int, help="only output N processes", default=None) - parser.add_argument("-e", help="show events for query results", action="store_true", default=False) - parser.add_argument("-c", help="show children for query results", action="store_true", default=False) - parser.add_argument("-p", help="show parents for query results", action="store_true", default=False) - parser.add_argument("-t", help="show tree for query results", action="store_true", default=False) - parser.add_argument("-S", type=str, help="sory by this field", required=False) - parser.add_argument("-D", help="return results in descending order", action="store_true") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - processes = cb.select(Process).where(args.q) - - direction = "ASC" - if args.D: - direction = "DESC" - - if args.S: - processes.sort_by(args.S, direction=direction) - - print("Number of processes: {}".format(len(processes))) - - if args.n: - processes = processes[0:args.n] - - for process in processes: - if args.f: - print(process) - else: - print("{} ({}): {}".format(process.process_name, process.process_guid, process.process_sha256)) - - if args.e: - print("=========== events ===========") - for event in process.events(): - if args.f: - print(event) - else: - print("\t{}".format(event.event_type)) - - if args.c: - print("========== children ==========") - for child in process.children: - if args.f: - print(child) - else: - print("\t{}: {}".format(child.process_name, child.process_sha256)) - - if args.p: - print("========== parents ==========") - for parent in process.parents: - if args.f: - print(parent) - else: - print("\t{}: {}".format(parent.process_name, parent.process_sha256)) - - - if args.t: - print("=========== tree =============") - tree = process.tree() - print(tree) - print(tree.nodes) - - print("===========================") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/threathunter/watchlist_operations.py b/examples/threathunter/watchlist_operations.py deleted file mode 100644 index 0fe95149..00000000 --- a/examples/threathunter/watchlist_operations.py +++ /dev/null @@ -1,324 +0,0 @@ -#!/usr/bin/env python -# - -import sys -from cbapi.psc.threathunter.models import Watchlist, Report, Feed -from cbapi.example_helpers import eprint, read_iocs, build_cli_parser, get_cb_threathunter_object -from cbapi.errors import ObjectNotFoundError -import logging -import json -import time -import hashlib - -log = logging.getLogger(__name__) - - -def get_watchlist(cb, watchlist_id=None, watchlist_name=None): - if watchlist_id: - return cb.select(Watchlist, watchlist_id) - elif watchlist_name: - feeds = [feed for feed in cb.select(Watchlist) if feed.name == watchlist_name] - - if not feeds: - eprint("No watchlist named {}".format(watchlist_name)) - sys.exit(1) - elif len(feeds) > 1: - eprint("More than one feed named {}, not continuing".format(watchlist_name)) - sys.exit(1) - - return feeds[0] - else: - raise ValueError("expected either watchlist_id or watchlist_name") - - -def get_report(watchlist, report_id=None, report_name=None): - if report_id: - reports = [report for report in watchlist.reports if report.id == report_id] - elif report_name: - reports = [report for report in watchlist.reports if report.title == report_name] - else: - raise ValueError("expected either report_id or report_name") - - if not reports: - eprint("No matching reports found.") - sys.exit(1) - if len(reports) > 1: - eprint("More than one matching report found.") - sys.exit(1) - - return reports[0] - - -def get_report_feed(watchlist, report_id=None, report_name=None): - reports = watchlist.feed.reports - - if report_id: - reports = [report for report in reports if report.id == report_id] - elif report_name: - reports = [report for report in reports if report.title == report_name] - else: - raise ValueError("expected either report_id or report_name") - - if not reports: - eprint("No matching reports found.") - sys.exit(1) - if len(reports) > 1: - eprint("More than one matching report found.") - sys.exit(1) - - return reports[0] - - -def list_watchlists(cb, parser, args): - watchlists = cb.select(Watchlist) - - for watchlist in watchlists: - print(watchlist) - if args.reports: - for report in watchlist.reports: - print(report) - if watchlist.feed: - for report in watchlist.feed.reports: - print(report) - - -def subscribe_watchlist(cb, parser, args): - try: - cb.select(Feed, args.feed_id) - except ObjectNotFoundError: - eprint("Nonexistent or private feed: {}".format(args.feed_id)) - sys.exit(1) - - classifier = { - "key": "feed_id", - "value": args.feed_id, - } - - watchlist_dict = { - "name": args.watchlist_name, - "description": args.description, - "tags_enabled": args.tags, - "alerts_enabled": args.alerts, - "create_timestamp": args.timestamp, - "last_update_timestamp": args.last_update, - "report_ids": [], - "classifier": classifier, - } - - watchlist = cb.create(Watchlist, watchlist_dict) - watchlist.save() - - -def create_watchlist(cb, parser, args): - watchlist_dict = { - "name": args.watchlist_name, - "description": args.description, - "tags_enabled": args.tags, - "alerts_enabled": args.alerts, - "create_timestamp": args.timestamp, - "last_update_timestamp": args.last_update, - "report_ids": [], - "classifier": None, - } - - rep_tags = [] - if args.rep_tags: - rep_tags = args.rep_tags.split(",") - - report_dict = { - "timestamp": args.rep_timestamp, - "title": args.rep_title, - "description": args.rep_desc, - "severity": args.rep_severity, - "link": args.rep_link, - "tags": rep_tags, - "iocs_v2": [], # NOTE(ww): The feed server will convert IOCs to v2s for us. - } - - report_id, iocs = read_iocs(cb) - - report_dict["id"] = report_id - report_dict["iocs"] = iocs - - report = cb.create(Report, report_dict) - report.save_watchlist() - - watchlist_dict["report_ids"].append(report.id) - watchlist = cb.create(Watchlist, watchlist_dict) - watchlist.save() - - -def delete_watchlist(cb, parser, args): - watchlist = get_watchlist(cb, watchlist_id=args.watchlist_id, watchlist_name=args.watchlist_name) - - if args.reports: - [report.delete() for report in watchlist.reports] - - watchlist.delete() - - -def alter_report(cb, parser, args): - watchlist = get_watchlist(cb, watchlist_id=args.watchlist_id) - - if watchlist.reports: - report = get_report(watchlist, report_id=args.report_id) - else: - report = get_report_feed(watchlist, report_id=args.report_id) - - if args.severity: - if watchlist.reports: - report.update(severity=args.severity) - else: - report.custom_severity = args.severity - - if args.activate: - report.unignore() - elif args.deactivate: - report.ignore() - - -def alter_ioc(cb, parser, args): - watchlist = get_watchlist(cb, watchlist_id=args.watchlist_id) - report = get_report(watchlist, report_id=args.report_id) - - iocs = [ioc for ioc in report.iocs_ if ioc.id == args.ioc_id] - - if not iocs: - eprint("No IOC with ID {} found.".format(args.ioc_id)) - sys.exit(1) - elif len(iocs) > 1: - eprint("More than one IOC with ID {} found.".format(args.ioc_id)) - sys.exit(1) - - if args.activate: - iocs[0].unignore() - elif args.deactivate: - iocs[0].ignore() - - -def export_watchlist(cb, parser, args): - watchlist = get_watchlist(cb, watchlist_id=args.watchlist_id, watchlist_name=args.watchlist_name) - exported = { - 'watchlist': watchlist._info, - } - - exported['reports'] = [report._info for report in watchlist.reports] - - print(json.dumps(exported)) - - -def import_watchlist(cb, parser, args): - imported = json.loads(sys.stdin.read()) - - # clear any report IDs, since we'll regenerate them - imported["watchlist"]["report_ids"].clear() - - watchlist = cb.create(Watchlist, imported['watchlist']) - watchlist.save() - - # import each report and extract its new ID - report_ids = [] - for rep_dict in imported["reports"]: - - # NOTE(ww): Previous versions of the CbTH Watchlist API weren't - # generating IOC IDs on the server side. If they don't show up - # in our import, generate them manually. - for ioc in rep_dict["iocs_v2"]: - if not ioc["id"]: - ioc_id = hashlib.md5() - ioc_id.update(str(time.time()).encode("utf-8")) - [ioc_id.update(value.encode("utf-8")) for value in ioc["values"]] - ioc["id"] = ioc_id.hexdigest() - report = cb.create(Report, rep_dict) - report.save_watchlist() - report_ids.append(report.id) - - # finally, update our new watchlist with the imported reports - if report_ids: - watchlist.update(report_ids=report_ids) - - -def main(): - parser = build_cli_parser() - commands = parser.add_subparsers(help="Feed commands", dest="command_name") - - list_command = commands.add_parser("list", help="List all configured watchlists") - list_command.add_argument("-r", "--reports", action="store_true", help="List reports for each watchlist", default=False) - - subscribe_command = commands.add_parser("subscribe", help="Create a watchlist with a feed") - subscribe_command.add_argument("-i", "--feed_id", type=str, help="The Feed ID", required=True) - subscribe_command.add_argument("-w", "--watchlist_name", type=str, help="Watchlist name", required=True) - subscribe_command.add_argument("-d", "--description", type=str, help="Watchlist description", required=True) - subscribe_command.add_argument("-t", "--tags", action="store_true", help="Enable tags", default=False) - subscribe_command.add_argument("-a", "--alerts", action="store_true", help="Enable alerts", default=False) - subscribe_command.add_argument("-T", "--timestamp", type=int, help="Creation timestamp", default=int(time.time())) - subscribe_command.add_argument("-U", "--last_update", type=int, help="Last update timestamp", default=int(time.time())) - - create_command = commands.add_parser("create", help="Create a watchlist with a report") - create_command.add_argument("-w", "--watchlist_name", type=str, help="Watchlist name", required=True) - create_command.add_argument("-d", "--description", type=str, help="Watchlist description", required=True) - create_command.add_argument("-t", "--tags", action="store_true", help="Enable tags", default=False) - create_command.add_argument("-a", "--alerts", action="store_true", help="Enable alerts", default=False) - create_command.add_argument("-T", "--timestamp", type=int, help="Creation timestamp", default=int(time.time())) - create_command.add_argument("-U", "--last_update", type=int, help="Last update timestamp", default=int(time.time())) - # Report metadata arguments. - create_command.add_argument("--rep_timestamp", type=int, help="Report timestamp", default=int(time.time())) - create_command.add_argument("--rep_title", type=str, help="Report title", required=True) - create_command.add_argument("--rep_desc", type=str, help="Report description", required=True) - create_command.add_argument("--rep_severity", type=int, help="Report severity", default=1) - create_command.add_argument("--rep_link", type=str, help="Report link") - create_command.add_argument("--rep_tags", type=str, help="Report tags, comma separated") - create_command.add_argument("--rep_visibility", type=str, help="Report visibility") - - delete_command = commands.add_parser("delete", help="Delete a watchlist") - delete_command.add_argument("-R", "--reports", action="store_true", help="Delete all associated reports too", default=False) - specifier = delete_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--watchlist_id", type=str, help="The watchlist ID") - specifier.add_argument("-w", "--watchlist_name", type=str, help="The watchlist name") - - alter_report_command = commands.add_parser("alter-report", help="Change the properties of a watchlist's report") - alter_report_command.add_argument("-i", "--watchlist_id", type=str, help="Watchlist ID", required=True) - alter_report_command.add_argument("-r", "--report_id", type=str, help="Report ID", required=True) - alter_report_command.add_argument("-s", "--severity", type=int, help="The report's severity", required=False) - specifier = alter_report_command.add_mutually_exclusive_group(required=False) - specifier.add_argument("-d", "--deactivate", action="store_true", help="Deactive alerts for this report") - specifier.add_argument("-a", "--activate", action="store_true", help="Activate alerts for this report") - - alter_ioc_command = commands.add_parser("alter-ioc", help="Change the properties of a watchlist's IOC") - alter_ioc_command.add_argument("-i", "--watchlist_id", type=str, help="Watchlist ID", required=True) - alter_ioc_command.add_argument("-r", "--report_id", type=str, help="Report ID", required=True) - alter_ioc_command.add_argument("-I", "--ioc_id", type=str, help="IOC ID", required=True) - specifier = alter_ioc_command.add_mutually_exclusive_group(required=False) - specifier.add_argument("-d", "--deactivate", action="store_true", help="Deactive alerts for this IOC") - specifier.add_argument("-a", "--activate", action="store_true", help="Activate alerts for this IOC") - - export_command = commands.add_parser("export", help="Export a watchlist into an importable format") - specifier = export_command.add_mutually_exclusive_group(required=True) - specifier.add_argument("-i", "--watchlist_id", type=str, help="Watchlist ID") - specifier.add_argument("-w", "--watchlist_name", type=str, help="Watchlist name") - - commands.add_parser("import", help="Import a previously exported watchlist") - - args = parser.parse_args() - cb = get_cb_threathunter_object(args) - - if args.command_name == "list": - return list_watchlists(cb, parser, args) - elif args.command_name == "subscribe": - return subscribe_watchlist(cb, parser, args) - elif args.command_name == "create": - return create_watchlist(cb, parser, args) - elif args.command_name == "delete": - return delete_watchlist(cb, parser, args) - elif args.command_name == "alter-report": - return alter_report(cb, parser, args) - elif args.command_name == "alter-ioc": - return alter_ioc(cb, parser, args) - elif args.command_name == "export": - return export_watchlist(cb, parser, args) - elif args.command_name == "import": - return import_watchlist(cb, parser, args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/runtests.bat b/runtests.bat index 7c9dceaf..6163babf 100755 --- a/runtests.bat +++ b/runtests.bat @@ -1,5 +1,5 @@ @echo off setlocal set PYTHONPATH=src -pytest test/ +pytest -v test/ endlocal diff --git a/runtests.sh b/runtests.sh index a46f46f2..1efaf470 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,2 +1,2 @@ #!/bin/bash -PYTHONPATH=src pytest test/ +PYTHONPATH=src pytest -v test/ diff --git a/setup.cfg b/setup.cfg index fe62be21..e78a0367 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,12 @@ description-file = README.md [bdist_wheel] universal = 1 + +[flake8] +exclude = + src/cbapi/response/sensor_events.py, + .svn,CVS,.bzr,.hg,.git,__pycache__,.tox +per-file-ignores = src/*/__init__.py:F401,F403 +max-doc-length = 120 +max-line-length = 120 + diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 7d20c2f3..ef4e12d5 --- a/setup.py +++ b/setup.py @@ -9,36 +9,35 @@ 'cbapi', 'cbapi.protection', 'cbapi.response', - 'cbapi.cache', - 'cbapi.psc', - 'cbapi.psc.defense', - 'cbapi.psc.threathunter', - 'cbapi.psc.livequery' + 'cbapi.cache' ] -install_requires=[ +install_requires = [ 'requests', - 'attrdict', 'cachetools', 'pyyaml', 'pika', 'prompt_toolkit', 'pygments', - 'pytest<=5.0', 'python-dateutil', 'protobuf', 'solrq', 'validators' ] +tests_requires = [ + 'pytest<=5.0' +] + if sys.version_info < (2, 7): install_requires.extend(['simplejson', 'total-ordering', 'ordereddict']) if sys.version_info < (3, 0): install_requires.extend(['futures']) - +if sys.version_info > (3, 6): + install_requires.extend(['packaging']) setup( name='cbapi', - version='1.5.4', + version='2.0.0', url='https://github.com/carbonblack/cbapi-python', license='MIT', author='Carbon Black', @@ -46,10 +45,11 @@ description='Carbon Black REST API Python Bindings', packages=packages, include_package_data=True, - package_dir = {'': 'src'}, + package_dir={'': 'src'}, zip_safe=False, platforms='any', install_requires=install_requires, + tests_requires=tests_requires, classifiers=[ 'Environment :: Web Environment', 'Intended Audience :: Developers', @@ -57,5 +57,5 @@ 'Programming Language :: Python', 'Topic :: Software Development :: Libraries :: Python Modules' ], - scripts=['bin/cbapi-response', 'bin/cbapi-protection', 'bin/cbapi-defense', 'bin/cbapi', 'bin/cbapi-psc'] + scripts=['bin/cbapi-response', 'bin/cbapi-protection', 'bin/cbapi'] ) diff --git a/setversion.py b/setversion.py new file mode 100755 index 00000000..73d65a5b --- /dev/null +++ b/setversion.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python + +# CbAPI Project Version Number Setting Script +# AGRB 11/7/2019 + +import sys +import os +import re +import argparse +from datetime import date + + +def readme_rewriter(line, ctxt): + expr = ctxt.get("readme_expr", None) + if not expr: + expr = re.compile(r"^\*\*Latest Version:") + ctxt["readme_expr"] = expr + if expr.match(line): + return "**Latest Version: {0}**\n".format(ctxt["version"]) + return None + + +def changelog_rewriter(line, ctxt): + expr = ctxt.get("changelog_expr", None) + if not expr: + expr = re.compile(r"^\.\. top-of-changelog") + ctxt["changelog_expr"] = expr + if expr.match(line): + datestr = date.today().strftime("%B %d, %Y") + cl1 = "CbAPI {0} - Released {1}".format(ctxt["version"], datestr) + cl2 = "".ljust(len(cl1), "-") + updates = "\n\nUpdates\n\n.. (add your updates here)\n" + return line + "\n" + cl1 + "\n" + cl2 + updates + return None + + +def doc_conf_rewriter(line, ctxt): + vexpr = ctxt.get("doc_version_expr", None) + if not vexpr: + vexpr = re.compile(r"^version = ") + ctxt["doc_version_expr"] = vexpr + rexpr = ctxt.get("doc_release_expr", None) + if not rexpr: + rexpr = re.compile(r"^release = ") + ctxt["doc_release_expr"] = rexpr + if vexpr.match(line): + t = re.match(r"^(\d+\.\d+)\.", ctxt["version"]) + vn = ctxt["version"] + if t: + vn = t.group(1) + return "version = u'{0}'\n".format(vn) + if rexpr.match(line): + return "release = u'{0}'\n".format(ctxt["version"]) + return None + + +def setup_rewriter(line, ctxt): + expr = ctxt.get("setup_expr", None) + if not expr: + expr = re.compile(r"^(\s*)version=") + ctxt["setup_expr"] = expr + m = expr.match(line) + if m: + return "{0}version='{1}',\n".format(m.group(1), ctxt["version"]) + return None + + +def init_rewriter(line, ctxt): + expr = ctxt.get("init_expr", None) + if not expr: + expr = re.compile(r"^__version__ = ") + ctxt["init_expr"] = expr + if expr.match(line): + return "__version__ = '{0}'\n".format(ctxt["version"]) + return None + + +def rewrite_file(infilename, rewritefunc, ctxt): + outfilename = infilename + ".new" + if not ctxt["accept"]: + infile = open(infilename, "r") + outfile = open(outfilename, "w") + try: + s = infile.readline() + while s: + s2 = rewritefunc(s, ctxt) + if s2: + outfile.write(s2) + else: + outfile.write(s) + s = infile.readline() + finally: + infile.close() + outfile.close() + else: + if not os.access(outfilename, os.F_OK): + print("warning: new file {0} does not exist to be renamed".format(outfilename)) + return + if not ctxt["nodelete"]: + if ctxt["backup"]: + os.rename(infilename, infilename + ".bak") + else: + os.remove(infilename) + os.rename(outfilename, infilename) + + +def main(): + parser = argparse.ArgumentParser(description="Set the version number in CbAPI source and documentation.\n" + "Execute this on a release or hotfix branch to update " + "the version numbers in the source.", + epilog="After running, edit docs/changelog.rst and add the new changelog " + "information under the new heading.") + parser.add_argument("version", help="New version number to add, or \"accept\" to accept changes to .new files") + parser.add_argument("-n", "--nodelete", action="store_true", + help="Do not delete existing files, leave new files with .new extension") + parser.add_argument("-b", "--backup", action="store_true", + help="Keep old versions of files around with a .bak extension") + + args = parser.parse_args() + + is_accept = False + if args.version == "accept": + if args.nodelete: + print("cannot specify --nodelete with accept command") + return 1 + is_accept = True + else: + vnexpr = re.compile(r"^[1-9]\d*\.\d+\.\d+$") + if not vnexpr.match(args.version): + print("Invalid version number {0}: must be three numeric values separated by dots\n".format(args.version)) + return 1 + + ctxt = {"version": args.version, "nodelete": args.nodelete, "backup": args.backup, "accept": is_accept} + rewrite_file("README.md", readme_rewriter, ctxt) + rewrite_file("docs/changelog.rst", changelog_rewriter, ctxt) + rewrite_file("docs/conf.py", doc_conf_rewriter, ctxt) + rewrite_file("setup.py", setup_rewriter, ctxt) + rewrite_file("src/cbapi/__init__.py", init_rewriter, ctxt) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/cbapi/__init__.py b/src/cbapi/__init__.py old mode 100755 new mode 100644 index db4780f0..aea6d6ff --- a/src/cbapi/__init__.py +++ b/src/cbapi/__init__.py @@ -5,15 +5,9 @@ __title__ = 'cbapi' __author__ = 'Carbon Black Developer Network' __license__ = 'MIT' -__copyright__ = 'Copyright 2019 Carbon Black' -__version__ = '1.5.4' +__copyright__ = 'Copyright 2018-2022 VMware Carbon Black' +__version__ = '2.0.0' # New API as of cbapi 0.9.0 from cbapi.response.rest_api import CbEnterpriseResponseAPI, CbResponseAPI from cbapi.protection.rest_api import CbEnterpriseProtectionAPI, CbProtectionAPI -from cbapi.psc.defense import CbDefenseAPI -from cbapi.psc.threathunter import CbThreatHunterAPI -from cbapi.psc.livequery import CbLiveQueryAPI - -# for compatibility with Cb Defense code from cbapi < 1.4.0 -import cbapi.psc.defense as defense diff --git a/src/cbapi/attrdict.py b/src/cbapi/attrdict.py new file mode 100644 index 00000000..c551397b --- /dev/null +++ b/src/cbapi/attrdict.py @@ -0,0 +1,273 @@ +# Copyright (c) 2013 Brendan Curran-Johnson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +"""Functions and Classes from attrdict, modified for Python3""" + +from abc import ABCMeta, abstractmethod +from collections.abc import Mapping, MutableMapping, Sequence +import re + +"""A right-favoring Mapping merge.""" + + +def merge(left, right): + """ + Merge two mappings objects together, combining overlapping Mappings, favoring right-values + left: The left Mapping object. + right: The right (favored) Mapping object. + NOTE: This is not commutative (merge(a,b) != merge(b,a)). + """ + merged = {} + + left_keys = frozenset(left) + right_keys = frozenset(right) + + # Items only in the left Mapping + for key in left_keys - right_keys: + merged[key] = left[key] + + # Items only in the right Mapping + for key in right_keys - left_keys: + merged[key] = right[key] + + # in both + for key in left_keys & right_keys: + left_value = left[key] + right_value = right[key] + + if (isinstance(left_value, Mapping) and isinstance(right_value, Mapping)): # recursive merge + merged[key] = merge(left_value, right_value) + else: # overwrite with right value + merged[key] = right_value + + return merged + + +""" +Mixin Classes for Attr-support. +""" + + +class Attr(Mapping): + """ + A mixin class for a mapping that allows for attribute-style access of values. + A key may be used as an attribute if: + * It is a string + * It matches /^[A-Za-z][A-Za-z0-9_]*$/ (i.e., a public attribute) + * The key doesn't overlap with any class attributes (for Attr, + those would be 'get', 'items', 'keys', 'values', 'mro', and + 'register'). + If a values which is accessed as an attribute is a Sequence-type + (and is not a string/bytes), it will be converted to a + _sequence_type with any mappings within it converted to Attrs. + NOTE: This means that if _sequence_type is not None, then a + sequence accessed as an attribute will be a different object + than if accessed as an attribute than if it is accessed as an + item. + """ + @abstractmethod + def _configuration(self): + """All required state for building a new instance with the same settings as the current object.""" + + @classmethod + def _constructor(cls, mapping, configuration): + """ + A standardized constructor used internally by Attr. + mapping: A mapping of key-value pairs. It is HIGHLY recommended + that you use this as the internal key-value pair mapping, as + that will allow nested assignment (e.g., attr.foo.bar = baz) + configuration: The return value of Attr._configuration + """ + raise NotImplementedError("You need to implement this") + + def __call__(self, key): + """ + Dynamically access a key-value pair. + key: A key associated with a value in the mapping. + This differs from __getitem__, because it returns a new instance + of an Attr (if the value is a Mapping object). + """ + if key not in self: + raise AttributeError( + "'{cls} instance has no attribute '{name}'".format( + cls=self.__class__.__name__, name=key + ) + ) + + return self._build(self[key]) + + def __getattr__(self, key): + """Access an item as an attribute.""" + if key not in self or not self._valid_name(key): + raise AttributeError( + "'{cls}' instance has no attribute '{name}'".format( + cls=self.__class__.__name__, name=key + ) + ) + + return self._build(self[key]) + + def __add__(self, other): + """ + Add a mapping to this Attr, creating a new, merged Attr. + other: A mapping. + NOTE: Addition is not commutative. a + b != b + a. + """ + if not isinstance(other, Mapping): + return NotImplemented + + return self._constructor(merge(self, other), self._configuration()) + + def __radd__(self, other): + """ + Add this Attr to a mapping, creating a new, merged Attr. + other: A mapping. + NOTE: Addition is not commutative. a + b != b + a. + """ + if not isinstance(other, Mapping): + return NotImplemented + + return self._constructor(merge(other, self), self._configuration()) + + def _build(self, obj): + """Conditionally convert an object to allow for recursive mapping access. + obj: An object that was a key-value pair in the mapping. If obj + is a mapping, self._constructor(obj, self._configuration()) + will be called. If obj is a non-string/bytes sequence, and + self._sequence_type is not None, the obj will be converted + to type _sequence_type and build will be called on its + elements. + """ + if isinstance(obj, Mapping): + obj = self._constructor(obj, self._configuration()) + elif (isinstance(obj, Sequence) and not isinstance(obj, (str, bytes))): + sequence_type = getattr(self, '_sequence_type', None) + + if sequence_type: + obj = sequence_type(self._build(element) for element in obj) + + return obj + + @classmethod + def _valid_name(cls, key): + """ + Check whether a key is a valid attribute name. + A key may be used as an attribute if: + * It is a string + * It matches /^[A-Za-z][A-Za-z0-9_]*$/ (i.e., a public attribute) + * The key doesn't overlap with any class attributes (for Attr, + those would be 'get', 'items', 'keys', 'values', 'mro', and + 'register'). + """ + return ( + isinstance(key, str) and + re.match('^[A-Za-z][A-Za-z0-9_]*$', key) and + not hasattr(cls, key) + ) + + +class MutableAttr(Attr, MutableMapping, metaclass=ABCMeta): + """A mixin class for a mapping that allows for attribute-style access of values.""" + def _setattr(self, key, value): + """Add an attribute to the object, without attempting to add it as a key to the mapping.""" + super(MutableAttr, self).__setattr__(key, value) + + def __setattr__(self, key, value): + """ + Add an attribute. + key: The name of the attribute + value: The attributes contents + """ + if self._valid_name(key): + self[key] = value + elif getattr(self, '_allow_invalid_attributes', True): + super(MutableAttr, self).__setattr__(key, value) + else: + raise TypeError( + "'{cls}' does not allow attribute creation.".format( + cls=self.__class__.__name__ + ) + ) + + def _delattr(self, key): + """Delete an attribute from the object, without attempting to remove it from the mapping.""" + super(MutableAttr, self).__delattr__(key) + + def __delattr__(self, key, force=False): + """ + Delete an attribute. + key: The name of the attribute + """ + if self._valid_name(key): + del self[key] + elif getattr(self, '_allow_invalid_attributes', True): + super(MutableAttr, self).__delattr__(key) + else: + raise TypeError( + "'{cls}' does not allow attribute deletion.".format( + cls=self.__class__.__name__ + ) + ) + + +""" +A dict that implements MutableAttr. +""" + + +class AttrDict(dict, MutableAttr): + """A dict that implements MutableAttr.""" + def __init__(self, *args, **kwargs): + """Initilize the AttrDict""" + super(AttrDict, self).__init__(*args, **kwargs) + + self._setattr('_sequence_type', tuple) + self._setattr('_allow_invalid_attributes', False) + + def _configuration(self): + """The configuration for an attrmap instance.""" + return self._sequence_type + + def __getstate__(self): + """Serialize the object.""" + return ( + self.copy(), + self._sequence_type, + self._allow_invalid_attributes + ) + + def __setstate__(self, state): + """Deserialize the object.""" + mapping, sequence_type, allow_invalid_attributes = state + self.update(mapping) + self._setattr('_sequence_type', sequence_type) + self._setattr('_allow_invalid_attributes', allow_invalid_attributes) + + def __repr__(self): + """Override offical string representation.""" + return f'AttrDict({super(AttrDict, self).__repr__()})' + + @classmethod + def _constructor(cls, mapping, configuration): + """A standardized constructor.""" + attr = cls(mapping) + attr._setattr('_sequence_type', configuration) + + return attr \ No newline at end of file diff --git a/src/cbapi/auth.py b/src/cbapi/auth.py index 5470050d..bfc6c163 100644 --- a/src/cbapi/auth.py +++ b/src/cbapi/auth.py @@ -1,6 +1,6 @@ from cbapi.six.moves.configparser import RawConfigParser import os -import attrdict +from cbapi import attrdict import logging import cbapi.six as six @@ -47,35 +47,38 @@ def __init__(self, *args, **kwargs): if isinstance(x, six.string_types) and x.lower() in _boolean_states: self[k] = _boolean_states[x.lower()] + class CredentialStoreFactory(object): - #CredentialStore(product_name, credential_file=credential_file) + # CredentialStore(product_name, credential_file=credential_file) @staticmethod - def getCredentialStore(product_name,credential_file): - if credential_file is None and os.environ.get('CBAPI_TOKEN',False) and os.environ.get('CBAPI_URL',False): + def getCredentialStore(product_name, credential_file): + if credential_file is None and os.environ.get('CBAPI_TOKEN', False) and os.environ.get('CBAPI_URL', False): log.debug("Using Envar credential store") return EnvarCredentialStore() else: log.debug("Using file credential store") - return FileCredentialStore(**{"product_name":product_name,"credential_file":credential_file}) + return FileCredentialStore(**{"product_name": product_name, "credential_file": credential_file}) + -#A CredentialStore backed by os.environ rather than by a file on disk +# A CredentialStore backed by os.environ rather than by a file on disk class EnvarCredentialStore(object): def __init__(self): - #`CBAPI_URL`, `CBAPI_TOKEN`, `CBAPI_SSL_VERIFY` + # `CBAPI_URL`, `CBAPI_TOKEN`, `CBAPI_SSL_VERIFY` environ = os.environ - self.cbapi_url = environ.get('CBAPI_URL',None) - self.cbapi_token = environ.get('CBAPI_TOKEN',None) - self.cbapi_ssl_verify = environ.get('CBAPI_SSL_VERIFY',True) + self.cbapi_url = environ.get('CBAPI_URL', None) + self.cbapi_token = environ.get('CBAPI_TOKEN', None) + self.cbapi_ssl_verify = environ.get('CBAPI_SSL_VERIFY', True) self.org_key = environ.get('CBAPI_ORG_KEY', None) - self.credentials = Credentials(url=self.cbapi_url,token=self.cbapi_token,ssl_verify=self.cbapi_ssl_verify) + self.credentials = Credentials(url=self.cbapi_url, token=self.cbapi_token, ssl_verify=self.cbapi_ssl_verify) - def get_credentials(self,profile=None): + def get_credentials(self, profile=None): return self.credentials + class FileCredentialStore(object): def __init__(self, product_name, **kwargs): - if product_name not in ("response", "protection", "psc"): + if product_name not in ("response", "protection"): raise CredentialError("Product name {0:s} not valid".format(product_name)) self.credential_search_path = [ diff --git a/src/cbapi/cache/lru.py b/src/cbapi/cache/lru.py index 47c6d617..3b2476b3 100644 --- a/src/cbapi/cache/lru.py +++ b/src/cbapi/cache/lru.py @@ -31,6 +31,7 @@ def wrapper(func): return LRUCachedFunction(func, LRUCacheDict(max_size, expiration)) return wrapper + def _lock_decorator(func): """ If the LRUCacheDict is concurrent, then we should lock in order to avoid @@ -45,6 +46,7 @@ def withlock(self, *args, **kwargs): withlock.__name__ == func.__name__ return withlock + class LRUCacheDict(object): """ A dictionary-like object, supporting LRU caching semantics. @@ -95,8 +97,10 @@ class EmptyCacheThread(threading.Thread): def __init__(self, cache, peek_duration=60): me = self + def kill_self(o): me + self.ref = weakref.ref(cache) self.peek_duration = peek_duration super(LRUCacheDict.EmptyCacheThread, self).__init__() @@ -136,7 +140,7 @@ def clear(self): self.__access_times.clear() def __contains__(self, key): - return self.has_key(key) + return key in self @_lock_decorator def has_key(self, key): @@ -218,6 +222,7 @@ def cleanup(self): else: return None + class LRUCachedFunction(object): """ A memoized function, backed by an LRU cache. @@ -247,7 +252,8 @@ class LRUCachedFunction(object): >>> f(6) Calling f(6) 6 - >>> f(4) #No longer in cache - 4 is the least recently used, and there are at least 3 others items in cache [3,4,5,6]. + >>> f(4) #No longer in cache - 4 is the least recently used, and there are at least 3 others + items in cache [3,4,5,6]. Calling f(4) 4 @@ -261,7 +267,8 @@ def __init__(self, function, cache=None): self.__name__ = self.function.__name__ def __call__(self, *args, **kwargs): - key = repr( (args, kwargs) ) + "#" + self.__name__ #In principle a python repr(...) should not return any # characters. + key = repr((args, kwargs)) + "#" + self.__name__ + # In principle a python repr(...) should not return any # characters. try: return self.cache[key] except KeyError: @@ -269,6 +276,7 @@ def __call__(self, *args, **kwargs): self.cache[key] = value return value + if __name__ == "__main__": import doctest doctest.testmod() diff --git a/src/cbapi/connection.py b/src/cbapi/connection.py index 5f3727f9..523f1101 100644 --- a/src/cbapi/connection.py +++ b/src/cbapi/connection.py @@ -1,10 +1,12 @@ #!/usr/bin/env python +"""Manages the CBAPI connection to the server.""" + from __future__ import absolute_import import requests import sys -from requests.adapters import HTTPAdapter, DEFAULT_POOLBLOCK, DEFAULT_RETRIES, DEFAULT_POOLSIZE +from requests.adapters import HTTPAdapter, DEFAULT_POOLBLOCK, DEFAULT_RETRIES, DEFAULT_POOLSIZE, DEFAULT_POOL_TIMEOUT try: from requests.packages.urllib3.util.ssl_ import create_urllib3_context @@ -23,8 +25,6 @@ except ImportError: MAX_RETRIES = 5 -from requests.packages.urllib3.poolmanager import PoolManager - import logging import json @@ -32,8 +32,8 @@ from cbapi.six.moves import urllib from .auth import CredentialStoreFactory, Credentials -from .errors import ServerError, TimeoutError, ApiError, ObjectNotFoundError, UnauthorizedError, CredentialError, \ - ConnectionError +from .errors import ClientError, QuerySyntaxError, ServerError, TimeoutError, ApiError, ObjectNotFoundError, \ + UnauthorizedError, ConnectionError from . import __version__ from .cache.lru import lru_cache_function @@ -43,10 +43,32 @@ log = logging.getLogger(__name__) +def try_json(resp): + """ + Return a parsed JSON representation of the input. + + Args: + resp (str): Input to be parsed. + + Returns: + object: The parsed JSON result, or an empty dict if the value is not valid JSON. + """ + try: + return resp.json() + except ValueError: + return dict() + + def check_python_tls_compatibility(): + """ + Verify which level of TLS/SSL that this version of the code is compatible with. + + Returns: + str: The maximum level of TLS/SSL that this version is compatible with. + """ try: - tls_adapter = CbAPISessionAdapter(force_tls_1_2=True) - except Exception as e: + CbAPISessionAdapter(force_tls_1_2=True) + except Exception: ret = "TLSv1.1" if "OP_NO_TLSv1_1" not in ssl.__dict__: @@ -64,7 +86,21 @@ def check_python_tls_compatibility(): class CbAPISessionAdapter(HTTPAdapter): + """Adapter object used to handle TLS connections to the CB server.""" + def __init__(self, verify_hostname=True, force_tls_1_2=False, max_retries=DEFAULT_RETRIES, **pool_kwargs): + """ + Initialize the CbAPISessionManager. + + Args: + verify_hostname (boolean): True if we want to verify the hostname. + force_tls_1_2 (boolean): True to force the use of TLS 1.2. + max_retries (int): Maximum number of retries. + **pool_kwargs: Additional arguments. + + Raises: + ApiError: If the library versions are too old to force the use of TLS 1.2. + """ self._cbapi_verify_hostname = verify_hostname self._cbapi_force_tls_1_2 = force_tls_1_2 @@ -74,6 +110,18 @@ def __init__(self, verify_hostname=True, force_tls_1_2=False, max_retries=DEFAUL super(CbAPISessionAdapter, self).__init__(max_retries=max_retries, **pool_kwargs) def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs): + """ + Initialize the connection pool manager. + + Args: + connections (int): Initial number of connections to be used. + maxsize (int): Maximum size of the connection pool. + block (object): Blocking policy. + **pool_kwargs: Additional arguments for the connection pool. + + Returns: + object: TBD + """ if self._cbapi_force_tls_1_2 and REQUESTS_HAS_URLLIB_SSL_CONTEXT: # Force the use of TLS v1.2 when talking to this Cb Response server. context = create_urllib3_context(ciphers=('TLSv1.2:!aNULL:!eNULL:!MD5')) @@ -93,7 +141,24 @@ def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool class Connection(object): - def __init__(self, credentials, integration_name=None, timeout=None, max_retries=None, **pool_kwargs): + """Object that encapsulates the HTTP connection to the CB server.""" + + def __init__(self, credentials, integration_name=None, timeout=None, max_retries=None, proxy_session=None, **pool_kwargs): + """ + Initialize the Connection object. + + Args: + credentials (object): The credentials to use for the connection. + integration_name (str): The integration name being used. + timeout (int): The timeout value to use for HTTP requests on this connection. + max_retries (int): The maximum number of times to retry a request. + proxy_session (requests.Session) custom session to be used + **pool_kwargs: Additional arguments to be used to initialize connection pooling. + + Raises: + ApiError: If there's an internal error initializing the connection. + ConnectionError: If there's a problem with the credentials. + """ if not credentials.url or not credentials.url.startswith("https://"): raise ConnectionError("Server URL must be a URL: eg. https://localhost") @@ -113,14 +178,19 @@ def __init__(self, credentials, integration_name=None, timeout=None, max_retries if credentials.ssl_cert_file: self.ssl_verify = credentials.ssl_cert_file - user_agent = "cbapi/{0:s} Python/{1:d}.{2:d}.{3:d}".format(__version__, - sys.version_info[0], sys.version_info[1], sys.version_info[2]) + user_agent = "cbapi/{0:s} Python/{1:d}.{2:d}.{3:d}" \ + .format(__version__, sys.version_info[0], sys.version_info[1], sys.version_info[2]) if integration_name: user_agent += " {}".format(integration_name) self.token = credentials.token self.token_header = {'X-Auth-Token': self.token, 'User-Agent': user_agent} - self.session = requests.Session() + if proxy_session: + self.session = proxy_session + credentials.use_custom_proxy_session = True + else: + self.session = requests.Session() + credentials.use_custom_proxy_session = False self._timeout = timeout @@ -140,7 +210,10 @@ def __init__(self, credentials, integration_name=None, timeout=None, max_retries self.session.mount(self.server, tls_adapter) self.proxies = {} - if credentials.ignore_system_proxy: # see https://github.com/kennethreitz/requests/issues/879 + if credentials.use_custom_proxy_session: + # get the custom session proxies + self.proxies = self.session.proxies + elif credentials.ignore_system_proxy: # see https://github.com/kennethreitz/requests/issues/879 # Unfortunately, requests will look for any proxy-related environment variables and use those anyway. The # only way to solve this without side effects, is passing in empty strings for 'http' and 'https': self.proxies = { @@ -154,6 +227,27 @@ def __init__(self, credentials, integration_name=None, timeout=None, max_retries self.proxies['https'] = credentials.proxy def http_request(self, method, url, **kwargs): + """ + Submit a HTTP request to the server. + + Args: + method (str): The method name to use for the HTTP request. + url (str): The URL to submit the request to. + **kwargs: Additional arguments for the request. + + Returns: + object: Result of the HTTP request. + + Raises: + ApiError: An unknown problem was detected. + ClientError: The server returned an error code in the 4xx range, indicating a problem with the request. + ConnectionError: A problem was seen with the HTTP connection. + ObjectNotFoundError: The specified object was not found on the server. + QuerySyntaxError: The query passed in had invalid syntax. + ServerError: The server returned an error code in the 5xx range, indicating a problem on the server side. + TimeoutError: The HTTP request timed out. + UnauthorizedError: The stored credentials do not permit access to the specified request. + """ method = method.upper() verify_ssl = kwargs.pop('verify', None) or self.ssl_verify @@ -180,41 +274,93 @@ def http_request(self, method, url, **kwargs): except requests.Timeout as timeout_error: raise TimeoutError(uri=uri, original_exception=timeout_error) except requests.ConnectionError as connection_error: - raise ApiError("Received a network connection error from {0:s}: {1:s}".format(self.server, - str(connection_error)), - original_exception=connection_error) + raise ConnectionError("Received a network connection error from {0:s}: {1:s}" + .format(self.server, str(connection_error)), + original_exception=connection_error) except Exception as e: raise ApiError("Unknown exception when connecting to server: {0:s}".format(str(e)), original_exception=e) else: - if r.status_code == 404: + if r.status_code >= 500: + raise ServerError(error_code=r.status_code, message=r.text) + elif r.status_code == 404: raise ObjectNotFoundError(uri=uri, message=r.text) elif r.status_code == 401: raise UnauthorizedError(uri=uri, action=method, message=r.text) + elif r.status_code == 400 and try_json(r).get('reason') == 'query_malformed_syntax': + raise QuerySyntaxError(uri=uri, message=r.text) elif r.status_code >= 400: - raise ServerError(error_code=r.status_code, message=r.text) + raise ClientError(error_code=r.status_code, message=r.text) return r def get(self, url, **kwargs): + """ + Submit a GET request on this connection. + + Args: + url (str): The URL to submit the request to. + **kwargs: Additional arguments for the request. + + Returns: + object: Result of the HTTP request. + """ return self.http_request("GET", url, **kwargs) def post(self, url, **kwargs): + """ + Submit a POST request on this connection. + + Args: + url (str): The URL to submit the request to. + **kwargs: Additional arguments for the request. + + Returns: + object: Result of the HTTP request. + """ return self.http_request("POST", url, **kwargs) def put(self, url, **kwargs): + """ + Submit a PUT request on this connection. + + Args: + url (str): The URL to submit the request to. + **kwargs: Additional arguments for the request. + + Returns: + object: Result of the HTTP request. + """ return self.http_request("PUT", url, **kwargs) def delete(self, url, **kwargs): + """ + Submit a DELETE request on this connection. + + Args: + url (str): The URL to submit the request to. + **kwargs: Additional arguments for the request. + + Returns: + object: Result of the HTTP request. + """ return self.http_request("DELETE", url, **kwargs) class BaseAPI(object): - """baseapi""" + """The base API object used by all CBAPI objects to communicate with the server.""" + def __init__(self, *args, **kwargs): + """ + Initialize the base API information. + + Args: + *args: TBD + **kwargs: Additional arguments. + """ product_name = kwargs.pop("product_name", None) credential_file = kwargs.pop("credential_file", None) integration_name = kwargs.pop("integration_name", None) - self.credential_store = CredentialStoreFactory.getCredentialStore(product_name,credential_file) + self.credential_store = CredentialStoreFactory.getCredentialStore(product_name, credential_file) url, token, org_key = kwargs.pop("url", None), kwargs.pop("token", None), kwargs.pop("org_key", None) if url and token: @@ -223,7 +369,7 @@ def __init__(self, *args, **kwargs): else: credentials = {"url": url, "token": token} - for k in ("ssl_verify",): + for k in ("ssl_verify", "proxy", "ssl_cert_file"): if k in kwargs: credentials[k] = kwargs.pop(k) self.credentials = Credentials(credentials) @@ -232,26 +378,48 @@ def __init__(self, *args, **kwargs): self.credential_profile_name = kwargs.pop("profile", None) self.credentials = self.credential_store.get_credentials(self.credential_profile_name) - timeout = kwargs.pop("timeout", None) - max_retries = kwargs.pop("max_retries", None) - pool_connections = kwargs.pop("pool_connections", DEFAULT_POOLSIZE) + timeout = kwargs.pop("timeout", DEFAULT_POOL_TIMEOUT) + max_retries = kwargs.pop("max_retries", DEFAULT_RETRIES) + proxy_session = kwargs.pop("proxy_session", None) + pool_connections = kwargs.pop("pool_connections", 1) pool_maxsize = kwargs.pop("pool_maxsize", DEFAULT_POOLSIZE) pool_block = kwargs.pop("pool_block", DEFAULT_POOLBLOCK) self.session = Connection(self.credentials, integration_name=integration_name, timeout=timeout, - max_retries=max_retries, pool_connections=pool_connections, + max_retries=max_retries, proxy_session=proxy_session, pool_connections=pool_connections, pool_maxsize=pool_maxsize, pool_block=pool_block) def raise_unless_json(self, ret, expected): + """ + Raise a ServerError unless we got back an HTTP 200 response with JSON containing all the expected values. + + Args: + ret (object): Return value to be checked. + expected (dict): Expected keys and values that need to be found in the JSON response. + + Raises: + ServerError: If the HTTP response is anything but 200, or if the expected values are not found. + """ if ret.status_code == 200: message = ret.json() for k, v in iteritems(expected): if k not in message or message[k] != v: raise ServerError(ret.status_code, message) else: - raise ServerError(ret.status_code, "".format(ret.content), ) + raise ServerError(ret.status_code, "{0}".format(ret.content), ) def get_object(self, uri, query_parameters=None, default=None): + """ + Submit a GET request to the server and parse the result as JSON before returning. + + Args: + uri (str): The URI to send the GET request to. + query_parameters (object): Parameters for the query. + default (object): What gets returned in the event of an empty response. + + Returns: + object: Result of the GET request. + """ if query_parameters: if isinstance(query_parameters, dict): query_parameters = convert_query_params(query_parameters) @@ -261,7 +429,7 @@ def get_object(self, uri, query_parameters=None, default=None): if result.status_code == 200: try: return result.json() - except: + except Exception: raise ServerError(result.status_code, "Cannot parse response as JSON: {0:s}".format(result.content)) elif result.status_code == 204: # empty response @@ -269,7 +437,49 @@ def get_object(self, uri, query_parameters=None, default=None): else: raise ServerError(error_code=result.status_code, message="Unknown error: {0}".format(result.content)) + def get_raw_data(self, uri, query_parameters=None, default=None, **kwargs): + """ + Submit a GET request to the server and return the result without parsing it. + + Args: + uri (str): The URI to send the GET request to. + query_parameters (object): Parameters for the query. + default (object): What gets returned in the event of an empty response. + **kwargs: + + Returns: + object: Result of the GET request. + """ + if query_parameters: + if isinstance(query_parameters, dict): + query_parameters = convert_query_params(query_parameters) + uri += '?%s' % (urllib.parse.urlencode(sorted(query_parameters))) + + hdrs = kwargs.pop("headers", {}) + result = self.api_json_request("GET", uri, headers=hdrs) + if result.status_code == 200: + return result.text + elif result.status_code == 204: + # empty response + return default + else: + raise ServerError(error_code=result.status_code, message="Unknown error: {0}".format(result.content)) + def api_json_request(self, method, uri, **kwargs): + """ + Submit a request to the server. + + Args: + method (str): HTTP method to use. + uri (str): URI to submit the request to. + **kwargs (dict): Additional arguments. + + Returns: + object: Result of the operation. + + Raises: + ServerError: If there's an error output from the server. + """ headers = kwargs.pop("headers", {}) raw_data = None @@ -283,7 +493,7 @@ def api_json_request(self, method, uri, **kwargs): try: resp = result.json() - except: + except ValueError: return result if "errorMessage" in resp: @@ -292,21 +502,57 @@ def api_json_request(self, method, uri, **kwargs): return result def post_object(self, uri, body, **kwargs): + """ + Send a POST request to the specified URI. + + Args: + uri (str): The URI to send the POST request to. + body (object): The data to be sent in the body of the POST request. + **kwargs: + + Returns: + object: The return data from the POST request. + """ return self.api_json_request("POST", uri, data=body, **kwargs) def put_object(self, uri, body, **kwargs): + """ + Send a PUT request to the specified URI. + + Args: + uri (str): The URI to send the PUT request to. + body (object): The data to be sent in the body of the PUT request. + **kwargs: + + Returns: + object: The return data from the PUT request. + """ return self.api_json_request("PUT", uri, data=body, **kwargs) def delete_object(self, uri): + """ + Send a DELETE request to the specified URI. + + Args: + uri (str): The URI to send the DELETE request to. + + Returns: + object: The return data from the DELETE request. + """ return self.api_json_request("DELETE", uri) def select(self, cls, unique_id=None, *args, **kwargs): - """Prepares a query against the Carbon Black data store. + """ + Prepare a query against the Carbon Black data store. - :param class cls: The Model class (for example, Computer, Process, Binary, FileInstance) to query - :param unique_id: (optional) The unique id of the object to retrieve, to retrieve a single object by ID + Args: + cls (class): The Model class (for example, Computer, Process, Binary, FileInstance) to query + unique_id (optional): The unique id of the object to retrieve, to retrieve a single object by ID + *args: + **kwargs: - :returns: An instance of the Model class if a unique_id is provided, otherwise a Query object + Returns: + object: An instance of the Model class if a unique_id is provided, otherwise a Query object """ if unique_id is not None: return select_instance(self, cls, unique_id, *args, **kwargs) @@ -314,12 +560,18 @@ def select(self, cls, unique_id=None, *args, **kwargs): return self._perform_query(cls, **kwargs) def create(self, cls, data=None): - """Creates a new object. + """ + Create a new object. + + Args: + cls (class): The Model class (only some models can be created, for example, Feed, Notification, ...) + data (object): The data used to initialize the new object - :param class cls: The Model class (only some models can be created, for example, Feed, Notification, ...) + Returns: + Model: An empty instance of the model class. - :returns: An empty instance of the Model class - :raises ApiError: if the Model cannot be created + Raises: + ApiError: If the Model cannot be created. """ if issubclass(cls, CreatableModelMixin): n = cls(self) @@ -335,6 +587,12 @@ def _perform_query(self, cls, **kwargs): @property def url(self): + """ + Return the connection URL. + + Returns: + str: The connection URL. + """ return self.session.server @@ -342,4 +600,17 @@ def url(self): # TODO: how does this interfere with mutable objects? @lru_cache_function(max_size=1024, expiration=1*60) def select_instance(api, cls, unique_id, *args, **kwargs): + """ + Select a cached instance of an object. + + Args: + api: TBD + cls: TBD + unique_id: TBD + *args: + **kwargs: + + Returns: + TBD + """ return cls(api, unique_id, *args, **kwargs) diff --git a/src/cbapi/defense.py b/src/cbapi/defense.py deleted file mode 100644 index 676dcd85..00000000 --- a/src/cbapi/defense.py +++ /dev/null @@ -1,2 +0,0 @@ -# Compatibility with old Defense API code -from cbapi.psc.defense import * diff --git a/src/cbapi/errors.py b/src/cbapi/errors.py index cadcf171..921bcb54 100644 --- a/src/cbapi/errors.py +++ b/src/cbapi/errors.py @@ -1,31 +1,126 @@ #!/usr/bin/env python -from cbapi.six import python_2_unicode_compatible - +"""Exceptions that are thrown by CBAPI operations.""" -class ConnectionError(Exception): - pass +from cbapi.six import python_2_unicode_compatible class ApiError(Exception): + """Base class for all CBAPI errors; also raised for generic internal errors.""" + def __init__(self, message=None, original_exception=None): + """ + Initialize the ApiError. + + Args: + message (str): The actual error message. + original_exception (Exception): The exception that caused this one to be raised. + """ self.original_exception = original_exception self.message = str(message) def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ return self.message +@python_2_unicode_compatible +class ClientError(ApiError): + """A ClientError is raised when an HTTP 4xx error code is returned from the Carbon Black server.""" + + def __init__(self, error_code, message, result=None, original_exception=None): + """ + Initialize the ClientError. + + Args: + error_code (int): The error code that was received from the server. + message (str): The actual error message. + result (object): The result of the operation from the server. + original_exception (Exception): The exception that caused this one to be raised. + """ + super(ClientError, self).__init__(message=message, original_exception=original_exception) + + self.error_code = error_code + self.result = result + + def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ + msg = "Received error code {0:d} from API".format(self.error_code) + if self.message: + msg += ": {0:s}".format(self.message) + else: + msg += " (No further information provided)" + + if self.result: + msg += ". {}".format(self.result) + return msg + + +@python_2_unicode_compatible +class QuerySyntaxError(ApiError): + """The request contains a query with malformed syntax.""" + + def __init__(self, uri, message=None, original_exception=None): + """ + Initialize the QuerySyntaxError. + + Args: + uri (str): The URI of the action that failed. + message (str): The error message. + original_exception (Exception): The exception that caused this one to be raised. + """ + super(QuerySyntaxError, self).__init__(message=message, original_exception=original_exception) + self.uri = uri + + def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ + msg = "Received query syntax error for {0:s}".format(self.uri) + if self.message: + msg += ": {0:s}".format(self.message) + + return msg + + @python_2_unicode_compatible class ServerError(ApiError): - """A ServerError is raised when an HTTP error code is returned from the Carbon Black server.""" + """A ServerError is raised when an HTTP 5xx error code is returned from the Carbon Black server.""" def __init__(self, error_code, message, result=None, original_exception=None): + """ + Initialize the ServerError. + + Args: + error_code (int): The error code that was received from the server. + message (str): The actual error message. + result (object): The result of the operation from the server. + original_exception (Exception): The exception that caused this one to be raised. + """ super(ServerError, self).__init__(message=message, original_exception=original_exception) self.error_code = error_code self.result = result def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ msg = "Received error code {0:d} from API".format(self.error_code) if self.message: msg += ": {0:s}".format(self.message) @@ -42,10 +137,24 @@ class ObjectNotFoundError(ApiError): """The requested object could not be found in the Carbon Black datastore.""" def __init__(self, uri, message=None, original_exception=None): + """ + Initialize the ObjectNotFoundError. + + Args: + uri (str): The URI of the action that failed. + message (str): The error message. + original_exception (Exception): The exception that caused this one to be raised. + """ super(ObjectNotFoundError, self).__init__(message=message, original_exception=original_exception) self.uri = uri def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ msg = "Received 404 (Object Not Found) for {0:s}".format(self.uri) if self.message: msg += ": {0:s}".format(self.message) @@ -55,12 +164,29 @@ def __str__(self): @python_2_unicode_compatible class TimeoutError(ApiError): + """A requested operation timed out.""" + def __init__(self, uri=None, error_code=None, message=None, original_exception=None): + """ + Initialize the TimeoutError. + + Args: + uri (str): The URI of the action that timed out. + error_code (int): The error code that was received from the server. + message (str): The error message. + original_exception (Exception): The exception that caused this one to be raised. + """ super(TimeoutError, self).__init__(message=message, original_exception=original_exception) self.uri = uri self.error_code = error_code def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ if self.uri: msg = "Timed out when requesting {0:s} from API".format(self.uri) if self.error_code: @@ -75,31 +201,60 @@ def __str__(self): @python_2_unicode_compatible class UnauthorizedError(ApiError): + """The action that was attempted was not authorized.""" + def __init__(self, uri, message=None, action="read", original_exception=None): + """ + Initialize the UnauthorizedError. + + Args: + uri (str): The URI of the action that was not authorized. + message (str): The error message. + action (str): The action that was being performed that was not authorized. + original_exception (Exception): The exception that caused this one to be raised. + """ super(UnauthorizedError, self).__init__(message=message, original_exception=original_exception) self.uri = uri self.action = action def __str__(self): + """ + Convert the exception to a string. + + Returns: + str: String equivalent of the exception. + """ if self.message: return "Check your API Credentials: " + str(self.message) return "Unauthorized (Check API creds): attempted to {0:s} {1:s}".format(self.action, self.uri) +class ConnectionError(ApiError): + """There was an error in the connection to the server.""" + + pass + + class CredentialError(ApiError): + """The credentials had an unspecified error.""" + pass class InvalidObjectError(ApiError): + """An invalid object was received by the server.""" + pass class InvalidHashError(Exception): + """An invalid hash value was used.""" + pass class MoreThanOneResultError(ApiError): """Only one object was requested, but multiple matches were found in the Carbon Black datastore.""" - pass + pass diff --git a/src/cbapi/event.py b/src/cbapi/event.py index fb1982da..48734d56 100644 --- a/src/cbapi/event.py +++ b/src/cbapi/event.py @@ -58,7 +58,7 @@ def run(self): try: callback["func"](*callback["args"], **kwargs) - except Exception as e: + except Exception: with self._error_lock: self._errors.append({"exception": traceback.format_exc(), "timestamp": time.time(), "callback_func": callback["func"].__name__, diff --git a/src/cbapi/example_helpers.py b/src/cbapi/example_helpers.py index fb8b2123..798b94cf 100644 --- a/src/cbapi/example_helpers.py +++ b/src/cbapi/example_helpers.py @@ -1,4 +1,5 @@ from __future__ import print_function +from cbapi.six import PY3 import sys import time import argparse @@ -14,21 +15,19 @@ import hashlib from cbapi.protection import CbEnterpriseProtectionAPI -from cbapi.psc.defense import CbDefenseAPI -from cbapi.psc.threathunter import CbThreatHunterAPI -from cbapi.psc.livequery import CbLiveQueryAPI from cbapi.response import CbEnterpriseResponseAPI log = logging.getLogger(__name__) # Example scripts: we want to make sure that sys.stdout is using utf-8 encoding. See issue #36. -from cbapi.six import PY3 if not PY3: sys.stdout = codecs.getwriter('utf8')(sys.stdout) + def eprint(*args, **kwargs): - _print(*args, file=sys.stderr, **kwargs) + print(*args, file=sys.stderr, **kwargs) + def build_cli_parser(description="Cb Example Script"): parser = argparse.ArgumentParser(description=description) @@ -65,7 +64,6 @@ def get_cb_response_object(args): def get_cb_protection_object(args): if args.verbose: - import logging logging.basicConfig() logging.getLogger("cbapi").setLevel(logging.DEBUG) logging.getLogger("__main__").setLevel(logging.DEBUG) @@ -78,50 +76,6 @@ def get_cb_protection_object(args): return cb -def get_cb_defense_object(args): - if args.verbose: - import logging - logging.basicConfig() - logging.getLogger("cbapi").setLevel(logging.DEBUG) - logging.getLogger("__main__").setLevel(logging.DEBUG) - - if args.cburl and args.apitoken: - cb = CbDefenseAPI(url=args.cburl, token=args.apitoken, ssl_verify=(not args.no_ssl_verify)) - else: - cb = CbDefenseAPI(profile=args.profile) - - return cb - - -def get_cb_threathunter_object(args): - if args.verbose: - logging.basicConfig() - logging.getLogger("cbapi").setLevel(logging.DEBUG) - logging.getLogger("__main__").setLevel(logging.DEBUG) - - if args.cburl and args.apitoken: - cb = CbThreatHunterAPI(url=args.cburl, token=args.apitoken, ssl_verify=(not args.no_ssl_verify)) - else: - cb = CbThreatHunterAPI(profile=args.profile) - - return cb - - -def get_cb_livequery_object(args): - if args.verbose: - logging.basicConfig() - logging.getLogger("cbapi").setLevel(logging.DEBUG) - logging.getLogger("__main__").setLevel(logging.DEBUG) - - if args.cburl and args.apitoken and args.orgkey: - cb = CbLiveQueryAPI(url=args.cburl, token=args.apitoken, org_key=args.orgkey, - ssl_verify=(not args.no_ssl_verify)) - else: - cb = CbLiveQueryAPI(profile=args.profile) - - return cb - - def get_object_by_name_or_id(cb, cls, name_field="name", id=None, name=None, force_init=True): clsname = cls.__name__ try: @@ -166,22 +120,28 @@ def read_iocs(cb, file=sys.stdin): eprint("line {}: invalid query".format(idx + 1)) return (report_id.hexdigest(), dict(iocs)) + # # Live Response # + class QuitException(Exception): pass + class CliArgsException(Exception): pass + class CliHelpException(Exception): pass + class CliAttachError(Exception): pass + def split_cli(line): ''' we'd like to use shlex.split() but that doesn't work well for @@ -193,7 +153,6 @@ def split_cli(line): parts = line.split(' ') final = [] - inQuotes = False while len(parts) > 0: tok = parts.pop(0) @@ -277,7 +236,7 @@ def cmdloop(self, intro=None): break except KeyboardInterrupt: break - except CliAttachError as e: + except CliAttachError: print("You must attach to a session") continue except CliArgsException as e: @@ -325,7 +284,6 @@ def _file_path_fixup(self, path): and performs the fixups. ''' - if (self._is_path_absolute(path)): return path elif (self._is_path_drive_relative(path)): @@ -490,11 +448,13 @@ def do_ps(self, line): self._needs_attached() p = CliArgs(usage='ps [OPTIONS]') - p.add_option('-v', '--verbose', default=False, action='store_true', help='Display verbose info about each process') + p.add_option('-v', '--verbose', default=False, action='store_true', + help='Display verbose info about each process') p.add_option('-p', '--pid', default=None, help='Display only the given pid') (opts, args) = p.parse_line(line) - if (opts.pid): opts.pid = int(opts.pid) + if opts.pid: + opts.pid = int(opts.pid) processes = self.lr_session.list_processes() @@ -563,7 +523,7 @@ def do_exec(self, line): exe = tok - #ok - now the command (exe) is in tok + # ok - now the command (exe) is in tok # we need to do some crappy path manipulation # to see what we are supposed to execute if (self._is_path_absolute(exe)): @@ -579,7 +539,7 @@ def do_exec(self, line): # then a file exist in the current working # directory that matches the exe name - execute it exe = ntpath.join(self.cwd, exe) - else : + else: # the cwd + exe does not exist - let windows # resolve the path pass @@ -589,7 +549,7 @@ def do_exec(self, line): cmdline = exe + ' ' + ' '.join(parts) else: cmdline = exe - #print "CMD: %s" % cmdline + # print "CMD: %s" % cmdline if not optWorkDir: optWorkDir = self.cwd @@ -637,7 +597,6 @@ def do_del(self, line): del ''' - self._needs_attached() if line is None or line == '': @@ -657,7 +616,6 @@ def do_mkdir(self, line): mdkir ''' - self._needs_attached() if line is None or line == '': diff --git a/src/cbapi/live_response_api.py b/src/cbapi/live_response_api.py index 6126fd32..7c05932c 100644 --- a/src/cbapi/live_response_api.py +++ b/src/cbapi/live_response_api.py @@ -1,3 +1,6 @@ + +"""The Live Response API and associated objects.""" + from __future__ import absolute_import import json @@ -10,9 +13,9 @@ import shutil -from cbapi.errors import TimeoutError, ObjectNotFoundError, ApiError, ServerError +from cbapi.errors import TimeoutError, ObjectNotFoundError, ApiError from cbapi.six import itervalues -from concurrent.futures import ThreadPoolExecutor, as_completed, _base, wait +from concurrent.futures import _base, wait from cbapi import winerror from cbapi.six.moves.queue import Queue @@ -24,7 +27,15 @@ class LiveResponseError(Exception): + """Exception raised for errors with Live Response.""" + def __init__(self, details): + """ + Initialize the LiveResponseError. + + Args: + details (object): Details of the specific error. + """ message_list = [] self.details = details @@ -32,9 +43,9 @@ def __init__(self, details): self.decoded_win32_error = "" # Details object: - # {u'status': u'error', u'username': u'admin', u'sensor_id': 9, u'name': u'kill', u'completion': 1464319733.190924, - # u'object': 1660, u'session_id': 7, u'result_type': u'WinHresult', u'create_time': 1464319733.171967, - # u'result_desc': u'', u'id': 22, u'result_code': 2147942487} + # {u'status': u'error', u'username': u'admin', u'sensor_id': 9, u'name': u'kill', + # u'completion': 1464319733.190924, u'object': 1660, u'session_id': 7, u'result_type': u'WinHresult', + # u'create_time': 1464319733.171967, u'result_desc': u'', u'id': 22, u'result_code': 2147942487} if self.details.get("status") == "error" and self.details.get("result_type") == "WinHresult": # attempt to decode the win32 error @@ -45,21 +56,38 @@ def __init__(self, details): self.decoded_win32_error = winerror.decode_hresult(self.win32_error) if self.decoded_win32_error: win32_error_text += " ({0})".format(self.decoded_win32_error) - except: + except Exception: pass finally: message_list.append(win32_error_text) - + self.message = ": ".join(message_list) def __str__(self): + """ + Return the string equivalent of this exception (the exception's message). + + Returns: + str: The exception's message. + """ return self.message class CbLRSessionBase(object): + """A Live Response session that interacts with a remote machine.""" + MAX_RETRY_COUNT = 5 def __init__(self, cblr_manager, session_id, sensor_id, session_data=None): + """ + Initialize the CbLRSessionBase. + + Args: + cblr_manager (CbLRManagerBase): The Live Response manager governing this session. + session_id (str): The ID of this session. + sensor_id (int): The ID of the sensor (remote machine) we're connected to. + session_data (dict): Additional session data. + """ self.session_id = session_id self.sensor_id = sensor_id self._cblr_manager = cblr_manager @@ -73,16 +101,32 @@ def __init__(self, cblr_manager, session_id, sensor_id, session_data=None): self.cblr_base = self._cblr_manager.cblr_base def __enter__(self): + """Enter the Live Response session context.""" return self def __exit__(self, exc_type, exc_val, exc_tb): + """ + Exit the Live Response session context. + + Args: + exc_type (str): Exception type, if any. + exc_val (Exception): Exception value, if any. + exc_tb (str): Exception traceback, if any. + """ self.close() def close(self): + """Close the Live Response session.""" self._cblr_manager.close_session(self.sensor_id, self.session_id) self._closed = True def get_session_archive(self): + """ + Get the archive data of the current session. + + Returns: + object: Contains the archive data of the current session. + """ response = self._cb.session.get("{cblr_base}/session/{0}/archive".format(self.session_id, cblr_base=self.cblr_base), stream=True) response.raw.decode_content = True @@ -92,6 +136,17 @@ def get_session_archive(self): # File operations # def get_raw_file(self, file_name, timeout=None, delay=None): + """ + Retrieve contents of the specified file on the remote machine. + + Args: + file_name (str): Name of the file to be retrieved. + timeout (int): Timeout for the operation. + delay (float): TBD + + Returns: + object: Contains the data of the file. + """ data = {"name": "get file", "object": file_name} resp = self._lr_post_command(data).json() @@ -108,11 +163,15 @@ def get_raw_file(self, file_name, timeout=None, delay=None): def get_file(self, file_name, timeout=None, delay=None): """ - Retrieve contents of the specified file name + Retrieve contents of the specified file on the remote machine. - :param str file_name: Name of the file - :return: Content of the specified file name - :rtype: str + Args: + file_name (str): Name of the file to be retrieved. + timeout (int): Timeout for the operation. + delay (float): TBD + + Returns: + str: Contents of the specified file. """ fp = self.get_raw_file(file_name, timeout=timeout, delay=delay) content = fp.read() @@ -122,10 +181,10 @@ def get_file(self, file_name, timeout=None, delay=None): def delete_file(self, filename): """ - Delete the specified file name + Delete the specified file name on the remote machine. - :param str filename: Name of the file - :return: None + Args: + filename (str): Name of the file to be deleted. """ data = {"name": "delete file", "object": filename} resp = self._lr_post_command(data).json() @@ -133,17 +192,16 @@ def delete_file(self, filename): self._poll_command(command_id) def put_file(self, infp, remote_filename): - """ - Create a new file on the remote endpoint with the specified data - - :Example: + r""" + Create a new file on the remote machine with the specified data. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... lr_session.put_file(open("test.txt", "rb"), r"c:\test.txt") - :param str infp: Python file-like containing data to upload to the remote endpoint - :param str remote_filename: File name to create on the remote endpoint - :return: None + Args: + infp (object): Python file-like containing data to upload to the remote endpoint. + remote_filename (str): File name to create on the remote endpoint. """ data = {"name": "put file", "object": remote_filename} file_id = self._upload_file(infp) @@ -154,11 +212,10 @@ def put_file(self, infp, remote_filename): self._poll_command(command_id) def list_directory(self, dir_name): - """ - List the contents of a directory - - :Example: + r""" + List the contents of a directory on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... pprint.pprint(lr_session.list_directory('C:\\\\temp\\\\')) [{u'attributes': [u'DIRECTORY'], @@ -180,9 +237,11 @@ def list_directory(self, dir_name): u'last_write_time': 1476390668, u'size': 0}] - :param str dir_name: Directory to list. This parameter should end with '\' - :return: Returns a directory listing - :rtype: list + Args: + dir_name (str): Directory to list. This parameter should end with the path separator. + + Returns: + list: A list of dicts, each one describing a directory entry. """ data = {"name": "directory list", "object": dir_name} resp = self._lr_post_command(data).json() @@ -191,10 +250,10 @@ def list_directory(self, dir_name): def create_directory(self, dir_name): """ - Create a directory on the remote endpoint + Create a directory on the remote machine. - :param str dir_name: New directory name - :return: None + Args: + dir_name (str): The new directory name. """ data = {"name": "create directory", "object": dir_name} resp = self._lr_post_command(data).json() @@ -202,6 +261,15 @@ def create_directory(self, dir_name): self._poll_command(command_id) def path_join(self, *dirnames): + """ + Join multiple directory names into a single one. + + Args: + *dirnames (list): List of directory names to join together. + + Returns: + str: The joined directory name. + """ if self.os_type == 1: # Windows return "\\".join(dirnames) @@ -210,27 +278,37 @@ def path_join(self, *dirnames): return "/".join(dirnames) def path_islink(self, fi): + """ + Determine if the path is a link. Not implemented. + + Args: + fi (str): File to check. + + Returns: + True if the file is a link, False if not. + """ # TODO: implement return False def walk(self, top, topdown=True, onerror=None, followlinks=False): - """ - Perform a full directory walk with recursion into subdirectories - - :Example: + r""" + Perform a full directory walk with recursion into subdirectories on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... for entry in lr_session.walk(directory_name): ... print(entry) ('C:\\temp\\', [u'dir1', u'dir2'], [u'file1.txt']) - :param str top: Directory to recurse - :param bool topdown: if True, start output from top level directory - :param bool onerror: Callback if an error occurs. - This function is called with one argument (the exception that occurred) - :param bool followlinks: Follow symbolic links - :return: Returns output in the follow tuple format: (Directory Name, [dirnames], [filenames]) - :rtype: tuple + Args: + top (str): Directory to recurse on. + topdown (bool): If True, start output from top level directory. + onerror (func): Callback if an error occurs. This function is called with one argument (the exception + that occurred). + followlinks (bool): True to follow symbolic links. + + Returns: + list: List of tuples containing directory name, subdirectory names, file names. """ try: allfiles = self.list_directory(self.path_join(top, "*")) @@ -260,17 +338,19 @@ def walk(self, top, topdown=True, onerror=None, followlinks=False): if not topdown: yield top, [fn["filename"] for fn in dirnames], [fn["filename"] for fn in filenames] - # # Process operations # + def kill_process(self, pid): """ - Terminate a process on the remote endpoint + Terminate a process on the remote machine. + + Args: + pid (int): Process ID to be terminated. - :param pid: Process ID to terminate - :return: True if success, False if failure - :rtype: bool + Returns: + bool: True if success, False if failure. """ data = {"name": "kill", "object": pid} resp = self._lr_post_command(data).json() @@ -286,24 +366,25 @@ def kill_process(self, pid): def create_process(self, command_string, wait_for_output=True, remote_output_file_name=None, working_directory=None, wait_timeout=30, wait_for_completion=True): """ - Create a new process with the specified command string. - - :Example: + Create a new process on the remote machine with the specified command string. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... print(lr_session.create_process(r'cmd.exe /c "ping.exe 192.168.1.1"')) Pinging 192.168.1.1 with 32 bytes of data: Reply from 192.168.1.1: bytes=32 time<1ms TTL=64 - :param str command_string: command string used for the create process operation - :param bool wait_for_output: Block on output from the new process (execute in foreground). This will also set - wait_for_completion (below). - :param str remote_output_file_name: The remote output file name used for process output - :param str working_directory: The working directory of the create process operation - :param int wait_timeout: Time out used for this live response command - :param bool wait_for_completion: Wait until the process is completed before returning - :return: returns the output of the command string - :rtype: str + Args: + command_string (str): Command string used for the create process operation. + wait_for_output (bool): True to block on output from the new process (execute in foreground). + This will also set wait_for_completion (below). + remote_output_file_name (str): The remote output file name used for process output. + working_directory (str): The working directory of the create process operation. + wait_timeout (int): Timeout used for this command. + wait_for_completion (bool): True to wait until the process is completed before returning. + + Returns: + str: The output of the process. """ # process is: # - create a temporary file name @@ -345,11 +426,10 @@ def create_process(self, command_string, wait_for_output=True, remote_output_fil return None def list_processes(self): - """ - List currently running processes - - :Example: + r""" + List currently running processes on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... print(lr_session.list_processes()[0]) {u'command_line': u'', @@ -362,8 +442,8 @@ def list_processes(self): u'sid': u's-1-5-18', u'username': u'NT AUTHORITY\\SYSTEM'} - :return: returns a list of running processes - :rtype: list + Returns: + list: A list of dicts describing the processes. """ data = {"name": "process list"} resp = self._lr_post_command(data).json() @@ -378,13 +458,12 @@ def list_processes(self): # "values" is a list containing a dictionary for each registry value in the key # "sub_keys" is a list containing one entry for each sub_key def list_registry_keys_and_values(self, regkey): - """ - Enumerate subkeys and values of the specified registry key. - - :Example: + r""" + Enumerate subkeys and values of the specified registry key on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: - >>> pprint.pprint(lr_session.list_registry_keys_and_values('HKLM\\SYSTEM\\CurrentControlSet\\services\\ACPI')) + >>> pprint.pprint(lr_session.list_registry_keys_and_values('HKLM\\SYSTEM\\CurrentControlSet\\services\\ACPI')) {'sub_keys': [u'Parameters', u'Enum'], 'values': [{u'value_data': 0, u'value_name': u'Start', @@ -411,9 +490,12 @@ def list_registry_keys_and_values(self, regkey): u'value_name': u'Tag', u'value_type': u'REG_DWORD'}]} - :param str regkey: The registry key to enumerate - :return: returns a dictionary with 2 keys (sub_keys and values) - :rtype: dict + Args: + regkey (str): The registry key to enumerate. + + Returns: + dict: A dictionary with two keys, 'sub_keys' (a list of subkey names) and 'values' (a list of dicts + containing value data, name, and type). """ data = {"name": "reg enum key", "object": regkey} resp = self._lr_post_command(data).json() @@ -426,11 +508,13 @@ def list_registry_keys_and_values(self, regkey): # returns a list containing a dictionary for each registry value in the key def list_registry_keys(self, regkey): """ - Enumerate all registry values from the specified registry key. + Enumerate all registry values from the specified registry key on the remote machine. - :param regkey: The registry key to enumearte - :return: returns a list of values - :rtype: list + Args: + regkey (str): The registry key to enumerate. + + Returns: + list: List of values for the registry key. """ data = {"name": "reg enum key", "object": regkey} resp = self._lr_post_command(data).json() @@ -440,18 +524,19 @@ def list_registry_keys(self, regkey): # returns a dictionary with the registry value def get_registry_value(self, regkey): - """ - Returns the associated value of the specified registry key - - :Example: + r""" + Return the associated value of the specified registry key on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: >>> pprint.pprint(lr_session.get_registry_value('HKLM\\SYSTEM\\CurrentControlSet\\services\\ACPI\\Start')) {u'value_data': 0, u'value_name': u'Start', u'value_type': u'REG_DWORD'} - :param str regkey: The registry key to retrieve - :return: Returns a dictionary with keys of: value_data, value_name, value_type - :rtype: dict + Args: + regkey (str): The registry key to retrieve. + + Returns: + dict: A dictionary with keys of: value_data, value_name, value_type. """ data = {"name": "reg query value", "object": regkey} resp = self._lr_post_command(data).json() @@ -460,19 +545,18 @@ def get_registry_value(self, regkey): return self._poll_command(command_id).get("value", {}) def set_registry_value(self, regkey, value, overwrite=True, value_type=None): - """ - Set a registry value of the specified registry key - - :Example: + r""" + Set a registry value on the specified registry key on the remote machine. + Example: >>> with c.select(Sensor, 1).lr_session() as lr_session: ... lr_session.set_registry_value('HKLM\\\\SYSTEM\\\\CurrentControlSet\\\\services\\\\ACPI\\\\testvalue', 1) - :param str regkey: They registry key to set - :param obj value: The value data - :param bool overwrite: Overwrite value if True - :param str value_type: The type of value. Examples: REG_DWORD, REG_MULTI_SZ, REG_SZ - :return: None + Args: + regkey (str): The registry key to set. + value (object): The value data. + overwrite (bool): If True, any existing value will be overwritten. + value_type (str): The type of value. Examples: REG_DWORD, REG_MULTI_SZ, REG_SZ """ if value_type is None: if type(value) == int: @@ -493,10 +577,10 @@ def set_registry_value(self, regkey, value, overwrite=True, value_type=None): def create_registry_key(self, regkey): """ - Create a new registry + Create a new registry key on the remote machine. - :param str regkey: The registry key to create - :return: None + Args: + regkey (str): The registry key to create. """ data = {"name": "reg create key", "object": regkey} resp = self._lr_post_command(data).json() @@ -505,10 +589,10 @@ def create_registry_key(self, regkey): def delete_registry_key(self, regkey): """ - Delete a registry key + Delete a registry key on the remote machine. - :param str regkey: The registry key to delete - :return: None + Args: + regkey (str): The registry key to delete. """ data = {"name": "reg delete key", "object": regkey} resp = self._lr_post_command(data).json() @@ -517,10 +601,10 @@ def delete_registry_key(self, regkey): def delete_registry_value(self, regkey): """ - Delete a registry value + Delete a registry value on the remote machine. - :param str regkey: the registry value to delete - :return: None + Args: + regkey (str): The registry value to delete. """ data = {"name": "reg delete value", "object": regkey} resp = self._lr_post_command(data).json() @@ -531,12 +615,30 @@ def delete_registry_value(self, regkey): # Physical memory capture # def memdump(self, local_filename, remote_filename=None, compress=False): + """ + Perform a memory dump operation on the remote machine. + + Args: + local_filename (str): Name of the file the memory dump will be transferred to on the local machine. + remote_filename (str): Name of the file the memory dump will be stored in on the remote machine. + compress (bool): True to compress the file on the remote system. + """ dump_object = self.start_memdump(remote_filename=remote_filename, compress=compress) dump_object.wait() dump_object.get(local_filename) dump_object.delete() def start_memdump(self, remote_filename=None, compress=True): + """ + Start a memory dump operation on the remote machine. + + Args: + remote_filename (str): Name of the file the memory dump will be stored in on the remote machine. + compress (bool): True to compress the file on the remote system. + + Returns: + LiveResponseMemdump: Controlling object for the memory dump operation. + """ if not remote_filename: remote_filename = self._random_file_name() @@ -588,10 +690,11 @@ def _lr_post_command(self, data): try: error_message = json.loads(e.message) if error_message["status"] == "NOT_FOUND": - self.session_id, self.session_data = self._cblr_manager._get_or_create_session(self.sensor_id) + self.session_id, self.session_data = \ + self._cblr_manager._get_or_create_session(self.sensor_id) retries -= 1 continue - except: + except Exception: pass raise ApiError("Received 404 error from server: {0}".format(e.message)) else: @@ -601,7 +704,17 @@ def _lr_post_command(self, data): class LiveResponseMemdump(object): + """Object managing a memory dump on a remote machine.""" + def __init__(self, lr_session, memdump_id, remote_filename): + """ + Initialize the LiveResponseMemdump. + + Args: + lr_session (Session): The Live Response session to the machine doing the memory dump. + memdump_id (str): The ID of the memory dump being performed. + remote_filename (str): The file name the memory dump will be stored in on the remote machine. + """ self.lr_session = lr_session self.memdump_id = memdump_id self.remote_filename = remote_filename @@ -609,6 +722,12 @@ def __init__(self, lr_session, memdump_id, remote_filename): self._error = None def get(self, local_filename): + """ + Retrieve the remote memory dump to a local file. + + Args: + local_filename (str): Filename locally that will receive the memory dump. + """ if not self._done: self.wait() if self._error: @@ -618,20 +737,42 @@ def get(self, local_filename): shutil.copyfileobj(src, dst) def wait(self): + """Wait for the remote memory dump to complete.""" self.lr_session._poll_command(self.memdump_id, timeout=3600, delay=5) self._done = True def delete(self): + """Delete the memory dump file.""" self.lr_session.delete_file(self.remote_filename) def jobrunner(callable, cb, sensor_id): + """ + Wrap a callable object with a live response session. + + Args: + callable (object): The object to be wrapped. + cb (BaseAPI): The CBAPI object reference. + sensor_id (int): The sensor ID to use to get the session. + + Returns: + object: The wrapped object. + """ with cb.select(Sensor, sensor_id).lr_session() as sess: return callable(sess) class WorkItem(object): + """Work item for scheduling.""" + def __init__(self, fn, sensor_id): + """ + Initialize the WorkItem. + + Args: + fn (func): The function to be called to do the actual work. + sensor_id (object): The sensor ID or Sensor object the work item is directed for. + """ self.fn = fn if isinstance(sensor_id, Sensor): self.sensor_id = sensor_id.id @@ -642,19 +783,47 @@ def __init__(self, fn, sensor_id): class CompletionNotification(object): + """The notification that an operation is complete.""" + def __init__(self, sensor_id): + """ + Initialize the CompletionNotification. + + Args: + sensor_id (int): The sensor ID this notification is for. + """ self.sensor_id = sensor_id class WorkerStatus(object): + """Holds the status of an individual worker.""" + def __init__(self, sensor_id, status="ready", exception=None): + """ + Initialize the WorkerStatus. + + Args: + sensor_id (int): The sensor ID this status is for. + status (str): The current status value. + exception (Exception): Any exception that happened. + """ self.sensor_id = sensor_id self.status = status self.exception = exception class JobWorker(threading.Thread): + """Thread object that executes individual Live Response jobs.""" + def __init__(self, cb, sensor_id, result_queue): + """ + Initialize the JobWorker. + + Args: + cb (BaseAPI): The CBAPI object reference. + sensor_id (int): The ID of the sensor being used. + result_queue (Queue): The queue where results are placed. + """ super(JobWorker, self).__init__() self.cb = cb self.sensor_id = sensor_id @@ -663,6 +832,7 @@ def __init__(self, cb, sensor_id, result_queue): self.result_queue = result_queue def run(self): + """Execute the job worker.""" try: self.lr_session = self.cb.live_response.request_session(self.sensor_id) self.result_queue.put(WorkerStatus(self.sensor_id, status="ready")) @@ -684,6 +854,12 @@ def run(self): self.result_queue.put(WorkerStatus(self.sensor_id, status="exiting")) def run_job(self, work_item): + """ + Execute an individual WorkItem. + + Args: + work_item (WorkItem): The work item to execute. + """ try: work_item.future.set_result(work_item.fn(self.lr_session)) except Exception as e: @@ -691,9 +867,18 @@ def run_job(self, work_item): class LiveResponseJobScheduler(threading.Thread): + """Thread that schedules Live Response jobs.""" + daemon = True def __init__(self, cb, max_workers=10): + """ + Initialize the LiveResponseJobScheduler. + + Args: + cb (BaseAPI): The CBAPI object reference. + max_workers (int): Maximum number of JobWorker threads to use. + """ super(LiveResponseJobScheduler, self).__init__() self._cb = cb self._job_workers = {} @@ -703,6 +888,7 @@ def __init__(self, cb, max_workers=10): self.schedule_queue = Queue() def run(self): + """Execute the job scheduler.""" log.debug("Starting Live Response Job Scheduler") while True: @@ -785,6 +971,12 @@ def _cleanup_unscheduled_jobs(self): del self._unscheduled_jobs[k] def submit_job(self, work_item): + """ + Submit a new job to be processed. + + Args: + work_item (WorkItem): New job to be processed. + """ self.schedule_queue.put(work_item) def _spawn_new_workers(self): @@ -793,10 +985,24 @@ def _spawn_new_workers(self): schedule_max = self._max_workers - len(self._job_workers) - sensors = [s for s in self._cb.select(Sensor) if s.id in self._unscheduled_jobs - and s.id not in self._job_workers - and s.status == "Online"] - sensors_to_schedule = sorted(sensors, key=lambda x: x.next_checkin_time)[:schedule_max] + sensors = [] + + for sensor_id in self._unscheduled_jobs: + sensor = self._cb.select(Sensor, sensor_id) + if sensor_id in self._job_workers: + log.debug("Skipping sensor {} already has a worker created".format(sensor_id)) + elif sensor.status == "Online": + sensors.append(sensor) + else: + log.warn("Sensor {} could not be found or is not Online".format(sensor_id)) + + # Non performant original code + # sensors = [s for s in self._cb.select(Sensor) if s.id in self._unscheduled_jobs + # and s.id not in self._job_workers + # and s.status == "Online"] + sensors_to_schedule = sorted(sensors, key=lambda x: ( + int(x.num_storefiles_bytes) + int(x.num_eventlog_bytes), x.next_checkin_time + ))[:schedule_max] log.debug("Spawning new workers to handle these sensors: {0}".format(sensors_to_schedule)) for sensor in sensors_to_schedule: @@ -806,10 +1012,20 @@ def _spawn_new_workers(self): class CbLRManagerBase(object): + """Live Response manager object.""" + cblr_base = "" # override in subclass for each product cblr_session_cls = NotImplemented # override in subclass for each product def __init__(self, cb, timeout=30, keepalive_sessions=False): + """ + Initialize the CbLRManagerBase object. + + Args: + cb (BaseAPI): The CBAPI object reference. + timeout (int): Timeout to use for requesus. + keepalive_sessions (bool): If True, "ping" sessions occasionally to ensure they stay alive. + """ self._timeout = timeout self._cb = cb self._sessions = {} @@ -824,6 +1040,16 @@ def __init__(self, cb, timeout=30, keepalive_sessions=False): self._job_scheduler = None def submit_job(self, job, sensor): + """ + Submit a new job to be executed as a Live Response. + + Args: + job (object): The job to be scheduled. + sensor (int): ID of the sensor to use for job execution. + + Returns: + Future: A reference to the running job. + """ if self._job_scheduler is None: # spawn the scheduler thread self._job_scheduler = LiveResponseJobScheduler(self._cb) @@ -850,9 +1076,9 @@ def _session_keepalive_thread(self): log.debug("Session {0} for sensor {1} not valid any longer, removing from cache" .format(session.session_id, session.sensor_id)) delete_list.append(session.sensor_id) - except: - log.debug("Keepalive on session {0} (sensor {1}) failed with unknown error, removing from cache" - .format(session.session_id, session.sensor_id)) + except Exception: + log.debug(("Keepalive on session {0} (sensor {1}) failed with unknown error, " + + "removing from cache").format(session.session_id, session.sensor_id)) delete_list.append(session.sensor_id) for sensor_id in delete_list: @@ -860,6 +1086,15 @@ def _session_keepalive_thread(self): del self._sessions[sensor_id] def request_session(self, sensor_id): + """ + Initiate a new Live Response session. + + Args: + sensor_id (int): The sensor ID to use. + + Returns: + CbLRSessionBase: The new Live Response session. + """ if self._keepalive_sessions: with self._session_lock: if sensor_id in self._sessions: @@ -876,6 +1111,13 @@ def request_session(self, sensor_id): return session def close_session(self, sensor_id, session_id): + """ + Close the specified Live Response session. + + Args: + sensor_id (int): ID of the sensor. + session_id (int): ID of the session. + """ if self._keepalive_sessions: with self._session_lock: try: @@ -891,15 +1133,48 @@ def _send_keepalive(self, session_id): class GetFileJob(object): + """Object that retrieves a file via Live Response.""" + def __init__(self, file_name): + """ + Initialize the GetFileJob. + + Args: + file_name (str): The name of the file to be fetched. + """ self._file_name = file_name def run(self, session): + """ + Execute the file transfer. + + Args: + session (CbLRSessionBase): The Live Response session being used. + + Returns: + TBD + """ return session.get_file(self._file_name) # TODO: adjust the polling interval and also provide a callback function to report progress def poll_status(cb, url, desired_status="complete", timeout=None, delay=None): + """ + Poll the status of a Live Response query. + + Args: + cb (BaseAPI): The CBAPI object reference. + url (str): The URL to poll. + desired_status (str): The status we're looking for. + timeout (int): The timeout value in seconds. + delay (float): The delay between attempts in seconds. + + Returns: + object: The result of the Live Response query that has the desired status. + + Raises: + LiveResponseError: If an error response was encountered. + """ start_time = time.time() status = None diff --git a/src/cbapi/models.py b/src/cbapi/models.py index 4143d899..539ef777 100644 --- a/src/cbapi/models.py +++ b/src/cbapi/models.py @@ -8,7 +8,7 @@ import base64 import os.path -from cbapi.six import iteritems, add_metaclass +from cbapi.six import iteritems, add_metaclass, integer_types from cbapi.six.moves import range from .response.utils import convert_from_cb, convert_to_cb import yaml @@ -35,8 +35,9 @@ def __new__(mcs, name, bases, clsdict): swagger_meta_file = clsdict.pop("swagger_meta_file", None) model_data = {} if swagger_meta_file: - model_data = yaml.safe_load( - open(os.path.join(mcs.model_base_directory, swagger_meta_file), 'rb').read()) + with open(os.path.join(mcs.model_base_directory, swagger_meta_file), 'rb') as f: + model_data = yaml.safe_load(f.read()) + clsdict["__doc__"] = "Represents a %s object in the Carbon Black server.\n\n" % (name,) for field_name, field_info in iteritems(model_data.get("properties", {})): @@ -144,7 +145,7 @@ def __init__(self, field_name, multiplier=1.0): def __get__(self, instance, instance_type=None): d = super(EpochDateTimeFieldDescriptor, self).__get__(instance, instance_type) - if type(d) is float or type(d) is int or type(d) is long: + if type(d) is float or type(d) in integer_types: epoch_seconds = d / self.multiplier return datetime.utcfromtimestamp(epoch_seconds) else: @@ -229,7 +230,7 @@ def new_object(cls, cb, item, **kwargs): def __getattr__(self, item): try: - val = super(NewBaseModel, self).__getattribute__(item) + super(NewBaseModel, self).__getattribute__(item) except AttributeError: pass # fall through to the rest of the logic... @@ -412,7 +413,7 @@ def _update_object(self): else: log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) http_method = self.__class__._change_object_http_method - ret = self._cb.api_json_request(http_method,self._build_api_request_uri(http_method=http_method), + ret = self._cb.api_json_request(http_method, self._build_api_request_uri(http_method=http_method), data=self._info) return self._refresh_if_needed(ret) @@ -423,7 +424,7 @@ def _refresh_if_needed(self, request_ret): if request_ret.status_code not in range(200, 300): try: message = json.loads(request_ret.text)[0] - except: + except Exception: message = request_ret.text raise ServerError(request_ret.status_code, message, @@ -448,7 +449,7 @@ def _refresh_if_needed(self, request_ret): refresh_required = True else: self._full_init = True - except: + except Exception: refresh_required = True self._dirty_attributes = {} @@ -486,7 +487,7 @@ def _delete_object(self): if ret.status_code not in (200, 204): try: message = json.loads(ret.text)[0] - except: + except Exception: message = ret.text raise ServerError(ret.status_code, message, result="Did not delete {0:s}.".format(str(self))) @@ -505,4 +506,3 @@ def __repr__(self): if self.is_dirty(): r += " (*)" return r - diff --git a/src/cbapi/oldmodels.py b/src/cbapi/oldmodels.py index 64f6ac88..defb4d41 100644 --- a/src/cbapi/oldmodels.py +++ b/src/cbapi/oldmodels.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +"""Base model objects for the CBAPI.""" + from __future__ import absolute_import from functools import wraps @@ -14,14 +16,28 @@ class CreatableModelMixin(object): + """TBD.""" + pass # TODO: this doesn't exactly do what I want... this needs to be cleaned up before release def immutable(cls): + """Decorate a model object in such a way that allows it to be specified as "immutable".""" cls.__frozen = False def frozensetattr(self, key, value): + """ + Set an attribute on this object, but disallow it if the object is frozen. + + Args: + self (object): The object being changed by this attribute setting operation. + key (str): The attribute being changed. + value (object): The new value being set for the attribute. + + Returns: + TBD + """ if self.__frozen and not hasattr(self, key): print("Class {} is frozen. Cannot set {} = {}" .format(cls.__name__, key, value)) @@ -29,6 +45,15 @@ def frozensetattr(self, key, value): object.__setattr__(self, key, value) def init_decorator(func): + """ + Initialize the decorator on a function. + + Args: + func (func): Function to be wrapped by this decorator. + + Returns: + func: The wrapped function. + """ @wraps(func) def wrapper(self, *args, **kwargs): func(self, *args, **kwargs) @@ -43,7 +68,18 @@ def wrapper(self, *args, **kwargs): @python_2_unicode_compatible class BaseModel(object): + """The base "model" class for all objects that the CBAPI works with.""" + def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False): + """ + Initialize the BaseModel object. + + Args: + cb (BaseAPI): Reference to the CBAPI object. + model_unique_id (object): Unique ID for the model object. + initial_data (dict): Initial data for the model object. + force_init (bool): True to force the object to be re-initialized from the server. + """ self._cb = cb if model_unique_id is not None: self._model_unique_id = str(model_unique_id) @@ -78,13 +114,22 @@ def _stat_titles(self): @classmethod def new_object(cls, cb, item): + """ + Create a new instance of the object from item data. + + Args: + cb (BaseAPI): Reference to the CBAPI object. + item (dict): Item data, as retrieved from the server. + + Returns: + object: New instance of the object. + """ # TODO: do we ever need to evaluate item['unique_id'] which is the id + segment id? # TODO: is there a better way to handle this? (see how this is called from Query._search()) return cb.select(cls, item['id'], initial_data=item) def refresh(self): - """Refresh the object from the Carbon Black server. - """ + """Refresh the object from the Carbon Black server.""" self._retrieve_cb_info() def _retrieve_cb_info(self, query_parameters=None): @@ -106,20 +151,37 @@ def _build_api_request_uri(self): @property def webui_link(self): - """Returns a link associated with this object in the Carbon Black user interface. + """ + Return a link associated with this object in the Carbon Black user interface. - :returns: URL that can be used to view the object in the Carbon Black web user interface or None if the Model - does not support generating a Web user interface URL + Returns: + str: URL that can be used to view the object in the Carbon Black web user interface, or None if the Model + does not support generating a Web user interface URL """ return None def __dir__(self): + """ + Fetch a list of all attribute names defined for this object. + + Returns: + list: A list of strings, each of which represents an attribute name on the object. + """ if not self._full_init: self._retrieve_cb_info() return list(set().union(self.__dict__.keys(), self._info.keys())) def __getattr__(self, attrname): + """ + Return the value of an attribute on this object. + + Args: + attrname (str): Name of the attribute to return. + + Returns: + object: The attribute value. + """ try: object.__getattribute__(self, "_base_initialized") except AttributeError: @@ -144,12 +206,28 @@ def _attribute(self, attrname, default=None): return default def get(self, attrname, default_val=None): + """ + Return the value of an attribute on this object. + + Args: + attrname (str): Name of the attribute to be returned. + default_val (object): The default value to be returned if the attribute is undefined. + + Returns: + object: The attribute value, which may be default_val. + """ try: - return self._attribute(attrname) + return self._attribute(attrname, default_val) except AttributeError: return default_val def __str__(self): + """ + Return a representation of this object as a string. + + Returns: + str: A representation of this object as a string. + """ ret = '{0:s}.{1:s}:\n'.format(self.__class__.__module__, self.__class__.__name__) if self.webui_link: ret += "-> available via web UI at %s\n" % self.webui_link @@ -160,17 +238,35 @@ def __str__(self): return ret def __repr__(self): + """ + Return a representation of this object as a string. + + Returns: + str: A representation of this object as a string. + """ return "<%s.%s: id %s> @ %s" % (self.__class__.__module__, self.__class__.__name__, self._model_unique_id, self._cb.session.server) @property def original_document(self): + """ + Return the original document associated with this object. + + Returns: + object: Original document associated with this object. + """ if not self._full_init: self._retrieve_cb_info() return self._info def to_html(self): + """ + Return a representation of this object as HTML. + + Returns: + str: HTML representation of the object. + """ ret = u"

%s

" % self.__class__.__name__ ret += u"\n" for a in self._stat_titles: @@ -186,7 +282,17 @@ def _repr_html_(self): class MutableModel(BaseModel): + """A model object that implements mutability, the ability to change its values.""" + def __init__(self, cb, model_unique_id, initial_data=None): + """ + Initialize the MutableModel. + + Args: + cb (BaseAPI): Reference to the CBAPI object. + model_unique_id (object): Unique ID for the model object. + initial_data (dict): Initial data for the model object. + """ super(MutableModel, self).__init__(cb, model_unique_id, initial_data) self._dirty_attributes = {} self._mutable_initialized = True @@ -199,6 +305,16 @@ def _target_val(self, attrname, val): return val def __setattr__(self, attrname, val): + """ + Set an attribute on the model object. + + Args: + attrname (str): Name of the attribute to be set. + val (object): Value of the object to be set. + + Returns: + TBD + """ try: object.__getattribute__(self, "_mutable_initialized") except AttributeError: @@ -238,8 +354,11 @@ def __setattr__(self, attrname, val): super(MutableModel, self).__setattr__(attrname, val) def is_dirty(self): - """Returns True if this object has unsaved changes. Use :py:meth:`MutableModel.save` to upload the changes to - the Carbon Black server.""" + """ + Return True if this object has unsaved changes. + + Use MutableModel.save to upload the changes to the Carbon Black server. + """ return len(self._dirty_attributes) > 0 def _update_object(self): @@ -253,7 +372,7 @@ def _update_object(self): if ret.status_code not in (200, 204): try: message = json.loads(ret.text)[0] - except: + except Exception: message = ret.text raise ServerError(ret.status_code, message, @@ -273,7 +392,7 @@ def _update_object(self): else: self._info = json.loads(ret.text) self._full_init = True - except: + except Exception: self.refresh() if not self._model_unique_id: @@ -283,6 +402,7 @@ def _update_object(self): return self._model_unique_id def refresh(self): + """Refresh the contents of this obejct from the server.""" super(MutableModel, self).refresh() self._dirty_attributes = {} @@ -295,14 +415,16 @@ def _delete_object(self): if ret.status_code not in (200, 204): try: message = json.loads(ret.text)[0] - except: + except Exception: message = ret.text raise ServerError(ret.status_code, message, result="Did not delete {0:s}.".format(str(self))) def save(self): - """Save changes to this object to the Carbon Black server. + """ + Save changes to this object to the Carbon Black server. - :raises ServerError: if an error was returned by the Carbon Black server + Raises: + ServerError: If an error was returned by the Carbon Black server. """ if not self.is_dirty(): return @@ -310,6 +432,7 @@ def save(self): return self._update_object() def reset(self): + """Reset all changes made to this object.""" for k, v in iteritems(self._dirty_attributes): self._info[k] = v @@ -317,6 +440,7 @@ def reset(self): # TODO: How do we delete this object from our LRU cache? def delete(self): + """Delete this object.""" return self._delete_object() def _join(self, join_cls, field_name): diff --git a/src/cbapi/protection/models.py b/src/cbapi/protection/models.py index 9eeee882..5b982767 100644 --- a/src/cbapi/protection/models.py +++ b/src/cbapi/protection/models.py @@ -3,7 +3,11 @@ from ..oldmodels import BaseModel, immutable, MutableModel from ..models import MutableBaseModel, CreatableModelMixin, NewBaseModel from contextlib import closing -from distutils.version import LooseVersion +import sys +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from zipfile import ZipFile import cbapi.six as six @@ -95,10 +99,11 @@ def _build_api_request_uri(self, http_method="GET"): for n in ["debugLevel", "kernelDebugLevel", "debugFlags", "debugDuration", "ccLevel", "ccFlags"]): args.append("changeDiagnostics=true") if any(n in self._dirty_attributes - for n in ["template", "templateCloneCleanupMode", "templateCloneCleanupTime", "templateCloneCleanupTimeScale", "templateTrackModsOnly"]): + for n in ["template", "templateCloneCleanupMode", "templateCloneCleanupTime", + "templateCloneCleanupTimeScale", "templateTrackModsOnly"]): args.append("changeTemplate=true") if "tamperProtectionActive" in self._dirty_attributes: - if self.get("tamperProtectionActive", True) == True: + if self.get("tamperProtectionActive", True): args.append("newTamperProtectionActive=true") else: args.append("newTamperProtectionActive=false") @@ -145,7 +150,7 @@ class DriftReport(NewBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class DriftReportContents(NewBaseModel): @@ -153,7 +158,7 @@ class DriftReportContents(NewBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class Event(NewBaseModel): @@ -279,7 +284,7 @@ class GrantedUserPolicyPermission(NewBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") @immutable @@ -373,7 +378,7 @@ class PublisherCertificate(NewBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class ScriptRule(MutableBaseModel): @@ -381,7 +386,7 @@ class ScriptRule(MutableBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") @immutable @@ -412,7 +417,7 @@ class TrustedDirectory(MutableBaseModel): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class TrustedUser(MutableBaseModel, CreatableModelMixin): @@ -421,7 +426,7 @@ class TrustedUser(MutableBaseModel, CreatableModelMixin): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class User(MutableBaseModel, CreatableModelMixin): @@ -430,7 +435,7 @@ class User(MutableBaseModel, CreatableModelMixin): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") + return parse("8.0") class UserGroup(MutableBaseModel, CreatableModelMixin): @@ -439,5 +444,4 @@ class UserGroup(MutableBaseModel, CreatableModelMixin): @classmethod def _minimum_server_version(cls): - return LooseVersion("8.0") - + return parse("8.0") diff --git a/src/cbapi/protection/rest_api.py b/src/cbapi/protection/rest_api.py index ee722277..628c0016 100644 --- a/src/cbapi/protection/rest_api.py +++ b/src/cbapi/protection/rest_api.py @@ -1,7 +1,11 @@ from ..utils import convert_query_params from ..query import PaginatedQuery from ..errors import UnauthorizedError, ApiError -from distutils.version import LooseVersion +import sys +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from cbapi.connection import BaseAPI @@ -11,15 +15,15 @@ class CbProtectionAPI(BaseAPI): - """The main entry point into the Carbon Black Enterprise Protection API. + """The main entry point into the Carbon Black App Control API. :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. Uses the profile named 'default' when not specified. Usage:: - >>> from cbapi import CbEnterpriseProtectionAPI - >>> cb = CbEnterpriseProtectionAPI(profile="production") + >>> from cbapi import CbProtectionAPI + >>> cb = CbProtectionAPI(profile="production") """ def __init__(self, *args, **kwargs): super(CbProtectionAPI, self).__init__(product_name="protection", *args, **kwargs) @@ -30,8 +34,9 @@ def __init__(self, *args, **kwargs): except UnauthorizedError: raise UnauthorizedError(uri=self.url, message="Invalid API token for server {0:s}.".format(self.url)) - log.debug('Connected to Cb server version %s at %s' % (self._server_info['ParityServerVersion'], self.session.server)) - self.cb_server_version = LooseVersion(self._server_info['ParityServerVersion']) + log.debug('Connected to Cb server version %s at %s' + % (self._server_info['ParityServerVersion'], self.session.server)) + self.cb_server_version = parse(self._server_info['ParityServerVersion']) def _perform_query(self, cls, **kwargs): if hasattr(cls, "_query_implementation"): @@ -40,7 +45,8 @@ def _perform_query(self, cls, **kwargs): return Query(cls, self, **kwargs) def _populate_server_info(self): - self._server_info = dict((sc['name'], sc['value']) for sc in self.get_object("/api/bit9platform/v1/serverConfig")) + self._server_info = dict((sc['name'], sc['value']) + for sc in self.get_object("/api/bit9platform/v1/serverConfig")) @property def info(self): @@ -74,11 +80,11 @@ class CbEnterpriseProtectionAPI(CbProtectionAPI): class Query(PaginatedQuery): - """Represents a prepared query to the Carbon Black Enterprise Protection server. + """Represents a prepared query to the Carbon Black App Control server. - This object is returned as part of a :py:meth:`CbEnterpriseProtectionAPI.select` + This object is returned as part of a :py:meth:`CbProtectionAPI.select` operation on models requested from the Carbon Black - Enterprise Protection server. You should not have to create this class yourself. + App Control server. You should not have to create this class yourself. The query is not executed on the server until it's accessed, either as an iterator (where it will generate values on demand as they're requested) or as a list (where it will retrieve the entire result set and save to a list). @@ -86,12 +92,12 @@ class Query(PaginatedQuery): the query. The syntax for query :py:meth:where and :py:meth:sort methods can be found in the - `Enterprise Protection API reference`_ posted on the Carbon Black Developer Network website. + `App Control REST API reference`_ posted on the Carbon Black Developer Network website. Examples:: - >>> from cbapi.protection import CbEnterpriseProtectionAPI, Computer - >>> cb = CbEnterpriseProtectionAPI() + >>> from cbapi.protection import CbProtectionAPI, Computer + >>> cb = CbProtectionAPI() >>> query = cb.select(Computer) # returns a Query object matching all Computers >>> query = query.where("ipAddress:10.201.2.*") # add a filter to this Query >>> query = query.sort("processorSpeed DESC") # sort by computer processor speed, descending @@ -106,7 +112,7 @@ class Query(PaginatedQuery): - You can chain where clauses together to create AND queries; only objects that match all ``where`` clauses will be returned. - .. _Enterprise Protection API reference: + .. _App Control REST API reference: https://developer.carbonblack.com/reference/enterprise-protection/8.0/rest-api/ """ def __init__(self, doc_class, cb, query=None): @@ -132,11 +138,11 @@ def _clone(self): def where(self, q): """Add a filter to this query. - :param str q: Query string - see the `Enterprise Protection API reference`_. + :param str q: Query string - see the `App Control REST API reference`_. :return: Query object :rtype: :py:class:`Query` - .. _Enterprise Protection API reference: + .. _App Control REST API reference: https://developer.carbonblack.com/reference/enterprise-protection/8.0/rest-api/ """ nq = self._clone() @@ -146,11 +152,11 @@ def where(self, q): def and_(self, q): """Add a filter to this query. Equivalent to calling :py:meth:`where` on this object. - :param str q: Query string - see the `Enterprise Protection API reference`_. + :param str q: Query string - see the `App Control REST API reference`_. :return: Query object :rtype: :py:class:`Query` - .. _Enterprise Protection API reference: + .. _App Control REST API reference: https://developer.carbonblack.com/reference/enterprise-protection/8.0/rest-api/ """ return self.where(q) @@ -158,11 +164,11 @@ def and_(self, q): def sort(self, new_sort): """Set the sort order for this query. - :param str new_sort: Sort order - see the `Enterprise Protection API reference`_. + :param str new_sort: Sort order - see the `App Control REST API reference`_. :return: Query object :rtype: :py:class:`Query` - .. _Enterprise Protection API reference: + .. _App Control REST API reference: https://developer.carbonblack.com/reference/enterprise-protection/8.0/rest-api/ """ new_sort = new_sort.strip() @@ -179,7 +185,8 @@ def _count(self): args['q'] = self._query query_args = convert_query_params(args) - self._total_results = int(self._cb.get_object(self._doc_class.urlobject, query_parameters=query_args).get("count", 0)) + self._total_results = int(self._cb.get_object(self._doc_class.urlobject, + query_parameters=query_args).get("count", 0)) self._count_valid = True return self._total_results diff --git a/src/cbapi/psc/__init__.py b/src/cbapi/psc/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/cbapi/psc/defense/__init__.py b/src/cbapi/psc/defense/__init__.py deleted file mode 100644 index 97378861..00000000 --- a/src/cbapi/psc/defense/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Exported public API for the Cb Defense API - -from __future__ import absolute_import - -from .rest_api import CbDefenseAPI -from cbapi.psc.defense.models import Device, Event, Policy \ No newline at end of file diff --git a/src/cbapi/psc/defense/cblr.py b/src/cbapi/psc/defense/cblr.py deleted file mode 100644 index 6341a9f0..00000000 --- a/src/cbapi/psc/defense/cblr.py +++ /dev/null @@ -1,241 +0,0 @@ -import logging -import threading -from cbapi.six.moves.queue import Queue -from collections import defaultdict -from concurrent.futures import _base - -from cbapi.errors import TimeoutError -from cbapi.live_response_api import CbLRManagerBase, CbLRSessionBase, poll_status -from cbapi.psc.defense.models import Device - -OS_LIVE_RESPONSE_ENUM = { - "WINDOWS": 1, - "LINUX": 2, - "MAC": 4 -} - -log = logging.getLogger(__name__) - -class LiveResponseSession(CbLRSessionBase): - def __init__(self, cblr_manager, session_id, sensor_id, session_data=None): - super(LiveResponseSession, self).__init__(cblr_manager, session_id, sensor_id, session_data=session_data) - device_info = self._cb.select(Device, self.sensor_id) - self.os_type = OS_LIVE_RESPONSE_ENUM.get(device_info.deviceType, None) - -class WorkItem(object): - def __init__(self, fn, sensor_id): - self.fn = fn - if isinstance(sensor_id, Device): - self.sensor_id = sensor_id.deviceId - else: - self.sensor_id = int(sensor_id) - - self.future = _base.Future() -class CompletionNotification(object): - def __init__(self, sensor_id): - self.sensor_id = sensor_id - - -class WorkerStatus(object): - def __init__(self, sensor_id, status="ready", exception=None): - self.sensor_id = sensor_id - self.status = status - self.exception = exception - - -class JobWorker(threading.Thread): - def __init__(self, cb, sensor_id, result_queue): - super(JobWorker, self).__init__() - self.cb = cb - self.sensor_id = sensor_id - self.job_queue = Queue() - self.lr_session = None - self.result_queue = result_queue - - def run(self): - try: - self.lr_session = self.cb.live_response.request_session(self.sensor_id) - self.result_queue.put(WorkerStatus(self.sensor_id, status="ready")) - - while True: - work_item = self.job_queue.get(block=True) - if not work_item: - self.job_queue.task_done() - return - - self.run_job(work_item) - self.result_queue.put(CompletionNotification(self.sensor_id)) - self.job_queue.task_done() - except Exception as e: - self.result_queue.put(WorkerStatus(self.sensor_id, status="error", exception=e)) - finally: - if self.lr_session: - self.lr_session.close() - self.result_queue.put(WorkerStatus(self.sensor_id, status="exiting")) - - def run_job(self, work_item): - try: - work_item.future.set_result(work_item.fn(self.lr_session)) - except Exception as e: - work_item.future.set_exception(e) - -class LiveResponseSessionManager(CbLRManagerBase): - cblr_base = "/integrationServices/v3/cblr" - cblr_session_cls = LiveResponseSession - - def submit_job(self, job, sensor): - if self._job_scheduler is None: - # spawn the scheduler thread - self._job_scheduler = LiveResponseJobScheduler(self._cb) - self._job_scheduler.start() - - work_item = WorkItem(job, sensor) - self._job_scheduler.submit_job(work_item) - return work_item.future - - def _get_or_create_session(self, sensor_id): - session_id = self._create_session(sensor_id) - - try: - res = poll_status(self._cb, "{cblr_base}/session/{0}".format(session_id, cblr_base=self.cblr_base), - desired_status="ACTIVE", delay=1, timeout=360) - except Exception: - # "close" the session, otherwise it will stay in a pending state - self._close_session(session_id) - - # the Cb server will return a 404 if we don't establish a session in time, so convert this to a "timeout" - raise TimeoutError(uri="{cblr_base}/session/{0}".format(session_id, cblr_base=self.cblr_base), - message="Could not establish session with sensor {0}".format(sensor_id), - error_code=404) - else: - return session_id, res - - def _close_session(self, session_id): - try: - self._cb.put_object("{cblr_base}/session".format(session_id, cblr_base=self.cblr_base), - {"session_id": session_id, "status": "CLOSE"}) - except: - pass - - def _create_session(self, sensor_id): - response = self._cb.post_object("{cblr_base}/session/{0}".format(sensor_id, cblr_base=self.cblr_base), - {"sensor_id": sensor_id}).json() - session_id = response["id"] - return session_id - -class LiveResponseJobScheduler(threading.Thread): - daemon = True - - def __init__(self, cb, max_workers=10): - super(LiveResponseJobScheduler, self).__init__() - self._cb = cb - self._job_workers = {} - self._idle_workers = set() - self._unscheduled_jobs = defaultdict(list) - self._max_workers = max_workers - self.schedule_queue = Queue() - - def run(self): - log.debug("Starting Live Response Job Scheduler") - - while True: - log.debug("Waiting for item on Scheduler Queue") - item = self.schedule_queue.get(block=True) - log.debug("Got item: {0}".format(item)) - if isinstance(item, WorkItem): - # new WorkItem available - self._unscheduled_jobs[item.sensor_id].append(item) - elif isinstance(item, CompletionNotification): - # job completed - self._idle_workers.add(item.sensor_id) - elif isinstance(item, WorkerStatus): - if item.status == "error": - log.error("Error encountered by JobWorker[{0}]: {1}".format(item.sensor_id, - item.exception)) - elif item.status == "exiting": - log.debug("JobWorker[{0}] has exited, waiting...".format(item.sensor_id)) - self._job_workers[item.sensor_id].join() - log.debug("JobWorker[{0}] deleted".format(item.sensor_id)) - del self._job_workers[item.sensor_id] - try: - self._idle_workers.remove(item.sensor_id) - except KeyError: - pass - elif item.status == "ready": - log.debug("JobWorker[{0}] now ready to accept jobs, session established".format(item.sensor_id)) - self._idle_workers.add(item.sensor_id) - else: - log.debug("Unknown status from JobWorker[{0}]: {1}".format(item.sensor_id, item.status)) - else: - log.debug("Received unknown item on the scheduler Queue, exiting") - # exiting the scheduler if we get None - # TODO: wait for all worker threads to exit - return - - self._schedule_jobs() - - def _schedule_jobs(self): - log.debug("Entering scheduler") - - # First, see if there are new jobs to schedule on idle workers. - self._schedule_existing_workers() - - # If we have jobs scheduled to run on sensors with no current associated worker, let's spawn new ones. - if set(self._unscheduled_jobs.keys()) - self._idle_workers: - self._cleanup_idle_workers() - self._spawn_new_workers() - self._schedule_existing_workers() - - def _cleanup_idle_workers(self, max=None): - if not max: - max = self._max_workers - - for sensor in list(self._idle_workers)[:max]: - log.debug("asking worker for sensor id {0} to exit".format(sensor)) - self._job_workers[sensor].job_queue.put(None) - - def _schedule_existing_workers(self): - log.debug("There are idle workers for sensor ids {0}".format(self._idle_workers)) - - intersection = self._idle_workers.intersection(set(self._unscheduled_jobs.keys())) - - log.debug("{0} jobs ready to execute in existing execution slots".format(len(intersection))) - - for sensor in intersection: - item = self._unscheduled_jobs[sensor].pop(0) - self._job_workers[sensor].job_queue.put(item) - self._idle_workers.remove(item.sensor_id) - - self._cleanup_unscheduled_jobs() - - def _cleanup_unscheduled_jobs(self): - marked_for_deletion = [] - for k in self._unscheduled_jobs.keys(): - if len(self._unscheduled_jobs[k]) == 0: - marked_for_deletion.append(k) - - for k in marked_for_deletion: - del self._unscheduled_jobs[k] - - def submit_job(self, work_item): - self.schedule_queue.put(work_item) - - def _spawn_new_workers(self): - if len(self._job_workers) >= self._max_workers: - return - - schedule_max = self._max_workers - len(self._job_workers) - ''' - sensors = [s for s in self._cb.select(Device) if s.deviceId in self._unscheduled_jobs - and s.deviceId not in self._job_workers - and "AVAILABLE" in s.sensorStates] - ''' - log.debug("spawning new workers to handle unscheduled jobs: {0}".format(self._unscheduled_jobs)) - sensors = [s for s in self._cb.select(Device) if s.deviceId in self._unscheduled_jobs - and s.deviceId not in self._job_workers] - sensors_to_schedule = sensors[:schedule_max] - log.debug("Spawning new workers to handle these sensors: {0}".format(sensors_to_schedule)) - for sensor in sensors_to_schedule: - log.debug("Spawning new JobWorker for sensor id {0}".format(sensor.deviceId)) - self._job_workers[sensor.deviceId] = JobWorker(self._cb, sensor.deviceId, self.schedule_queue) - self._job_workers[sensor.deviceId].start() \ No newline at end of file diff --git a/src/cbapi/psc/defense/models.py b/src/cbapi/psc/defense/models.py deleted file mode 100644 index b68f34e0..00000000 --- a/src/cbapi/psc/defense/models.py +++ /dev/null @@ -1,164 +0,0 @@ -from cbapi.models import MutableBaseModel, CreatableModelMixin, NewBaseModel - -from copy import deepcopy -import logging -import json - -from cbapi.errors import ServerError - -log = logging.getLogger(__name__) - - -class DefenseMutableModel(MutableBaseModel): - _change_object_http_method = "PATCH" - _change_object_key_name = None - - def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=False): - super(DefenseMutableModel, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=force_init, full_doc=full_doc) - if not self._change_object_key_name: - self._change_object_key_name = self.primary_key - - def _parse(self, obj): - if type(obj) == dict and self.info_key in obj: - return obj[self.info_key] - - def _update_object(self): - if self._change_object_http_method != "PATCH": - return self._update_entire_object() - else: - return self._patch_object() - - def _update_entire_object(self): - if self.__class__.primary_key in self._dirty_attributes.keys() or self._model_unique_id is None: - new_object_info = deepcopy(self._info) - try: - if not self._new_object_needs_primary_key: - del(new_object_info[self.__class__.primary_key]) - except Exception: - pass - log.debug("Creating a new {0:s} object".format(self.__class__.__name__)) - ret = self._cb.api_json_request(self.__class__._new_object_http_method, self.urlobject, - data={self.info_key: new_object_info}) - else: - log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) - ret = self._cb.api_json_request(self.__class__._change_object_http_method, - self._build_api_request_uri(), data={self.info_key: self._info}) - - return self._refresh_if_needed(ret) - - def _patch_object(self): - if self.__class__.primary_key in self._dirty_attributes.keys() or self._model_unique_id is None: - log.debug("Creating a new {0:s} object".format(self.__class__.__name__)) - ret = self._cb.api_json_request(self.__class__._new_object_http_method, self.urlobject, - data=self._info) - else: - updates = {} - for k in self._dirty_attributes.keys(): - updates[k] = self._info[k] - log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) - ret = self._cb.api_json_request(self.__class__._change_object_http_method, - self._build_api_request_uri(), data=updates) - - return self._refresh_if_needed(ret) - - def _refresh_if_needed(self, request_ret): - refresh_required = True - - if request_ret.status_code not in range(200, 300): - try: - message = json.loads(request_ret.text)[0] - except: - message = request_ret.text - - raise ServerError(request_ret.status_code, message, - result="Did not update {} record.".format(self.__class__.__name__)) - else: - try: - message = request_ret.json() - log.debug("Received response: %s" % message) - if not isinstance(message, dict): - raise ServerError(request_ret.status_code, message, - result="Unknown error updating {0:s} record.".format(self.__class__.__name__)) - else: - if message.get("success", False): - if isinstance(message.get(self.info_key, None), dict): - self._info = message.get(self.info_key) - self._full_init = True - refresh_required = False - else: - if self._change_object_key_name in message.keys(): - # if all we got back was an ID, try refreshing to get the entire record. - log.debug("Only received an ID back from the server, forcing a refresh") - self._info[self.primary_key] = message[self._change_object_key_name] - refresh_required = True - else: - # "success" is False - raise ServerError(request_ret.status_code, message.get("message", ""), - result="Did not update {0:s} record.".format(self.__class__.__name__)) - except: - pass - - self._dirty_attributes = {} - if refresh_required: - self.refresh() - return self._model_unique_id - - -class Device(DefenseMutableModel): - urlobject = "/integrationServices/v3/device" - primary_key = "deviceId" - info_key = "deviceInfo" - swagger_meta_file = "psc/defense/models/deviceInfo.yaml" - - def __init__(self, cb, model_unique_id, initial_data=None): - super(Device, self).__init__(cb, model_unique_id, initial_data) - - def lr_session(self): - """ - Retrieve a Live Response session object for this Device. - - :return: Live Response session object - :rtype: :py:class:`cbapi.defense.cblr.LiveResponseSession` - :raises ApiError: if there is an error establishing a Live Response session for this Device - - """ - return self._cb._request_lr_session(self._model_unique_id) - - -class Event(NewBaseModel): - urlobject = "/integrationServices/v3/event" - primary_key = "eventId" - info_key = "eventInfo" - - def _parse(self, obj): - if type(obj) == dict and self.info_key in obj: - return obj[self.info_key] - - def __init__(self, cb, model_unique_id, initial_data=None): - super(Event, self).__init__(cb, model_unique_id, initial_data) - - -class Policy(DefenseMutableModel, CreatableModelMixin): - urlobject = "/integrationServices/v3/policy" - info_key = "policyInfo" - swagger_meta_file = "psc/defense/models/policyInfo.yaml" - _change_object_http_method = "PUT" - _change_object_key_name = "policyId" - - @property - def rules(self): - return dict([(r.get("id"), r) for r in self.policy.get("rules", [])]) - - def add_rule(self, new_rule): - self._cb.post_object("{0}/rule".format(self._build_api_request_uri()), {"ruleInfo": new_rule}) - self.refresh() - - def delete_rule(self, rule_id): - self._cb.delete_object("{0}/rule/{1}".format(self._build_api_request_uri(), rule_id)) - self.refresh() - - def replace_rule(self, rule_id, new_rule): - self._cb.put_object("{0}/rule/{1}".format(self._build_api_request_uri(), rule_id), - {"ruleInfo": new_rule}) - self.refresh() diff --git a/src/cbapi/psc/defense/models/deviceInfo.yaml b/src/cbapi/psc/defense/models/deviceInfo.yaml deleted file mode 100644 index 7d7106a7..00000000 --- a/src/cbapi/psc/defense/models/deviceInfo.yaml +++ /dev/null @@ -1,221 +0,0 @@ -type: object -properties: - osVersion: - type: string - activationCode: - type: string - organizationId: - type: integer - format: int64 - deviceId: - type: integer - format: int64 - deviceSessionId: - type: integer - format: int64 - deviceOwnerId: - type: integer - format: int64 - deviceGuid: - type: string - format: uuid - email: - type: string - format: email - assignedToId: - type: integer - format: int64 - assignedToName: - type: string - deviceType: - type: string - x-nullable: true - enum: - - "MAC" - - "WINDOWS" - firstName: - type: string - lastName: - type: string - middleName: - type: string - createTime: - type: integer - format: epoch-ms-date-time - policyId: - type: integer - format: int64 - policyName: - type: string - quarantined: - type: boolean - targetPriorityType: - type: string - x-nullable: true - enum: - - "HIGH" - - "LOW" - - "MEDIUM" - - "MISSION_CRITICAL" - lastVirusActivityTime: - type: integer - format: epoch-ms-date-time - firstVirusActivityTime: - type: integer - format: epoch-ms-date-time - activationCodeExpiryTime: - type: integer - format: epoch-ms-date-time - organizationName: - type: string - sensorVersion: - type: string - registeredTime: - type: integer - format: epoch-ms-date-time - lastContact: - type: integer - format: epoch-ms-date-time - lastReportedTime: - type: integer - format: epoch-ms-date-time - windowsPlatform: - type: string - x-nullable: true - enum: - - "CLIENT_X64" - - "CLIENT_X86" - - "SERVER_X6" - - "SERVER_X86" - vdiBaseDevice: - type: integer - format: int64 - avStatus: - type: array - items: - type: string - x-nullable: true - enum: - - "AV_ACTIVE" - - "AV_BYPASS" - - "AV_DEREGISTERED" - - "AV_NOT_REGISTERED" - - "AV_REGISTERED" - - "FULLY_DISABLED" - - "FULLY_ENABLED" - - "INSTALLED" - - "INSTALLED_SERVER" - - "NOT_INSTALLED" - - "ONACCESS_SCAN_DISABLED" - - "ONDEMAND_SCAN_DISABLED" - - "ONDEMOND_SCAN_DISABLED" - - "PRODUCT_UPDATE_DISABLED" - - "SIGNATURE_UPDATE_DISABLED" - - "UNINSTALLED" - - "UNINSTALLED_SERVER" - deregisteredTime: - type: integer - format: epoch-ms-date-time - sensorStates: - type: array - items: - type: string - x-nullable: true - enum: - - "ACTIVE" - - "CSR_ACTION" - - "DB_CORRUPTION_DETECTED" - - "DRIVER_INIT_ERROR" - - "LOOP_DETECTED" - - "PANICS_DETECTED" - - "REMGR_INIT_ERROR" - - "REPUX_ACTION" - - "SENSOR_MAINTENANCE" - - "SENSOR_RESET_IN_PROGRESS" - - "SENSOR_SHUTDOWN" - - "SENSOR_UNREGISTERED" - - "SENSOR_UPGRADE_IN_PROGRESS" - - "UNSUPPORTED_OS" - - "WATCHDOG" - messages: - type: array - items: - type: object - properties: - message: - type: string - time: - type: integer - format: epoch-ms-date-time - rootedBySensor: - type: boolean - rootedBySensorTime: - type: integer - format: epoch-ms-date-time - lastInternalIpAddress: - type: string - lastExternalIpAddress: - type: string - lastLocation: - type: string - x-nullable: true - enum: - - "OFFSITE" - - "ONSITE" - - "UNKNOWN" - avUpdateServers: - type: array - items: - type: string - passiveMode: - type: boolean - lastResetTime: - type: integer - format: epoch-ms-date-time - lastShutdownTime: - type: integer - format: epoch-ms-date-time - scanStatus: - type: string - scanLastActionTime: - type: integer - format: epoch-ms-date-time - scanLastCompleteTime: - type: integer - format: epoch-ms-date-time - linuxKernelVersion: - type: string - avEngine: - type: string - avLastScanTime: - type: integer - format: epoch-ms-date-time - rootedByAnalytics: - type: boolean - rootedByAnalyticsTime: - type: integer - format: epoch-ms-date-time - testId: - type: integer - avMaster: - type: boolean - uninstalledTime: - type: integer - format: epoch-ms-date-time - name: - type: string - status: - type: string - x-nullable: true - enum: - - "ACTIVE" - - "ALL" - - "BYPASS" - - "BYPASS_ON" - - "DEREGISTERED" - - "ERROR" - - "INACTIVE" - - "PENDING" - - "QUARANTINE" - - "REGISTERED" - - "UNINSTALLED" diff --git a/src/cbapi/psc/defense/models/policyInfo.yaml b/src/cbapi/psc/defense/models/policyInfo.yaml deleted file mode 100644 index b13dc207..00000000 --- a/src/cbapi/psc/defense/models/policyInfo.yaml +++ /dev/null @@ -1,25 +0,0 @@ -type: object -required: -- description -- name -- policy -- priorityLevel -- version -properties: - description: - type: string - id: - type: integer - latestRevision: - type: integer - name: - type: string - policy: - type: object - priorityLevel: - type: string - systemPolicy: - type: boolean - version: - type: integer - diff --git a/src/cbapi/psc/defense/rest_api.py b/src/cbapi/psc/defense/rest_api.py deleted file mode 100644 index cf28e67b..00000000 --- a/src/cbapi/psc/defense/rest_api.py +++ /dev/null @@ -1,189 +0,0 @@ -from cbapi.utils import convert_query_params -from cbapi.query import PaginatedQuery -from .cblr import LiveResponseSessionManager - -from cbapi.connection import BaseAPI -import logging -import time - -log = logging.getLogger(__name__) - - -def convert_to_kv_pairs(q): - k, v = q.split(':', 1) - return k, v - - -class CbDefenseAPI(BaseAPI): - """The main entry point into the Cb Defense API. - - :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. - Uses the profile named 'default' when not specified. - - Usage:: - - >>> from cbapi import CbDefenseAPI - >>> cb = CbDefenseAPI(profile="production") - """ - def __init__(self, *args, **kwargs): - super(CbDefenseAPI, self).__init__(product_name="psc", *args, **kwargs) - self._lr_scheduler = None - - def _perform_query(self, cls, query_string=None): - return Query(cls, self, query_string) - - def notification_listener(self, interval=60): - """Generator to continually poll the Cb Defense server for notifications (alerts). Note that this can only - be used with a 'SIEM' key generated in the Cb Defense console. - """ - while True: - for notification in self.get_notifications(): - yield notification - time.sleep(interval) - - def get_notifications(self): - """Retrieve queued notifications (alerts) from the Cb Defense server. Note that this can only be used - with a 'SIEM' key generated in the Cb Defense console. - - :returns: list of dictionary objects representing the notifications, or an empty list if none available. - """ - res = self.get_object("/integrationServices/v3/notification") - return res.get("notifications", []) - - @property - def live_response(self): - if self._lr_scheduler is None: - self._lr_scheduler = LiveResponseSessionManager(self) - - return self._lr_scheduler - - def _request_lr_session(self, sensor_id): - return self.live_response.request_session(sensor_id) - - -class Query(PaginatedQuery): - """Represents a prepared query to the Cb Defense server. - - This object is returned as part of a :py:meth:`CbDefenseAPI.select` - operation on models requested from the Cb Defense server. You should not have to create this class yourself. - - The query is not executed on the server until it's accessed, either as an iterator (where it will generate values - on demand as they're requested) or as a list (where it will retrieve the entire result set and save to a list). - You can also call the Python built-in ``len()`` on this object to retrieve the total number of items matching - the query. - - Examples:: - - >>> from cbapi.psc.defense import CbDefenseAPI - >>> cb = CbDefenseAPI() - - Notes: - - The slicing operator only supports start and end parameters, but not step. ``[1:-1]`` is legal, but - ``[1:2:-1]`` is not. - - You can chain where clauses together to create AND queries; only objects that match all ``where`` clauses - will be returned. - """ - def __init__(self, doc_class, cb, query=None): - super(Query, self).__init__(doc_class, cb, None) - if query: - self._query = [query] - else: - self._query = [] - - self._sort_by = None - self._group_by = None - self._batch_size = 100 - - def _clone(self): - nq = self.__class__(self._doc_class, self._cb) - nq._query = self._query[::] - nq._sort_by = self._sort_by - nq._group_by = self._group_by - nq._batch_size = self._batch_size - return nq - - def where(self, q): - """Add a filter to this query. - - :param str q: Query string - :return: Query object - :rtype: :py:class:`Query` - """ - nq = self._clone() - nq._query.append(q) - return nq - - def and_(self, q): - """Add a filter to this query. Equivalent to calling :py:meth:`where` on this object. - - :param str q: Query string - :return: Query object - :rtype: :py:class:`Query` - """ - return self.where(q) - - def prepare_query(self, args): - if self._query: - for qe in self._query: - k, v = convert_to_kv_pairs(qe) - args[k] = v - - return args - - def _count(self): - args = {'limit': 0} - args = self.prepare_query(args) - - query_args = convert_query_params(args) - self._total_results = int(self._cb.get_object(self._doc_class.urlobject, query_parameters=query_args) - .get("totalResults", 0)) - self._count_valid = True - return self._total_results - - def _search(self, start=0, rows=0): - # iterate over total result set, 1000 at a time - args = {} - if start != 0: - args['start'] = start - args['rows'] = self._batch_size - - current = start - numrows = 0 - - args = self.prepare_query(args) - still_querying = True - - while still_querying: - query_args = convert_query_params(args) - result = self._cb.get_object(self._doc_class.urlobject, query_parameters=query_args) - - self._total_results = result.get("totalResults", 0) - self._count_valid = True - - results = result.get('results', []) - - if results is None: - log.debug("Results are None") - if current >= 100000: - log.info("Max result size exceeded. Truncated to 100k.") - break - - for item in results: - yield item - current += 1 - numrows += 1 - if rows and numrows == rows: - still_querying = False - break - - args['start'] = current + 1 # as of 6/2017, the indexing on the Cb Defense backend is still 1-based - - if current >= self._total_results: - break - - if not results: - log.debug("server reported total_results overestimated the number of results for this query by {0}" - .format(self._total_results - current)) - log.debug("resetting total_results for this query to {0}".format(current)) - self._total_results = current - break diff --git a/src/cbapi/psc/livequery/__init__.py b/src/cbapi/psc/livequery/__init__.py deleted file mode 100644 index d48b5b1d..00000000 --- a/src/cbapi/psc/livequery/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# Exported public API for the Cb LiveQuery API - -from __future__ import absolute_import - -from .rest_api import CbLiveQueryAPI -from cbapi.psc.livequery.models import Run, RunHistory, Result, DeviceSummary -from cbapi.psc.livequery.models import ResultFacet, DeviceSummaryFacet diff --git a/src/cbapi/psc/livequery/models.py b/src/cbapi/psc/livequery/models.py deleted file mode 100644 index 82b501da..00000000 --- a/src/cbapi/psc/livequery/models.py +++ /dev/null @@ -1,291 +0,0 @@ -from __future__ import absolute_import -from cbapi.models import UnrefreshableModel, NewBaseModel -from cbapi.errors import ApiError, ServerError -from .query import RunQuery, RunHistoryQuery, ResultQuery, FacetQuery -import logging -import time - -log = logging.getLogger(__name__) - - -class Run(NewBaseModel): - """ - Represents a LiveQuery run. - - Example:: - - >>> run = cb.select(Run, run_id) - >>> print(run.name, run.sql, run.create_time) - >>> print(run.status, run.match_count) - >>> run.refresh() - """ - primary_key = "id" - swagger_meta_file = "psc/livequery/models/run.yaml" - urlobject = "/livequery/v1/orgs/{}/runs" - urlobject_single = "/livequery/v1/orgs/{}/runs/{}" - _is_deleted = False - - def __init__(self, cb, model_unique_id=None, initial_data=None): - if initial_data is not None: - item = initial_data - elif model_unique_id is not None: - url = self.urlobject_single.format(cb.credentials.org_key, model_unique_id) - item = cb.get_object(url) - - model_unique_id = item.get("id") - - super(Run, self).__init__( - cb, - model_unique_id=model_unique_id, - initial_data=item, - force_init=False, - full_doc=True, - ) - - @classmethod - def _query_implementation(cls, cb): - return RunQuery(cls, cb) - - def _refresh(self): - if self._is_deleted: - raise ApiError("cannot refresh a deleted query") - url = self.urlobject_single.format(self._cb.credentials.org_key, self.id) - resp = self._cb.get_object(url) - self._info = resp - self._last_refresh_time = time.time() - return True - - def stop(self): - if self._is_deleted: - raise ApiError("cannot stop a deleted query") - url = self.urlobject_single.format(self._cb.credentials.org_key, self.id) + "/status" - result = self._cb.put_object(url, {'status': 'CANCELLED'}) - if (result.status_code == 200): - try: - self._info = result.json() - self._last_refresh_time = time.time() - return True - except: - raise ServerError(result.status_code, "Cannot parse response as JSON: {0:s}".format(result.content)) - return False - - def delete(self): - if self._is_deleted: - return True # already deleted - url = self.urlobject_single.format(self._cb.credentials.org_key, self.id) - result = self._cb.delete_object(url) - if result.status_code == 200: - self._is_deleted = True - return True - return False - -class RunHistory(Run): - """ - Represents a historical LiveQuery ``Run``. - """ - urlobject_history = "/livequery/v1/orgs/{}/runs/_search" - - def __init__(self, cb, initial_data=None): - item = initial_data - model_unique_id = item.get("id") - super(Run, self).__init__(cb, - model_unique_id, initial_data=item, - force_init=False, full_doc=True) - - @classmethod - def _query_implementation(cls, cb): - return RunHistoryQuery(cls, cb) - -class Result(UnrefreshableModel): - """ - Represents a single result from a LiveQuery ``Run``. - """ - primary_key = "id" - swagger_meta_file = "psc/livequery/models/result.yaml" - urlobject = "/livequery/v1/orgs/{}/runs/{}/results/_search" - - class Device(UnrefreshableModel): - """ - Represents device information for a result. - """ - primary_key = "id" - - def __init__(self, cb, initial_data): - super(Result.Device, self).__init__( - cb, - model_unique_id=initial_data["id"], - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - - class Fields(UnrefreshableModel): - """ - Represents the fields of a result. - """ - def __init__(self, cb, initial_data): - super(Result.Fields, self).__init__( - cb, - model_unique_id=None, - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - - class Metrics(UnrefreshableModel): - """ - Represents the metrics for a result. - """ - def __init__(self, cb, initial_data): - super(Result.Metrics, self).__init__( - cb, - model_unique_id=None, - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - - @classmethod - def _query_implementation(cls, cb): - return ResultQuery(cls, cb) - - def __init__(self, cb, initial_data): - super(Result, self).__init__( - cb, - model_unique_id=initial_data["id"], - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - self._run_id = initial_data["id"] - self._device = Result.Device(cb, initial_data=initial_data["device"]) - self._fields = Result.Fields(cb, initial_data=initial_data["fields"]) - if "metrics" in initial_data: - self._metrics = Result.Metrics(cb, initial_data=initial_data["metrics"]) - else: - self._metrics = Result.Metrics(cb, initial_data=None) - - @property - def device_(self): - """ - Returns the reified ``Result.Device`` for this result. - """ - return self._device - - @property - def fields_(self): - """ - Returns the reified ``Result.Fields`` for this result. - """ - return self._fields - - @property - def metrics_(self): - """ - Returns the reified ``Result.Metrics`` for this result. - """ - return self._metrics - - def query_device_summaries(self): - return self._cb.select(DeviceSummary).run_id(self._run_id) - - def query_result_facets(self): - return self._cb.select(ResultFacet).run_id(self._run_id) - - def query_device_summary_facets(self): - return self._cb.select(DeviceSummaryFacet).run_id(self._run_id) - - -class DeviceSummary(UnrefreshableModel): - """ - Represents the summary of results from a single device during a single LiveQuery ``Run``. - """ - primary_key = "id" - swagger_meta_file = "psc/livequery/models/device_summary.yaml" - urlobject = "/livequery/v1/orgs/{}/runs/{}/results/device_summaries/_search" - - class Metrics(UnrefreshableModel): - """ - Represents the metrics for a result. - """ - def __init__(self, cb, initial_data): - super(DeviceSummary.Metrics, self).__init__( - cb, - model_unique_id=None, - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - - @classmethod - def _query_implementation(cls, cb): - return ResultQuery(cls, cb) - - def __init__(self, cb, initial_data): - super(DeviceSummary, self).__init__( - cb, - model_unique_id=initial_data["id"], - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - self._metrics = DeviceSummary.Metrics(cb, initial_data=initial_data["metrics"]) - - @property - def metrics_(self): - """ - Returns the reified ``DeviceSummary.Metrics`` for this result. - """ - return self._metrics - -class ResultFacet(UnrefreshableModel): - """ - Represents the summary of results for a single field in a LiveQuery ``Run``. - """ - primary_key = "field" - swagger_meta_file = "psc/livequery/models/facet.yaml" - urlobject = "/livequery/v1/orgs/{}/runs/{}/results/_facet" - - class Values(UnrefreshableModel): - """ - Represents the values associated with a field. - """ - def __init__(self, cb, initial_data): - super(ResultFacet.Values, self).__init__( - cb, - model_unique_id=None, - initial_data=initial_data, - force_init=False, - full_doc=True, - ) - - @classmethod - def _query_implementation(cls, cb): - return FacetQuery(cls, cb) - - def __init__(self, cb, initial_data): - super(ResultFacet, self).__init__( - cb, - model_unique_id=None, - initial_data=initial_data, - force_init=False, - full_doc=True - ) - self._values = ResultFacet.Values(cb, initial_data=initial_data["values"]) - - @property - def values_(self): - """ - Returns the reified ``ResultFacet.Values`` for this result. - """ - return self._values - -class DeviceSummaryFacet(ResultFacet): - """ - Represents the summary of results for a single device summary in a LiveQuery ``Run``. - """ - urlobject = "/livequery/v1/orgs/{}/runs/{}/results/device_summaries/_facet" - - def __init__(self, cb, initial_data): - super(DeviceSummaryFacet, self).__init__(cb, initial_data) - \ No newline at end of file diff --git a/src/cbapi/psc/livequery/models/device_summary.yaml b/src/cbapi/psc/livequery/models/device_summary.yaml deleted file mode 100755 index a4b32c88..00000000 --- a/src/cbapi/psc/livequery/models/device_summary.yaml +++ /dev/null @@ -1,52 +0,0 @@ -type: object -required: [] # TODO(ww): Find out which result fields are required -properties: - id: - type: string - description: The result's unique ID - total_results: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Number of results returned for this particular device - device_id: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The device's unique ID - device_name: - type: string - description: The device's name - time_received: - type: string - description: The time at which this result was received # NOTE(ww): This is a guess - format: date-time - status: - type: string - description: The result's status - device_message: - type: string - description: Placeholder # TODO(ww): Needs description - os: - type: string - description: The device's operating system - policy_id: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The device's policy ID - policy_name: - type: string - description: The device's policy name - metrics: - type: array - description: Metrics associated with the device - items: - type: object - description: Individual metrics entries - properties: - key: - type: string - description: The name of the particular metric - value: - type: number - format: float - description: The value of the particular metric - \ No newline at end of file diff --git a/src/cbapi/psc/livequery/models/facet.yaml b/src/cbapi/psc/livequery/models/facet.yaml deleted file mode 100755 index b20151b3..00000000 --- a/src/cbapi/psc/livequery/models/facet.yaml +++ /dev/null @@ -1,22 +0,0 @@ -type: object -required: [] # TODO(ww): Find out which result fields are required -properties: - field: - type: string - description: The name of the field being summarized - values: - type: array - items: - type: object - properties: - total: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The total number of times this value appears in the query output - id: - type: string - description: The ID of the value being enumerated - name: - type: string - description: The name of the value being enumerated - \ No newline at end of file diff --git a/src/cbapi/psc/livequery/models/result.yaml b/src/cbapi/psc/livequery/models/result.yaml deleted file mode 100644 index ff1d3bc8..00000000 --- a/src/cbapi/psc/livequery/models/result.yaml +++ /dev/null @@ -1,41 +0,0 @@ -type: object -required: [] # TODO(ww): Find out which result fields are required -properties: - id: - type: string - description: The result's unique ID - device: - type: object - description: The device associated with the result - properties: - id: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The device's unique ID - name: - type: string - description: The device's name - policy_id: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The device's policy ID - policy_name: - type: string - description: The device's policy name - status: - type: string - description: The result's status - time_received: - type: string - description: The time at which this result was received # NOTE(ww): This is a guess - device_message: - type: string - description: Placeholder # TODO(ww): Needs description - fields: - type: object - description: The fields returned by the backing osquery query - metrics: # TODO(ww): Document each field in metrics? - type: object - description: Metrics associated with the result's host - - diff --git a/src/cbapi/psc/livequery/models/run.yaml b/src/cbapi/psc/livequery/models/run.yaml deleted file mode 100644 index 089407be..00000000 --- a/src/cbapi/psc/livequery/models/run.yaml +++ /dev/null @@ -1,86 +0,0 @@ -type: object -required: # TODO(ww): Find out which run fields are required - - id -properties: - template_id: - type: string # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - org_key: - type: string - description: The organization key for this run - name: - type: string - description: The name of the LiveQuery run - id: - type: string - description: The run's unique ID - sql: - type: string - description: The LiveQuery query - created_by: - type: string - description: Placeholder # TODO(ww): Needs description - create_time: - type: string - description: When this run was created - status_update_time: - type: string - description: When the status of this run was last updated - timeout_time: - type: string - description: Placeholder # TODO(ww): Needs description - cancellation_time: - type: string - description: Placeholder # TODO(ww): Needs description - cancelled_by: - type: string # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - archive_time: - type: string - description: Placeholder # TODO(ww): Needs description - archived_by: - type: string # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - notify_on_finish: - type: boolean - description: Whether or not to send an email on query completion - active_org_devices: - type: integer - format: int32 # NOTE(ww): This is a guess - description: The number of devices active in the organization - status: - type: string - description: The run status - device_filter: - type: object - description: Any device filter rules associated with the run - schedule: - type: string # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - last_result_time: - type: string - description: When the most recent result for this run was reported - total_results: - type: integer - format: int32 - description: Placeholder # TODO(ww): Needs description - match_count: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - no_match_count: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - error_count: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - not_supported_count: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description - cancelled_count: - type: integer - format: int32 # NOTE(ww): This is a guess - description: Placeholder # TODO(ww): Needs description diff --git a/src/cbapi/psc/livequery/query.py b/src/cbapi/psc/livequery/query.py deleted file mode 100644 index 4ee5df99..00000000 --- a/src/cbapi/psc/livequery/query.py +++ /dev/null @@ -1,746 +0,0 @@ -from cbapi.errors import ApiError, MoreThanOneResultError -import logging -import functools -from six import string_types -from solrq import Q - -log = logging.getLogger(__name__) - - -class QueryBuilder(object): - """ - Provides a flexible interface for building prepared queries for the CB - LiveQuqery backend. - - This object can be instantiated directly, or can be managed implicitly - through the :py:meth:`CbLiveQuqeryAPI.select` API. - """ - - def __init__(self, **kwargs): - if kwargs: - self._query = Q(**kwargs) - else: - self._query = None - self._raw_query = None - - def _guard_query_params(func): - """Decorates the query construction methods of *QueryBuilder*, preventing - them from being called with parameters that would result in an intetnally - inconsistent query. - """ - - @functools.wraps(func) - def wrap_guard_query_change(self, q, **kwargs): - if self._raw_query is not None and (kwargs or isinstance(q, Q)): - raise ApiError("Cannot modify a raw query with structured parameters") - if self._query is not None and isinstance(q, string_types): - raise ApiError("Cannot modify a structured query with a raw parameter") - return func(self, q, **kwargs) - - return wrap_guard_query_change - - @_guard_query_params - def where(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - if self._raw_query is None: - self._raw_query = [] - self._raw_query.append(q) - elif isinstance(q, Q) or kwargs: - if self._query is not None: - raise ApiError("Use .and_() or .or_() for an extant solrq.Q object") - if kwargs: - q = Q(**kwargs) - self._query = q - else: - raise ApiError(".where() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def and_(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - self.where(q) - elif isinstance(q, Q) or kwargs: - if kwargs: - q = Q(**kwargs) - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".and_() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def or_(self, q, **kwargs): - """Adds a disjunctive filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - q = Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query | q - else: - raise ApiError(".or_() only accepts solrq.Q objects") - - return self - - @_guard_query_params - def not_(self, q, **kwargs): - """Adds a negative filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - q = ~Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".not_() only accepts solrq.Q objects") - - def _collapse(self): - """The query can be represented by either an array of strings - (_raw_query) which is concatenated and passed directly to Solr, or - a solrq.Q object (_query) which is then converted into a string to - pass to Solr. This function will perform the appropriate conversions to - end up with the 'q' string sent into the POST request to the - PSC-R query endpoint.""" - if self._raw_query is not None: - return " ".join(self._raw_query) - elif self._query is not None: - return str(self._query) - else: - return None # return everything - - -class LiveQueryBase: - """ - Represents the base of all LiveQuery query classes. - """ - - def __init__(self, doc_class, cb): - self._doc_class = doc_class - self._cb = cb - - -class IterableQueryMixin: - """ - A mix-in to provide iterability to a query. - """ - - def all(self): - return self._perform_query() - - def first(self): - allres = list(self) - res = allres[:1] - if not len(res): - return None - return res[0] - - def one(self): - allres = list(self) - res = allres[:2] - if len(res) == 0: - raise MoreThanOneResultError( - message="0 results for query {0:s}".format(self._query) - ) - if len(res) > 1: - raise MoreThanOneResultError( - message="{0:d} results found for query {1:s}".format( - len(self), self._query - ) - ) - return res[0] - - def __len__(self): - return 0 - - def __getitem__(self, item): - return None - - def __iter__(self): - return self._perform_query() - - -class RunQuery(LiveQueryBase): - """ - Represents a query that either creates or retrieves the - status of a LiveQuery run. - """ - - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_token = None - self._query_body = {"device_filter": {}} - self._device_filter = self._query_body["device_filter"] - - def device_ids(self, device_ids): - """ - Restricts the devices that this LiveQuery run is performed on - to the given IDs. - - :param device_ids: list of ints - :return: This instance - """ - if not all(isinstance(device_id, int) for device_id in device_ids): - raise ApiError("One or more invalid device IDs") - self._device_filter["device_ids"] = device_ids - return self - - def device_types(self, device_types): - """ - Restricts the devices that this LiveQuery run is performed on - to the given device types. - - :param device_types: list of strs - :return: This instance - """ - if not all(isinstance(device_type, str) for device_type in device_types): - raise ApiError("One or more invalid device types") - self._device_filter["device_types"] = device_types - return self - - def policy_ids(self, policy_ids): - """ - Restricts this LiveQuery run to the given policy IDs. - - :param policy_ids: list of ints - :return: This instance - """ - if not all(isinstance(policy_id, int) for policy_id in policy_ids): - raise ApiError("One or more invalid policy IDs") - self._device_filter["policy_ids"] = policy_ids - return self - - def where(self, sql): - """ - Sets this LiveQuery run's underlying SQL. - - :param sql: The SQL to execute - :return: This instance - """ - self._query_body["sql"] = sql - return self - - def name(self, name): - """ - Sets this LiveQuery run's name. If no name is explicitly set, - the run is named after its SQL. - - :param name: The run name - :return: This instance - """ - self._query_body["name"] = name - return self - - def notify_on_finish(self): - """ - Sets the notify-on-finish flag on this LiveQuery run. - - :return: This instance - """ - self._query_body["notify_on_finish"] = True - return self - - def submit(self): - """ - Submits this LiveQuery run. - - :return: A new ``Run`` instance containing the run's status - """ - if self._query_token is not None: - raise ApiError( - "Query already submitted: token {0}".format(self._query_token) - ) - - if "sql" not in self._query_body: - raise ApiError("Missing LiveQuery SQL") - - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) - resp = self._cb.post_object(url, body=self._query_body) - - return self._doc_class(self._cb, initial_data=resp.json()) - - -class RunHistoryQuery(LiveQueryBase, IterableQueryMixin): - """ - Represents a query that retrieves historic LiveQuery runs. - """ - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_builder = QueryBuilder() - self._sort = {} - - def where(self, q=None, **kwargs): - """Add a filter to this query. - - :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q: - return self - if isinstance(q, QueryBuilder): - self._query_builder = q - else: - self._query_builder.where(q, **kwargs) - return self - - def and_(self, q=None, **kwargs): - """Add a conjunctive filter to this query. - - :param q: Query string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") - - self._query_builder.and_(q, **kwargs) - return self - - def or_(self, q=None, **kwargs): - """Add a disjunctive filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".or_() expects a solrq.Q or kwargs") - - self._query_builder.or_(q, **kwargs) - return self - - def not_(self, q=None, **kwargs): - """Adds a negated filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q and not kwargs: - raise ApiError(".not_() expects a solrq.Q, or kwargs") - - self._query_builder.not_(q, **kwargs) - return self - - def sort_by(self, key, direction="ASC"): - """Sets the sorting behavior on a query's results. - - Example:: - - >>> cb.select(Result).run_id(my_run).where(username="foobar").sort_by("uid") - - :param key: the key in the schema to sort by - :param direction: the sort order, either "ASC" or "DESC" - :rtype: :py:class:`ResultQuery` - """ - self._sort.update({"field": key, "order": direction}) - return self - - def _build_request(self, start, rows): - request = {"start": start } - - if self._query_builder: - request["query"] = self._query_builder._collapse(); - if rows != 0: - request["rows"] = rows - if self._sort: - request["sort"] = [self._sort] - - return request - - def _count(self): - if self._count_valid: - return self._total_results - - url = self._doc_class.urlobject_history.format( - self._cb.credentials.org_key - ) - request = self._build_request(start=0, rows=0) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - return self._total_results - - def _perform_query(self, start=0, rows=0): - url = self._doc_class.urlobject_history.format( - self._cb.credentials.org_key - ) - current = start - numrows = 0 - still_querying = True - while still_querying: - request = self._build_request(start, rows) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - results = result.get("results", []) - for item in results: - yield self._doc_class(self._cb, item) - current += 1 - numrows += 1 - - if rows and numrows == rows: - still_querying = False - break - - start = current - if current >= self._total_results: - still_querying = False - break - - -class ResultQuery(LiveQueryBase, IterableQueryMixin): - """ - Represents a query that retrieves results from a LiveQuery run. - """ - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_builder = QueryBuilder() - self._criteria = {} - self._sort = {} - self._batch_size = 100 - self._run_id = None - - def where(self, q=None, **kwargs): - """Add a filter to this query. - - :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError( - ".where() expects a string, a QueryBuilder, a solrq.Q, or kwargs" - ) - - if isinstance(q, QueryBuilder): - self._query_builder = q - else: - self._query_builder.where(q, **kwargs) - return self - - def and_(self, q=None, **kwargs): - """Add a conjunctive filter to this query. - - :param q: Query string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") - - self._query_builder.and_(q, **kwargs) - return self - - def or_(self, q=None, **kwargs): - """Add a disjunctive filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".or_() expects a solrq.Q or kwargs") - - self._query_builder.or_(q, **kwargs) - return self - - def not_(self, q=None, **kwargs): - """Adds a negated filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q and not kwargs: - raise ApiError(".not_() expects a solrq.Q, or kwargs") - - self._query_builder.not_(q, **kwargs) - return self - - def criteria(self, **kwargs): - """Sets the filter criteria on a query's results. - - Example:: - - >>> cb.select(Result).run_id(my_run).criteria(device_id=[123, 456]) - - """ - self._criteria.update(kwargs) - return self - - def sort_by(self, key, direction="ASC"): - """Sets the sorting behavior on a query's results. - - Example:: - - >>> cb.select(Result).run_id(my_run).where(username="foobar").sort_by("uid") - - :param key: the key in the schema to sort by - :param direction: the sort order, either "ASC" or "DESC" - :rtype: :py:class:`ResultQuery` - """ - self._sort.update({"field": key, "order": direction}) - return self - - def run_id(self, run_id): - """Sets the run ID to query results for. - - Example:: - - >>> cb.select(Result).run_id(my_run) - """ - self._run_id = run_id - return self - - def _build_request(self, start, rows): - request = {"start": start, "query": self._query_builder._collapse()} - - if rows != 0: - request["rows"] = rows - if self._criteria: - request["criteria"] = self._criteria - if self._sort: - request["sort"] = [self._sort] - - return request - - def _count(self): - if self._count_valid: - return self._total_results - - if self._run_id is None: - raise ApiError("Can't retrieve count without a run ID") - - url = self._doc_class.urlobject.format( - self._cb.credentials.org_key, self._run_id - ) - request = self._build_request(start=0, rows=0) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - return self._total_results - - def _perform_query(self, start=0, rows=0): - if self._run_id is None: - raise ApiError("Can't retrieve results without a run ID") - - url = self._doc_class.urlobject.format( - self._cb.credentials.org_key, self._run_id - ) - current = start - numrows = 0 - still_querying = True - while still_querying: - request = self._build_request(start, rows) - resp = self._cb.post_object(url, body=request) - result = resp.json() - - self._total_results = result["num_found"] - self._count_valid = True - - results = result.get("results", []) - for item in results: - yield self._doc_class(self._cb, item) - current += 1 - numrows += 1 - - if rows and numrows == rows: - still_querying = False - break - - start = current - if current >= self._total_results: - still_querying = False - break - - -class FacetQuery(LiveQueryBase, IterableQueryMixin): - """ - Represents a query that receives facet information from a LiveQuery run. - """ - def __init__(self, doc_class, cb): - super().__init__(doc_class, cb) - self._query_builder = QueryBuilder() - self._facet_fields = [] - self._criteria = {} - self._run_id = None - - def where(self, q=None, **kwargs): - """Add a filter to this query. - - :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError( - ".where() expects a string, a QueryBuilder, a solrq.Q, or kwargs" - ) - - if isinstance(q, QueryBuilder): - self._query_builder = q - else: - self._query_builder.where(q, **kwargs) - return self - - def and_(self, q=None, **kwargs): - """Add a conjunctive filter to this query. - - :param q: Query string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") - - self._query_builder.and_(q, **kwargs) - return self - - def or_(self, q=None, **kwargs): - """Add a disjunctive filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".or_() expects a solrq.Q or kwargs") - - self._query_builder.or_(q, **kwargs) - return self - - def not_(self, q=None, **kwargs): - """Adds a negated filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q and not kwargs: - raise ApiError(".not_() expects a solrq.Q, or kwargs") - - self._query_builder.not_(q, **kwargs) - return self - - def facet_field(self, field): - """Sets the facet fields to be received by this query. - - Example:: - - >>> cb.select(ResultFacet).run_id(my_run).facet_field(["device.policy_name", "device.os"]) - - :param field: Field(s) to be received, either single string or list of strings - :return: Query object - :rtype: :py:class:`Query` - """ - if isinstance(field, string_types): - self._facet_fields.append(field) - else: - for name in field: - self._facet_fields.append(name) - return self - - def criteria(self, **kwargs): - """Sets the filter criteria on a query's results. - - Example:: - - >>> cb.select(ResultFacet).run_id(my_run).criteria(device_id=[123, 456]) - - """ - self._criteria.update(kwargs) - return self - - def run_id(self, run_id): - """Sets the run ID to query results for. - - Example:: - - >>> cb.select(ResultFacet).run_id(my_run) - """ - self._run_id = run_id - return self - - def _build_request(self, rows): - terms = { "fields": self._facet_fields } - if rows != 0: - terms["rows"] = rows - request = {"query": self._query_builder._collapse(), "terms": terms} - if self._criteria: - request["criteria"] = self._criteria - return request - - def _perform_query(self, rows=0): - if self._run_id is None: - raise ApiError("Can't retrieve results without a run ID") - - url = self._doc_class.urlobject.format( - self._cb.credentials.org_key, self._run_id - ) - request = self._build_request(rows) - resp = self._cb.post_object(url, body=request) - result = resp.json() - results = result.get("terms", []) - for item in results: - yield self._doc_class(self._cb, item) - \ No newline at end of file diff --git a/src/cbapi/psc/livequery/rest_api.py b/src/cbapi/psc/livequery/rest_api.py deleted file mode 100644 index 9e50fc14..00000000 --- a/src/cbapi/psc/livequery/rest_api.py +++ /dev/null @@ -1,37 +0,0 @@ -from cbapi.psc.livequery.models import Run, RunHistory -from cbapi.connection import BaseAPI -from cbapi.errors import CredentialError, ApiError -import logging - -log = logging.getLogger(__name__) - - -class CbLiveQueryAPI(BaseAPI): - """The main entry point into the Cb PSC LiveQuery API. - - :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. - Uses the profile named 'default' when not specified. - - Usage:: - - >>> from cbapi.psc.livequery import CbLiveQueryAPI - >>> cb = CbLiveQueryAPI(profile="production") - """ - def __init__(self, *args, **kwargs): - super(CbLiveQueryAPI, self).__init__(product_name="psc", *args, **kwargs) - self._lr_scheduler = None - - if not self.credentials.get("org_key", None): - raise CredentialError("No organization key specified") - - def _perform_query(self, cls, **kwargs): - if hasattr(cls, "_query_implementation"): - return cls._query_implementation(self) - else: - raise ApiError("All LiveQuery models should provide _query_implementation") - - def query(self, sql): - return self.select(Run).where(sql=sql) - - def query_history(self, query=None): - return self.select(RunHistory).where(query) diff --git a/src/cbapi/psc/threathunter/__init__.py b/src/cbapi/psc/threathunter/__init__.py deleted file mode 100644 index d1b0e338..00000000 --- a/src/cbapi/psc/threathunter/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# Exported public API for the Cb ThreatHunter API - -from __future__ import absolute_import - -from .rest_api import CbThreatHunterAPI -from cbapi.psc.threathunter.models import ( - Process, Event, Tree, Feed, Report, IOC, IOC_V2, Watchlist, Binary, Downloads -) -from cbapi.psc.threathunter.query import QueryBuilder diff --git a/src/cbapi/psc/threathunter/models.py b/src/cbapi/psc/threathunter/models.py deleted file mode 100644 index 1c2e7e5c..00000000 --- a/src/cbapi/psc/threathunter/models.py +++ /dev/null @@ -1,1111 +0,0 @@ -from __future__ import absolute_import -from cbapi.errors import ApiError, InvalidObjectError -from cbapi.models import CreatableModelMixin, MutableBaseModel, UnrefreshableModel -import logging -from cbapi.psc.threathunter.query import Query, AsyncProcessQuery, TreeQuery, FeedQuery, ReportQuery, WatchlistQuery -import validators -import time - -log = logging.getLogger(__name__) - - -class FeedModel(UnrefreshableModel, CreatableModelMixin, MutableBaseModel): - """A common base class for models used by the Feed and Watchlist APIs. - """ - pass - - -class Process(UnrefreshableModel): - """Represents a process retrieved by one of the CbTH endpoints. - """ - default_sort = 'last_update desc' - primary_key = "process_guid" - validation_url = "/threathunter/search/v1/orgs/{}/processes/search_validation" - - class Summary(UnrefreshableModel): - """Represents a summary of organization-specific information for - a process. - """ - default_sort = "last_update desc" - primary_key = "process_guid" - urlobject_single = "/threathunter/search/v1/orgs/{}/processes/summary" - - def __init__(self, cb, model_unique_id): - url = self.urlobject_single.format(cb.credentials.org_key) - summary = cb.get_object(url, query_parameters={"process_guid": model_unique_id}) - - while summary["incomplete_results"]: - log.debug("summary incomplete, requesting again") - summary = self._cb.get_object( - url, query_parameters={"process_guid": self.process_guid} - ) - - super(Process.Summary, self).__init__(cb, model_unique_id=model_unique_id, - initial_data=summary, force_init=False, - full_doc=True) - - @classmethod - def _query_implementation(cls, cb): - # This will emulate a synchronous process query, for now. - return AsyncProcessQuery(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=True): - super(Process, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=force_init, full_doc=full_doc) - - @property - def summary(self): - """Returns organization-specific information about this process. - """ - return self._cb.select(Process.Summary, self.process_guid) - - def events(self, **kwargs): - """Returns a query for events associated with this process's process GUID. - - :param kwargs: Arguments to filter the event query with. - :return: Returns a Query object with the appropriate search parameters for events - :rtype: :py:class:`cbapi.psc.threathunter.query.Query` - - Example:: - - >>> [print(event) for event in process.events()] - >>> [print(event) for event in process.events(event_type="modload")] - """ - query = self._cb.select(Event).where(process_guid=self.process_guid) - - if kwargs: - query = query.and_(**kwargs) - - return query - - def tree(self): - """Returns a :py:class:`Tree` of children (and possibly siblings) - associated with this process. - - :return: Returns a :py:class:`Tree` object - :rtype: :py:class:`Tree` - - Example: - - >>> tree = process.tree() - """ - data = self._cb.select(Tree).where(process_guid=self.process_guid).all() - return Tree(self._cb, initial_data=data) - - @property - def parents(self): - """Returns a query for parent processes associated with this process. - - :return: Returns a Query object with the appropriate search parameters for parent processes, or None if the process has no recorded parent - :rtype: :py:class:`cbapi.psc.threathunter.query.AsyncProcessQuery` or None - """ - if "parent_guid" in self._info: - return self._cb.select(Process).where(process_guid=self.parent_guid) - else: - return [] - - @property - def children(self): - """Returns a list of child processes for this process. - - :return: Returns a list of process objects - :rtype: list of :py:class:`Process` - """ - return [ - Process(self._cb, initial_data=child) - for child in self.summary.children - ] - - @property - def siblings(self): - """Returns a list of sibling processes for this process. - - :return: Returns a list of process objects - :rtype: list of :py:class:`Process` - """ - return [ - Process(self._cb, initial_data=sibling) - for sibling in self.summary.siblings - ] - - @property - def process_md5(self): - """Returns a string representation of the MD5 hash for this process. - - :return: A string representation of the process's MD5. - :rtype: str - """ - # NOTE: We have to check _info instead of poking the attribute directly - # to avoid the missing attrbute login in NewBaseModel. - if "process_hash" in self._info: - return next((hsh for hsh in self.process_hash if len(hsh) == 32), None) - else: - return None - - @property - def process_sha256(self): - """Returns a string representation of the SHA256 hash for this process. - - :return: A string representation of the process's SHA256. - :rtype: str - """ - if "process_hash" in self._info: - return next((hsh for hsh in self.process_hash if len(hsh) == 64), None) - else: - return None - - @property - def process_pids(self): - """Returns a list of PIDs associated with this process. - - :return: A list of PIDs - :rtype: list of ints - """ - # NOTE(ww): This exists because the API returns the list as "process_pid", - # which is misleading. We just give a slightly clearer name. - return self.process_pid - - -class Event(UnrefreshableModel): - """Events can be queried for via ``CbThreatHunterAPI.select`` - or though an already selected process with ``Process.events()``. - """ - urlobject = '/threathunter/search/v1/orgs/{}/events/_search' - validation_url = '/threathunter/search/v1/orgs/{}/events/search_validation' - default_sort = 'last_update desc' - primary_key = "process_guid" - - @classmethod - def _query_implementation(cls, cb): - return Query(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=True): - super(Event, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=force_init, full_doc=full_doc) - - -class Tree(UnrefreshableModel): - """The preferred interface for interacting with Tree models - is ``Process.tree()``. - """ - urlobject = '/threathunter/search/v1/orgs/{}/processes/tree' - primary_key = 'process_guid' - - @classmethod - def _query_implementation(cls, cb): - return TreeQuery(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None, force_init=False, full_doc=True): - super(Tree, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=force_init, full_doc=full_doc) - - @property - def children(self): - """Returns all of the children of the process that this tree is centered around. - - :return: A list of :py:class:`Process` instances - :rtype: list of :py:class:`Process` - """ - return [Process(self._cb, initial_data=child) for child in self.nodes["children"]] - - -class Feed(FeedModel): - """Represents a ThreatHunter feed's metadata. - """ - urlobject = "/threathunter/feedmgr/v2/orgs/{}/feeds" - urlobject_single = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}" - primary_key = "id" - swagger_meta_file = "psc/threathunter/models/feed.yaml" - - @classmethod - def _query_implementation(cls, cb): - return FeedQuery(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None): - item = {} - reports = [] - - if initial_data: - # NOTE(ww): Some endpoints give us the full Feed, others give us just the FeedInfo. - if "feedinfo" in initial_data: - item = initial_data["feedinfo"] - reports = initial_data.get("reports", []) - else: - item = initial_data - elif model_unique_id: - url = self.urlobject_single.format( - cb.credentials.org_key, model_unique_id - ) - resp = cb.get_object(url) - item = resp.get("feedinfo", {}) - reports = resp.get("reports", []) - - feed_id = item.get("id") - - super(Feed, self).__init__(cb, model_unique_id=feed_id, initial_data=item, - force_init=False, full_doc=True) - - self._reports = [Report(cb, initial_data=report, feed_id=feed_id) for report in reports] - - def save(self, public=False): - """Saves this feed on the ThreatHunter server. - - :param public: Whether to make the feed publicly available - :return: The saved feed - :rtype: :py:class:`Feed` - """ - self.validate() - - body = { - 'feedinfo': self._info, - 'reports': [report._info for report in self._reports], - } - - url = "/threathunter/feedmgr/v2/orgs/{}/feeds".format( - self._cb.credentials.org_key - ) - if public: - url = url + "/public" - - new_info = self._cb.post_object(url, body).json() - self._info.update(new_info) - return self - - def validate(self): - """Validates this feed's state. - - :raise InvalidObjectError: if the feed's state is invalid - """ - super(Feed, self).validate() - - if self.access not in ["public", "private"]: - raise InvalidObjectError("access should be public or private") - - if not validators.url(self.provider_url): - raise InvalidObjectError("provider_url should be a valid URL") - - for report in self._reports: - report.validate() - - def delete(self): - """Deletes this feed from the ThreatHunter server. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing feed ID") - - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.delete_object(url) - - def update(self, **kwargs): - """Update this feed's metadata with the given arguments. - - >>> feed.update(access="private") - - :param kwargs: The fields to update - :type kwargs: dict(str, str) - :raise InvalidObjectError: if `id` is missing or :py:meth:`validate` fails - :raise ApiError: if an invalid field is specified - """ - if not self.id: - raise InvalidObjectError("missing feed ID") - - for key, value in kwargs.items(): - if key in self._info: - self._info[key] = value - - self.validate() - - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/feedinfo".format( - self._cb.credentials.org_key, - self.id, - ) - new_info = self._cb.put_object(url, self._info).json() - self._info.update(new_info) - - return self - - @property - def reports(self): - """Returns a list of :py:class:`Report` associated with this feed. - - :return: a list of reports - :rtype: list(:py:class:`Report`) - """ - return self._cb.select(Report).where(feed_id=self.id) - - def replace_reports(self, reports): - """Replace this feed's reports with the given reports. - - :param reports: the reports to replace with - :type reports: list(:py:class:`Report`) - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing feed ID") - - rep_dicts = [report._info for report in reports] - body = {"reports": rep_dicts} - - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/reports".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.post_object(url, body) - - def append_reports(self, reports): - """Append the given reports to this feed's current reports. - - :param reports: the reports to append - :type reports: list(:py:class:`Report`) - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing feed ID") - - rep_dicts = [report._info for report in reports] - rep_dicts += [report._info for report in self.reports] - body = {"reports": rep_dicts} - - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/reports".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.post_object(url, body) - - -class Report(FeedModel): - """Represents reports retrieved from a ThreatHunter feed. - """ - urlobject = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/reports" - primary_key = "id" - swagger_meta_file = "psc/threathunter/models/report.yaml" - - @classmethod - def _query_implementation(cls, cb): - return ReportQuery(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None, - feed_id=None, from_watchlist=False): - - super(Report, self).__init__(cb, model_unique_id=initial_data.get("id"), - initial_data=initial_data, - force_init=False, full_doc=True) - - # NOTE(ww): Warn instead of failing since we allow Watchlist reports - # to be created via create(), but we don't actually know that the user - # intends to use them with a watchlist until they call save(). - if not feed_id and not from_watchlist: - log.warning("Report created without feed ID or not from watchlist") - - self._feed_id = feed_id - self._from_watchlist = from_watchlist - - if self.iocs: - self._iocs = IOC(cb, initial_data=self.iocs, report_id=self.id) - if self.iocs_v2: - self._iocs_v2 = [IOC_V2(cb, initial_data=ioc, report_id=self.id) for ioc in self.iocs_v2] - - def save_watchlist(self): - """Saves this report *as a watchlist report*. - - .. NOTE:: - This method **cannot** be used to save a feed report. To - save feed reports, create them with `cb.create` and use - :py:meth:`Feed.replace`. - - :raise InvalidObjectError: if :py:meth:`validate` fails - """ - self.validate() - - # NOTE(ww): Once saved, this object corresponds to a watchlist report. - # As such, we need to tell the model to route calls like update() - # and delete() to the correct (watchlist) endpoints. - self._from_watchlist = True - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports".format( - self._cb.credentials.org_key - ) - new_info = self._cb.post_object(url, self._info).json() - self._info.update(new_info) - return self - - def validate(self): - """Validates this report's state. - - :raise InvalidObjectError: if the report's state is invalid - """ - super(Report, self).validate() - - if self.link and not validators.url(self.link): - raise InvalidObjectError("link should be a valid URL") - - if self.iocs_v2: - [ioc.validate() for ioc in self._iocs_v2] - - def update(self, **kwargs): - """Update this report with the given arguments. - - .. NOTE:: - The report's timestamp is always updated, regardless of whether - passed explicitly. - - >>> report.update(title="My new report title") - - :param kwargs: The fields to update - :type kwargs: dict(str, str) - :return: The updated report - :rtype: :py:class:`Report` - :raises InvalidObjectError: if `id` is missing, or `feed_id` is missing - and this report is a feed report, or :py:meth:`validate` fails - """ - - if not self.id: - raise InvalidObjectError("missing Report ID") - - if self._from_watchlist: - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}".format( - self._cb.credentials.org_key, - self.id - ) - else: - if not self._feed_id: - raise InvalidObjectError("missing Feed ID") - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/reports/{}".format( - self._cb.credentials.org_key, - self._feed_id, - self.id - ) - - for key, value in kwargs.items(): - if key in self._info: - self._info[key] = value - - # NOTE(ww): Updating reports on the watchlist API appears to require - # updated timestamps. - self.timestamp = int(time.time()) - self.validate() - - new_info = self._cb.put_object(url, self._info).json() - self._info.update(new_info) - return self - - def delete(self): - """Deletes this report from the ThreatHunter server. - - >>> report.delete() - - :raises InvalidObjectError: if `id` is missing, or `feed_id` is missing - and this report is a feed report - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - - if self._from_watchlist: - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}".format( - self._cb.credentials.org_key, - self.id - ) - else: - if not self._feed_id: - raise InvalidObjectError("missing Feed ID") - url = "/threathunter/feedmgr/v2/orgs/{}/feeds/{}/reports/{}".format( - self._cb.credentials.org_key, - self._feed_id, - self.id - ) - - self._cb.delete_object(url) - - @property - def ignored(self): - """Returns the ignore status for this report. - - Only watchlist reports have an ignore status. - - >>> if report.ignored: - ... report.unignore() - - :return: whether or not this report is ignored - :rtype: bool - :raises InvalidObjectError: if `id` is missing or this report is not from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - if not self._from_watchlist: - raise InvalidObjectError("ignore status only applies to watchlist reports") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/ignore".format( - self._cb.credentials.org_key, - self.id - ) - resp = self._cb.get_object(url) - return resp["ignored"] - - def ignore(self): - """Sets the ignore status on this report. - - Only watchlist reports have an ignore status. - - :raises InvalidObjectError: if `id` is missing or this report is not from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - - if not self._from_watchlist: - raise InvalidObjectError("ignoring only applies to watchlist reports") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/ignore".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.put_object(url, None) - - def unignore(self): - """Removes the ignore status on this report. - - Only watchlist reports have an ignore status. - - :raises InvalidObjectError: if `id` is missing or this report is not from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - - if not self._from_watchlist: - raise InvalidObjectError("ignoring only applies to watchlist reports") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/ignore".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.delete_object(url) - - @property - def custom_severity(self): - """Returns the custom severity for this report. - - :return: The custom severity for this report, if it exists - :rtype: :py:class:`ReportSeverity` - :raise InvalidObjectError: if `id` is missing or this report is from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing report ID") - if self._from_watchlist: - raise InvalidObjectError("watchlist reports don't have custom severities") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/severity".format( - self._cb.credentials.org_key, - self.id - ) - resp = self._cb.get_object(url) - return ReportSeverity(self._cb, initial_data=resp) - - @custom_severity.setter - def custom_severity(self, sev_level): - """Sets or removed the custom severity for this report - - :param int sev_level: the new severity, or None to remove the custom severity - :return: The new custom severity, or None if removed - :rtype: :py:class:`ReportSeverity` or None - :raise InvalidObjectError: if `id` is missing or this report is from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing report ID") - if self._from_watchlist: - raise InvalidObjectError("watchlist reports don't have custom severities") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/severity".format( - self._cb.credentials.org_key, - self.id - ) - - if sev_level is None: - self._cb.delete_object(url) - return - - args = { - "report_id": self.id, - "severity": sev_level, - } - - resp = self._cb.put_object(url, args).json() - return ReportSeverity(self._cb, initial_data=resp) - - @property - def iocs_(self): - """Returns a list of :py:class:`IOC_V2` associated with this report. - - >>> for ioc in report.iocs_: - ... print(ioc.values) - - :return: a list of IOCs - :rtype: list(:py:class:`IOC_V2`) - """ - if not self.iocs_v2: - return [] - - # NOTE(ww): This name is underscored because something in the model - # hierarchy is messing up method resolution -- self.iocs and self.iocs_v2 - # are resolving to the attributes rather than the attribute-ified - # methods. - return self._iocs_v2 - - -class IOC(FeedModel): - """Represents a collection of categorized IOCs. - """ - swagger_meta_file = "psc/threathunter/models/iocs.yaml" - - def __init__(self, cb, model_unique_id=None, initial_data=None, report_id=None): - """Creates a new IOC instance. - - :raise ApiError: if `initial_data` is `None` - """ - if not initial_data: - raise ApiError("IOC can only be initialized from initial_data") - - super(IOC, self).__init__(cb, model_unique_id=model_unique_id, initial_data=initial_data, - force_init=False, full_doc=True) - - self._report_id = report_id - - def validate(self): - """Validates this IOC structure's state. - - :raise InvalidObjectError: if the IOC structure's state is invalid - """ - super(IOC, self).validate() - - for md5 in self.md5: - if not validators(md5): - raise InvalidObjectError("invalid MD5 checksum: {}".format(md5)) - for ipv4 in self.ipv4: - if not validators(ipv4): - raise InvalidObjectError("invalid IPv4 address: {}".format(ipv4)) - for ipv6 in self.ipv6: - if not validators(ipv6): - raise InvalidObjectError("invalid IPv6 address: {}".format(ipv6)) - for dns in self.dns: - if not validators(dns): - raise InvalidObjectError("invalid domain: {}".format(dns)) - for query in self.query: - if not self._cb.validate(query["search_query"]): - raise InvalidObjectError("invalid search query: {}".format(query["search_query"])) - - -class IOC_V2(FeedModel): - """Represents a collection of IOCs of a particular type, plus matching criteria and metadata. - """ - primary_key = "id" - swagger_meta_file = "psc/threathunter/models/ioc_v2.yaml" - - def __init__(self, cb, model_unique_id=None, initial_data=None, report_id=None): - """Creates a new IOC_V2 instance. - - :raise ApiError: if `initial_data` is `None` - """ - if not initial_data: - raise ApiError("IOC_V2 can only be initialized from initial_data") - - super(IOC_V2, self).__init__(cb, model_unique_id=initial_data.get(self.primary_key), - initial_data=initial_data, force_init=False, - full_doc=True) - - self._report_id = report_id - - def validate(self): - """Validates this IOC_V2's state. - - :raise InvalidObjectError: if the IOC_V2's state is invalid - """ - super(IOC_V2, self).validate() - - if self.link and not validators.url(self.link): - raise InvalidObjectError("link should be a valid URL") - - @property - def ignored(self): - """Returns whether or not this IOC is ignored - - >>> if ioc.ignored: - ... ioc.unignore() - - :return: the ignore status - :rtype: bool - :raise InvalidObjectError: if this IOC is missing an `id` or is not a watchlist IOC - """ - if not self.id: - raise InvalidObjectError("missing IOC ID") - if not self._report_id: - raise InvalidObjectError("ignore status only applies to watchlist IOCs") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/iocs/{}/ignore".format( - self._cb.credentials.org_key, - self._report_id, - self.id - ) - resp = self._cb.get_object(url) - return resp["ignored"] - - def ignore(self): - """Sets the ignore status on this IOC. - - Only watchlist IOCs have an ignore status. - - :raises InvalidObjectError: if `id` is missing or this IOC is not from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - if not self._report_id: - raise InvalidObjectError("ignoring only applies to watchlist IOCs") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/iocs/{}/ignore".format( - self._cb.credentials.org_key, - self._report_id, - self.id - ) - self._cb.put_object(url, None) - - def unignore(self): - """Removes the ignore status on this IOC. - - Only watchlist IOCs have an ignore status. - - :raises InvalidObjectError: if `id` is missing or this IOC is not from a watchlist - """ - if not self.id: - raise InvalidObjectError("missing Report ID") - if not self._report_id: - raise InvalidObjectError("ignoring only applies to watchlist IOCs") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}/iocs/{}/ignore".format( - self._cb.credentials.org_key, - self._report_id, - self.id - ) - self._cb.delete_object(url) - - -class Watchlist(FeedModel): - """Represents a ThreatHunter watchlist. - """ - # NOTE(ww): Not documented. - urlobject = "/threathunter/watchlistmgr/v2/watchlist" - urlobject_single = "/threathunter/watchlistmgr/v2/watchlist/{}" - swagger_meta_file = "psc/threathunter/models/watchlist.yaml" - - @classmethod - def _query_implementation(cls, cb): - return WatchlistQuery(cls, cb) - - def __init__(self, cb, model_unique_id=None, initial_data=None): - item = {} - - if initial_data: - item = initial_data - elif model_unique_id: - item = cb.get_object(self.urlobject_single.format(model_unique_id)) - - feed_id = item.get("id") - - super(Watchlist, self).__init__(cb, model_unique_id=feed_id, initial_data=item, - force_init=False, full_doc=True) - - def save(self): - """Saves this watchlist on the ThreatHunter server. - - :return: The saved watchlist - :rtype: :py:class:`Watchlist` - :raise InvalidObjectError: if :py:meth:`validate` fails - """ - self.validate() - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists".format( - self._cb.credentials.org_key - ) - new_info = self._cb.post_object(url, self._info).json() - self._info.update(new_info) - return self - - def validate(self): - """Validates this watchlist's state. - - :raise InvalidObjectError: if the watchlist's state is invalid - """ - super(Watchlist, self).validate() - - def update(self, **kwargs): - """Updates this watchlist with the given arguments. - - >>> watchlist.update(name="New Name") - - :param kwargs: The fields to update - :type kwargs: dict(str, str) - :raise InvalidObjectError: if `id` is missing or :py:meth:`validate` fails - :raise ApiError: if `report_ids` is given *and* is empty - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - # NOTE(ww): Special case, according to the docs. - if "report_ids" in kwargs and not kwargs["report_ids"]: - raise ApiError("can't update a watchlist to have an empty report list") - - for key, value in kwargs.items(): - if key in self._info: - self._info[key] = value - - self.validate() - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}".format( - self._cb.credentials.org_key, - self.id - ) - new_info = self._cb.put_object(url, self._info).json() - self._info.update(new_info) - - @property - def classifier_(self): - """Returns the classifier key and value, if any, for this watchlist. - - :rtype: tuple(str, str) or None - """ - classifier_dict = self._info.get("classifier") - - if not classifier_dict: - return None - - return (classifier_dict["key"], classifier_dict["value"]) - - def delete(self): - """Deletes this watchlist from the ThreatHunter server. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.delete_object(url) - - def enable_alerts(self): - """Enable alerts for this watchlist. Alerts are not retroactive. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}/alert".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.put_object(url, None) - - def disable_alerts(self): - """Disable alerts for this watchlist. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}/alert".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.delete_object(url) - - def enable_tags(self): - """Enable tagging for this watchlist. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}/tag".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.put_object(url, None) - - def disable_tags(self): - """Disable tagging for this watchlist. - - :raise InvalidObjectError: if `id` is missing - """ - if not self.id: - raise InvalidObjectError("missing Watchlist ID") - - url = "/threathunter/watchlistmgr/v3/orgs/{}/watchlists/{}/tag".format( - self._cb.credentials.org_key, - self.id - ) - self._cb.delete_object(url) - - @property - def feed(self): - """Returns the feed linked to this watchlist, if there is one. - - :return: the feed linked to this watchlist, if any - :rtype: :py:class:`Feed` or None - """ - if not self.classifier: - return None - if self.classifier["key"] != "feed_id": - log.warning("Unexpected classifier type: {}".format(self.classifier["key"])) - return None - - return self._cb.select(Feed, self.classifier["value"]) - - @property - def reports(self): - """Returns a list of :py:class:`Report` instances associated with this watchlist. - - .. NOTE:: - If this watchlist is a classifier (i.e. feed-linked) watchlist, - `reports` will be empty. To get the reports associated with the linked - feed, use :py:attr:`feed` like: - - >>> for report in watchlist.feed.reports: - ... print(report.title) - - :return: A list of reports - :rtype: list(:py:class:`Report`) - """ - if not self.report_ids: - return [] - - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/{}" - reports_ = [] - for rep_id in self.report_ids: - path = url.format(self._cb.credentials.org_key, rep_id) - resp = self._cb.get_object(path) - reports_.append(Report(self._cb, initial_data=resp, from_watchlist=True)) - - return reports_ - - -class ReportSeverity(FeedModel): - """Represents severity information for a watchlist report. - """ - primary_key = "report_id" - swagger_meta_file = "psc/threathunter/models/report_severity.yaml" - - def __init__(self, cb, initial_data=None): - if not initial_data: - raise ApiError("ReportSeverity can only be initialized from initial_data") - - super(ReportSeverity, self).__init__(cb, model_unique_id=initial_data.get(self.primary_key), - initial_data=initial_data, force_init=False, - full_doc=True) - - -class Binary(UnrefreshableModel): - """Represents a retrievable binary. - """ - primary_key = "sha256" - swagger_meta_file = "psc/threathunter/models/binary.yaml" - urlobject_single = "/ubs/v1/orgs/{}/sha256/{}/metadata" - - class Summary(UnrefreshableModel): - """Represents a summary of organization-specific information - for a retrievable binary. - """ - primary_key = "sha256" - urlobject_single = "/ubs/v1/orgs/{}/sha256/{}/summary/device" - - def __init__(self, cb, model_unique_id): - if not validators.sha256(model_unique_id): - raise ApiError("model_unique_id must be a valid SHA256") - - url = self.urlobject_single.format(cb.credentials.org_key, model_unique_id) - item = cb.get_object(url) - - super(Binary.Summary, self).__init__(cb, model_unique_id=model_unique_id, - initial_data=item, force_init=False, - full_doc=True) - - def __init__(self, cb, model_unique_id): - if not validators.sha256(model_unique_id): - raise ApiError("model_unique_id must be a valid SHA256") - - url = self.urlobject_single.format(cb.credentials.org_key, model_unique_id) - item = cb.get_object(url) - - super(Binary, self).__init__(cb, model_unique_id=model_unique_id, - initial_data=item, force_init=False, - full_doc=True) - - @property - def summary(self): - """Returns organization-specific information about this binary. - """ - return self._cb.select(Binary.Summary, self.sha256) - - @property - def download_url(self, expiration_seconds=3600): - """Returns a URL that can be used to download the file - for this binary. Returns None if no download can be found. - - :param expiration_seconds: How long the download should be valid for - :raise InvalidObjectError: if URL retrieval should be retried - :return: A pre-signed AWS download URL - :rtype: str - """ - downloads = self._cb.select(Downloads, [self.sha256], - expiration_seconds=expiration_seconds) - - if self.sha256 in downloads.not_found: - return None - elif self.sha256 in downloads.error: - raise InvalidObjectError("{} should be retried".format(self.sha256)) - else: - return next((item.url - for item in downloads.found() - if self.sha256 == item.sha256), None) - - -class Downloads(UnrefreshableModel): - """Represents download information for a list of process hashes. - """ - urlobject = "/ubs/v1/orgs/{}/file/_download" - - class FoundItem(UnrefreshableModel): - """Represents the download URL and process hash for a successfully - located binary. - """ - primary_key = "sha256" - - def __init__(self, cb, item): - super(Downloads.FoundItem, self).__init__(cb, model_unique_id=item["sha256"], - initial_data=item, force_init=False, - full_doc=True) - - def __init__(self, cb, shas, expiration_seconds=3600): - body = { - "sha256": shas, - "expiration_seconds": expiration_seconds, - } - - url = self.urlobject.format(cb.credentials.org_key) - item = cb.post_object(url, body).json() - - super(Downloads, self).__init__(cb, model_unique_id=None, - initial_data=item, force_init=False, - full_doc=True) - - @property - def found(self): - """Returns a list of :py:class:`Downloads.FoundItem`, one - for each binary found in the binary store. - """ - return [Downloads.FoundItem(self._cb, item) for item in self._info["found"]] diff --git a/src/cbapi/psc/threathunter/models/binary.yaml b/src/cbapi/psc/threathunter/models/binary.yaml deleted file mode 100644 index c19d7b14..00000000 --- a/src/cbapi/psc/threathunter/models/binary.yaml +++ /dev/null @@ -1,79 +0,0 @@ -type: object -required: - - sha256 - - md5 - - file_available - - available_file_size - - file_size - - os_type - - architecture -properties: - sha256: - type: string - description: The SHA-256 hash of the file - md5: - type: string - description: The MD5 hash of the file - file_available: - type: boolean - description: If true, the file is available for download - available_file_size: - type: integer - format: int64 # NOTE(ww): docs say long integer - description: The size of the file available for download - file_size: - type: integer - format: int64 - description: The size of the actual file (represented by the hash) - os_type: - type: string - description: The OS that this file is designed for - architecture: - type: array - items: - type: string - description: The set of architectures that this file was compiled for - lang_id: - type: integer - format: int32 # NOTE(ww): Swagger doesn't have a (u)int16 - description: The Language ID value for the Windows VERSIONINFO resource - charset_id: - type: integer - format: int32 - description: The Character set ID value for the Windows VERSIONINFO resource - internal_name: - type: string - description: The internal name from FileVersionInformation - product_name: - type: string - description: The product name from FileVersionInformation - company_name: - type: string - description: The company name from FileVersionInformation - trademark: - type: string - description: The trademark from FileVersionInformation - file_description: - type: string - description: The file description from FileVersionInformation - file_version: - type: string - description: The file version from FileVersionInformation - comments: - type: string - description: Comments from FileVersionInformation - original_filename: - type: string - description: The original filename from FileVersionInformation - product_description: - type: string - description: The product description from FileVersionInformation - product_version: - type: string - description: The product version from FileVersionInformation - private_build: - type: string - description: The private build from FileVersionInformation - special_build: - type: string - description: The special build from FileVersionInformation diff --git a/src/cbapi/psc/threathunter/models/feed.yaml b/src/cbapi/psc/threathunter/models/feed.yaml deleted file mode 100644 index 27c0e4f4..00000000 --- a/src/cbapi/psc/threathunter/models/feed.yaml +++ /dev/null @@ -1,33 +0,0 @@ -type: object -required: - - name - - owner - - provider_url - - summary - - category - - access -properties: - name: - type: string - description: A human-friendly name for this feed - owner: - type: string - description: The feed owner's connector ID - provider_url: - type: string - description: A URL supplied by the feed's provider - summary: - type: string - description: A human-friendly summary for the feed - category: - type: string - description: The feed's category - source_label: - type: string - description: The feed's source label - access: - type: string - description: The feed's access (public or private) - id: - type: string - description: The feed's unique ID diff --git a/src/cbapi/psc/threathunter/models/ioc_v2.yaml b/src/cbapi/psc/threathunter/models/ioc_v2.yaml deleted file mode 100644 index a65f50cb..00000000 --- a/src/cbapi/psc/threathunter/models/ioc_v2.yaml +++ /dev/null @@ -1,23 +0,0 @@ -type: object -required: - - id - - match_type - - values -properties: - id: - type: string - description: The IOC_V2's unique ID - match_type: - type: string - description: How IOCs in this IOC_V2 are matched - values: - type: array - items: - type: string - description: A list of IOCs - field: - type: string - description: The kind of IOCs contained in this IOC_V2 - link: - type: string - description: A URL for some reference for this IOC_V2 diff --git a/src/cbapi/psc/threathunter/models/iocs.yaml b/src/cbapi/psc/threathunter/models/iocs.yaml deleted file mode 100644 index 274cb7a8..00000000 --- a/src/cbapi/psc/threathunter/models/iocs.yaml +++ /dev/null @@ -1,32 +0,0 @@ -type: object -properties: - md5: - type: array - items: - type: string - description: A list of MD5 checksums - ipv4: - type: array - items: - type: string - description: A list of IPv4 addresses - ipv6: - type: array - items: - type: string - description: A list of IPv6 addresses - dns: - type: array - items: - type: string - description: A list of domain names - query: - type: array - items: - type: object # QueryIOC - properties: - index_type: - type: string - search_query: - type: string - description: A list of dicts, each containing an IOC query diff --git a/src/cbapi/psc/threathunter/models/report.yaml b/src/cbapi/psc/threathunter/models/report.yaml deleted file mode 100644 index 6ef3bad1..00000000 --- a/src/cbapi/psc/threathunter/models/report.yaml +++ /dev/null @@ -1,45 +0,0 @@ -type: object -required: - - id - - timestamp - - title - - description - - severity -properties: - id: - type: string - description: The report's unique ID - timestamp: - type: integer - format: int32 - description: When this report was created - title: - type: string - description: A human-friendly title for this report - description: - type: string - description: A human-friendly description for this report - severity: - type: integer - format: int32 - description: The severity of the IOCs within this report - link: - type: string - description: A URL for some reference for this report - tags: - type: array - items: - type: string - description: A list of tags for this report - iocs: - type: object - # NOTE(ww): Explicitly not documented, since we do almost everything - # through IOC_V2 - iocs_v2: - type: array - items: - type: object - description: A list of IOC_V2 dicts associated with this report - visibility: - type: string - description: The visibility of this report diff --git a/src/cbapi/psc/threathunter/models/report_severity.yaml b/src/cbapi/psc/threathunter/models/report_severity.yaml deleted file mode 100644 index da53e10e..00000000 --- a/src/cbapi/psc/threathunter/models/report_severity.yaml +++ /dev/null @@ -1,12 +0,0 @@ -type: object -required: - - report_id - - severity -properties: - report_id: - type: string - description: The unique ID for the corresponding report - severity: - type: integer - format: int32 - description: The severity level diff --git a/src/cbapi/psc/threathunter/models/watchlist.yaml b/src/cbapi/psc/threathunter/models/watchlist.yaml deleted file mode 100644 index 34990997..00000000 --- a/src/cbapi/psc/threathunter/models/watchlist.yaml +++ /dev/null @@ -1,43 +0,0 @@ -type: object -required: - - name - - description - - create_timestamp - - last_update_timestamp -properties: - name: - type: string - description: A human-friendly name for the watchlist - description: - type: string - description: A short description of the watchlist - id: - type: string - description: The watchlist's unique id - tags_enabled: - type: boolean - description: Whether tags are currently enabled - alerts_enabled: - type: boolean - description: Whether alerts are currently enabled - create_timestamp: - type: integer - format: int32 - description: When this watchlist was created - last_update_timestamp: - type: integer - format: int32 - description: Report IDs associated with this watchlist - report_ids: - type: array - items: - type: string - description: Report IDs associated with this watchlist - classifier: - type: object # ClassifierKeyValue - properties: - key: - type: string - value: - type: string - description: A key, value pair specifying an associated feed diff --git a/src/cbapi/psc/threathunter/query.py b/src/cbapi/psc/threathunter/query.py deleted file mode 100644 index ae7997af..00000000 --- a/src/cbapi/psc/threathunter/query.py +++ /dev/null @@ -1,612 +0,0 @@ -from cbapi.query import PaginatedQuery, BaseQuery, SimpleQuery -from cbapi.errors import ServerError, ApiError, TimeoutError -import time -from solrq import Q -from six import string_types -import logging -import functools - - -log = logging.getLogger(__name__) - - -class QueryBuilder(object): - """ - Provides a flexible interface for building prepared queries for the CB - ThreatHunter backend. - - This object can be instantiated directly, or can be managed implicitly - through the :py:meth:`CbThreatHunterAPI.select` API. - - Examples:: - - >>> from cbapi.psc.threathunter import QueryBuilder - >>> # build a query with chaining - >>> query = QueryBuilder().where(process_name="malicious.exe").and_(device_name="suspect") - >>> # start with an initial query, and chain another condition to it - >>> query = QueryBuilder(device_os="WINDOWS").or_(process_username="root") - - """ - def __init__(self, **kwargs): - if kwargs: - self._query = Q(**kwargs) - else: - self._query = None - self._raw_query = None - self._process_guid = None - - def _guard_query_params(func): - """Decorates the query construction methods of *QueryBuilder*, preventing - them from being called with parameters that would result in an intetnally - inconsistent query. - """ - @functools.wraps(func) - def wrap_guard_query_change(self, q, **kwargs): - if self._raw_query is not None and (kwargs or isinstance(q, Q)): - raise ApiError("Cannot modify a raw query with structured parameters") - if self._query is not None and isinstance(q, string_types): - raise ApiError("Cannot modify a structured query with a raw parameter") - return func(self, q, **kwargs) - return wrap_guard_query_change - - @_guard_query_params - def where(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - if self._raw_query is None: - self._raw_query = [] - self._raw_query.append(q) - elif isinstance(q, Q) or kwargs: - if self._query is not None: - raise ApiError("Use .and_() or .or_() for an extant solrq.Q object") - if kwargs: - self._process_guid = self._process_guid or kwargs.get("process_guid") - q = Q(**kwargs) - self._query = q - else: - raise ApiError(".where() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def and_(self, q, **kwargs): - """Adds a conjunctive filter to a query. - - :param q: string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if isinstance(q, string_types): - self.where(q) - elif isinstance(q, Q) or kwargs: - if kwargs: - self._process_guid = self._process_guid or kwargs.get("process_guid") - q = Q(**kwargs) - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".and_() only accepts strings or solrq.Q objects") - - return self - - @_guard_query_params - def or_(self, q, **kwargs): - """Adds a disjunctive filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - self._process_guid = self._process_guid or kwargs.get("process_guid") - q = Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query | q - else: - raise ApiError(".or_() only accepts solrq.Q objects") - - return self - - @_guard_query_params - def not_(self, q, **kwargs): - """Adds a negative filter to a query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: QueryBuilder object - :rtype: :py:class:`QueryBuilder` - """ - if kwargs: - q = ~ Q(**kwargs) - - if isinstance(q, Q): - if self._query is None: - self._query = q - else: - self._query = self._query & q - else: - raise ApiError(".not_() only accepts solrq.Q objects") - - def _collapse(self): - """The query can be represented by either an array of strings - (_raw_query) which is concatenated and passed directly to Solr, or - a solrq.Q object (_query) which is then converted into a string to - pass to Solr. This function will perform the appropriate conversions to - end up with the 'q' string sent into the POST request to the - PSC-R query endpoint.""" - if self._raw_query is not None: - return " ".join(self._raw_query) - elif self._query is not None: - return str(self._query) - else: - return "*:*" # return everything - - -class Query(PaginatedQuery): - """Represents a prepared query to the Cb ThreatHunter backend. - - This object is returned as part of a :py:meth:`CbThreatHunterPI.select` - operation on models requested from the Cb ThreatHunter backend. You should not have to create this class yourself. - - The query is not executed on the server until it's accessed, either as an iterator (where it will generate values - on demand as they're requested) or as a list (where it will retrieve the entire result set and save to a list). - You can also call the Python built-in ``len()`` on this object to retrieve the total number of items matching - the query. - - Examples:: - - >>> from cbapi.psc.threathunter import CbThreatHunterAPI - >>> cb = CbThreatHunterAPI() - >>> query = cb.select(Process) - >>> query = query.where(process_name="notepad.exe") - >>> # alternatively: - >>> query = query.where("process_name:notepad.exe") - - Notes: - - The slicing operator only supports start and end parameters, but not step. ``[1:-1]`` is legal, but - ``[1:2:-1]`` is not. - - You can chain where clauses together to create AND queries; only objects that match all ``where`` clauses - will be returned. - """ - - def __init__(self, doc_class, cb): - super(Query, self).__init__(doc_class, cb, None) - - self._query_builder = QueryBuilder() - self._sort_by = None - self._group_by = None - self._batch_size = 100 - self._default_args = {} - - def where(self, q=None, **kwargs): - """Add a filter to this query. - - :param q: Query string, :py:class:`QueryBuilder`, or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".where() expects a string, a QueryBuilder, a solrq.Q, or kwargs") - - if isinstance(q, QueryBuilder): - self._query_builder = q - else: - self._query_builder.where(q, **kwargs) - return self - - def and_(self, q=None, **kwargs): - """Add a conjunctive filter to this query. - - :param q: Query string or `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".and_() expects a string, a solrq.Q, or kwargs") - - self._query_builder.and_(q, **kwargs) - return self - - def or_(self, q=None, **kwargs): - """Add a disjunctive filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - if not q and not kwargs: - raise ApiError(".or_() expects a solrq.Q or kwargs") - - self._query_builder.or_(q, **kwargs) - return self - - def not_(self, q=None, **kwargs): - """Adds a negated filter to this query. - - :param q: `solrq.Q` object - :param kwargs: Arguments to construct a `solrq.Q` with - :return: Query object - :rtype: :py:class:`Query` - """ - - if not q and not kwargs: - raise ApiError(".not_() expects a solrq.Q, or kwargs") - - self._query_builder.not_(q, **kwargs) - return self - - def _get_query_parameters(self): - args = self._default_args.copy() - args['q'] = self._query_builder._collapse() - if self._query_builder._process_guid is not None: - args["cb.process_guid"] = self._query_builder._process_guid - args["fl"] = "*,parent_hash,parent_name,process_cmdline,backend_timestamp,device_external_ip,device_group,device_internal_ip,device_os,process_effective_reputation,process_reputation,ttp" - - return args - - def _count(self): - args = {"search_params": self._get_query_parameters()} - - log.debug("args: {}".format(str(args))) - - self._total_results = int(self._cb.post_object(self._doc_class.urlobject, body=args) - .json().get("response_header", {}).get("num_available", 0)) - self._count_valid = True - return self._total_results - - def _validate(self, args): - if not self._doc_class.validation_url: - return - - url = self._doc_class.validation_url.format(self._cb.credentials.org_key) - validated = self._cb.get_object(url, query_parameters=args) - - if not validated.get("valid"): - raise ApiError("Invalid query: {}: {}".format(args, validated["invalid_message"])) - - def _search(self, start=0, rows=0): - # iterate over total result set, 100 at a time - args = self._get_query_parameters() - self._validate(args) - - if start != 0: - args['start'] = start - args['rows'] = self._batch_size - - args = {"search_params": args} - - current = start - numrows = 0 - - still_querying = True - - while still_querying: - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) - resp = self._cb.post_object(url, body=args) - result = resp.json() - - self._total_results = result.get("response_header", {}).get("num_available", 0) - self._count_valid = True - - results = result.get('docs', []) - - for item in results: - yield item - current += 1 - numrows += 1 - if rows and numrows == rows: - still_querying = False - break - - args['start'] = current + 1 # as of 6/2017, the indexing on the Cb Defense backend is still 1-based - - if current >= self._total_results: - break - if not results: - log.debug("server reported total_results overestimated the number of results for this query by {0}" - .format(self._total_results - current)) - log.debug("resetting total_results for this query to {0}".format(current)) - self._total_results = current - break - - -class AsyncProcessQuery(Query): - """Represents the query logic for an asychronous Process query. - - This class specializes :py:class:`Query` to handle the particulars of - process querying. - """ - def __init__(self, doc_class, cb): - super(AsyncProcessQuery, self).__init__(doc_class, cb) - self._query_token = None - self._timeout = 0 - self._timed_out = False - self._sort_by = "backend_timestamp" # Requires default to prevent unstable fetching of results - self._sort_direction = "ASC" - - def sort_by(self, key, direction="ASC"): - """Sets the sorting behavior on a query's results. - - Example:: - - >>> cb.select(Process).where(process_name="cmd.exe").sort_by("device_timestamp") - - :param key: the key in the schema to sort by - :param direction: the sort order, either "ASC" or "DESC" - :rtype: :py:class:`AsyncProcessQuery` - """ - self._sort_by = key - self._sort_direction = direction - - # Append to search_job query - self._default_args['sort'] = '{} {}'.format(key, direction) - return self - - def timeout(self, msecs): - """Sets the timeout on a process query. - - Example:: - - >>> cb.select(Process).where(process_name="foo.exe").timeout(5000) - - :param: msecs: the timeout duration, in milliseconds - :return: AsyncProcessQuery object - :rtype: :py:class:`AsyncProcessQuery` - """ - self._timeout = msecs - return self - - def _submit(self): - if self._query_token: - raise ApiError("Query already submitted: token {0}".format(self._query_token)) - - args = self._get_query_parameters() - self._validate(args) - - url = "/threathunter/search/v1/orgs/{}/processes/search_jobs".format(self._cb.credentials.org_key) - query_start = self._cb.post_object(url, body={"search_params": args}) - - self._query_token = query_start.json().get("query_id") - self._timed_out = False - self._submit_time = time.time() * 1000 - - def _still_querying(self): - if not self._query_token: - self._submit() - - status_url = "/threathunter/search/v1/orgs/{}/processes/search_jobs/{}".format( - self._cb.credentials.org_key, - self._query_token, - ) - result = self._cb.get_object(status_url) - - searchers_contacted = result.get("contacted", 0) - searchers_completed = result.get("completed", 0) - log.debug("contacted = {}, completed = {}".format(searchers_contacted, searchers_completed)) - if searchers_contacted == 0: - return True - if searchers_completed < searchers_contacted: - if self._timeout != 0 and (time.time() * 1000) - self._submit_time > self._timeout: - self._timed_out = True - return False - return True - - return False - - def _count(self): - if self._count_valid: - return self._total_results - - while self._still_querying(): - time.sleep(.5) - - if self._timed_out: - raise TimeoutError(message="user-specified timeout exceeded while waiting for results") - - result_url = "/threathunter/search/v1/orgs/{}/processes/search_jobs/{}/results".format( - self._cb.credentials.org_key, - self._query_token, - ) - result = self._cb.get_object(result_url) - - self._total_results = result.get('response_header', {}).get('num_available', 0) - self._count_valid = True - - return self._total_results - - def _search(self, start=0, rows=0): - if not self._query_token: - self._submit() - - while self._still_querying(): - time.sleep(.5) - - if self._timed_out: - raise TimeoutError(message="user-specified timeout exceeded while waiting for results") - - log.debug("Pulling results, timed_out={}".format(self._timed_out)) - - current = start - rows_fetched = 0 - still_fetching = True - result_url_template = "/threathunter/search/v1/orgs/{}/processes/search_jobs/{}/results".format( - self._cb.credentials.org_key, - self._query_token - ) - query_parameters = {} - while still_fetching: - result_url = '{}?start={}&rows={}'.format( - result_url_template, - current, - 10 # Batch gets to reduce API calls - ) - - result = self._cb.get_object(result_url, query_parameters=query_parameters) - - self._total_results = result.get('response_header', {}).get('num_available', 0) - self._count_valid = True - - results = result.get('data', []) - - for item in results: - yield item - current += 1 - rows_fetched += 1 - - if rows and rows_fetched >= rows: - still_fetching = False - break - - if current >= self._total_results: - still_fetching = False - - log.debug("current: {}, total_results: {}".format(current, self._total_results)) - - -class TreeQuery(BaseQuery): - """ Represents the logic for a Tree query. - """ - def __init__(self, doc_class, cb): - super(TreeQuery, self).__init__() - self._doc_class = doc_class - self._cb = cb - self._args = {} - - def where(self, **kwargs): - """Adds a conjunctive filter to this *TreeQuery*. - - Example:: - - >>> cb.select(Tree).where(process_guid="...") - - :param: kwargs: Arguments to invoke the *TreeQuery* with. - :return: this *TreeQuery* - :rtype: :py:class:`TreeQuery` - """ - self._args = dict(self._args, **kwargs) - return self - - def and_(self, **kwargs): - """Adds a conjunctive filter to this *TreeQuery*. - - :param: kwargs: Arguments to invoke the *TreeQuery* with. - :return: this *TreeQuery* - :rtype: :py:class:`TreeQuery` - """ - self.where(**kwargs) - return self - - def or_(self, **kwargs): - """Unsupported. Will raise if called. - - :raise: :py:class:`ApiError` - """ - raise ApiError(".or_() cannot be called on Tree queries") - - def _perform_query(self): - if "process_guid" not in self._args: - raise ApiError("required parameter process_guid missing") - - log.debug("Fetching process tree") - - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) - results = self._cb.get_object(url, query_parameters=self._args) - - while results["incomplete_results"]: - result = self._cb.get_object(self._doc_class.urlobject, query_parameters=self._args) - results["nodes"]["children"].extend(result["nodes"]["children"]) - results["incomplete_results"] = result["incomplete_results"] - - return results - - -class FeedQuery(SimpleQuery): - """Represents the logic for a :py:class:`Feed` query. - - >>> cb.select(Feed) - >>> cb.select(Feed, id) - >>> cb.select(Feed).where(include_public=True) - """ - def __init__(self, doc_class, cb): - super(FeedQuery, self).__init__(doc_class, cb) - self._args = {} - - def where(self, **kwargs): - self._args = dict(self._args, **kwargs) - return self - - @property - def results(self): - log.debug("Fetching all feeds") - url = self._doc_class.urlobject.format(self._cb.credentials.org_key) - resp = self._cb.get_object(url, query_parameters=self._args) - results = resp.get("results", []) - return [self._doc_class(self._cb, initial_data=item) for item in results] - - -class ReportQuery(SimpleQuery): - """Represents the logic for a :py:class:`Report` query. - - >>> cb.select(Report).where(feed_id=id) - - .. NOTE:: - Only feed reports can be queried. Watchlist reports - should be interacted with via :py:meth:`Watchlist.reports`. - """ - def __init__(self, doc_class, cb): - super(ReportQuery, self).__init__(doc_class, cb) - self._args = {} - - def where(self, **kwargs): - self._args = dict(self._args, **kwargs) - return self - - @property - def results(self): - if "feed_id" not in self._args: - raise ApiError("required parameter feed_id missing") - - feed_id = self._args["feed_id"] - - log.debug("Fetching all reports") - url = self._doc_class.urlobject.format( - self._cb.credentials.org_key, - feed_id, - ) - resp = self._cb.get_object(url) - results = resp.get("results", []) - return [self._doc_class(self._cb, initial_data=item, feed_id=feed_id) for item in results] - - -class WatchlistQuery(SimpleQuery): - """Represents the logic for a :py:class:`Watchlist` query. - - >>> cb.select(Watchlist) - """ - def __init__(self, doc_class, cb): - super(WatchlistQuery, self).__init__(doc_class, cb) - - @property - def results(self): - log.debug("Fetching all watchlists") - - resp = self._cb.get_object(self._doc_class.urlobject) - results = resp.get("results", []) - return [self._doc_class(self._cb, initial_data=item) for item in results] diff --git a/src/cbapi/psc/threathunter/rest_api.py b/src/cbapi/psc/threathunter/rest_api.py deleted file mode 100644 index 463d6966..00000000 --- a/src/cbapi/psc/threathunter/rest_api.py +++ /dev/null @@ -1,116 +0,0 @@ -from cbapi.psc.threathunter.query import Query -from cbapi.connection import BaseAPI -from cbapi.psc.threathunter.models import ReportSeverity -from cbapi.errors import CredentialError -import logging - -log = logging.getLogger(__name__) - - -class CbThreatHunterAPI(BaseAPI): - """The main entry point into the Cb ThreatHunter PSC API. - - :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. - Uses the profile named 'default' when not specified. - - Usage:: - - >>> from cbapi.psc.threathunter import CbThreatHunterAPI - >>> cb = CbThreatHunterAPI(profile="production") - """ - def __init__(self, *args, **kwargs): - super(CbThreatHunterAPI, self).__init__(product_name="psc", *args, **kwargs) - self._lr_scheduler = None - - if not self.credentials.get("org_key", None): - raise CredentialError("No organization key specified") - - def _perform_query(self, cls, **kwargs): - if hasattr(cls, "_query_implementation"): - return cls._query_implementation(self) - else: - return Query(cls, self, **kwargs) - - def create(self, cls, data=None): - """Creates a new model. - - >>> feed = cb.create(Feed, feed_data) - - :param cls: The model being created - :param data: The data to pre-populate the model with - :type data: dict(str, object) - :return: an instance of `cls` - """ - return cls(self, initial_data=data) - - def validate_query(self, query): - """Validates the given IOC query. - - >>> cb.validate_query("process_name:chrome.exe") # True - - :param str query: the query to validate - :return: whether or not the query is valid - :rtype: bool - """ - args = {"q": query} - url = "/threathunter/search/v1/orgs/{}/processes/search_validation".format( - self.credentials.org_key - ) - resp = self.get_object(url, query_parameters=args) - - return resp.get("valid", False) - - def convert_query(self, query): - """Converts a legacy CB Response query to a ThreatHunter query. - - :param str query: the query to convert - :return: the converted query - :rtype: str - """ - args = {"query": query} - resp = self.post_object("/threathunter/feedmgr/v2/query/translate", args).json() - - return resp.get("query") - - @property - def custom_severities(self): - """Returns a list of active :py:class:`ReportSeverity` instances - - :rtype: list[:py:class:`ReportSeverity`] - """ - # TODO(ww): There's probably a better place to put this. - url = "/threathunter/watchlistmgr/v3/orgs/{}/reports/severity".format( - self.credentials.org_key - ) - resp = self.get_object(url) - items = resp.get("results", []) - return [self.create(ReportSeverity, item) for item in items] - - def queries(self): - """Retrieves a list of queries, active or complete, known by - the ThreatHunter server. - - :return: a list of query ids - :rtype: list(str) - """ - url = "/threathunter/search/v1/orgs/{}/processes/search_jobs".format( - self.credentials.org_key - ) - ids = self.get_object(url) - return ids.get("query_ids", []) - - def limits(self): - """Returns a dictionary containing API limiting information. - - Example: - - >>> cb.limits() - {u'status_code': 200, u'time_bounds': {u'upper': 1545335070095, u'lower': 1542779216139}} - - :return: a dict of limiting information - :rtype: dict(str, str) - """ - url = "/threathunter/search/v1/orgs/{}/processes/limits".format( - self.credentials.org_key - ) - return self.get_object(url) diff --git a/src/cbapi/query.py b/src/cbapi/query.py index a08b29da..fcb6f937 100644 --- a/src/cbapi/query.py +++ b/src/cbapi/query.py @@ -32,7 +32,8 @@ def one(self): res = self[:2] if len(res) == 0 or len(res) > 1: - raise MoreThanOneResultError(message="{0:d} results found for query {1:s}".format(len(self), str(self._query))) + raise MoreThanOneResultError(message="{0:d} results found for query {1:s}" + .format(len(self), str(self._query))) return res[0] diff --git a/src/cbapi/response/cblr.py b/src/cbapi/response/cblr.py index c17a571c..bded6f7d 100644 --- a/src/cbapi/response/cblr.py +++ b/src/cbapi/response/cblr.py @@ -14,7 +14,8 @@ class LiveResponseSessionManager(CbLRManagerBase): cblr_session_cls = LiveResponseSession def _get_or_create_session(self, sensor_id): - sensor_sessions = [s for s in self._cb.get_object("{cblr_base}/session?active_only=true".format(cblr_base=self.cblr_base)) + sensor_sessions = [s for s in self._cb.get_object("{cblr_base}/session?active_only=true" + .format(cblr_base=self.cblr_base)) if s["sensor_id"] == sensor_id and s["status"] in ("pending", "active")] if len(sensor_sessions) > 0: @@ -41,7 +42,7 @@ def _close_session(self, session_id): session_data = self._cb.get_object("{cblr_base}/session/{0}".format(session_id, cblr_base=self.cblr_base)) session_data["status"] = "close" self._cb.put_object("{cblr_base}/session/{0}".format(session_id, cblr_base=self.cblr_base), session_data) - except: + except Exception: pass def _create_session(self, sensor_id): diff --git a/src/cbapi/response/event.py b/src/cbapi/response/event.py index 9dbe59ac..c26c9ffd 100644 --- a/src/cbapi/response/event.py +++ b/src/cbapi/response/event.py @@ -33,18 +33,18 @@ def __init__(self, cb, filename): def run(self): while not self._fp.closed: - l = self._fp.readline() - if l == "": + line = self._fp.readline() + if line == "": # wait for new events sleep(1) continue try: - msg = json.loads(l) + msg = json.loads(line) routing_key = msg.get("type") # log.debug("Received message with routing key %s" % routing_key) - registry.eval_callback(routing_key, l, self._cb) - except: + registry.eval_callback(routing_key, line, self._cb) + except Exception: pass def stop(self): @@ -76,8 +76,8 @@ def __init__(self, cb, listening_address, **kwargs): certfile = kwargs.get("certfile", None) if not keyfile or not certfile: raise CredentialError("Need to specify 'keyfile' and 'certfile' for HTTPS") - self.httpd.socket = ssl.wrap_socket (self.httpd.socket, certfile=certfile, keyfile=keyfile, - server_side=True) + self.httpd.socket = ssl.wrap_socket(self.httpd.socket, certfile=certfile, keyfile=keyfile, + server_side=True) def run(self): self.httpd.serve_forever() @@ -168,7 +168,7 @@ def on_connection_closed(self, connection, reply_code, reply_text): self._connection.ioloop.stop() else: log.warning('Connection closed, reopening in 5 seconds: (%s) %s', - reply_code, reply_text) + reply_code, reply_text) self._connection.add_timeout(5, self.reconnect) def reconnect(self): @@ -231,7 +231,7 @@ def on_channel_closed(self, channel, reply_code, reply_text): """ log.warning('Channel %i was closed: (%s) %s', - channel, reply_code, reply_text) + channel, reply_code, reply_text) self._connection.close() def setup_exchange(self, exchange_name): @@ -282,7 +282,7 @@ def on_queue_declareok(self, method_frame): """ for routing_key in self.ROUTING_KEYS: log.debug('Binding %s to %s with %s', - self.EXCHANGE, self.QUEUE, routing_key) + self.EXCHANGE, self.QUEUE, routing_key) self._channel.queue_bind(self.on_bindok, self.QUEUE, self.EXCHANGE, routing_key) @@ -331,7 +331,7 @@ def on_consumer_cancelled(self, method_frame): """ log.debug('Consumer was cancelled remotely, shutting down: %r', - method_frame) + method_frame) if self._channel: self._channel.close() @@ -419,7 +419,7 @@ def on_message(self, unused_channel, basic_deliver, properties, body): """ log.debug('Received message # %s with properties %s', - basic_deliver.delivery_tag, properties) + basic_deliver.delivery_tag, properties) registry.eval_callback(basic_deliver.routing_key, body, self._cb) diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py old mode 100644 new mode 100755 index 44351708..0f177f13 --- a/src/cbapi/response/models.py +++ b/src/cbapi/response/models.py @@ -1,11 +1,13 @@ #!/usr/bin/env python - from __future__ import absolute_import -import contextlib import copy import json -from distutils.version import LooseVersion +import sys +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from collections import namedtuple, defaultdict import base64 from datetime import datetime, timedelta @@ -19,33 +21,32 @@ import time from cbapi.utils import convert_query_params -from ..errors import InvalidObjectError, ApiError, TimeoutError -from ..oldmodels import BaseModel, MutableModel, immutable - -if six.PY3: - long = int - from io import BytesIO as StringIO -else: - from cStringIO import StringIO +from ..errors import InvalidObjectError, ApiError, TimeoutError, MoreThanOneResultError +from ..oldmodels import BaseModel, immutable from ..models import NewBaseModel, MutableBaseModel, CreatableModelMixin -from ..oldmodels import BaseModel, immutable from .utils import convert_from_cb, convert_from_solr, parse_42_guid, convert_event_time, parse_process_guid, \ convert_to_solr from ..errors import ServerError, InvalidHashError, ObjectNotFoundError from ..query import SimpleQuery, PaginatedQuery from .query import Query -try: - from functools import total_ordering -except ImportError: - from total_ordering import total_ordering - from cbapi.six import python_2_unicode_compatible, iteritems # Get constants for decoding the Netconn events import socket +if six.PY3: + long = int + from io import BytesIO as StringIO +else: + from cStringIO import StringIO + +try: + from functools import total_ordering +except ImportError: + from total_ordering import total_ordering + log = logging.getLogger(__name__) @@ -102,7 +103,7 @@ class StoragePartitionQuery(SimpleQuery): def results(self): if not self._full_init: self._results = [] - for k,v in iteritems(self._cb.get_object(self._urlobject, default={})): + for k, v in iteritems(self._cb.get_object(self._urlobject, default={})): t = self._doc_class.new_object(self._cb, v, full_doc=True) if self._match_query(t): self._results.append(t) @@ -257,8 +258,9 @@ def _bulk_update(self, payload): return None - def set_ignored(self, ignored_flag=True): - payload = {"updates": {"is_ignored": ignored_flag, "requested_status": "False Positive"}} + def set_ignored(self, ignored_flag=True, status="False Positive"): + """Ignore all future Alerts from the Report that triggered this Alert.""" + payload = {"set_ignored": ignored_flag, "requested_status": status} return self._bulk_update(payload) def assign(self, target): @@ -266,6 +268,9 @@ def assign(self, target): return self._bulk_update(payload) def change_status(self, new_status): + allowed_statuses = ["In Progress", "Unresolved", "Resolved", "False Positive"] + if new_status not in allowed_statuses: + raise ApiError("Alert status must be one of {0}".format(allowed_statuses)) payload = {"requested_status": new_status} return self._bulk_update(payload) @@ -282,11 +287,42 @@ def _query_implementation(cls, cb): def __init__(self, cb, alert_id, initial_data=None): super(Alert, self).__init__(cb, alert_id, initial_data) + if alert_id is not None and initial_data is None: + self.refresh() - def _refresh(self): - # there is no GET method for an Alert. + def refresh(self): + url = '{}?cb.fq.unique_id={}'.format(self.urlobject, self.unique_id) + resp = self._cb.get_object(url) + result = resp.get("results", []) + if len(result) > 1: + raise MoreThanOneResultError("More than one Alert matched the unique_id") + elif len(result) == 0: + raise ObjectNotFoundError("Alert could not be found by unique_id") + else: + self._info = result[0] + self._last_refresh_time = time.time() + self._full_init = True return True + def set_ignored(self, ignored_flag=True, status="False Positive"): + """Ignore all future Alerts from the Report that triggered this Alert.""" + payload = {"set_ignored": ignored_flag, "requested_status": status} + payload["alert_ids"] = [self.unique_id] + return self._cb.post_object("/api/v1/alerts", payload) + + def assign(self, target): + payload = {"assigned_to": target, "requested_status": "In Progress"} + payload["alert_ids"] = [self.unique_id] + return self._cb.post_object("/api/v1/alerts", payload) + + def change_status(self, new_status): + allowed_statuses = ["In Progress", "Unresolved", "Resolved", "False Positive"] + if new_status not in allowed_statuses: + raise ApiError("Alert status must be one of {0}".format(allowed_statuses)) + payload = {"status": new_status} + payload["unique_id"] = self.unique_id + return self._cb.post_object("/api/v1/alert/{0}".format(self.unique_id), payload) + @property def process(self): if 'process' in self.alert_type: @@ -311,6 +347,17 @@ def feed(self): def trigger_ioc(self): return self.ioc_attr + # override the Alert API URL to /api/v1/alert when performing POST/PUTs + def _build_api_request_uri(self, http_method="GET"): + if http_method == "GET": + return super(Alert, self)._build_api_request_uri(http_method) + else: + baseuri = "/api/v1/alert" + if self._model_unique_id is not None: + return baseuri + "/%s" % self._model_unique_id + else: + return baseuri + class Feed(MutableBaseModel, CreatableModelMixin): swagger_meta_file = "response/models/feed.yaml" @@ -345,7 +392,8 @@ def search_processes(self, min_score=None, max_score=None): :param min_score: minimum feed score :param max_score: maximum feed score - :return: Returns a :py:class:`response.rest_api.Query` object with the appropriate search parameters for processes + :return: Returns a :py:class:`response.rest_api.Query` object with the appropriate + search parameters for processes :rtype: :py:class:`response.rest_api.Query` """ return self._search(Process, min_score, max_score) @@ -355,7 +403,8 @@ def search_binaries(self, min_score=None, max_score=None): Perform a *Binary* search within this feed that satisfies min_score and max_score :param min_score: minimum feed score :param max_score: maximum feed score - :return: Returns a :py:class:`response.rest_api.Query` object within the appropriate search parameters for binaries + :return: Returns a :py:class:`response.rest_api.Query` object within the appropriate + search parameters for binaries :rtype: :py:class:`response.rest_api.Query` """ return self._search(Binary, min_score, max_score) @@ -545,6 +594,8 @@ def _search(self, start=0, rows=0): else: args = {} + args.update({"sort.col":"computer_name", "sort.dir":"asc"}) + args['start'] = start if rows: @@ -609,11 +660,11 @@ def __init__(self, *args, **kwargs): def _query_implementation(cls, cb): # ** Disable the paginated query implementation for now ** - # if cb.cb_server_version >= LooseVersion("5.2.0"): + # if cb.cb_server_version >= parse("5.2.0"): # return SensorPaginatedQuery(cls, cb) # else: # return SensorQuery(cls, cb) - return SensorQuery(cls, cb) + return SensorPaginatedQuery(cls, cb) @property def group(self): @@ -655,7 +706,7 @@ def network_interfaces(self): for adapter in getattr(self, 'network_adapters', '').split('|'): parts = adapter.split(',') if len(parts) == 2: - out.append(Sensor.NetworkAdapter._make([':'.join(a+b for a,b in zip(parts[1][::2], parts[1][1::2])), + out.append(Sensor.NetworkAdapter._make([':'.join(a+b for a, b in zip(parts[1][::2], parts[1][1::2])), parts[0]])) return out @@ -669,21 +720,21 @@ def os(self): @property def registration_time(self): """ - Returns the time the sensor registered with the Cb Response Server + Returns the time the sensor registered with the EDR Server """ return convert_from_cb(getattr(self, 'registration_time', -1)) @property def sid(self): """ - Security Identifier being used by the Cb Response Sensor + Security Identifier being used by the EDR Sensor """ return getattr(self, 'computer_sid') @property def webui_link(self): """ - Returns the Cb Response Web UI link associated with this Sensor + Returns the Carbon Black EDR Web UI link associated with this Sensor """ return '{0:s}/#/host/{1}'.format(self._cb.url, self._model_unique_id) @@ -691,7 +742,7 @@ def webui_link(self): @property def queued_stats(self): """ - Returns a list of status and size of the queued event logs from the associated Cb Response Sensor + Returns a list of status and size of the queued event logs from the associated EDR Sensor :example: @@ -710,14 +761,14 @@ def queued_stats(self): @property def activity_stats(self): """ - Returns a list of activity statistics from the associated Cb Response Sensor + Returns a list of activity statistics from the associated EDR Sensor """ return self._cb.get_object("{0}/activity".format(self._build_api_request_uri()), default=[]) @property def resource_status(self): """ - Returns a list of memory statistics used by the Cb Response Sensor + Returns a list of memory statistics used by the EDR Sensor """ return self._cb.get_object("{0}/resourcestatus".format(self._build_api_request_uri()), default=[]) @@ -737,12 +788,11 @@ def lr_session(self): def flush_events(self): """ - Performs a flush of events for this Cb Response Sensor + Performs a flush of events for this EDR Sensor - :warning: This may cause a significant amount of network traffic from this sensor to the Cb Response Server + :warning: This may cause a significant amount of network traffic from this sensor to the EDR Server """ - # Note that Cb Response 6 requires the date/time stamp to be sent in RFC822 format (not ISO 8601). # since the date/time stamp just needs to be far in the future, we just fake a GMT timezone. self.event_log_flush_time = datetime.now() + timedelta(days=365) @@ -752,7 +802,7 @@ def restart_sensor(self): """ Restarts the Carbon Black sensor (*not* the underlying endpoint operating system). - This simply sets the flag to ask the sensor to restart the next time it checks into the Cb Response server, + This simply sets the flag to ask the sensor to restart the next time it checks into the EDR server, it does not wait for the sensor to restart. """ self.restart_queued = True @@ -760,7 +810,7 @@ def restart_sensor(self): def isolate(self, timeout=None): """ - Turn on network isolation for this Cb Response Sensor. + Turn on network isolation for this EDR Sensor. This function will block and only return when the isolation is complete, or if a timeout is reached. By default, there is no timeout. You can specify a timeout period (in seconds) in the "timeout" parameter to this @@ -785,7 +835,7 @@ def isolate(self, timeout=None): def unisolate(self, timeout=None): """ - Turn off network isolation for this Cb Response Sensor. + Turn off network isolation for this EDR Sensor. This function will block and only return when the isolation is removed, or if a timeout is reached. By default, there is no timeout. You can specify a timeout period (in seconds) in the "timeout" parameter to this @@ -809,15 +859,21 @@ def unisolate(self, timeout=None): return True def _update_object(self): - # Workarounds for issuing a sensor queue flush in Cb Response 6.0 + # 1st Workarounds for issuing a sensor queue flush in Cb Response 6.0 # - We only want to reflect back the event_log_flush_time if the user explicitly set it to a new value # (therefore, set the event_log_flush_time to None if it isn't marked dirty) # - The event_log_flush_time must be sent in RFC822 format (not ISO 8601) for Cb Response 6.x servers. # # Note that even though we delete the event_log_flush_time here, it'll get re-initialized when we GET # the sensor after sending the PUT request. - if "event_log_flush_time" in self._dirty_attributes and self._info.get("event_log_flush_time", None) is not None: - if self._cb.cb_server_version > LooseVersion("6.0.0"): + # + # 2nd Workarounds for updating a sensor's ability to update attributes in Cb Response 7.1 + # - Only allowed fields are: ['network_isolation_enabled', 'restart_queued', 'uninstall', 'liveresponse_init', 'group_id', 'notes', 'event_log_flush_time'] + # - Instead of sending in entire sensorObject fields, we will only send in the above 7 + + if "event_log_flush_time" in self._dirty_attributes and self._info.get("event_log_flush_time", + None) is not None: + if self._cb.cb_server_version > parse("6.0.0"): # since the date/time stamp just needs to be far in the future, we just fake a GMT timezone. try: self._info["event_log_flush_time"] = self.event_log_flush_time.strftime("%a, %d %b %Y %H:%M:%S GMT") @@ -827,12 +883,38 @@ def _update_object(self): else: self._info["event_log_flush_time"] = None - return super(Sensor, self)._update_object() + if self.__class__.primary_key in self._dirty_attributes.keys() or self._model_unique_id is None: + new_object_info = deepcopy(self._info) + try: + if not self._new_object_needs_primary_key: + del(new_object_info[self.__class__.primary_key]) + except Exception: + pass + log.debug("Creating a new {0:s} object".format(self.__class__.__name__)) + ret = self._cb.api_json_request(self.__class__._new_object_http_method, self.urlobject, + data=new_object_info) + else: + log.debug("Updating {0:s} with unique ID {1:s}".format(self.__class__.__name__, str(self._model_unique_id))) + http_method = self.__class__._change_object_http_method + + allowed_fields = { + 'network_isolation_enabled': self._info['network_isolation_enabled'], + 'restart_queued': self._info['restart_queued'], + 'uninstall': self._info['uninstall'], + 'group_id': self._info['group_id'], + 'notes': self._info['notes'], + 'event_log_flush_time': self._info['event_log_flush_time'] + } + + ret = self._cb.api_json_request(http_method, self._build_api_request_uri(http_method=http_method), + data=allowed_fields) + + return self._refresh_if_needed(ret) class SensorGroup(MutableBaseModel, CreatableModelMixin): swagger_meta_file = "response/models/group-modify.yaml" - urlobject = '/api/group' + urlobject = '/api/v3/group' @classmethod def _query_implementation(cls, cb): @@ -862,6 +944,53 @@ def site(self): def site(self, new_site): self.site_id = new_site.id +class SensorBuilds(MutableBaseModel): + swagger_meta_file = "response/models/sensor-builds.yaml" + urlobject = '/api/v2/builds' + + def __init__(self, cb, initial_data=None): + super(SensorBuilds, self).__init__(cb) + if initial_data is not None: + temp_list = [] + for b in initial_data['Windows']: + temp_list.append(b['version_string']) + self.Windows = temp_list + temp_list = [] + for v in initial_data['Linux']: + temp_list.append(v['version_string']) + self.Linux = temp_list + temp_list = [] + for v in initial_data['OSX']: + temp_list.append(v['version_string']) + self.OSX = temp_list + + @classmethod + def new_object(self, cb, initial_data): + o = self(cb, initial_data) + return o + + @classmethod + def _query_implementation(self, cb): + return BuildsQuery(self, cb) + + def _refresh(self): + self._info = {'Windows': [], 'Linux': [], 'OSX': []} + return True + +class BuildsQuery(SimpleQuery): + + def __init__(self, doc_class, cb): + super(BuildsQuery, self).__init__(doc_class, cb) + + @property + def results(self): + if not self._full_init: + api_results = self._cb.get_object(self._urlobject, default={}) + self._results = self._doc_class.new_object(self._cb, api_results) + self._full_init = True + + return self._results + class SensorQuery(SimpleQuery): valid_field_names = ['ip', 'hostname', 'groupid'] @@ -882,12 +1011,12 @@ def where(self, new_query): @property def results(self): if not self._full_init: - #ZE CB-15681 - REMOVE BLOCK WHEN BUG IS FIXED + # ZE CB-15681 - REMOVE BLOCK WHEN BUG IS FIXED try: full_results = self._cb.get_object(self._urlobject, query_parameters=convert_query_params(self._query)) - except ServerError as se: + except ServerError: full_results = False - #ZE CB-15681 - REMOVE BLOCK WHEN BUG IS FIXED + # ZE CB-15681 - REMOVE BLOCK WHEN BUG IS FIXED if not full_results: self._results = [] else: @@ -945,7 +1074,7 @@ def _retrieve_cb_info(self): return info def _update_object(self): - if self._cb.cb_server_version < LooseVersion("6.1.0") or self._info.get("id", None) == None: + if self._cb.cb_server_version < parse("6.1.0") or self._info.get("id", None) is None: # only include IDs of the teams and not the entire dictionary # - applies to Cb Response server < 6.0 as well as Cb Response servers >= 6.0 where the user hasn't # been created yet. @@ -1074,7 +1203,7 @@ def query(self, new_query): self._reset_query() qt = list(self._query) qt.append(("q", new_query)) - self.search_query = "&".join(("{0}={1}".format(k, urllib.parse.quote(v)) for k,v in qt)) + self.search_query = "&".join(("{0}={1}".format(k, urllib.parse.quote(v)) for k, v in qt)) @property def facets(self): @@ -1257,7 +1386,7 @@ def tag_info(self, tag_name): class ThreatReportQuery(Query): def set_ignored(self, ignored_flag=True): qt = (("cb.urlver", "1"), ("q", self._query)) - search_query = "&".join(("{0}={1}".format(k, urllib.parse.quote(v)) for k,v in qt)) + search_query = "&".join(("{0}={1}".format(k, urllib.parse.quote(v)) for k, v in qt)) payload = {"updates": {"is_ignored": ignored_flag}, "query": search_query} self._cb.post_object("/api/v1/threat_report", payload) @@ -1269,7 +1398,7 @@ class ThreatReport(MutableBaseModel): @classmethod def _query_implementation(cls, cb): - if cb.cb_server_version >= LooseVersion('5.1.0'): + if cb.cb_server_version >= parse('5.1.0'): return ThreatReportQuery(cls, cb) else: return Query(cls, cb) @@ -1304,8 +1433,8 @@ def __init__(self, cb, full_id, initial_data=None): try: # fill in feed_id and id (feed_id, report_id) = full_id.split(":") - initial_data = { "feed_id": feed_id, "id": report_id } - except: + initial_data = {"feed_id": feed_id, "id": report_id} + except Exception: raise ApiError("ThreatReport ID must be in form ':'") super(ThreatReport, self).__init__(cb, full_id, initial_data) @@ -1315,7 +1444,7 @@ def feed(self): return self._join(Feed, "feed_id") def _update_object(self): - update_content = { "ids": { str(self.feed_id): [str(self.id)] }, "updates": {}} + update_content = {"ids": {str(self.feed_id): [str(self.id)]}, "updates": {}} for k in self._dirty_attributes.keys(): update_content["updates"][k] = getattr(self, k) @@ -1324,7 +1453,7 @@ def _update_object(self): if ret.status_code not in (200, 204): try: message = json.loads(ret.text)[0] - except: + except Exception: message = ret.text raise ServerError(ret.status_code, message, @@ -1343,7 +1472,7 @@ def _update_object(self): else: self._info = json.loads(ret.text) self._full_init = True - except: + except Exception: self.refresh() self._dirty_attributes = {} @@ -1392,14 +1521,14 @@ def group_by(self, field_name): """Set the group-by field name for this query. Typically, you will want to set this to 'id' if you only want one result per process. - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :param str field_name: Field name to group the result set by. :return: Query object :rtype: :py:class:`ProcessQuery` """ - if self._cb.cb_server_version >= LooseVersion('6.0.0'): + if self._cb.cb_server_version >= parse('6.0.0'): nq = self._clone() nq._default_args["cb.group"] = field_name return nq @@ -1410,8 +1539,8 @@ def group_by(self, field_name): def max_children(self, num_children): """Sets the number of children to fetch with the process - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :default: 15 :param int num_children: Number of children to fetch with process @@ -1441,14 +1570,14 @@ def min_last_update(self, v): This option will limit the number of Solr cores that need to be searched for events that match the query. - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :param str v: Timestamp (either string or datetime object). :return: Query object :rtype: :py:class:`ProcessQuery` """ - if self._cb.cb_server_version >= LooseVersion('6.0.0'): + if self._cb.cb_server_version >= parse('6.0.0'): nq = self._clone() try: v = v.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -1467,14 +1596,14 @@ def min_last_server_update(self, v): This option will limit the number of Solr cores that need to be searched for events that match the query. - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :param str v: Timestamp (either string or datetime object). :return: Query object :rtype: :py:class:`ProcessQuery` """ - if self._cb.cb_server_version >= LooseVersion('6.0.0'): + if self._cb.cb_server_version >= parse('6.0.0'): nq = self._clone() try: v = v.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -1493,14 +1622,14 @@ def max_last_update(self, v): This option will limit the number of Solr cores that need to be searched for events that match the query. - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :param str v: Timestamp (either string or datetime object). :return: Query object :rtype: :py:class:`ProcessQuery` """ - if self._cb.cb_server_version >= LooseVersion('6.0.0'): + if self._cb.cb_server_version >= parse('6.0.0'): nq = self._clone() try: v = v.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -1519,14 +1648,14 @@ def max_last_server_update(self, v): This option will limit the number of Solr cores that need to be searched for events that match the query. - This method is only available for Cb Response servers 6.0 and above. Calling this on a Query object connected - to a Cb Response 5.x server will simply result in a no-op. + This method is only available for EDR servers 6.0 and above. Calling this on a Query object connected + to a EDR 5.x server will simply result in a no-op. :param str v: Timestamp (either string or datetime object). :return: Query object :rtype: :py:class:`ProcessQuery` """ - if self._cb.cb_server_version >= LooseVersion('6.0.0'): + if self._cb.cb_server_version >= parse('6.0.0'): nq = self._clone() try: v = v.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -1631,7 +1760,7 @@ def _build_api_request_uri(self): @property def webui_link(self): """ - Returns the Cb Response Web UI link associated with this Binary object + Returns the Carbon Black EDR Web UI link associated with this Binary object """ return '{0:s}/#binary/{1:s}'.format(self._cb.url, self.md5sum) @@ -1715,8 +1844,8 @@ def endpoints(self): @property def version_info(self): """ - Returns a :class:`.VersionInfo` object containing detailed information: File Descritpion, File Version, Product Name, - Product Version, Company Name, Legal Copyright, and Original FileName + Returns a :class:`.VersionInfo` object containing detailed information: File Descritpion, File Version, + Product Name, Product Version, Company Name, Legal Copyright, and Original FileName """ return Binary.VersionInfo._make([self._attribute('file_desc', ""), self._attribute('file_version', ""), self._attribute('product_name', ""), self._attribute('product_version', ""), @@ -1802,7 +1931,6 @@ def is_executable_image(self): """ return self._attribute('is_executable_image', False) - @property def icon(self): """ @@ -1813,7 +1941,7 @@ def icon(self): icon = self._attribute('icon') if not icon: icon = '' - except: + except Exception: pass return base64.b64decode(icon) @@ -1821,7 +1949,7 @@ def icon(self): @property def banned(self): """ - Returns *BannedHash* object if this Binary's hash has been whitelisted (Banned), otherwise returns *False* + Returns *BannedHash* object if this Binary's hash has been banned, otherwise returns *False* """ try: bh = self._cb.select(BannedHash, self.md5sum.lower()) @@ -1917,7 +2045,7 @@ def parse_netconn(self, seq, netconn): timestamp = convert_event_time(parts[0]) try: new_conn['remote_ip'] = socket.inet_ntoa(struct.pack('>i', int(parts[1]))) - except: + except Exception: new_conn['remote_ip'] = '0.0.0.0' new_conn['remote_port'] = int(parts[2]) new_conn['proto'] = protocols[int(parts[3])] @@ -2021,7 +2149,7 @@ def _lookup_privilege(privilege_code): if parts[8] == 'true': new_crossproc['is_target'] = True - if new_crossproc['is_target'] == True: + if new_crossproc['is_target']: new_crossproc['target_procguid'] = self.process_model.id new_crossproc['target_md5'] = self.process_model.process_md5 new_crossproc['target_path'] = self.process_model.path @@ -2038,6 +2166,22 @@ def _lookup_privilege(privilege_code): return CbCrossProcEvent(self.process_model, timestamp, seq, new_crossproc) + def parse_processblock(self, seq, processblock): + processblock_json = json.loads(processblock) + new_processblock = {} + timestamp = processblock_json.get("timestamp", None) + new_processblock["type"] = processblock_json.get("block_type", None) + new_processblock["event"] = processblock_json.get("block_event", None) + new_processblock["result"] = processblock_json.get("block_result", None) + new_processblock["error"] = processblock_json.get("block_error", None) + new_processblock["md5"] = processblock_json.get("blocked_md5", None) + new_processblock["path"] = processblock_json.get("blocked_path", None) + new_processblock["cmdline"] = processblock_json.get("blocked_cmdline", None) + new_processblock["uid"] = processblock_json.get("blocked_uid", None) + new_processblock["username"] = processblock_json.get("blocked_username", None) + + return CbProcessBlockEvent(self.process_model, timestamp, seq, new_processblock) + class ProcessV2Parser(ProcessV1Parser): def __init__(self, process_model): @@ -2056,7 +2200,7 @@ def parse_netconn(self, seq, netconn): for ipfield in ('remote_ip', 'local_ip', 'proxy_ip'): try: new_conn[ipfield] = socket.inet_ntoa(struct.pack('>i', int(netconn.get(ipfield, 0)))) - except: + except Exception: new_conn[ipfield] = netconn.get(ipfield, '0.0.0.0') for portfield in ('remote_port', 'local_port', 'proxy_port'): @@ -2134,7 +2278,8 @@ def _query_implementation(cls, cb): @classmethod def new_object(cls, cb, item, max_children=15): # 'id' did not exist in some process documents from a 5.2 -> 6.1 upgrade - return cb.select(Process, item['id'] or item['unique_id'], long(item['segment_id']), max_children, initial_data=item) + return cb.select(Process, item['id'] or item['unique_id'], long(item['segment_id']), + max_children, initial_data=item) def parse_guid(self, procguid): try: @@ -2144,12 +2289,13 @@ def parse_guid(self, procguid): # new 5.x process IDs are hex strings with optional segment IDs. if len(procguid) == 45: return procguid[:36], int(procguid[38:], 16) - elif len(procguid) == 49 and self._cb.cb_server_version >= LooseVersion('6.0.0'): + elif len(procguid) == 49 and self._cb.cb_server_version >= parse('6.0.0'): return procguid[:36], int(procguid[38:], 16) else: return None, None - def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=None, force_init=False, suppressed_process=False): + def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=None, force_init=False, + suppressed_process=False): self.max_children = max_children self.current_segment = segment self.suppressed_process = suppressed_process @@ -2167,7 +2313,7 @@ def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=Non self.__children_info = None self.__sibling_info = None - if cb.cb_server_version < LooseVersion('6.0.0'): + if cb.cb_server_version < parse('6.0.0'): self._default_segment = 1 else: self._default_segment = 0 @@ -2180,7 +2326,7 @@ def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=Non if len(procguid) == 45: self.id = procguid[:36] self.current_segment = int(procguid[38:], 16) - elif len(procguid) == 49 and cb.cb_server_version >= LooseVersion('6.0.0'): + elif len(procguid) == 49 and cb.cb_server_version >= parse('6.0.0'): self.id = procguid[:36] self.current_segment = int(procguid[38:], 16) else: @@ -2197,14 +2343,14 @@ def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=Non self._process_summary_api = 'v1' - if cb.cb_server_version >= LooseVersion('6.0.0'): + if cb.cb_server_version >= parse('6.0.0'): self._process_summary_api = 'v2' self._process_event_api = 'v4' self._event_parser = ProcessV4Parser(self) - elif cb.cb_server_version >= LooseVersion('5.2.0'): + elif cb.cb_server_version >= parse('5.2.0'): self._process_event_api = 'v3' self._event_parser = ProcessV3Parser(self) - elif cb.cb_server_version >= LooseVersion('5.1.0'): + elif cb.cb_server_version >= parse('5.1.0'): # CbER 5.1.0 introduced an extended event API self._process_event_api = 'v2' self._event_parser = ProcessV2Parser(self) @@ -2255,7 +2401,7 @@ def _attribute(self, attrname, default=None): # Relaxing this a bit to allow for cases where the information is there if attrname in ['parent_unique_id', 'parent_name'] and not self._full_init: - if attrname in self._info and self._info[attrname] != None and self._info[attrname] != "": + if attrname in self._info and self._info[attrname] is not None and self._info[attrname] != "": return self._info[attrname] else: self._retrieve_cb_info() @@ -2300,7 +2446,7 @@ def _parse(self, obj): else: log.debug("Unable to recover start time and process PID for process %s, marking invalid" % self.id) self.valid_process = False - except Exception as e: + except Exception: log.debug("Unable to parse process GUID %s, marking invalid" % self.id) self.valid_process = False @@ -2387,7 +2533,7 @@ def walk_children(self, callback, max_depth=0, depth=0): @property def parent_md5(self): """ - Workaround since parent_md5 silently disappeared in ~Cb Response 6.x + Workaround since parent_md5 silently disappeared in EDR 6.x """ return self.parent.process_md5 @@ -2412,12 +2558,12 @@ def end(self): if self._info.get("end") is not None: return convert_from_solr(self._info.get('end', -1)) - if self.get("terminated", False) == True and self.get("last_update") is not None: + if self.get("terminated", False) and self.get("last_update") is not None: return convert_from_solr(self._attribute('last_update', -1)) def require_events(self): event_key_list = ['filemod_complete', 'regmod_complete', 'modload_complete', 'netconn_complete', - 'crossproc_complete', 'childproc_complete'] + 'crossproc_complete', 'childproc_complete', 'processblock_complete'] if not self.valid_process or self.suppressed_process: return @@ -2455,6 +2601,18 @@ def refresh(self): self._segments = [] super(Process, self).refresh() + @property + def processblocks(self): + """ + Generator that returns :py:class:`CbProcessBlockEvent` objects associated with this process + """ + self.require_events() + + i = 0 + for raw_processblock in self._events.get(self.current_segment, {}).get('processblock_complete', []): + yield self._event_parser.parse_processblock(i, raw_processblock) + i += 1 + @property def segment(self): log.debug("The .segment attribute will be deprecated in future versions of the cbapi Python module.") @@ -2529,17 +2687,18 @@ def crossprocs(self): @property def parents(self): - current_process = self - while True: - try : - parent = current_process.parent - if not(parent) or parent.get('process_pid',-1) == -1: - break - yield parent - current_process = parent - except ObjectNotFoundError: - return - return + current_process = self + while True: + try: + parent = current_process.parent + if not(parent) or parent.get('process_pid', -1) == -1: + break + yield parent + current_process = parent + except ObjectNotFoundError: + return + return + @property def children(self): """ @@ -2560,14 +2719,15 @@ def children(self): timestamp = convert_event_time(child.get("start") or "1970-01-01T00:00:00Z") yield CbChildProcEvent(self, timestamp, i, { - "procguid": child.get("unique_id",None), - "md5": child.get("process_md5",None), - "pid": child.get("process_pid",None), - "path": child.get("path",None), + "procguid": child.get("unique_id", None), + "md5": child.get("process_md5", None), + "pid": child.get("process_pid", None), + "path": child.get("path", None), "terminated": False }, is_suppressed=child.get("is_suppressed", False), - proc_data=child) + proc_data=child, + max_children=self.max_children) else: for cp in self.childprocs: yield cp @@ -2593,13 +2753,13 @@ def all_events_segment(self): :return: list of CbEvent objects """ segment_events = list(self.modloads) + list(self.netconns) + list(self.filemods) + \ - list(self.children) + list(self.regmods) + list(self.crossprocs) + list(self.children) + list(self.regmods) + list(self.crossprocs) segment_events.sort() return segment_events def get_segments(self): if not self._segments: - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): log.debug("using process_id search for cb response server < 6.0") segment_query = Query(Process, self._cb, query="process_id:{0}".format(self.id)).sort("") proclist = sorted([res["segment_id"] for res in segment_query._search()]) @@ -2665,9 +2825,10 @@ def depth(self): @property def threat_intel_hits(self): try: - hits = self._cb.get_object("/api/v1/process/{0}/{1}/threat_intel_hits".format(self.id, self.current_segment)) + hits = self._cb.get_object("/api/v1/process/{0}/{1}/threat_intel_hits".format(self.id, + self.current_segment)) return hits - except ServerError as e: + except ServerError: raise ApiError("Sharing IOCs not set up in Cb server. See {}/#/share for more information." .format(self._cb.credentials.url)) @@ -2707,11 +2868,11 @@ def binary(self): @property def comms_ip(self): """ - Returns ascii representation of the ip address used to communicate with the Cb Response Server + Returns ascii representation of the ip address used to communicate with the EDR Server """ try: ip_address = socket.inet_ntoa(struct.pack('>i', self._attribute('comms_ip', 0))) - except: + except Exception: ip_address = self._attribute('comms_ip', 0) return ip_address @@ -2719,12 +2880,12 @@ def comms_ip(self): @property def interface_ip(self): """ - Returns ascii representation of the ip address of the interface used to communicate with the Cb Response server. + Returns ascii representation of the ip address of the interface used to communicate with the EDR server. If using NAT, this will be the "internal" IP address of the sensor. """ try: ip_address = socket.inet_ntoa(struct.pack('>i', self._attribute('interface_ip', 0))) - except: + except Exception: ip_address = self._attribute('interface_ip', 0) return ip_address @@ -2738,13 +2899,12 @@ def process_md5(self): except StopIteration: return None - @property def path(self): # Some processes don't have a path associated with them. Try and use the first modload as the file path # otherwise, return None. (tested with Cb Response server 5.2.0.161004.1206) try: - return self._attribute("path","") or next(self.modloads).path + return self._attribute("path", "") or next(self.modloads).path except StopIteration: return None @@ -2799,7 +2959,7 @@ def sensor(self): @property def webui_link(self): """ - Returns the Cb Response Web UI link associated with this process + Returns the Carbon Black EDR Web UI link associated with this process """ if not self.suppressed_process: return '%s/#analyze/%s/%s' % (self._cb.url, self.id, self.current_segment) @@ -2874,17 +3034,19 @@ def require_all_events(self): 'modload_count': 'modload_complete', 'netconn_count': 'netconn_complete', 'crossproc_count': 'crossproc_complete', - 'childproc_count': 'childproc_complete' + 'childproc_count': 'childproc_complete', + 'processblock_count': 'processblock_complete' } if not self.valid_process or self.suppressed_process: return # Get all events - res = self._cb.get_object("/api/{0}/process/{1}/{2}/event".format(self._process_event_api, self.id, 0)).get("process", {}) + res = self._cb.get_object("/api/{0}/process/{1}/{2}/event".format(self._process_event_api, + self.id, 0)).get("process", {}) self._events['all'] = {} - for (event_count,event_complete) in event_types.items(): + for (event_count, event_complete) in event_types.items(): complete_events = res.get(event_complete, 0) if complete_events: self._info[event_count] = len(complete_events) @@ -2892,7 +3054,7 @@ def require_all_events(self): self._events['all'][event_count] = self._info[event_count] self._events['all'][event_complete] = self._info[event_complete] - #total_events = sum([self._info[event_count] for event_count in event_types]) + # total_events = sum([self._info[event_count] for event_count in event_types]) # Delete references in res. if not self._full_init: @@ -2907,9 +3069,8 @@ def require_all_events(self): self.all_events_loaded = True - def all_childprocs(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -2931,7 +3092,7 @@ def all_childprocs(self): i += 1 def all_modloads(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -2953,7 +3114,7 @@ def all_modloads(self): i += 1 def all_filemods(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -2974,8 +3135,30 @@ def all_filemods(self): yield self._event_parser.parse_filemod(i, raw_filemod) i += 1 + def all_processblocks(self): + if self._cb.cb_server_version < parse('6.0.0'): + self.get_segments() + segments = self._segments + + i = 0 + for segment in segments: + self.current_segment = segment + self.require_events() + + for raw_processblock in self._events.get(self.current_segment, {}).get('processblock_complete', []): + yield self._event_parser.parse_processblock(i, raw_processblock) + i += 1 + else: + if not self.all_events_loaded: + self.require_all_events() + + i = 0 + for raw_processblock in self._events.get('all', {}).get('processblock_complete', []): + yield self._event_parser.parse_processblock(i, raw_processblock) + i += 1 + def all_regmods(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -2997,7 +3180,7 @@ def all_regmods(self): i += 1 def all_crossprocs(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -3019,7 +3202,7 @@ def all_crossprocs(self): i += 1 def all_netconns(self): - if self._cb.cb_server_version < LooseVersion('6.0.0'): + if self._cb.cb_server_version < parse('6.0.0'): self.get_segments() segments = self._segments @@ -3040,11 +3223,12 @@ def all_netconns(self): yield self._event_parser.parse_netconn(i, raw_netconn) i += 1 + def get_constants(prefix): return dict((getattr(socket, n), n) for n in dir(socket) if n.startswith(prefix) - ) + ) protocols = get_constants("IPPROTO_") @@ -3102,7 +3286,7 @@ def tamper_event(self): class CbModLoadEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data, binary_data=None): - super(CbModLoadEvent,self).__init__(parent_process, timestamp, sequence, event_data) + super(CbModLoadEvent, self).__init__(parent_process, timestamp, sequence, event_data) self.event_type = u'Cb Module Load event' self.stat_titles.extend(['md5', 'path']) @@ -3119,30 +3303,38 @@ def is_signed(self): class CbFileModEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data): - super(CbFileModEvent,self).__init__(parent_process, timestamp, sequence, event_data) + super(CbFileModEvent, self).__init__(parent_process, timestamp, sequence, event_data) self.event_type = u'Cb File Modification event' self.stat_titles.extend(['type', 'path', 'filetype', 'md5']) class CbRegModEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data): - super(CbRegModEvent,self).__init__(parent_process, timestamp, sequence, event_data) + super(CbRegModEvent, self).__init__(parent_process, timestamp, sequence, event_data) self.event_type = u'Cb Registry Modification event' self.stat_titles.extend(['type', 'path']) class CbNetConnEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data, version=1): - super(CbNetConnEvent,self).__init__(parent_process, timestamp, sequence, event_data) + super(CbNetConnEvent, self).__init__(parent_process, timestamp, sequence, event_data) self.event_type = u'Cb Network Connection event' self.stat_titles.extend(['domain', 'remote_ip', 'remote_port', 'proto', 'direction']) if version == 2: self.stat_titles.extend(['local_ip', 'local_port', 'proxy_ip', 'proxy_port']) +class CbProcessBlockEvent(CbEvent): + def __init__(self, parent_process, timestamp, sequence, event_data): + super(CbProcessBlockEvent, self).__init__(parent_process, timestamp, sequence, event_data) + self.event_type = u'Cb Process Block event' + self.stat_titles.extend(['type', 'event', 'result', 'error', 'md5', 'path', 'uid', 'username']) + + class CbChildProcEvent(CbEvent): - def __init__(self, parent_process, timestamp, sequence, event_data, is_suppressed=False, proc_data=None): - super(CbChildProcEvent,self).__init__(parent_process, timestamp, sequence, event_data) + def __init__(self, parent_process, timestamp, sequence, event_data, is_suppressed=False, proc_data=None, max_children=15): + super(CbChildProcEvent, self).__init__(parent_process, timestamp, sequence, event_data) + self.max_children = max_children self.event_type = u'Cb Child Process event' self.stat_titles.extend(['procguid', 'pid', 'path', 'md5']) self.is_suppressed = is_suppressed @@ -3184,12 +3376,12 @@ def process(self): child_unique_id = self.procguid return self.parent._cb.select(Process, child_unique_id, initial_data=proc_data, - suppressed_process=self.is_suppressed) + suppressed_process=self.is_suppressed, max_children=self.max_children) class CbCrossProcEvent(CbEvent): def __init__(self, parent_process, timestamp, sequence, event_data): - super(CbCrossProcEvent,self).__init__(parent_process, timestamp, sequence, event_data) + super(CbCrossProcEvent, self).__init__(parent_process, timestamp, sequence, event_data) self.event_type = u'Cb Cross Process event' self.stat_titles.extend(['type', 'privileges', 'target_md5', 'target_path']) diff --git a/src/cbapi/response/models/group-modify.yaml b/src/cbapi/response/models/group-modify.yaml index ad12e971..0d57430e 100644 --- a/src/cbapi/response/models/group-modify.yaml +++ b/src/cbapi/response/models/group-modify.yaml @@ -2,6 +2,7 @@ type: "object" required: - name - site_id + - sensorbackend_server properties: id: type: "integer" @@ -21,6 +22,12 @@ properties: type: "integer" description: "Site id to add group to" default: "1" + address_resolution_hint: + type: string + enum: + - 'ipv4' + - 'ipv6' + - '' team_access: type: "array" description: "Team access array" @@ -50,10 +57,17 @@ properties: format: "int32" description: "Set the tamper level, default 0" default: "0" - sensor_version: + sensor_version_windows: + type: "string" + description: "The version of the sensor for Windows hosts in this group, default Manual" + default: "Manual" + sensor_version_osx: type: "string" - description: "The version of the sensor in this group, default Manual" + description: "The version of the sensor for OSX hosts in this group, default Manual" default: "Manual" + sensor_version_linux: + type: "string" + description: "The version of the sensor for Linux hosts in this group, default Manual" datastore_server: type: "string" description: "Datastore server address if different from sensorbackend server, default null" @@ -67,6 +81,10 @@ properties: type: "boolean" description: "Enable/Disable banning, default true" default: "true" + collect_amsi: + type: "boolean" + description: "Enable/Disable amsi collection, default true" + default: "false" collect_emet_events: type: "boolean" description: "Enable/Disable emet events collection, default true" @@ -119,6 +137,9 @@ properties: type: "boolean" description: "Enable/Disable filter known dll, default false" default: "false" + number_of_hosts: + type: "integer" + description: "Number of hosts" process_filter_level: type: "integer" format: "int32" @@ -144,6 +165,62 @@ properties: format: "int32" description: "Percent of the storefile, default 1" default: "1" + server_cert_id: + type: "integer" + description: "Server certificate ID to use" + default: "2" + isolation_exclusions: + type: "array" + description: "isolation exclusions" + items: + title: "modIsolationExclusions" + type: "object" + properties: + id: + type: "integer" + description: "id of exclusion" + group_id: + type: "integer" + description: "ID of groupy to add exclusions to" + name: + type: "string" + description: "name of exclusions" + ip_address: + type: "string" + description: "IP address of exclusion" + enabled: + type: "boolean" + description: "status of exclusion" + uploader_id: + type: "integer" + description: "id of uploader" + added_time: + type: "string" + format: "iso-date-time" + description: "time when added" + changed_time: + type: "string" + format: "iso-date-time" + description: "time when changed" + uploader_username: + type: "string" + description: "username of uploader" + event_exclusions: + type: "array" + description: "Configured set of events to exclude from event collection for a group of sensors" + items: + type: "object" + properties: + eventsToIgnore: + type: "array" + description: "Set of event strings, one or more of 'crossproc', 'filemod', 'filemod-non-binary', 'modload', 'netconn', 'process-info' or 'regmod'" + items: + type: "string" + paths: + type: "array" + description: 'Set of path strings (e.g. "c:\\Windows\\system32\\regsvr32.exe")' + items: + type: "string" vdi_enabled: type: "boolean" description: "Enable/Disable vdi, default false" diff --git a/src/cbapi/response/models/process.yaml b/src/cbapi/response/models/process.yaml index 71f1befe..44a605e9 100644 --- a/src/cbapi/response/models/process.yaml +++ b/src/cbapi/response/models/process.yaml @@ -58,7 +58,6 @@ properties: format: int32 group: type: string - type: string netconn_count: type: integer format: int32 diff --git a/src/cbapi/response/models/sensor-builds.yaml b/src/cbapi/response/models/sensor-builds.yaml new file mode 100644 index 00000000..3b31077b --- /dev/null +++ b/src/cbapi/response/models/sensor-builds.yaml @@ -0,0 +1,29 @@ +type: "object" +properties: + Windows: + type: "array" + description: 'Sensor versions for Windows' + items: + type: "object" + properties: + version_string: + type: "string" + description: "Sensor build version" + Linux: + type: "array" + description: 'Sensor versions for Linux' + items: + type: "object" + properties: + version_string: + type: "string" + description: "Sensor build version" + OSX: + type: "array" + description: 'Sensor versions for OSX' + items: + type: "object" + properties: + version_string: + type: "string" + description: "Sensor build version" diff --git a/src/cbapi/response/models/user.yaml b/src/cbapi/response/models/user.yaml index 6758453e..33915583 100644 --- a/src/cbapi/response/models/user.yaml +++ b/src/cbapi/response/models/user.yaml @@ -41,3 +41,6 @@ properties: email: type: string description: '' + current_password: + type: string + description: '' diff --git a/src/cbapi/response/query.py b/src/cbapi/response/query.py index 97fab3a7..e02b11df 100644 --- a/src/cbapi/response/query.py +++ b/src/cbapi/response/query.py @@ -1,9 +1,12 @@ from ..utils import convert_query_params from ..query import PaginatedQuery from cbapi.six.moves import urllib -from distutils.version import LooseVersion +import sys +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from ..errors import ApiError -from ..utils import convert_query_params import copy import logging @@ -12,11 +15,11 @@ class Query(PaginatedQuery): - """Represents a prepared query to the Carbon Black Enterprise Response server. + """Represents a prepared query to the Carbon Black EDR server. - This object is returned as part of a :py:meth:`CbEnterpriseResponseAPI.select` + This object is returned as part of a :py:meth:`CbResponseAPI.select` operation on Process and Binary objects from the Carbon Black - Enterprise Response server. You should not have to create this class yourself. + EDR server. You should not have to create this class yourself. The query is not executed on the server until it's accessed, either as an iterator (where it will generate values on demand as they're requested) or as a list (where it will retrieve the entire result set and save to a list). @@ -29,7 +32,7 @@ class Query(PaginatedQuery): Examples:: - >>> cb = CbEnterpriseResponseAPI() + >>> cb = CbResponseAPI() >>> query = cb.select(Process) # returns a Query object matching all Processes >>> query = query.where("process_name:notepad.exe") # add a filter to this Query >>> query = query.sort("last_update desc") # sort by last update time, most recent first @@ -57,7 +60,7 @@ def __init__(self, doc_class, cb, query=None, raw_query=None): # FIX: Cb Response server version 5.1.0-3 throws an exception after returning HTTP 504 when facet=false in the # HTTP request. Work around this by only setting facet=false on 5.1.1 and above server versions. - if self._cb.cb_server_version >= LooseVersion('5.1.1'): + if self._cb.cb_server_version >= parse('5.1.1'): self._default_args["facet"] = "false" def _clone(self): @@ -75,7 +78,8 @@ def _clone(self): def sort(self, new_sort): """Set the sort order for this query. - :param str new_sort: New sort order - see the `Query Reference `_. + :param str new_sort: New sort order - see the + `Query Reference `_. :return: Query object :rtype: :py:class:`Query` """ @@ -96,7 +100,8 @@ def webui_link(self): def and_(self, new_query): """Add a filter to this query. Equivalent to calling :py:meth:`where` on this object. - :param str new_query: Query string - see the `Query Reference `_. + :param str new_query: Query string - see the + `Query Reference `_. :return: Query object :rtype: :py:class:`Query` """ @@ -105,7 +110,8 @@ def and_(self, new_query): def where(self, new_query): """Add a filter to this query. - :param str new_query: Query string - see the `Query Reference `_. + :param str new_query: Query string - see the + `Query Reference `_. :return: Query object :rtype: :py:class:`Query` """ @@ -162,7 +168,8 @@ def _count(self): qargs = convert_query_params(args) - self._total_results = self._cb.get_object(self._doc_class.urlobject, query_parameters=qargs).get('total_results', 0) + self._total_results = self._cb.get_object(self._doc_class.urlobject, + query_parameters=qargs).get('total_results', 0) self._count_valid = True return self._total_results diff --git a/src/cbapi/response/rest_api.py b/src/cbapi/response/rest_api.py index 6022bc7e..e172ab90 100644 --- a/src/cbapi/response/rest_api.py +++ b/src/cbapi/response/rest_api.py @@ -4,10 +4,14 @@ from cbapi.six.moves import urllib -from distutils.version import LooseVersion +import sys +if sys.version_info <= (3, 6): + from distutils.version import LooseVersion as parse +else: + from packaging.version import parse from ..connection import BaseAPI from .models import Process, Binary, Watchlist, Investigation, Alert, ThreatReport, StoragePartition -from ..errors import UnauthorizedError, ApiError +from ..errors import UnauthorizedError, ApiError, ClientError from .cblr import LiveResponseSessionManager from .query import Query @@ -17,20 +21,21 @@ class CbResponseAPI(BaseAPI): - """The main entry point into the Carbon Black Enterprise Response API. + """The main entry point into the Carbon Black EDR API. Note that calling this will automatically connect to the Carbon Black server in order to verify connectivity and get the server version. - :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black server. Uses the - profile named 'default' when not specified. - :param str url: (optional, discouraged) Instead of using a credential profile, pass URL and API token to the constructor. + :param str profile: (optional) Use the credentials in the named profile when connecting to the Carbon Black + server. Uses the profile named 'default' when not specified. + :param str url: (optional, discouraged) Instead of using a credential profile, pass URL and API token to + the constructor. :param str token: (optional, discouraged) API token :param bool ssl_verify: (optional, discouraged) Enable or disable SSL certificate verification Usage:: - >>> from cbapi import CbEnterpriseResponseAPI - >>> cb = CbEnterpriseResponseAPI(profile="production") + >>> from cbapi import CbResponseAPI + >>> cb = CbResponseAPI(profile="production") """ def __init__(self, *args, **kwargs): timeout = kwargs.pop("timeout", 120) # set default timeout period to two minutes, 2x the default nginx timeout @@ -43,15 +48,24 @@ def __init__(self, *args, **kwargs): raise UnauthorizedError(uri=self.url, message="Invalid API token for server {0:s}.".format(self.url)) log.debug('Connected to Cb server version %s at %s' % (self.server_info['version'], self.session.server)) - self.cb_server_version = LooseVersion(self.server_info['version']) - if self.cb_server_version < LooseVersion('5.0'): + self.cb_server_version = parse(self.server_info['version']) + if self.cb_server_version < parse('5.0'): raise ApiError("CbEnterpriseResponseAPI only supports Cb servers version >= 5.0.0") self._has_legacy_partitions = False - if self.cb_server_version >= LooseVersion('6.0'): - legacy_partitions = [p for p in self.select(StoragePartition) if p.info.get("isLegacy", False)] - if legacy_partitions: - self._has_legacy_partitions = True + try: + if self.cb_server_version >= parse('6.0'): + legacy_partitions = [p for p in self.select(StoragePartition) if p.info.get("isLegacy", False)] + if legacy_partitions: + self._has_legacy_partitions = True + except ClientError as ce: + # If we get a 403 on this endpoint, ignore during init, + # as we will not be able to work with StoragePartitions regardless + # https://github.com/carbonblack/cbapi-python/issues/303 + if ce.error_code == 403: + pass + else: + raise ce # no intervention self._lr_scheduler = None @@ -65,7 +79,7 @@ def live_response(self): return self._lr_scheduler def info(self): - """Retrieve basic version information from the Carbon Black Enterprise Response server. + """Retrieve basic version information from the Carbon Black DER server. :return: Dictionary with information retrieved from the ``/api/info`` API route :rtype: dict @@ -74,7 +88,7 @@ def info(self): return r.json() def dashboard_statistics(self): - """Retrieve dashboard statistics from the Carbon Black Enterprise Response server. + """Retrieve dashboard statistics from the Carbon Black EDR server. :return: Dictionary with information retrieved from the ``/api/v1/dashboard/statistics`` API route :rtype: dict @@ -83,7 +97,7 @@ def dashboard_statistics(self): return r.json() def license_request(self): - """Retrieve license request block from the Carbon Black Enterprise Response server. + """Retrieve license request block from the Carbon Black EDR server. :return: License request block :rtype: str @@ -92,7 +106,7 @@ def license_request(self): return r.json().get("license_request_block", "") def update_license(self, license_block): - """Upload new license to the Carbon Black Enterprise Response server. + """Upload new license to the Carbon Black EDR server. :param str license_block: Licence block provided by Carbon Black support :raises ServerError: if the license is not accepted by the Carbon Black server @@ -107,21 +121,20 @@ def _perform_query(self, cls, **kwargs): return Query(cls, self, **kwargs) def from_ui(self, uri): - """Retrieve a Carbon Black Enterprise Response object based on URL from the Carbon Black Enterprise Response - web user interface. + """Retrieve a Carbon Black EDR object based on URL from the Carbon Black EDR web user interface. For example, calling this function with ``https://server/#/analyze/00000001-0000-0554-01d1-3bc4553b8c9f/1`` as the ``uri`` argument will return a new :py:class: cbapi.response.models.Process class initialized with the process GUID from the URL. - :param str uri: Web browser URL from the Cb web interface + :param str uri: Web browser URL from the CB web interface :return: the appropriate model object for the URL provided :raises ApiError: if the URL does not correspond to a recognized model object """ o = urllib.parse.urlparse(uri) - if self._parsed_url.scheme != o.scheme or \ - self._parsed_url.hostname != o.hostname or \ - self._parsed_url.port != o.port: + if self._parsed_url.scheme != o.scheme \ + or self._parsed_url.hostname != o.hostname \ + or self._parsed_url.port != o.port: raise ApiError("Invalid URL provided") frag = o.fragment.lstrip('/') @@ -165,7 +178,7 @@ def _request_lr_session(self, sensor_id): return self.live_response.request_session(sensor_id) def create_new_partition(self): - """Create a new Solr time partition for event storage. Available in Cb Response 6.1 and above. + """Create a new Solr time partition for event storage. Available in Carbon Black EDR 6.1 and above. This will force roll-over current hot partition into warm partition (by renaming it to a time-stamped name) and create a new hot partition ("writer"). @@ -181,5 +194,3 @@ class CbEnterpriseResponseAPI(CbResponseAPI): Backwards compatibility for previous scripts """ pass - - diff --git a/src/cbapi/response/utils.py b/src/cbapi/response/utils.py index f6562724..6288befc 100644 --- a/src/cbapi/response/utils.py +++ b/src/cbapi/response/utils.py @@ -30,7 +30,7 @@ def parse_42_guid(guid): def parse_process_guid(guid): sensor_id, proc_pid, proc_createtime = parse_42_guid(guid) - return sensor_id, proc_pid, datetime(1601,1,1) + timedelta(microseconds=proc_createtime / 10) + return sensor_id, proc_pid, datetime(1601, 1, 1) + timedelta(microseconds=proc_createtime / 10) def convert_to_solr(dt): @@ -55,7 +55,7 @@ def convert_from_cb(s): if s is None: return dateutil.parser.parse("1970-01-01T00:00:00Z") else: - return dateutil.parser.parse(s) + return dateutil.parser.parse(s) def convert_event_time(s): @@ -64,6 +64,7 @@ def convert_event_time(s): # is included, and sometimes it isn't... so we normalize it by stripping off the TZ data (unfortunately) return convert_from_cb(s).replace(tzinfo=None) + def convert_to_cb(dt): return dt.strftime(cb_datetime_format) @@ -73,7 +74,7 @@ def get_constants(prefix): return dict((getattr(socket, n), n) for n in dir(socket) if n.startswith(prefix) - ) + ) protocols = get_constants("IPPROTO_") diff --git a/src/cbapi/six.py b/src/cbapi/six.py index 190c0239..1b0eaf56 100644 --- a/src/cbapi/six.py +++ b/src/cbapi/six.py @@ -1,6 +1,4 @@ -"""Utilities for writing code that runs on Python 2 and 3""" - -# Copyright (c) 2010-2015 Benjamin Peterson +# Copyright (c) 2010-2020 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -20,6 +18,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +"""Utilities for writing code that runs on Python 2 and 3""" + from __future__ import absolute_import import functools @@ -29,7 +29,7 @@ import types __author__ = "Benjamin Peterson " -__version__ = "1.10.0" +__version__ = "1.16.0" # Useful for very coarse version differentiation. @@ -46,10 +46,10 @@ MAXSIZE = sys.maxsize else: - string_types = basestring, - integer_types = (int, long) + string_types = basestring, # noqa: F821 + integer_types = (int, long) # noqa: F821 class_types = (type, types.ClassType) - text_type = unicode + text_type = unicode # noqa: F821 binary_type = str if sys.platform.startswith("java"): @@ -71,6 +71,11 @@ def __len__(self): MAXSIZE = int((1 << 63) - 1) del X +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + def _add_doc(func, doc): """Add documentation to a function.""" @@ -186,6 +191,11 @@ def find_module(self, fullname, path=None): return self return None + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + def __get_module(self, fullname): try: return self.known_modules[fullname] @@ -223,6 +233,12 @@ def get_code(self, fullname): return None get_source = get_code # same as get_code + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + _importer = _SixMetaPathImporter(__name__) @@ -241,12 +257,13 @@ class _MovedItems(_LazyModule): MovedAttribute("map", "itertools", "builtins", "imap", "map"), MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), MovedAttribute("reduce", "__builtin__", "functools"), MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), MovedAttribute("StringIO", "StringIO", "io"), - MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserDict", "UserDict", "collections", "IterableUserDict", "UserDict"), MovedAttribute("UserList", "UserList", "collections"), MovedAttribute("UserString", "UserString", "collections"), MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), @@ -254,18 +271,21 @@ class _MovedItems(_LazyModule): MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), + MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"), MovedModule("copyreg", "copy_reg"), MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"), MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), MovedModule("http_cookies", "Cookie", "http.cookies"), MovedModule("html_entities", "htmlentitydefs", "html.entities"), MovedModule("html_parser", "HTMLParser", "html.parser"), MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), - MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), @@ -337,10 +357,12 @@ class Module_six_moves_urllib_parse(_LazyModule): MovedAttribute("quote_plus", "urllib", "urllib.parse"), MovedAttribute("unquote", "urllib", "urllib.parse"), MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), MovedAttribute("urlencode", "urllib", "urllib.parse"), MovedAttribute("splitquery", "urllib", "urllib.parse"), MovedAttribute("splittag", "urllib", "urllib.parse"), MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), MovedAttribute("uses_params", "urlparse", "urllib.parse"), @@ -416,6 +438,8 @@ class Module_six_moves_urllib_request(_LazyModule): MovedAttribute("URLopener", "urllib", "urllib.request"), MovedAttribute("FancyURLopener", "urllib", "urllib.request"), MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), ] for attr in _urllib_request_moved_attributes: setattr(Module_six_moves_urllib_request, attr.name, attr) @@ -631,20 +655,23 @@ def u(s): import io StringIO = io.StringIO BytesIO = io.BytesIO + del io _assertCountEqual = "assertCountEqual" if sys.version_info[1] <= 1: _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" else: _assertRaisesRegex = "assertRaisesRegex" _assertRegex = "assertRegex" + _assertNotRegex = "assertNotRegex" else: def b(s): return s # Workaround for standalone backslash def u(s): - return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") # noqa: F821 unichr = unichr int2byte = chr @@ -659,6 +686,7 @@ def indexbytes(buf, i): _assertCountEqual = "assertItemsEqual" _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" _add_doc(b, """Byte literal""") _add_doc(u, """Text literal""") @@ -675,15 +703,23 @@ def assertRegex(self, *args, **kwargs): return getattr(self, _assertRegex)(*args, **kwargs) +def assertNotRegex(self, *args, **kwargs): + return getattr(self, _assertNotRegex)(*args, **kwargs) + + if PY3: exec_ = getattr(moves.builtins, "exec") def reraise(tp, value, tb=None): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None else: def exec_(_code_, _globs_=None, _locs_=None): @@ -699,19 +735,19 @@ def exec_(_code_, _globs_=None, _locs_=None): exec("""exec _code_ in _globs_, _locs_""") exec_("""def reraise(tp, value, tb=None): - raise tp, value, tb + try: + raise tp, value, tb + finally: + tb = None """) -if sys.version_info[:2] == (3, 2): +if sys.version_info[:2] > (3,): exec_("""def raise_from(value, from_value): - if from_value is None: - raise value - raise value from from_value -""") -elif sys.version_info[:2] > (3, 2): - exec_("""def raise_from(value, from_value): - raise value from from_value + try: + raise value from from_value + finally: + value = None """) else: def raise_from(value, from_value): @@ -727,11 +763,11 @@ def print_(*args, **kwargs): return def write(data): - if not isinstance(data, basestring): + if not isinstance(data, basestring): # noqa: F821 data = str(data) # If the file has an encoding, encode unicode with it. - if (isinstance(fp, file) and - isinstance(data, unicode) and + if (isinstance(fp, file) and # noqa: F821 + isinstance(data, unicode) and # noqa: F821 fp.encoding is not None): errors = getattr(fp, "errors", None) if errors is None: @@ -741,13 +777,13 @@ def write(data): want_unicode = False sep = kwargs.pop("sep", None) if sep is not None: - if isinstance(sep, unicode): + if isinstance(sep, unicode): # noqa: F821 want_unicode = True elif not isinstance(sep, str): raise TypeError("sep must be None or a string") end = kwargs.pop("end", None) if end is not None: - if isinstance(end, unicode): + if isinstance(end, unicode): # noqa: F821 want_unicode = True elif not isinstance(end, str): raise TypeError("end must be None or a string") @@ -755,12 +791,12 @@ def write(data): raise TypeError("invalid keyword arguments to print()") if not want_unicode: for arg in args: - if isinstance(arg, unicode): + if isinstance(arg, unicode): # noqa: F821 want_unicode = True break if want_unicode: - newline = unicode("\n") - space = unicode(" ") + newline = unicode("\n") # noqa: F821 + space = unicode(" ") # noqa: F821 else: newline = "\n" space = " " @@ -786,13 +822,33 @@ def print_(*args, **kwargs): _add_doc(reraise, """Reraise an exception.""") if sys.version_info[0:2] < (3, 4): + # This does exactly the same what the :func:`py3:functools.update_wrapper` + # function does on Python versions after 3.2. It sets the ``__wrapped__`` + # attribute on ``wrapper`` object and it doesn't raise an error if any of + # the attributes mentioned in ``assigned`` and ``updated`` are missing on + # ``wrapped`` object. + def _update_wrapper(wrapper, wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + wrapper.__wrapped__ = wrapped + return wrapper + _update_wrapper.__doc__ = functools.update_wrapper.__doc__ + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, updated=functools.WRAPPER_UPDATES): - def wrapper(f): - f = functools.wraps(wrapped, assigned, updated)(f) - f.__wrapped__ = wrapped - return f - return wrapper + return functools.partial(_update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) + wraps.__doc__ = functools.wraps.__doc__ + else: wraps = functools.wraps @@ -802,10 +858,22 @@ def with_metaclass(meta, *bases): # This requires a bit of explanation: the basic idea is to make a dummy # metaclass for one level of class instantiation that replaces itself with # the actual metaclass. - class metaclass(meta): + class metaclass(type): def __new__(cls, name, this_bases, d): - return meta(name, bases, d) + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d['__orig_bases__'] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) return type.__new__(metaclass, 'temporary_class', (), {}) @@ -821,13 +889,75 @@ def wrapper(cls): orig_vars.pop(slots_var) orig_vars.pop('__dict__', None) orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, binary_type): + return s + if isinstance(s, text_type): + return s.encode(encoding, errors) + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + # Optimization: Fast return for the common case. + if type(s) is str: + return s + if PY2 and isinstance(s, text_type): + return s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + return s.decode(encoding, errors) + elif not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + def python_2_unicode_compatible(klass): """ - A decorator that defines __unicode__ and __str__ methods under Python 2. + A class decorator that defines __unicode__ and __str__ methods under Python 2. Under Python 3 it does nothing. To support Python 2 and 3 with a single code base, define a __str__ method diff --git a/src/cbapi/winerror.py b/src/cbapi/winerror.py index e63310cb..88811885 100644 --- a/src/cbapi/winerror.py +++ b/src/cbapi/winerror.py @@ -2568,6 +2568,7 @@ class Win32Error(ErrorBaseClass): ERROR_DS_INVALID_SEARCH_FLAG_TUPLE = 8627 ERROR_DS_HIERARCHY_TABLE_TOO_DEEP = 8628 + SEVERITY_SUCCESS = 0 SEVERITY_ERROR = 1 @@ -2996,29 +2997,43 @@ class CommDlgError(ErrorBaseClass): def HRESULT_FROM_WIN32(scode): return -2147024896 | (scode & 65535) + def SUCCEEDED(Status): return ((Status) >= 0) -def FAILED(Status): return (Status<0) + +def FAILED(Status): return (Status < 0) + def HRESULT_CODE(hr): return ((hr) & 65535) + def SCODE_CODE(sc): return ((sc) & 65535) + def HRESULT_FACILITY(hr): return (((hr) >> 16) & 8191) + def SCODE_FACILITY(sc): return (((sc) >> 16) & 8191) + def HRESULT_SEVERITY(hr): return (((hr) >> 31) & 1) + def SCODE_SEVERITY(sc): return (((sc) >> 31) & 1) + FACILITY_NT_BIT = 268435456 + + def HRESULT_FROM_NT(x): return x | FACILITY_NT_BIT + def GetScode(hr): return hr + def ResultFromScode(sc): return sc + def decode_hresult(hresult): if HRESULT_FACILITY(hresult) == Facility.FACILITY_WIN32: return Win32Error.lookup_error(HRESULT_CODE(hresult)) diff --git a/test/cbapi/__init__.py b/test/cbapi/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/test/cbapi/psc/__init__.py b/test/cbapi/psc/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/test/cbapi/psc/livequery/__init__.py b/test/cbapi/psc/livequery/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/test/cbapi/psc/livequery/test_models.py b/test/cbapi/psc/livequery/test_models.py deleted file mode 100755 index bbda0539..00000000 --- a/test/cbapi/psc/livequery/test_models.py +++ /dev/null @@ -1,294 +0,0 @@ -import pytest -from cbapi.psc.livequery.rest_api import CbLiveQueryAPI -from cbapi.psc.livequery.models import Run, Result -from cbapi.psc.livequery.query import ResultQuery, FacetQuery -from cbapi.errors import ApiError -from test.mocks import MockResponse, ConnectionMocks - - -def test_run_refresh(monkeypatch): - _was_called = False - - def mock_get_object(url, parms=None, default=None): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" - assert parms is None - assert default is None - _was_called = True - return {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"COMPLETE"} - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"ACTIVE"}) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - rc = run.refresh() - assert _was_called - assert rc == True - assert run.org_key == "Z100" - assert run.name == "FoobieBletch" - assert run.id == "abcdefg" - assert run.status == "COMPLETE" - - -def test_run_stop(monkeypatch): - _was_called = False - - def mock_put_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/status" - assert body["status"] == "CANCELLED" - _was_called = True - return MockResponse({"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"CANCELLED"}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"ACTIVE"}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", mock_put_object) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - rc = run.stop() - assert _was_called - assert rc == True - assert run.org_key == "Z100" - assert run.name == "FoobieBletch" - assert run.id == "abcdefg" - assert run.status == "CANCELLED" - - -def test_run_stop_failed(monkeypatch): - _was_called = False - - def mock_put_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/status" - assert body["status"] == "CANCELLED" - _was_called = True - return MockResponse({"error_message":"The query is not presently running."}, 409) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"CANCELLED"}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", mock_put_object) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - rc = run.stop() - assert _was_called - assert rc == False - - -def test_run_delete(monkeypatch): - _was_called = False - - def mock_delete_object(url): - nonlocal _was_called - if _was_called: - pytest.fail("delete should not be called twice!") - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" - _was_called = True - return MockResponse(None) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"ACTIVE"}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", mock_delete_object) - rc = run.delete() - assert _was_called - assert rc == True - assert run._is_deleted - # Now ensure that certain operations that don't make sense on a deleted object raise ApiError - with pytest.raises(ApiError): - run.refresh() - with pytest.raises(ApiError): - run.stop() - # And make sure that deleting a deleted object returns True immediately - rc = run.delete() - assert rc == True - - -def test_run_delete_failed(monkeypatch): - _was_called = False - - def mock_delete_object(url): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" - _was_called = True - return MockResponse(None, 403) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - run = Run(api, "abcdefg", {"org_key":"Z100", "name":"FoobieBletch", - "id":"abcdefg", "status":"ACTIVE"}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", mock_delete_object) - rc = run.delete() - assert _was_called - assert rc == False - assert not run._is_deleted - - -def test_result_device_summaries(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/results/device_summaries/_search" - assert body["query"] == "foo" - t = body["criteria"] - assert t["device_name"] == ["AxCx", "A7X"] - t = body["sort"][0] - assert t["field"] == "device_name" - assert t["order"] == "ASC" - _was_called = True - metrics = [{"key":"aaa", "value":0.0}, {"key":"bbb", "value":0.0}] - res1 = {"id":"ghijklm", "total_results":2, "device_id":314159, "metrics":metrics} - res2 = {"id":"mnopqrs", "total_results":3, "device_id":271828, "metrics":metrics} - return MockResponse({"org_key":"Z100", "num_found":2, "results":[res1, res2]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - tmp_id = {"id":"abcdefg"} - result = Result(api, {"id": "abcdefg", "device":tmp_id, "fields":{}, "metrics":{}}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = result.query_device_summaries().where("foo").criteria(device_name=["AxCx", "A7X"]) - query = query.sort_by("device_name") - assert isinstance(query, ResultQuery) - count = 0 - for item in query.all(): - if item.id == "ghijklm": - assert item.total_results == 2 - assert item.device_id == 314159 - elif item.id == "mnopqrs": - assert item.total_results == 3 - assert item.device_id == 271828 - else: - pytest.fail("Invalid object with ID %s seen" % item.id) - count = count + 1 - assert _was_called - assert count == 2 - - -def test_result_query_result_facets(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/results/_facet" - assert body["query"] == "xyzzy" - t = body["criteria"] - assert t["device_name"] == ["AxCx", "A7X"] - t = body["terms"] - assert t["fields"] == ["alpha", "bravo", "charlie"] - _was_called = True - v1 = {"total":1, "id":"alpha1", "name":"alpha1"} - v2 = {"total":2, "id":"alpha2", "name":"alpha2"} - term1 = {"field":"alpha", "values":[v1, v2]} - v1 = {"total":1, "id":"bravo1", "name":"bravo1"} - v2 = {"total":2, "id":"bravo2", "name":"bravo2"} - term2 = {"field":"bravo", "values":[v1, v2]} - v1 = {"total":1, "id":"charlie1", "name":"charlie1"} - v2 = {"total":2, "id":"charlie2", "name":"charlie2"} - term3 = {"field":"charlie", "values":[v1, v2]} - return MockResponse({"terms":[term1, term2, term3]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - tmp_id = {"id":"abcdefg"} - result = Result(api, {"id": "abcdefg", "device":tmp_id, "fields":{}, "metrics":{}}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = result.query_result_facets().where("xyzzy") - query = query.facet_field("alpha").facet_field(["bravo", "charlie"]) - query = query.criteria(device_name=["AxCx", "A7X"]) - assert isinstance(query, FacetQuery) - count = 0 - for item in query.all(): - vals = item.values - if item.field == "alpha": - assert vals[0]["id"] == "alpha1" - assert vals[1]["id"] == "alpha2" - elif item.field == "bravo": - assert vals[0]["id"] == "bravo1" - assert vals[1]["id"] == "bravo2" - elif item.field == "charlie": - assert vals[0]["id"] == "charlie1" - assert vals[1]["id"] == "charlie2" - else: - pytest.fail("Unknown field name %s seen" % item.field) - count = count + 1 - assert _was_called - assert count == 3 - - -def test_result_query_device_summary_facets(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg/results/device_summaries/_facet" - assert body["query"] == "xyzzy" - t = body["criteria"] - assert t["device_name"] == ["AxCx", "A7X"] - t = body["terms"] - assert t["fields"] == ["alpha", "bravo", "charlie"] - _was_called = True - v1 = {"total":1, "id":"alpha1", "name":"alpha1"} - v2 = {"total":2, "id":"alpha2", "name":"alpha2"} - term1 = {"field":"alpha", "values":[v1, v2]} - v1 = {"total":1, "id":"bravo1", "name":"bravo1"} - v2 = {"total":2, "id":"bravo2", "name":"bravo2"} - term2 = {"field":"bravo", "values":[v1, v2]} - v1 = {"total":1, "id":"charlie1", "name":"charlie1"} - v2 = {"total":2, "id":"charlie2", "name":"charlie2"} - term3 = {"field":"charlie", "values":[v1, v2]} - return MockResponse({"terms":[term1, term2, term3]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - tmp_id = {"id":"abcdefg"} - result = Result(api, {"id": "abcdefg", "device":tmp_id, "fields":{}, "metrics":{}}) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = result.query_device_summary_facets().where("xyzzy") - query = query.facet_field("alpha").facet_field(["bravo", "charlie"]) - query = query.criteria(device_name=["AxCx", "A7X"]) - assert isinstance(query, FacetQuery) - count = 0 - for item in query.all(): - vals = item.values - if item.field == "alpha": - assert vals[0]["id"] == "alpha1" - assert vals[1]["id"] == "alpha2" - elif item.field == "bravo": - assert vals[0]["id"] == "bravo1" - assert vals[1]["id"] == "bravo2" - elif item.field == "charlie": - assert vals[0]["id"] == "charlie1" - assert vals[1]["id"] == "charlie2" - else: - pytest.fail("Unknown field name %s seen" % item.field) - count = count + 1 - assert _was_called - assert count == 3 diff --git a/test/cbapi/psc/livequery/test_rest_api.py b/test/cbapi/psc/livequery/test_rest_api.py deleted file mode 100755 index 97a60b5b..00000000 --- a/test/cbapi/psc/livequery/test_rest_api.py +++ /dev/null @@ -1,196 +0,0 @@ -import pytest -from cbapi.psc.livequery.rest_api import CbLiveQueryAPI -from cbapi.psc.livequery.models import Run -from cbapi.psc.livequery.query import RunQuery, RunHistoryQuery -from cbapi.errors import ApiError, CredentialError -from test.mocks import MockResponse, ConnectionMocks - -def test_no_org_key(): - with pytest.raises(CredentialError): - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - ssl_verify=True) # note: no org_key - - -def test_simple_get(monkeypatch): - _was_called = False - - def mock_get_object(url, parms=None, default=None): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/abcdefg" - assert parms is None - assert default is None - _was_called = True - return {"org_key":"Z100", "name":"FoobieBletch", "id":"abcdefg"} - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", mock_get_object) - monkeypatch.setattr(api, "post_object", ConnectionMocks.get("POST")) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - run = api.select(Run, "abcdefg") - assert _was_called - assert run.org_key == "Z100" - assert run.name == "FoobieBletch" - assert run.id == "abcdefg" - - -def test_query(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs" - assert body["sql"] == "select * from whatever;" - _was_called = True - return MockResponse({"org_key":"Z100", "name":"FoobieBletch", "id":"abcdefg"}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.query("select * from whatever;"); - assert isinstance(query, RunQuery) - run = query.submit() - assert _was_called - assert run.org_key == "Z100" - assert run.name == "FoobieBletch" - assert run.id == "abcdefg" - - -def test_query_with_everything(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs" - assert body["sql"] == "select * from whatever;" - assert body["name"] == "AmyWasHere" - assert body["notify_on_finish"] == True - df = body["device_filter"] - assert df["device_ids"] == [1, 2, 3] - assert df["device_types"] == ["Alpha", "Bravo", "Charlie"] - assert df["policy_ids"] == [16, 27, 38] - _was_called = True - return MockResponse({"org_key":"Z100", "name":"FoobieBletch", "id":"abcdefg"}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.query("select * from whatever;").device_ids([1, 2, 3]) - query = query.device_types(["Alpha", "Bravo", "Charlie"]); - query = query.policy_ids([16, 27, 38]) - query = query.name("AmyWasHere").notify_on_finish() - assert isinstance(query, RunQuery) - run = query.submit() - assert _was_called - assert run.org_key == "Z100" - assert run.name == "FoobieBletch" - assert run.id == "abcdefg" - - -def test_query_device_ids_broken(): - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - query = api.query("select * from whatever;"); - with pytest.raises(ApiError): - query = query.device_ids(["Bogus"]) - - -def test_query_device_types_broken(): - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - query = api.query("select * from whatever;"); - with pytest.raises(ApiError): - query = query.device_types([420]); - - -def test_query_policy_ids_broken(): - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - query = api.query("select * from whatever;"); - with pytest.raises(ApiError): - query = query.policy_ids(["Bogus"]) - - -def test_query_history(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/_search" - assert body["query"] == "xyzzy" - _was_called = True - run1 = {"org_key":"Z100", "name":"FoobieBletch", "id":"abcdefg"} - run2 = {"org_key":"Z100", "name":"Aoxomoxoa", "id":"cdefghi"} - run3 = {"org_key":"Z100", "name":"Read_Me", "id":"efghijk"} - return MockResponse({"org_key":"Z100", "num_found":3, "results":[run1, run2, run3]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.query_history("xyzzy") - assert isinstance(query, RunHistoryQuery) - count = 0 - for item in query.all(): - assert item.org_key == "Z100" - if item.id == "abcdefg": - assert item.name == "FoobieBletch" - elif item.id == "cdefghi": - assert item.name == "Aoxomoxoa" - elif item.id == "efghijk": - assert item.name == "Read_Me" - else: - pytest.fail("Unknown item ID: %s" % item.id) - count = count + 1 - assert _was_called - assert count == 3 - - -def test_query_history_with_everything(monkeypatch): - _was_called = False - - def mock_post_object(url, body, **kwargs): - nonlocal _was_called - assert url == "/livequery/v1/orgs/Z100/runs/_search" - assert body["query"] == "xyzzy" - t = body["sort"][0] - assert t["field"] == "id" - assert t["order"] == "ASC" - _was_called = True - run1 = {"org_key":"Z100", "name":"FoobieBletch", "id":"abcdefg"} - run2 = {"org_key":"Z100", "name":"Aoxomoxoa", "id":"cdefghi"} - run3 = {"org_key":"Z100", "name":"Read_Me", "id":"efghijk"} - return MockResponse({"org_key":"Z100", "num_found":3, "results":[run1, run2, run3]}) - - api = CbLiveQueryAPI(url="https://example.com", token="ABCD/1234", - org_key="Z100", ssl_verify=True) - monkeypatch.setattr(api, "get_object", ConnectionMocks.get("GET")) - monkeypatch.setattr(api, "post_object", mock_post_object) - monkeypatch.setattr(api, "put_object", ConnectionMocks.get("PUT")) - monkeypatch.setattr(api, "delete_object", ConnectionMocks.get("DELETE")) - query = api.query_history("xyzzy").sort_by("id") - assert isinstance(query, RunHistoryQuery) - count = 0 - for item in query.all(): - assert item.org_key == "Z100" - if item.id == "abcdefg": - assert item.name == "FoobieBletch" - elif item.id == "cdefghi": - assert item.name == "Aoxomoxoa" - elif item.id == "efghijk": - assert item.name == "Read_Me" - else: - pytest.fail("Unknown item ID: %s" % item.id) - count = count + 1 - assert _was_called - assert count == 3 - \ No newline at end of file diff --git a/test/cbtest.py b/test/cbtest.py new file mode 100755 index 00000000..cdfe9467 --- /dev/null +++ b/test/cbtest.py @@ -0,0 +1,38 @@ +import pytest + + +class StubResponse(object): + def __init__(self, contents, scode=200): + self._contents = contents + self.status_code = scode + + def json(self): + return self._contents + + +def _failing_get_object(url, parms=None, default=None): + pytest.fail("GET called for %s when it shouldn't be" % url) + + +def _failing_get_raw_data(url, query_params, **kwargs): + pytest.fail("Raw GET called for %s when it shouldn't be" % url) + + +def _failing_post_object(url, body, **kwargs): + pytest.fail("POST called for %s when it shouldn't be" % url) + + +def _failing_put_object(url, body, **kwargs): + pytest.fail("PUT called for %s when it shouldn't be" % url) + + +def _failing_delete_object(url): + pytest.fail("DELETE called for %s when it shouldn't be" % url) + + +def patch_cbapi(monkeypatch, api, **kwargs): + monkeypatch.setattr(api, "get_object", kwargs.get('GET', _failing_get_object)) + monkeypatch.setattr(api, "get_raw_data", kwargs.get('RAW_GET', _failing_get_raw_data)) + monkeypatch.setattr(api, "post_object", kwargs.get('POST', _failing_post_object)) + monkeypatch.setattr(api, "put_object", kwargs.get('PUT', _failing_put_object)) + monkeypatch.setattr(api, "delete_object", kwargs.get('DELETE', _failing_delete_object)) diff --git a/test/mocks.py b/test/mocks.py deleted file mode 100755 index 8d5f24df..00000000 --- a/test/mocks.py +++ /dev/null @@ -1,29 +0,0 @@ -import pytest - -class MockResponse(object): - def __init__(self, contents, scode=200): - self._contents = contents - self.status_code = scode - - def json(self): - return self._contents - -def failing_mock_get_object(url, parms=None, default=None): - pytest.fail("GET called for %s when it shouldn't be" % url) - -def failing_mock_post_object(url, body, **kwargs): - pytest.fail("POST called for %s when it shouldn't be" % url) - -def failing_mock_put_object(url, body, **kwargs): - pytest.fail("PUT called for %s when it shouldn't be" % url) - -def failing_mock_delete_object(url): - pytest.fail("DELETE called for %s when it shouldn't be" % url) - -_methods = {"GET":failing_mock_get_object, "POST":failing_mock_post_object, - "PUT":failing_mock_put_object, "DELETE":failing_mock_delete_object} - -class ConnectionMocks: - @classmethod - def get(cls, name): - return _methods[name] diff --git a/tests/requests_cache/backends/storage/dbdict.py b/tests/requests_cache/backends/storage/dbdict.py index 2c75264b..513db8bc 100755 --- a/tests/requests_cache/backends/storage/dbdict.py +++ b/tests/requests_cache/backends/storage/dbdict.py @@ -6,7 +6,6 @@ Dictionary-like objects for saving large data sets to `sqlite` database """ -from collections import MutableMapping import sqlite3 as sqlite from contextlib import contextmanager try: @@ -17,6 +16,10 @@ import cPickle as pickle except ImportError: import pickle +try: + from collections.abc import MutableMapping +except ImportError: + from collections import MutableMapping from requests_cache.compat import bytes diff --git a/tests/test_defense_policy.py b/tests/test_defense_policy.py deleted file mode 100644 index 48653764..00000000 --- a/tests/test_defense_policy.py +++ /dev/null @@ -1,53 +0,0 @@ -import sys -import time - -import glob -import json -import os -import unittest - -from cbapi.psc.defense import * - - -sys.path.append(os.path.dirname(__file__)) -import requests_cache - -@unittest.skip("temporarily disabled") -def test_policy(rulefiles): - requests_cache.uninstall_cache() - defense_api = CbDefenseAPI(profile="test") - - - default_policies = [policy for policy in defense_api.select(Policy) if policy.name == "default"] - new_policy = defense_api.create(Policy) - new_policy.policy = default_policies[0].policy - new_policy.name = "cbapi-python-test-%d" % time.time() - new_policy.priorityLevel = "LOW" - new_policy.description = "Test policy" - new_policy.version = 2 - new_policy.save() - - for t in rulefiles: - try: - test_rule(new_policy, t) - print("Added rule %s" % t) - except Exception as e: - print("Exception adding rule %s: %s" % (t, e)) - - new_policy.delete() - - -@unittest.skip("temporarily disabled") -def test_rule(new_policy, fn): - new_rule = json.load(open(fn, "r")) - new_policy.add_rule(new_rule) - - -if __name__ == '__main__': - rulefiles = glob.glob(os.path.join(os.path.dirname(__file__), "data", "defense", "policy_rules", "*.json")) - print(rulefiles) - - test_policy(rulefiles) - - - unittest.main()
KeyValue