Skip to content

Commit 2a2d163

Browse files
bitdiscoverybitdiscovery
committed
initial commit, enjoy!
0 parents  commit 2a2d163

16 files changed

+1135
-0
lines changed

‎README.md‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Bit Discovery Python scripts
2+
3+
These scripts demonstrate how you can integrate your work with the Bit Discovery API.
4+
5+
To run these scripts, you have to have [Python 3.6+](https://www.python.org/downloads/) installed on your computer and
6+
your Bit Discovery API keys for an inventory. (You can get this on
7+
your [Bit Discovery profile page](https://dev.bitdiscovery.com/user/profile).) The best way is to save your API key to a
8+
variable and reuse it for every script.
9+
10+
```shell
11+
APIKEY=eyJh...hUFs
12+
python pdf-report.py $APIKEY
13+
```
14+
15+
If you need more information about the options of any script, just see the help message:
16+
17+
```shell
18+
python pdf-report.py --help
19+
```
20+
21+
## PDF Report
22+
23+
The `pdf-report.py` script exports the assets from one or all of your inventories (`--multiple` flag), and creates a PDF
24+
file analysis based on the inventory data.
25+
26+
### Usage
27+
28+
> For now, this script requires Python 3.6, because the fpdf dependency doesn't work with newer versions.
29+
30+
Install the dependencies:
31+
32+
```shell
33+
pip install argparse datetime fpdf matplotlib pypdf2 requests
34+
```
35+
36+
Create a pdf report by passing the inventory api key:
37+
38+
```shell
39+
python3 pdf-report.py $APIKEY
40+
```
41+
42+
If you want to create a pdf report for every inventory you own, you can pass the `--multiple` flag:
43+
44+
```shell
45+
python3 pdf-report.py $APIKEY --multiple
46+
```
47+
48+
## Auto add assets
49+
50+
The `auto-add-assets.py` script can search your cloud provider, AWS, Google Cloud or Azure (using their respective
51+
command-line tools), and add your running instances to the provided inventory.
52+
53+
### Usage
54+
55+
For this script you need to have and be signed in to the command-line interfaces of your cloud provider of choice. For
56+
AWS, install the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html), for GCP Cloud
57+
the [gcloud CLI](https://cloud.google.com/sdk/docs/install) and for
58+
Azure [az](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli).
59+
60+
After that, install the Python dependencies:
61+
62+
```shell
63+
pip install argparse datetime requests sh
64+
```
65+
66+
Read your running EC2 instances and buckets from your AWS account:
67+
68+
```shell
69+
python3 auto-add-assets.py amazon-ec2 $APIKEY
70+
```
71+
72+
Similarly for Google Cloud and Azure:
73+
74+
```shell
75+
# GCP
76+
python3 auto-add-assets.py google-cloud $APIKEY
77+
# Azure
78+
python3 auto-add-assets.py azure $APIKEY
79+
```
80+
81+
## Delete ip or source
82+
83+
The `delete-ip.py` script deletes one specific IP or source from your inventory.
84+
85+
### Usage
86+
87+
First, install the Python dependencies:
88+
89+
```shell
90+
pip install argparse requests sh
91+
```
92+
93+
Delete an IP or a source (with its id) from the given inventory:
94+
95+
```shell
96+
python3 delete-ip.py ip 1.1.1.1 $APIKEY
97+
python3 delete-ip.py source 13 $APIKEY
98+
```

‎auto-add-assets.py‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
#!/usr/bin/python3
2+
import sys
3+
from argparse import ArgumentParser
4+
from typing import Dict, Any, Optional, List
5+
from bitdiscovery.api import BitDiscoveryApi, try_multiple_times, get_lastid
6+
from bitdiscovery.cloud import get_provider, remove_matches, CloudProvider, AWSProvider
7+
8+
parser = ArgumentParser(description="Add your cloud provider assets to your Bit Discovery inventory.")
9+
parser.add_argument('cloudprovider', metavar="PROVIDER", type=str, choices=['amazon-ec2', 'google-cloud', 'azure'],
10+
help="The cloud provider to add assets from, either amazon-ec2, google-cloud or azure.")
11+
parser.add_argument('apikey', metavar="APIKEY", type=str, help="Your Bit Discovery API key.")
12+
parser.add_argument('--env', choices=['dev', 'staging', 'prod'], default="dev",
13+
help="The Bit Discovery environment (by default 'dev')")
14+
parser.add_argument('--offset', type=int, default=0, help="Offset to the API request data (by default 0).")
15+
parser.add_argument('--limit', type=int, default=5000, help="Limit to the API request data (by default 500).")
16+
args = parser.parse_args()
17+
18+
APIKEY: str = args.apikey
19+
CLOUD_PROVIDER: str = args.cloudprovider
20+
APIURL: str = "https://bitdiscovery.com/api/1.0"
21+
OFFSET: int = args.offset
22+
LIMIT: int = args.limit
23+
24+
25+
# Takes two dicts and safely merges them into a copy
26+
def merge_two_dicts(x: Dict, y: Dict) -> Dict:
27+
z = x.copy() # start with x's keys and values
28+
z.update(y) # modifies z with y's keys and values & returns None
29+
return z
30+
31+
32+
# Find all IPs belonging in Bit Discovery
33+
print("Initializing and pulling assets from Bit Discovery...")
34+
35+
api = BitDiscoveryApi(APIURL, APIKEY)
36+
inventories_json: Dict[str, Any] = {}
37+
try:
38+
inventories_json = api.find_inventories(OFFSET, LIMIT)
39+
except:
40+
print("API call failed. Try again later.")
41+
exit(1)
42+
43+
# TODO: maybe remove iteration if we cannot add to multiple inventories
44+
inventories: Dict[str, str] = {inventories_json['actualInventory']['inventory_name']: APIKEY}
45+
46+
for entityname in inventories:
47+
jsondata = []
48+
# TODO: this is never used, why do we have to keep this (merge_two_dicts too)
49+
inventoryips = {}
50+
51+
print(f"Starting sources for: {entityname}.")
52+
53+
sourcesdata: List[Dict[str, Any]] = []
54+
55+
# Collect every source from Bit Discovery inventory with pagination
56+
lastid: str = ''
57+
offset: int = OFFSET
58+
while True:
59+
result: Optional[Dict[str, Any]] = try_multiple_times(
60+
lambda: api.search_for_source(LIMIT, lastid, ""),
61+
max_tries=5
62+
)
63+
64+
if result is None:
65+
print("\tAPI call failed too many times. Try again later.")
66+
exit(1)
67+
68+
sourcesdata.append(result)
69+
lastid = get_lastid(result)
70+
offset += LIMIT
71+
total: int = int(sourcesdata[0]['total'])
72+
73+
if offset > total:
74+
break
75+
76+
# Collect all source IPs that aren't CIDRs
77+
sourceips: Dict[str, int] = {}
78+
for sources in sourcesdata:
79+
for source in sources.get('searches', []):
80+
if 'search_type' in source and source['search_type'] == 'iprange':
81+
ipcandidate = source['keyword'].lower()
82+
if '-' in ipcandidate or '/' in ipcandidate:
83+
continue
84+
else:
85+
# Number 2 indicates it's a source, not an asset
86+
sourceips[ipcandidate] = 2
87+
88+
# TODO: if inventoryips is to be deleted this can be removed as well
89+
superset: Dict[str, int] = merge_two_dicts(sourceips, inventoryips)
90+
for ips in superset:
91+
if ips in sourceips and ips in inventoryips:
92+
# Number 3 saying the IP address is found in both
93+
superset[ips] = 3
94+
95+
# Find all IPs in cloud
96+
addednum = 0
97+
98+
provider: CloudProvider = get_provider(CLOUD_PROVIDER)
99+
100+
# Get provider IP ranges from the provider
101+
# TODO: why is this read? we don't use this for anything
102+
print(f"\tWe're on {provider.name}, so processing accordingly")
103+
print(f"\t\tGetting and parsing all of {provider.name}'s public IP space")
104+
prefixes: Dict[str, int] = provider.get_ip_ranges()
105+
106+
# Get your ips from the provider
107+
print("\t\tGetting and parsing your public IPs")
108+
ips: Dict[str, int] = provider.get_instance_ips()
109+
110+
# If IPs in cloud match Bit Discovery remove them from list to do further checks on (they haven't changed)
111+
print("\t\tIgnorning assets that haven't changed.")
112+
ips_new, old_ips = remove_matches(superset, inventoryips, sourceips, ips)
113+
114+
# If IPs are not in Bit Discovery but they are in cloud add them
115+
print("\t\tAdding new IPs")
116+
for new_ip in ips_new:
117+
# Try to add the new IPs to the Bit Discovery inventory
118+
result: Optional[bool] = try_multiple_times(
119+
lambda: api.add_ip(new_ip),
120+
max_tries=5
121+
)
122+
123+
if result is None:
124+
print("\tAPI call failed too many times. Try again later.")
125+
exit(1)
126+
127+
# Increment added count
128+
addednum += 1
129+
130+
print(f"\tAdded a total of {str(addednum)} {provider.name} IPs.")
131+
132+
# If provider is AWS, then we can also retrieve the buckets
133+
if type(provider) == AWSProvider:
134+
print("\t\tFinding s3 buckets.")
135+
buckets = AWSProvider().find_s3_buckets()
136+
137+
print("\t\tAdding s3 buckets.")
138+
addedbucket = 0
139+
for bucket in buckets:
140+
url = AWSProvider().find_s3_region(bucket)
141+
142+
# Try to add bucket URLs to the Bit Discovery inventory
143+
success: Optional[bool] = try_multiple_times(
144+
lambda: api.add_source(url),
145+
max_tries=5
146+
)
147+
148+
if result is None:
149+
print("\tAPI call failed too many times. Try again later.")
150+
exit(1)
151+
152+
# Increment bucket count
153+
addedbucket += 1
154+
155+
print("\tAdded a total of " + str(addedbucket) + " S3 buckets.")
156+
157+
print("Done.")

‎bitdiscovery/api.py‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
import requests
2+
from typing import Any, Callable, Dict, Optional, List
3+
4+
5+
def try_multiple_times(fn: Callable[..., Any], max_tries: int) -> Optional[Any]:
6+
"""
7+
Retry a function multiple times if it fails.
8+
9+
:param fn: The function to run (it should return on success, throw on failure)
10+
:param max_tries: The number of times after failure is registered, and None returned.
11+
:return: Either the returned value on success, or None on failure.
12+
"""
13+
result = None
14+
i = 0
15+
while result is None and i < max_tries:
16+
try:
17+
result = fn()
18+
except Exception as e:
19+
print("ERROR: " + str(e))
20+
result = None
21+
i += 1
22+
23+
return result
24+
25+
26+
def get_lastid(assets: Dict[str, List[Dict[str, str]]]) -> str:
27+
"""
28+
Finds the last asset's ID so you know where you left off
29+
30+
:param assets: The list of assets as got back from the API.
31+
:return: The last asset from the list.
32+
"""
33+
lastid = ''
34+
if "assets" in assets:
35+
for asset in assets["assets"]:
36+
if "id" in asset:
37+
lastid = str(asset["id"])
38+
return lastid
39+
40+
41+
class BitDiscoveryApi:
42+
"""
43+
Initializes an object to call the Bit Discovery API with a base URL and API key.
44+
"""
45+
apiurl: str
46+
apikey: str
47+
48+
def __init__(self, apiurl: str, apikey: str):
49+
self.apiurl = apiurl
50+
self.apikey = apikey
51+
52+
def find_inventories(self, offset: int, limit: int) -> Dict[str, Any]:
53+
url = f'{self.apiurl}/inventories/list?offset={str(offset)}&limit={str(limit)}&forcescreenshots=false'
54+
headers = {'Accept': 'application/json', 'Authorization': self.apikey}
55+
r = requests.get(url, headers=headers)
56+
return r.json()
57+
58+
def get_dashboard(self, querytypes: str) -> Dict[str, Any]:
59+
url = f'{self.apiurl}/dashboard?columns={str(querytypes)}'
60+
payload = '[ { "column": "bd.original_hostname", "type": "ends with", "value": "" } ]'
61+
headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': self.apikey}
62+
63+
r = requests.post(url, data=payload, headers=headers)
64+
return r.json()
65+
66+
def search_inventory(self, limit: int, after: str) -> Dict[str, Any]:
67+
payload = '[ { "column": "bd.original_hostname", "type": "ends with", "value": "" } ]'
68+
69+
if after == '':
70+
url = f'{self.apiurl}/inventory?limit={str(limit)}&offset=0&sortorder=true&inventory=false'
71+
else:
72+
url = f'{self.apiurl}/inventory?limit={str(limit)}&after={str(after)}&sortorder=true&inventory=false'
73+
headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': self.apikey}
74+
75+
r = requests.post(url, data=payload, headers=headers)
76+
return r.json()
77+
78+
def search_for_ip_address(self, limit: int, after: str, ip: str) -> Dict[str, Any]:
79+
payload = '[ {"column": "bd.ip_address", "type": "is", "value": "' + str(ip) + '" } ]'
80+
headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': self.apikey}
81+
82+
if after == '':
83+
url = f'{self.apiurl}/inventory?limit={limit}&sortorder=true&columns=id,bd.ip_address'
84+
else:
85+
url = f'{self.apiurl}/inventory?limit={limit}&after={after}&sortorder=true&columns=id,bd.ip_address'
86+
87+
r = requests.post(url, data=payload, headers=headers)
88+
return r.json()
89+
90+
def search_for_source(self, limit: int, after: str, search: str) -> Dict[str, Any]:
91+
headers = {'Accept': 'application/json', 'Authorization': self.apikey}
92+
93+
if after == '':
94+
url = f'{self.apiurl}/sources?offset=0&limit={limit}&search={search}'
95+
else:
96+
url = f'{self.apiurl}/sources?offset=0&offset={after}&limit={limit}&search={search}'
97+
98+
r = requests.get(url, headers=headers)
99+
return r.json()
100+
101+
def add_ip(self, new_ip: str) -> bool:
102+
payload = '{ "ip": "' + str(new_ip) + '" }'
103+
url = f'{self.apiurl}/source/ip/add'
104+
headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': self.apikey}
105+
requests.post(url, data=payload, headers=headers)
106+
return True
107+
108+
def add_source(self, new_source: str) -> bool:
109+
payload = '{ "keyword": "' + str(new_source) + '" }'
110+
url = f'{self.apiurl}/source/add?as_subdomain=true&dont_discover=true'
111+
headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': self.apikey}
112+
requests.post(url, data=payload, headers=headers)
113+
return True
114+
115+
def archive_ip(self, old_id: str) -> bool:
116+
payload = '[ {"id": "' + old_id + '", "hidden": true } ]'
117+
url = f'{self.apiurl}/asset/hide'
118+
headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': self.apikey}
119+
requests.post(url, data=payload, headers=headers)
120+
return True
121+
122+
def delete_source(self, old_source_id: str) -> bool:
123+
url = f'{self.apiurl}/source/{old_source_id}/delete'
124+
headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': self.apikey}
125+
requests.post(url, headers=headers)
126+
return True

0 commit comments

Comments
 (0)