Upload STIX files through the REST API to Reveal(x)

If the name of the file matches the name of a threat collection on the appliance, the script overwrites the threat collection with the file contents. If there are no threat collection names that match the file name, the script uploads the file to create a new threat collection.

Each row of the CSV file must contain the following columns in the specified order:

The script includes the following configuration variables:

#!/usr/bin/env python2

import ssl
import json
import os
import requests
import sys
import csv

APPLIANCE_LIST = 'appliances.csv'
STIX_DIR = 'stix_dir'

# Get appliance hostnames and Read through appliance URL and API key list
appliances = []
with open(APPLIANCE_LIST, 'rb') as f:
    reader = csv.reader(f)
    for row in reader:
        appliance = {
                      'host': row[0],
                      'api_key': row[1]
                    }
        appliances.append(appliance)

# Get a json object of every threat collection on the appliance
def getCollections():
    url = host + 'api/v1/threatcollections'
    r = requests.get(url, headers=headers, verify=False)
    return r.json()

# Check which STIX files have already been uploaded
def check_files(collections):
    update_list = []
    skip_list = []
    # Get a list of all stix files in the STIX_DIR directory
    names = []
    for dir, subdirs, files in os.walk(STIX_DIR):
        for file in files:
            if file.endswith(('.tar', '.tar.gz', '.tgz')):
                name = file.split('.')[0]
                names.append(name)
    # Check each threat collection for names that match the STIX file names
    for c in collections:
        c_name = c['name']
        if c_name in names:
            update_list.append(c)
            skip_list.append(c_name)
    return update_list, skip_list

# Process each file in the STIX_DIR directory
# If it is for a new threat collection, add the threat collection
# If there is already a threat collection for the file, update the threat collection
def process_files(update_files, skip_list):
    for dir, subdirs, files in os.walk(STIX_DIR):
        for file in files:
            name = file.split('.')[0]
            if file.endswith(('.tar', '.tar.gz', '.tgz')) and name not in skip_list:
               upload_new(file, dir)
            else:
               for c in update_files:
                   if c['name'] == name:
                       update_old(file, dir, c)

# Upload a new threat collection
def upload_new(file, dir):
    print 'Uploading ' + file + ' on ' + host
    url = host + 'api/v1/threatcollections'
    file_path = os.path.join(dir, file)
    name = file.split('.')[0]
    files = {'file': open(file_path, 'rb')}
    values = {'name': name}
    r = requests.post(url, data=values, files=files, headers=headers, verify=False)
    print r.status_code
    print r.text

# Update an existing threat collection
def update_old(file, dir, c):
    print 'Updating ' + file + ' on ' + host
    url = host + 'api/v1/threatcollections/~' + str(c['user_key'])
    file_path = os.path.join(dir, file)
    files = {'file': open(file_path, 'rb')}
    r = requests.put(url, files=files, headers=headers, verify=False)
    print r.status_code
    print r.text

# Process STIX files for each appliance
for appliance in appliances:
    host = appliance['host']
    api_key = appliance['api_key']
    context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
    headers = {'Authorization': 'ExtraHop apikey=%s' % api_key}
    collections = getCollections()
    update_files, skip_list = check_files(collections)
    process_files(update_files, skip_list)