The following script will upload a CSV to NetApp Cloud Insights, so that it can be processed by the Post ETL script, in order to update the Cloud Insights Data Warehouse dwh_custom.
Save as say ci_upload_csv.py and run as:
python ci_upload_csv.py TENANT_URL API_TOKEN CSV_FILE
Note 1: The following script doesn't include code for being behind a proxy, if you need this then hints are here: Using Python with Proxy and Authentication. Or you could enter proxy details into the Command Prompt like:
CMD> set https_proxy = http://USERNAME:PASSWORD@PROXYFQDN:PORT
Note 2: This is based on the NetApp CI module, just with the upload to DataWarehouse bit extracted and turned into a simplified, standalone tool. Also added verify=False in there (since where I'm working the SSL cert gets intercepted and modified.)
The Script
#!/usr/bin/env python
''' Upload csv file to be processed by Data Warehouse custom script '''
import argparse
from pathlib import Path
from datetime import datetime
from enum import Enum
import requests
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
''' START OF CLASS Client '''
class Client:
def __init__(self, instance, token, version='v1', session=None):
self.host = instance.rstrip('/') # Trim trailing slash
self.version = version
self.base_path = f'/rest/{version}'
if session is None:
session = requests.Session()
session.headers.update({
'Content-Type': 'application/json',
'X-CloudInsights-ApiKey': token,
})
self.session = session
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
return False
def _convert_to_fully_qualified_url(self, api_path):
if api_path.startswith(self.host):
# Fully qualified URL
return api_path
if api_path.startswith(self.base_path):
# URL that is only missing the schema and server
return self.host + api_path
# The URL is relative - needs to add prefix
return f'{self.host}{self.base_path}/{api_path.lstrip("/")}'
def _request(self, method, path, **kwargs):
return_response = kwargs.pop('return_response', None)
response = self.session.request(
method,
self._convert_to_fully_qualified_url(path),
**kwargs,
verify=False
)
self._raise_for_status(response)
if return_response:
return response
# Certain calls like /dwh-management/upload/csvs return an empty result
if not response.text:
return {}
return response.json()
def post(self, path, json=None, **kwargs):
return self._request('POST', path, json=json, **kwargs)
@staticmethod
def _raise_for_status(response):
''' Improve the original raise_for_status in the requests library to add some logging '''
if 400 <= response.status_code < 500:
# For client error, raise a regular HTTP error
raise requests.HTTPError(f'{response.status_code} Client Error: {response.reason} {response.text}', response=response)
elif 500 <= response.status_code < 600:
# For server-side errors, we need further parsing to extract OCI specific error information
raise APIError(f'{response.status_code} Server Error: {response.reason}', response=response)
def upload_dwh_csv(self, filename):
''' Upload a CSV file to be processed by Data Warehouse. '''
return self.post(
'dwh-management/upload/csvs',
files={'customFile': open(filename, 'rb')},
headers={'Content-Type': None}, # Reset Content-Type from default
return_response=True # Get the response object, as no JSON is returned for this call
)
def close(self):
''' Close the session. '''
self.session = None
''' END OF CLASS Client '''
''' UPLOAD CSV CODE '''
def upload_file(client, csv_file):
if not Path(csv_file).is_file():
print(f'File "{csv_file}" does not exist or is not a file.')
return
try:
response = client.upload_dwh_csv(csv_file)
if response.ok: print('File uploaded successfully!')
else: print(f'Error posting file: {response.text}')
except Exception as e: print(e)
''' MAIN PROGRAM '''
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('instance', help='Cloud Insights Instance')
parser.add_argument('token', help='Cloud Insights Token')
parser.add_argument('csv', help='CSV upload file')
options = parser.parse_args()
with Client(options.instance, options.token) as client:
upload_file(client, options.csv)
Comments
Post a Comment