File Management Automation for File Management

[intapiuser]

On article Uploading Files Locally Using Rest (Linux) we saw examples of how to automate the upload of files to File Management. This article describe methods to automate the management of files on File Manager.

Compatible with Linux Version <= L2021.3

import requests

import json

import re

import time

from datetime import datetime

import websocket

from websocket import create_connection

# Sisense API token

apiToken = 'Bearer YOUR_API_TOKEN'

# Sisense hostname

sisenseHost = 'https://test.sisense.com'

# Retreive Droppy session token

url = sisenseHost + '/app/explore/!/token'

headers = {'content-type': 'application/x-www-form-urlencoded',

'accept': '*/*',

'accept-encoding' : 'gzip, deflate, br',

'accept-language' : 'en-US,en;q=0.9,ru;q=0.8',

'dnt' : '1',

'Authorization': apiToken,

'sec-fetch-dest' : 'empty',

'sec-fetch-mode' : 'same-origin',

'sec-fetch-site' : 'same-origin',

'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',

'x-app': 'droppy'

}

r = requests.get(url, headers=headers)

token = r.content

token = str(token, 'utf-8')

def downloadFolder(externalPath, localPath):

 # Downloads folder content as ZIP archive

 headers = {

 'Authorization': apiToken,

 }

 if localPath[-1] != '\\':

  localPath += '\\'

 if externalPath[0] == '/':

  externalPath = externalPath[1:]

 url = '%s/app/explore/!/zip/%s'%(sisenseHost, externalPath)

 r = requests.get(url, headers=headers, allow_redirects=True)

 m = re.search('filename="(.*?)"', r.headers.get('content-disposition'))

 if m:

  filename = m.group(1)

 open(localPath+filename, 'wb').write(r.content)

def downloadFile(externalPath, localPath):

 headers = {

 'Authorization': apiToken,

 }

 if localPath[-1] != '\\':

  localPath += '\\'

 if externalPath[0] == '/':

  externalPath = externalPath[1:]

 url = '%s/app/explore/!/dl/%s'%(sisenseHost, externalPath)

 r = requests.get(url, headers=headers, allow_redirects=True)

 m = re.search('filename="(.*?)"', r.headers.get('content-disposition'))

 if m:

  filename = m.group(1)

 open(localPath+filename, 'wb').write(r.content)

def deleteObject(ws, objectPath):

 command = {"vId":0,"type":"DELETE_FILE","data":objectPath,"token":token}

 ws.send(json.dumps(command))

def renameObject(ws, oldPath, newPath):

 command = {"vId":0,"type":"RENAME","data":{"src":oldPath,"dst": newPath},"token":token}

 ws.send(json.dumps(command))

def copyObject(ws, oldPath, newPath):

 command = {"vId":0,"type":"CLIPBOARD","data":{"type":"copy","src":oldPath,"dst":newPath},"token":token}

 ws.send(json.dumps(command))

def moveObject(ws, oldPath, newPath):

 command = {"vId":0,"type":"CLIPBOARD","data":{"type":"cut","src":oldPath,"dst":newPath},"token":token}

 ws.send(json.dumps(command))

def listFolder(ws, folderPath):

 command = {"vId":0,"type":"REQUEST_UPDATE","data":folderPath,"token":token}

 ws.send(json.dumps(command))

 result = json.loads(ws.recv())

 # Following code is for pretty print folder content

 for k, v in result.get('data').items():

  val = v.split('|')

  timeModified = datetime.utcfromtimestamp(int(val[1])).strftime('%Y-%m-%d %H:%M:%S')

  if val[0] == 'f':

   print ('File %s; Modified(UTC) %s; Size %s B'%(k, timeModified, val[2]))

  elif val[0] == 'd':

   print ('Directory %s; Modified(UTC) %s; Size %s B'%(k, timeModified, val[2]))

websocket.enableTrace(False)

ws = create_connection('wss%s/app/explore/!/socket'%(sisenseHost.replace('https','')))

# Function for deleting file/folder

#deleteObject(ws, '/data/Test/wetwt.txt')

# Function for copying file/folder

#copyObject(ws, '/data/Test/asd.txt', '/data/Test/New folder/asd.txt')

# Function for moving file/folder

#moveObject(ws, '/data/Test/asd1.txt', '/data/Test/New folder/asd1.txt')

# Function for viewing folder content

# listFolder(ws, '/connectors/excel')

# Function for renaming file/folder

#renameObject(ws, '/data/Test/asd.txt', '/data/Test/asd1.txt')

# Function for downloading folder

#downloadFolder('/data/Test/New folder', r'C:\Users\vladislav.kokh\Documents')

# Function for downloading file

#downloadFile('/data/Test/New folder/assssd.txt', r'C:\Users\vladislav.kokh\Documents')

Compatible with Linux Version >= L2021.5

# Install dependency:
# pip3 install requests
​
# Sisense hostname
url = 'https://test.sisense.com'
​
​
# Sisense API token
# token = 'Bearer YOUR_API_TOKEN'
​
import requests
import json
import urllib.parse
import os
def get_auth(token):
 headers = {
 'Authorization': token,
 }
​
​
 x_auth = requests.post(f'{url}/app/explore/api/login', headers=headers)
 if x_auth.status_code==200:
  return x_auth.text
 else:
  print ('Cannot get x-auth token. Check API token validity')
  return False
  
# # SERVICE FUNCTION
# # Do NOT modify
x_auth = get_auth(token)
​
​
def get_files(token, x_auth, path):
​
​
 headers = {
 'Authorization': token,
 'x-auth': x_auth
 }
 if path:
  x = requests.get(f'{url}/app/explore/api/resources{path}', headers=headers)
 else:
  print (f'Wrong path {path}')
  return False
 if x.status_code==200:
  return json.loads(x.text).get('items')
 else:
  print ('Wrong API token')
  return False
​
​
def upload_files(token, x_auth, src_path, dst_path):
​
 headers = {
 'Authorization': token,
 'x-auth': x_auth,
 }
​
 if dst_path:
  if dst_path[-1] != '/':
   dst_path += '/'
  dst_path = f'{dst_path}{os.path.basename(src_path)}'
  with open(src_path,'rb') as ff:
   x = requests.post(f'{url}/app/explore/api/resources{dst_path}?override=true', headers=headers,data=ff)
​
 else:
  print (f'Wrong path {dst_path}')
  return False
 if x.status_code==200:
  return x.text
def download_file(token, x_auth,path, dst_folder):
​
 headers = {
 'Authorization': token,
 'x-auth': x_auth,
 'content-type': 'application/x-www-form-urlencoded'
 }
 local_filename = path.split('/')[-1]
 if dst_folder[-1] != '\\':
  dst_folder += '\\'
 with requests.get(f'{url}/app/explore/api/raw{path}', headers=headers, stream=True) as r:
  r.raise_for_status()
  with open(dst_folder+local_filename, 'wb') as f:
   for chunk in r.iter_content(chunk_size=8192): 
    # If you have chunk encoded response uncomment if
    # and set chunk_size parameter to None.
    #if chunk: 
    f.write(chunk)
 return local_filename
def download_folder(token, x_auth,src_folder, dst_path):
 headers = {
 'Authorization': token,
 'x-auth': x_auth,
 'content-type': 'application/x-www-form-urlencoded'
 }
​
 
 with requests.get(f'{url}/app/explore/api/raw{src_folder}/?algo=zip', headers=headers, stream=True) as r:
  r.raise_for_status()
  with open(dst_path, 'wb') as f:
   for chunk in r.iter_content(chunk_size=8192): 
    f.write(chunk)
 return dst_path
def download_zipped_files(token, x_auth,src_files, dst_path):
 headers = {
 'Authorization': token,
 'x-auth': x_auth,
 }
 files_url_string = urllib.parse.quote_plus(','.join(src_files))
 with requests.get(f'{url}/app/explore/api/raw/?files={files_url_string}&algo=zip', headers=headers, stream=True) as r:
  r.raise_for_status()
  with open(dst_path, 'wb') as f:
   for chunk in r.iter_content(chunk_size=8192): 
    f.write(chunk)
 return dst_path
​
def copy_object(token, x_auth, src_path, dst_path):
 headers = {
 'Authorization': token,
 'x-auth': x_auth,
 }
 if src_path and dst_path:
  dst_path = urllib.parse.quote_plus(dst_path)
​
  x = requests.patch(f'{url}/app/explore/api/resources{src_path}?action=copy&destination={dst_path}&override=true&rename=false', headers=headers)
 else:
  print (f'Wrong path {src_path}  {dst_path}')
  return False
 if x.status_code==200:
  return x.text
​
​
​
def move_object(token, x_auth, src_path, dst_path):
 headers = {
 'Authorization': token,
 'x-auth': x_auth,
 }
 if src_path and dst_path:
  dst_path = urllib.parse.quote_plus(dst_path)
​
  x = requests.patch(f'{url}/app/explore/api/resources{src_path}?action=rename&destination={dst_path}&override=true&rename=false', headers=headers)
 else:
  print (f'Wrong path {src_path}  {dst_path} ')
  return False
 if x.status_code==200:
  return x.text
​
​
def delete_object(token, x_auth, path):
 headers = {
 'Authorization': token,
 'x-auth': x_auth,
 }
 if path:
​
  x = requests.delete(f'{url}/app/explore/api/resources{path}', headers=headers)
 else:
  print (f'Wrong path {path}')
  return False
 if x.status_code==200:
  return x.text
​

# Function for viewing folder content
# print (get_files(token, x_auth, path='/'))
​
# Function for uploading file to Sisense
# print(upload_files(token, x_auth, src_path=r'C:\Users\user\Documents\aaaaa.zip', dst_path='/usage'))
​
# Function for download file from Sisense
# print (download_file(token, x_auth, path='/usage/regex.txt', dst_folder=r'C:\Users\user\Documents'))
​
# Function for download folder from Sisense. Downloads in ZIP format. 
# print (download_folder(token, x_auth, src_folder='/usage', dst_path=r'C:\Users\user\Documents\aaaaa.zip'))
​
# Function for download multiple files from Sisense. Downloads in ZIP format. Must be minimum 2 files provided.
# print( download_zipped_files(token, x_auth, src_files=['/usage/regex.txt','/usage/values2.yaml'], dst_path=r'C:\Users\user\Documents\aaaaa.zip') )
​
# Function to copy file inside Sisense
# print (copy_object(token, x_auth, src_path='/usage/regex.txt', dst_path='/styles/regex.txt'))
​
# Function to move or rename file inside Sisense
# print (move_object(token, x_auth, src_path='/styles/regex.txt', dst_path='/usage/regex.txt'))
​
# Function to delete file inside Sisense
# print (delete_object(token, x_auth, path='/styles/regex.txt'))

File Management is using an open source application name Droppy, while the upload is done on REST the management itself is done on a web socket. Below examples in Python

Python script to manage files via Droppy socket interface

• Script requires installed websocket client library Install via pip: pip install websocket_client
• Get Sisense user API token and paste it to apiToken variable(row 12)
• Token format should be following 'Bearer TOKEN' (ex. 'Bearer eyJhbGciOiJIUzI1NiIsInR5cC...')
• Set sisenseHost variable(row 15) with your Sisense URL (ex. 'https://test.sisense.com')
• Starting from row 124 we provide Droppy management function examples. You can find function for deleting, moving, renaming, etc..
• Each function can be run independently or together with others (To run a couple functions you need to uncomment all of them)

Compatible with Linux Version <= L2021.3

DELETING FILE OR FOLDER(OBJECT):

• Copy object location from Droppy (For ex. '/connectors/genericjdbc/manifest.json')
• Note that object path should NOT end with '/'(ex. '/connectors/genericjdbc/manifest.json/' is wrong)
• Find deleteObject function commented (row 125)
• Uncomment this line (Remove # from the beggining of the row)
• Paste the object location to the function as a second argument ( deleteObject(ws,'/connectors/genericjdbc/manifest.json') )
• Run python script (Open CMD and type 'python droppyFileManager.py')
• Object should be deleted
• Close droppyFileManager.py without saving changes

GET FOLDER CONTENT:

• Copy folder location from Droppy (For ex. '/connectors/genericjdbc')
• Note that object path should NOT end with '/'(ex. '/connectors/genericjdbc/' is wrong)
• Find listFolder function commented (row 131)
• Uncomment this line (Remove # from the beggining of the row)
• Paste the folder location to the function as a second argument ( deleteObject(ws,'/connectors/genericjdbc') )
• Run python script (Open CMD and type 'python droppyFileManager.py')
• Folder content should be printed to console (ex. 'File description.json; Modified(UTC) 2020-03-06 15:39:41; Size 472 B')
• Close droppyFileManager.py without saving changes

[riskpeep]

This script works great for Sisense servers using local authn.

Our server uses SSO though (Okta), and the authn fails in the get_auth method with the call to login.  Is there a way to authn to File Browser via the API for SSO based configurations?