-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmario.py
executable file
·119 lines (93 loc) · 3.34 KB
/
mario.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
#! /usr/bin/env python
##################################################################
# Mario - The CDAP Pipeline Extraction Tool
#
# Author: Tony Hajdari, [email protected]
#
# Use: Allows you to extract all the deployed
# pipelines from your CDAP instance.
##################################################################
import requests,json,logging, sys, os
#from logging.config import fileConfig
#from logging.handlers import RotatingFileHandler
from logging import handlers
# #fileConfig("log.conf")
log = logging.getLogger('')
log.setLevel(logging.DEBUG)
format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(format)
log.addHandler(ch)
# fh = handlers.RotatingFileHandler(LOGFILE, maxBytes=(1048576*5), backupCount=7)
# fh.setFormatter(format)
# log.addHandler(fh)
host = 'localhost'
port = '11015'
cdap = 'http://'+ host + ':' + port
namespaces = '/v3/namespaces'
drafts = '/v3/configuration/user'
p = { 'name':'','description':'','artifact':'','config':'' }
output = 'Pipelines'
def getJSON(url):
r = requests.get(url)
d = r.json()
#d = json.loads(r.text)
return d #returns a dict
#Example of an App collection endpoint
# http://localhost:11015/v3/namespaces/default/apps
def getApps(ns):
return getJSON(cdap + namespaces + '/' + ns + '/apps')
#Example of an individual App endpoint
#http://localhost:11015/v3/namespaces/default/apps/MyAppName
def getApp(ns, id):
return getJSON(cdap + namespaces + '/' + ns + '/apps' + '/' + id)
#Get the available namespaces for this CDAP instance
def getNamespaces():
return getJSON(cdap + namespaces)
# Get pipeline drafts
def getDrafts():
return getJSON(cdap + drafts)
#Write the pipelline config out to a file
def exportPipeline(ns, id, data):
fileName = id + '.json'
directory = output + '/' + ns
path = directory + '/' + fileName
if not os.path.exists(directory):
os.makedirs(directory)
with open(path, 'w') as f:
f.write(data)
#Get the draft pipelines -- this is NOT namespace specific
#will retrieve the drafts in ALL namespaces
drafts = getDrafts()
#loop through all namespaces
for namespace in getNamespaces():
#set the global namespace name
ns = namespace.get('name')
log.debug('Namespace: %s', ns)
#get all the drafts per namespace
d = drafts.get('property').get('hydratorDrafts').get(ns)
name = d.itervalues().next().get('name')
p['name'] = name
p['artifact'] = d.itervalues().next().get('description')
p['artifact'] = d.itervalues().next().get('artifact')
p['config'] = d.itervalues().next().get('config')
spec = json.dumps(p)
#log.debug('Draft Pipeline: %s', spec)
exportPipeline(ns, name, spec)
#Export the deployed pipelines in this namespace
for i in getApps(ns):
#filer out anything other than cdap-data-pipeline
artifactType = i.get('artifact').get('name')
if "cdap-data-pipeline" in artifactType:
id = i['id']
if not id in ('_Tracker', 'dataprep'):
log.debug('App Namespace: %s', ns)
log.debug('pipeline name: %s', id)
app = getApp(ns,id)
#log.debug('Pipeline = %s', json.dumps(app, sort_keys=True, indent=4))
p['name'] = app.get('name')
p['description'] = app.get('description')
p['artifact'] = app.get('artifact')
p['config'] = json.loads(app.get('configuration'))
spec = json.dumps(p, sort_keys=True, indent=4)
exportPipeline(ns, id, spec)