Merge pull request #13 from Chaffelson/multicloudtesting

v0.1.0 Release

docker/v2/localdev/Dockerfile

@@ -44,4 +44,4 @@ CMD ["whoville/mayor.py"]
 
 
 # Build & Run Example from project root /whoville
-# docker build -f="./docker/v2/localdev/Dockerfile" -t whoville:hdp3cbd . && docker run -v /whoville:/whoville --env PROFILE=/whoville/profile.yml --name whoville whoville:hdp3cbd
+# docker build -f="./docker/v2/localdev/Dockerfile" -t whoville:next . && docker run -v /whoville:/whoville --env PROFILE=/whoville/profile.yml --name whoville whoville:next

docs/history.rst

@@ -1,11 +1,38 @@
 History
 -------
 
+0.1.0
+    7 Nov 2018
+
+- Added Azure and GCP Support
+- Cloudbreak install script is more efficient
+- Fixed DNS bug when Docker is installed in a Definition
+- AWS Role based access no longer preferred due to support complexity
+- Refactored out hard coded passwords, all passwords now taken from Profile or randomly generated
+- Refactored Definitions to support multicloud deployment
+- Simplified Definition Sequences
+- Enhanced Profile validation for easier user experience
+- Reworked all Tags to match multicloud requirements
+- Incremented Profile to ver2 for new Tags
+- All wait times doubled to allow for Azure
+
+
+
+0.0.1-rc4
+    28 Sept 2018
+
+- refactored naming to be clearer
+- Added copy_def and merge_def primitives for future demo script usage
+- Added cmdline menu for Quickstart and Docker consumption
+- Tidied up exemplar definitions
+
+
 0.0.1-rc1
     24 Sept 2018
 
 - Initial Release for beta testing
 
+
 0.0.1-rc2
     25 Sept 2018
 
@@ -26,13 +53,4 @@ History
 - refactored sequencing primitives to remove name clash and be more readable - please update definitions
 - Moved sequencing primitives into actions.py
 - Refactored borg singleton to support iteraction and basic parameter setting, more to come
-- Documentation updates
-
-
-0.0.1-rc4
-    28 Sept 2018
-
-- refactored naming to be clearer
-- Added copy_def and merge_def primitives for future demo script usage
-- Added cmdline menu for Quickstart and Docker consumption
-- Tidied up exemplar definitions
+- Documentation updates

profile.RENAME.yml

@@ -34,7 +34,7 @@ sshkey_pub: ssh-rsa CHANGEME
 
 
 # ------------ DEFAULTS ---------------
-
+profilever: 2
 # URIs for additional resources to be loaded and processed by the orchestrator.
 # Loc's can be either local or github
 # URI should point to the folder root of the resources
@@ -47,11 +47,12 @@ resources:
 # and users and environments
 # Empty tags will be autopopulated with useful defaults
 tags:
-  Owner: [email protected]
-  Service: Whoville Ephemeral Cluster
-  BusinessUnit: DevOps
-  StartDate:
-  EndDate:
+  deployer: dchaffelson
+  deploytool: whoville
+  service: whovilleephemeralcluster
+  businessunit: se
+  startdate:
+  enddate:
 
 
 # The base name of your objectsore bucket if using objectstore definitions, 

requirements.txt

@@ -5,4 +5,12 @@ pygithub>=1.4
 apache-libcloud>=2.3.0
 boto3>=1.9
 urllib3>=1.22
-ruamel.yaml>=0.15.37
+ruamel.yaml>=0.15.37
+azure-common>=1.0.0
+azure-mgmt-resource>=1.1.0
+azure-mgmt-network>=1.0.0
+azure-mgmt-compute>=2.0.0
+azure-mgmt-storage>=2.0.0
+azure-mgmt-consumption>=1.0.0
+google-api-python-client>=1.7
+pycrypto>=0.1

resources/v2/inf-cda30-single/inf-cda30-single.yaml

@@ -12,7 +12,6 @@ seq:
 - prep_deps: [cda30-s, cda30-s]
 - prep_spec: [cda30-s, cda30-s]
 - do_builds: [cda30-s]
-- wait_event: [inf-cda30-single, cda30-s, event_type, BILLING_STARTED]
 - open_port: [tcp, 8080, 8080, '0.0.0.0/0']
 input:
   yarn.vcores: 12
@@ -24,7 +23,7 @@ blueprint:
   name: blueprint.json
 group:
   master:
-    machine: m5.2xlarge
+    machine: 4x28-8x32
 mpack:
 - name: hdf320
   desc: HDF 3.2.0

resources/v2/inf-hcp16-multi/inf-hcp16-multi.yaml

@@ -3,7 +3,6 @@ seq:
 - prep_deps: [inf-hcp16-multi, hcp16-m]
 - prep_spec: [inf-hcp16-multi, hcp16-m]
 - do_builds: [hcp16-m]
-- wait_event: [inf-hcp16-multi, hcp16-m, event_type, BILLING_STARTED]
 - open_port: [tcp, 8080, 8080, '0.0.0.0/0']
 deploywait: 1800
 purge: true
@@ -14,9 +13,6 @@ input:
   nifi.registry.security.encrypt.configuration.password: CALL:security.generate_password
   nifi.registry.sensitive.props.key: CALL:security.generate_passphrase
   ranger.kms.password: CALL:security.generate_password
-infra:
-  disktypes: [gp2, standard]
-  os: redhat7
 blueprint:
   name: blueprint.json
 recipe:
@@ -36,62 +32,62 @@ group:
   master2:
     recipe: [noexec-tmp]
     type: CORE
-    machine: m5.2xlarge
+    machine: 4x28-8x32
     nodes: 1
   master1:
     recipe: [noexec-tmp]
     type: CORE
-    machine: m5.2xlarge
+    machine: 4x28-8x32
     nodes: 1
   solr:
     recipe: [noexec-tmp]
     type: CORE
-    machine: m5.xlarge
+    machine: 4x14-4x16
     nodes: 1
   ambari:
     recipe: [noexec-tmp, metron-db-setup, mysql-client, requests-upgrade, allow-renewable]
     type: GATEWAY
-    machine: m5.2xlarge
+    machine: 4x28-8x32
     nodes: 1
   worker:
     recipe: [noexec-tmp]
     type: CORE
-    machine: m5.xlarge
+    machine: 4x14-4x16
     nodes: 1
   metron:
     recipe: [noexec-tmp, nodejs, requests-upgrade, mysql-client]
     type: CORE
-    machine: m5.xlarge
+    machine: 4x14-4x16
     nodes: 1
   hbase:
     recipe: [noexec-tmp]
     type: CORE
-    machine: m5.xlarge
+    machine: 4x14-4x16
     nodes: 1
   kafka:
     recipe: [noexec-tmp]
     type: CORE
-    machine: m5.2xlarge
+    machine: 4x28-8x32
     nodes: 1
   streaming:
     recipe: [noexec-tmp]
     type: CORE
-    machine: m5.xlarge
+    machine: 4x14-4x16
     nodes: 1
   zookeeper:
     recipe: [noexec-tmp]
     type: CORE
-    machine: m5.2xlarge
+    machine: 4x28-8x32
     nodes: 1
   master3:
     recipe: [noexec-tmp]
     type: CORE
-    machine: m5.2xlarge
+    machine: 4x28-8x32
     nodes: 1
   nifi:
     recipe: [noexec-tmp]
     type: CORE
-    machine: m5.xlarge
+    machine: 4x14-4x16
     nodes: 1
 mpack:
 - name: hdp-search

resources/v2/inf-hdp30-single/inf-hdp30-single.yaml

@@ -1,15 +1,12 @@
 desc: Single node cluster with minimal install of HDP-3.0 
-purge: false 
+purge: true 
 deploywait: 1800
 seq:
 - prep_deps: [inf-hdp30-single, hdp30-s]
 - prep_spec: [inf-hdp30-single, hdp30-s] 
 - do_builds: [hdp30-s]
-- wait_event: [inf-hdp30-single, hdp30-s, event_type, BILLING_STARTED]
 - open_port: [tcp, 8080, 8080, '0.0.0.0/0']
 infra:
-  disktypes: [gp2, standard]
-  os: redhat7
   ambarirepo:
     version: 2.7.1.0
     baseUrl: http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.7.1.0
@@ -30,5 +27,5 @@ group:
   master:
     recipe: [remount-tmpfs, prepare-metastores-30, phoenix-client]
     type: GATEWAY
-    machine: m5.xlarge
+    machine: 4x14-4x16
     nodes: 1

resources/v2/inf-hdp30-single/remount-tmpfs.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
 #remount tmpfs to ensure NOEXEC is disabled
-mount -o remount,size=10G /tmp
-mount -o remount,exec /tmp
+if grep -Fxq "/etc/fstab" /tmp
+then
+    mount -o remount,size=10G /tmp
+    mount -o remount,exec /tmp
+fi

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.1
+current_version = 0.1.0
 commit = True
 tag = True
 

setup.py

@@ -14,7 +14,7 @@
 with open('requirements.txt') as reqs_file:
     requirements = reqs_file.read().splitlines()
 
-proj_version = '0.0.1'
+proj_version = '0.1.0'
 
 setup(
     name='whoville',

whoville/__init__.py

@@ -5,6 +5,6 @@
 """
 __author__ = """Daniel Chaffelson"""
 __email__ = '[email protected]'
-__version__ = '0.0.1'
+__version__ = '0.1.0'
 __all__ = ['config', 'deploy', 'utils', 'security', 'mayor', 'infra',
            'cloudbreak', 'actions']

whoville/actions.py

@@ -11,9 +11,7 @@
 from __future__ import absolute_import as _absolute_import
 import logging as _logging
 from datetime import datetime as _datetime
-from whoville import deploy
-from whoville.utils import get_val as _get_val
-from whoville.utils import set_val as _set_val
+from whoville import deploy, utils
 
 _horton = deploy.Horton()
 
@@ -39,6 +37,13 @@ def do_builds(args):
             fullname,
             purge=False
         )
+        deploy.wait_for_event(
+            fullname,
+            'event_type',
+            'BILLING_STARTED',
+            _datetime.utcnow(),
+            600
+        )
 
 
 def wait_event(args):
@@ -83,10 +88,19 @@ def replace_str(args):
     cache_key = args[2]
     log.info("Replacing string [%s] with [%s] in Resource [%s] in def [%s]",
              cache_key, _horton.cache[cache_key], res_name, def_key)
-    s = _horton.resources[def_key][res_name].replace(
-        cache_key, _horton.cache[cache_key]
-    )
-    _horton.resources[def_key][res_name] = s
+    resource = _horton.resources[def_key][res_name]
+    # Read
+    if isinstance(resource, dict):
+        source = utils.dump(resource)
+    else:
+        source = resource
+    # Replace String
+    target = source.replace(cache_key, _horton.cache[cache_key])
+    # Write
+    if isinstance(resource, dict):
+        _horton.resources[def_key][res_name] = utils.load(target)
+    else:
+        _horton.resources[def_key][res_name] = target
 
 
 def copy_def(args):

whoville/config.py

@@ -26,6 +26,10 @@
 if not profile_loc:
     profile_loc = 'profile.yml'
 
+
+# --- Profile Versioning ------
+min_profile_ver = 2
+
 # --- Logging ------
 logging.basicConfig(level=logging.INFO)