using openstack driver for libcloud and added cpsupdate for quickly u…

…pdated a single or multiple ConPaaS componenets

conpaas-director/cpsdirector/iaas/clouds/openstack.py

@@ -1,7 +1,8 @@
-import math
+import math, time
 
 from libcloud.compute.types import Provider
 from libcloud.compute.providers import get_driver
+from libcloud.common.exceptions import BaseHTTPError
 
 from .base import Cloud
 
@@ -10,32 +11,69 @@ class OpenStackCloud(Cloud):
     def __init__(self, cloud_name, iaas_config):
         Cloud.__init__(self, cloud_name)
 
+        self.config = iaas_config
+        self.cloud_name = cloud_name
         cloud_params = [
             'USER', 'PASSWORD', 'HOST',
             'IMAGE_ID', 'SIZE_ID',
             'KEY_NAME',
             'SECURITY_GROUP_NAME',
         ]
 
-        self._check_cloud_params(iaas_config, cloud_params)
+        self._check_cloud_params(self.config, cloud_params)
 
-        self.user = iaas_config.get(cloud_name, 'USER')
-        self.passwd = iaas_config.get(cloud_name, 'PASSWORD')
-        self.host = iaas_config.get(cloud_name, 'HOST')
-        self.img_id = iaas_config.get(cloud_name, 'IMAGE_ID')
-        self.size_id = iaas_config.get(cloud_name, 'SIZE_ID')
-        self.key_name = iaas_config.get(cloud_name, 'KEY_NAME')
-        self.sg = iaas_config.get(cloud_name, 'SECURITY_GROUP_NAME')
+        self.user = self.config.get(self.cloud_name, 'USER')
+        self.passwd = self.config.get(self.cloud_name, 'PASSWORD')
+        self.host = self.config.get(self.cloud_name, 'HOST')
+
+    def get_by_id_or_name(self, label, id_or_name, rlist):
+        by_id = filter(lambda x: x.id==id_or_name, rlist)
+        if len(by_id) > 0:
+            return by_id[0]
+        by_name = filter(lambda x: x.name==id_or_name, rlist)
+        if len(by_name) > 0:
+            return by_name[0]
+        raise ValueError('%s is not a valid value for %s' % (id_or_name, label))
+
+    def set_instance_attributes(self):
+        self.img = self.get_by_id_or_name('IMAGE_ID', 
+                                          self.config.get(self.cloud_name, 'IMAGE_ID'), 
+                                          self.driver.list_images())
+
+        self.size = self.get_by_id_or_name('SIZE_ID', 
+                                          self.config.get(self.cloud_name, 'SIZE_ID'), 
+                                          self.driver.list_sizes())
+
+        self.network = None
+        if self.config.has_option(self.cloud_name, 'NETWORK_ID'):
+            self.network = self.get_by_id_or_name('NETWORK_ID', 
+                                              self.config.get(self.cloud_name, 'NETWORK_ID'), 
+                                              self.driver.ex_list_networks())
+
+        self.floating_pool = self.driver.ex_list_floating_ip_pools()[0]    
+        self.key_name = self.config.get(self.cloud_name, 'KEY_NAME')
+        self.sg = self.config.get(self.cloud_name, 'SECURITY_GROUP_NAME')
+        self.auto_assing_floating = True
+        if self.config.has_option(self.cloud_name, 'AUTO_ASSIGN_FLOATING_IP'):
+            self.auto_assing_floating = self.config.getboolean(self.cloud_name, 'AUTO_ASSIGN_FLOATING_IP')
 
     def get_cloud_type(self):
         return 'openstack'
 
     # connect to openstack cloud
     def _connect(self):
-        Driver = get_driver(Provider.EUCALYPTUS)
+        # Driver = get_driver(Provider.EUCALYPTUS)
+        Driver = get_driver(Provider.OPENSTACK)
 
+        # self.driver = Driver(self.user, self.passwd, secure=False,
+        #     host=self.host, port=8773, path='/services/Cloud')
         self.driver = Driver(self.user, self.passwd, secure=False,
-            host=self.host, port=8773, path='/services/Cloud')
+                   ex_force_auth_url='http://%s:5000' % self.host,
+                   ex_force_auth_version='2.0_password', 
+                   ex_tenant_name=self.user)
+
+        self.set_instance_attributes()
+
         self.connected = True
 
     def config(self, config_params={}, context=None):
@@ -46,33 +84,59 @@ def new_instances(self, count, name='conpaas', inst_type=None):
         if self.connected is False:
             self._connect()
 
-        if inst_type is None:
-            inst_type = self.size_id
-
-        class size:
-            id = inst_type
-
-        class img:
-            id = self.img_id
+        flavor = self.size
+        if inst_type is not None:
+            flavor = self.get_by_id_or_name('SIZE_ID', inst_type, self.driver.list_sizes())
 
         kwargs = {
-            'size': size,
-            'image': img,
+            'size': flavor,
+            'image': self.img,
             'name': name,
             'ex_mincount': str(count),
             'ex_maxcount': str(count),
             'ex_securitygroup': self.sg,
             'ex_keyname': self.key_name,
             'ex_userdata': self.get_context()
         }
+
+        if self.network:
+            kwargs['networks'] = [self.network]
+
+        lc_nodes = self.driver.create_node(**kwargs)
+        if not self.auto_assing_floating:
+            self.associate_floating_ips(lc_nodes)
 
-        nodes = self._create_service_nodes(self.driver.create_node(**kwargs))
+        nodes = self._create_service_nodes(lc_nodes)
 
         if count > 1:
             return nodes
 
         return [ nodes ]
 
+    def associate_floating_ips(self, instances):
+
+        if type(instances) is not list:
+            instances = [instances]
+
+        nr_attempts = 3 * len(instances)
+
+        for instance in instances:
+            while nr_attempts > 0:
+                try: 
+                    self.driver.ex_attach_floating_ip_to_node(instance, self.get_floating_ip().ip_address)
+                    break
+                except BaseHTTPError:
+                    self.logger.debug('Attaching failed probly because the VM was not on a network yet, let\'s wait a bit and retry')
+                    time.sleep(1)
+                    nr_attempts -= 1
+
+        if nr_attempts == 0:
+            raise Exception('Error assigning floating IPs')
+
+    def get_floating_ip(self):
+        free_fips = filter(lambda x: x.node_id==None, self.driver.ex_list_floating_ips())
+        return free_fips[0] if len(free_fips) > 0 else self.floating_pool.create_floating_ip()
+
     def create_volume(self, size, name, vm_id=None):
         # OpenStack expects volume size in GiB.
         size /= 1024.0

conpaas-director/director.cfg.example

@@ -198,6 +198,7 @@ SUPPORT_EXTERNAL_IDP = false
 
 # KEY_NAME = 
 
+# TODO: this is not true when not using eucaliptus driver!
 # This variable contains the identifier of the ConPaaS image, as should be used
 # in the EC2 API. The value starts with "ami-" and can be obtained using the
 # "euca-describe-images" command. For more information on how to register a new
@@ -213,6 +214,23 @@ SUPPORT_EXTERNAL_IDP = false
 
 # SIZE_ID = m1.small
 
+# This variable contains the name or the id of the network where the VM (container)
+# will be booted. If there is only one network, it is not neccessary to set this 
+# parameter, Openstack will pick the default one 
+
+# NETWORK_ID = conpaas-net
+
+# The following variable should match the ability of the network component of Openscack to auto-assigning
+# public (floating) IPs to the newly created VMs. For example if you use nova-network, auto
+# assignment is possible, therefore this flag should be set to "true". However, neutron
+# does not support auto-assignment of floating ips, hence that has to be done by ConPaaS. In order
+# to instruct ConPaaS to do that, this flag has to be set to "false". If not set the system
+# will take "true" as the default value. Note that if this flag is left "true" while the network 
+# component does not support auto-assignment will result in the director waiting forever for the manager to start
+
+# AUTO_ASSIGN_FLOATING_IP = true
+
+
 ##############
 # OpenNebula #
 ##############

conpaas-director/setup.py

@@ -9,7 +9,7 @@
 from setuptools import setup, find_packages
 from pkg_resources import Requirement, resource_filename
 
-CPSVERSION = '204'
+CPSVERSION = '100'
 
 CONFDIR = '/etc/cpsdirector'
 LOGDIR = '/var/log/cpsdirector'
@@ -81,4 +81,4 @@
             getgrnam('www-data').gr_gid)
     except OSError:
         print "W: 'chown www-data:www-data %s' failed" % CONFDIR + '/data'
-print 'hellasdfldflsdf fadfasdf  %s' %sys.argv[1]
+

conpaas-frontend/www/__init__.php

@@ -13,7 +13,7 @@ class Conf {
 	const CONF_DIR = CONPAAS_CONF_DIR;
 	const HOST = CONPAAS_HOST;
 	const DIRECTOR = DIRECTOR_URL;
-
+	const DEBUG = DEBUG;
 	/*
 	 * time, in seconds, elapsed since the creation of a service, after which
 	 * that service is turned to ERROR state due to a prolongue unreachable INIT

conpaas-frontend/www/config-example.php

@@ -11,6 +11,7 @@
 // be accessible
 const CONPAAS_HOST = 'conpaas.yourdomain.com';
 
+const DEBUG = false;
 // These parameters, if specified, are used for performing CAPTCHA[1]
 // operations and they are issued for a specific domain. To generate a pair of
 // keys for your domain, please go to the reCAPTCHA admin page[2] (it's hosted

conpaas-frontend/www/login.php

@@ -106,13 +106,13 @@
                 <tr>
                     <td class="name">username</td>
                     <td class="input">
-                        <input type="text" id="username" />
+                        <input type="text" id="username" <?php if (Conf::DEBUG) {?> value="test"  <?php } ?> />
                     </td>
                 </tr>
                 <tr>
                     <td class="name">password</td>
                     <td class="input">
-                        <input type="password" id="password" />
+                        <input type="password" id="password" <?php if (Conf::DEBUG) {?> value="password"  <?php } ?> />
                     </td>
                 </tr>
                 <tr class="register_form" style="display: none">

conpaas-services/config/cloud/clouds-template.cfg

@@ -36,3 +36,4 @@ OPENNEBULA_NETWORK_ID =
 OPENNEBULA_NETWORK_GATEWAY = 
 OPENNEBULA_NETWORK_NAMESERVER = 
 
+

cpsupdate.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+# supported_comps=('director' 'manager' 'lib' 'tools' 'frontend')
+CPSVERSION="100"
+director="cpsdirector-$CPSVERSION.tar.gz" 
+manager="ConPaaS-$CPSVERSION.tar.gz" 
+lib="cpslib-$CPSVERSION.tar.gz"
+tools="cps-tools-$CPSVERSION.tar.gz"
+frontend="cpsfrontend-$CPSVERSION.tar.gz"
+
+taballs=""
+
+IP_PREFIX=192.168.13
+
+rm -f *$CPSVERSION*.tar.gz
+
+./mkdist.sh $CPSVERSION "$@" 
+
+# TMPFILE=$(mktemp)
+TMPFILE='/tmp/cpsinstall'
+
+cat <<EOT >> $TMPFILE
+#!/bin/bash
+
+IP_ADDRESS="\$(ip addr show | perl -ne 'print "\$1\n" if /inet ([\d.]+).*scope global/' | grep "$IP_PREFIX" | head -1)"
+#DIRECTOR_URL="https://\${IP_ADDRESS}:5555"
+#echo \$DIRECTOR_URL
+
+EOT
+
+
+ls $director > /dev/null 2>&1 && 
+{
+cat <<EOT >> $TMPFILE
+echo "#installing director"
+#cp /etc/cpsdirector/director.cfg .
+rm -rf /usr/local/lib/python2.7/dist-packages/cpsdirector-*-py2.7.egg
+tar -xaf cpsdirector-*.tar.gz
+rm -f cpsdirector-*.tar.gz
+cd cpsdirector-* 
+echo \$IP_ADDRESS |  make install
+cd ..
+rm -rf cpsdirector-*
+#mv director.cfg /etc/cpsdirector/
+sqlite3 /etc/cpsdirector/director.db 'delete from resource; delete from service'
+service apache2 restart
+EOT
+taballs+="$director "
+}
+
+ls $manager > /dev/null 2>&1 && 
+{
+cat <<EOT >> $TMPFILE
+mv $manager /etc/cpsdirector/ConPaaS.tar.gz
+EOT
+taballs+="$manager "
+}
+
+ls $lib > /dev/null 2>&1 && 
+{
+cat <<EOT >> $TMPFILE
+rm -r /usr/local/lib/python2.7/dist-packages/cpslib-*-py2.7.egg/
+easy_install cpslib-*.tar.gz
+rm cpslib-*.tar.gz
+EOT
+taballs+="$lib "
+}
+
+ls $tools > /dev/null 2>&1 && 
+{
+cat <<EOT >> $TMPFILE
+rm -rf /usr/local/lib/python2.7/dist-packages/cps_tools/
+cd cps-tools-*
+./configure --sysconf=/etc
+make install
+#mkdir -p $HOME/.conpaas
+cd ..
+rm -rf cps-tools*
+
+EOT
+taballs+="$tools "
+}
+
+
+ls $frontend > /dev/null 2>&1 && 
+{
+cat <<EOT >> $TMPFILE
+tar -xaf cpsfrontend*.tar.gz
+rm cpsfrontend*.tar.gz
+cp -r cpsfrontend-*/www/* /var/www/html/
+rm -rf cpsfrontend*
+EOT
+taballs+="$frontend "
+}
+
+
+
+echo ""
+echo "Uploading tarballs and execution script:"
+
+scp $TMPFILE $taballs conpaas:
+ssh conpaas 'bash cpsinstall; rm cpsinstall'
+# ssh conpaas 'cat cpsinstall; rm cpsinstall'
+
+rm -f ${TMPFILE}