work on swift, syslog, midonet-api, bugfixes

Makefile

@@ -100,10 +100,19 @@ stage7: $(PREREQUISITES)
 	$(RUNSTAGE)
 
 #
-# do not run this yet, it is still experimental
+# tempest, experimental
 #
-#stage8: sshconfig
-#	$(RUNSTAGE)
+stage8: sshconfig
+	$(RUNSTAGE)
+
+#
+# swift, experimental
+#
+# swift is needed for letting an openstack cloud be a juju environments platform target,
+# it is used for storing the deployment state
+#
+stage9: sshconfig
+	$(RUNSTAGE)
 
 start:
 	mkdir -pv $(TMPDIR)

bin/mkpwcache.sh

@@ -39,7 +39,8 @@ for PASS in MYSQL_DATABASE_PASSWORD \
   TROVE_DBPASS \
   TROVE_PASS \
   MIDONET_PASS \
-  NEUTRON_METADATA_SHARED_SECRET
+  NEUTRON_METADATA_SHARED_SECRET \
+  SWIFT_PASS
 do
   if [[ ! "$(env | grep "^${PASS}")" == "" ]]; then
     echo "export $(env | grep "^${PASS}")"

lib/orizuru/config.py

@@ -126,10 +126,17 @@ def __set_services(cls):
         services["neutron"]["type"] = "network"
         services["neutron"]["description"] = "OpenStack Networking"
 
+        services["swift"] = {}
+        services["swift"]["publicurl"] = "8080"
+        services["swift"]["internalurl"] = "8080"
+        services["swift"]["adminurl"] = "8080"
+        services["swift"]["type"] = "object-store"
+        services["swift"]["description"] = "OpenStack Object Store"
+
         services["midonet"] = {}
-        services["midonet"]["publicurl"] = "8080"
-        services["midonet"]["internalurl"] = "8080"
-        services["midonet"]["adminurl"] = "8080"
+        services["midonet"]["publicurl"] = "8081/midonet-api"
+        services["midonet"]["internalurl"] = "8081/midonet-api"
+        services["midonet"]["adminurl"] = "8081/midonet-api"
         services["midonet"]["type"] = "midonet"
         services["midonet"]["description"] = "MidoNet API Service"
 

lib/orizuru/operations.py

@@ -235,6 +235,7 @@ def __init__(self, metadata):
         self._metadata = metadata
 
     def install(self):
+        self.rsyslog()
         self.screen()
         self.login_stuff()
         self.apt_get_update()
@@ -355,6 +356,74 @@ def apt_get_update(cls):
     def common_packages(self):
         cuisine.package_ensure(self._metadata.config["common_packages"])
 
+    def rsyslog(self):
+
+        cuisine.package_ensure("rsyslog")
+
+        controller_name = self._metadata.roles["openstack_controller"][0]
+        controller_ip_suffix = self._metadata.config["idx"][controller_name]
+        controller_ip = "%s.%s" % (self._metadata.config["vpn_base"], controller_ip_suffix)
+
+        if env.host_string <> controller_name:
+            cuisine.file_write("/etc/rsyslog.conf", """
+
+$KLogPermitNonKernelFacility on
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+$RepeatedMsgReduction on
+$FileOwner syslog
+$FileGroup adm
+$FileCreateMode 0640
+$DirCreateMode 0755
+$Umask 0022
+$PrivDropToUser syslog
+$PrivDropToGroup syslog
+$WorkDirectory /var/spool/rsyslog
+$IncludeConfig /etc/rsyslog.d/*.conf
+$ModLoad imuxsock
+$ModLoad imklog
+
+*.* @%s:514
+*.* @@%s:514
+""" % (controller_ip, controller_ip))
+
+        else:
+            cuisine.file_write("/etc/rsyslog.conf", """
+
+$ModLoad imuxsock # provides support for local system logging
+$ModLoad imklog   # provides kernel logging support
+
+$KLogPermitNonKernelFacility on
+
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+
+$RepeatedMsgReduction on
+
+$FileOwner syslog
+$FileGroup adm
+$FileCreateMode 0640
+$DirCreateMode 0755
+$Umask 0022
+$PrivDropToUser syslog
+$PrivDropToGroup syslog
+
+$WorkDirectory /var/spool/rsyslog
+
+$IncludeConfig /etc/rsyslog.d/*.conf
+
+$ModLoad imudp
+
+$UDPServerRun 514
+
+$template FILENAME,"/var/log/%fromhost-ip%/syslog.log"
+
+*.* ?FILENAME
+
+""")
+
+        run("service rsyslog restart")
+
+        run("logger ping")
+
     def newrelic(self):
         if env.host_string not in self._metadata.containers:
             run("rm -fv /etc/newrelic/nrsysmond.cfg* || true")
@@ -395,6 +464,8 @@ def newrelic(self):
     ))
 
     def cloud_repository(self):
+        run("rm -rf /etc/apt/sources.list.d/cloudarchive-*")
+
         cuisine.package_ensure(["python-software-properties", "software-properties-common", "ubuntu-cloud-keyring"])
 
         self.dist_upgrade()
@@ -423,6 +494,8 @@ def cloud_repository(self):
         self._metadata.config["apt-cacher"]
     ))
 
+        self.dist_upgrade()
+
     @classmethod
     def dist_upgrade(cls):
         run("""

puppet/modules/midonet-neutron-plugin/manifests/midonet.pp

@@ -18,7 +18,7 @@
   $core_plugin                = 'midonet.neutron.plugin.MidonetPluginV2',
   $plugin_path                = '/etc/neutron/plugins/midonet/midonet.ini'
   $midonet_api_address        = '127.0.0.1',
-  $midonet_api_port           = '8080',
+  $midonet_api_port           = '8081',
   $midonet_keystone_username,
   $midonet_keystone_password,
   $keystone_admin_tenant_name = 'admin'

puppet/modules/midonet_api/templates/usr/share/midonet-api/WEB-INF/web.xml.erb

@@ -42,20 +42,6 @@
     <param-value>admin</param-value>
   </context-param>
 
-  <!-- Mock auth configuration -->
-  <context-param>
-    <param-name>mock_auth-admin_token</param-name>
-    <param-value>999888777666</param-value>
-  </context-param>
-  <context-param>
-    <param-name>mock_auth-tenant_admin_token</param-name>
-    <param-value>999888777666</param-value>
-  </context-param>
-  <context-param>
-    <param-name>mock_auth-tenant_user_token</param-name>
-    <param-value>999888777666</param-value>
-  </context-param>
-
   <!-- Keystone configuration -->
   <context-param>
     <param-name>keystone-service_protocol</param-name>
@@ -81,24 +67,6 @@
     <param-value><%= @keystone_tenant_name -%></param-value>
   </context-param>
 
-  <!-- CloudStack auth configuration -->
-  <context-param>
-    <param-name>cloudstack-api_base_uri</param-name>
-    <param-value>http://127.0.0.1:8080</param-value>
-  </context-param>
-  <context-param>
-    <param-name>cloudstack-api_path</param-name>
-    <param-value>/client/api?</param-value>
-  </context-param>
-  <context-param>
-    <param-name>cloudstack-api_key</param-name>
-    <param-value></param-value>
-  </context-param>
-  <context-param>
-    <param-name>cloudstack-secret_key</param-name>
-    <param-value></param-value>
-  </context-param>
-
   <!-- Zookeeper configuration -->
   <!-- The following parameters should match the ones in midolman.conf
        except 'use_mock' -->
@@ -137,14 +105,12 @@
   <!-- Servlet Listner -->
   <listener>
      <listener-class>
-         <!-- Use Jersey's Guice compatible context listener -->
          org.midonet.api.servlet.JerseyGuiceServletContextListener
      </listener-class>
   </listener>
 
   <!-- Servlet filter -->
   <filter>
-     <!-- Filter to enable Guice -->
     <filter-name>Guice Filter</filter-name>
     <filter-class>com.google.inject.servlet.GuiceFilter</filter-class>
   </filter>

stages/cleanup/fabfile.py

@@ -66,11 +66,18 @@ def fabric_docker_rm_role_containers_and_cleanup():
 
 rm -fv /var/run/netns/docker_*_"${SERVER_NAME}"
 
+""" %(env.host_string, container))
+
+    run("""
+
 rm -fv /etc/newrelic/nrsysmond.cfg
+rm -fv /etc/apt/sources.list.d/cloudarchive*
+rm -fv /etc/apt/sources.list.d/newrelic*
+rm -fv /etc/apt/sources.list.d/mido*
 
-exit 0
+apt-get update
 
-""" % (env.host_string, container))
+""")
 
     puts(red("destroying all containers"))
     run("""

stages/info/fabfile.py

@@ -85,4 +85,11 @@ def info(admin_password="test"):
                 )))
         puts("")
 
+    puts("log in to %s (%s) and tail /var/log/syslog to see syslog of all hosts and containers" % (
+        metadata.roles["openstack_controller"][0],
+        metadata.servers[metadata.roles["openstack_controller"][0]]["ip"]
+        ))
+
+    puts("")
+
     sys.exit(0)

stages/stage4/fabfile.py

@@ -375,12 +375,12 @@ def docker_containers_for_roles_stage4():
 #
 if [[ "midonet_api" == "${CONTAINER_ROLE}" ]]; then
     for IP in "${MIDONET_API_OUTER_IP}" "${DEFAULT_GW_IFACE_IP}"; do
-        iptables -t nat -I PREROUTING -i "${DEFAULT_GW_IFACE}" -p tcp -d "${IP}" --dport 8080 -j DNAT --to "${CONTAINER_IP}:8080"
-        iptables -I FORWARD -p tcp -d "${IP}" --dport 8080 -j ACCEPT
+        iptables -t nat -I PREROUTING -i "${DEFAULT_GW_IFACE}" -p tcp -d "${IP}" --dport 8081 -j DNAT --to "${CONTAINER_IP}:8081"
+        iptables -I FORWARD -p tcp -d "${IP}" --dport 8081 -j ACCEPT
     done
 else
-    iptables -t nat -I PREROUTING -i dockertinc -p tcp -d "${MIDONET_API_OUTER_IP}" --dport 8080 -j DNAT --to "${MIDONET_API_IP}:8080"
-    iptables -I FORWARD -p tcp -d "${MIDONET_API_IP}" --dport 8080 -j ACCEPT
+    iptables -t nat -I PREROUTING -i dockertinc -p tcp -d "${MIDONET_API_OUTER_IP}" --dport 8081 -j DNAT --to "${MIDONET_API_IP}:8081"
+    iptables -I FORWARD -p tcp -d "${MIDONET_API_IP}" --dport 8081 -j ACCEPT
 fi
 
 """ % (

stages/stage6/fabfile.py

@@ -492,6 +492,8 @@ def stage6_container_openstack_neutron():
 
 MIDONET_API="%s"
 
+MIDONET_API_URL="%s"
+
 source /etc/keystone/KEYSTONERC_ADMIN
 
 SERVICE_TENANT_ID="$(keystone tenant-list | grep 'service' | awk -F'|' '{print $2;}' | xargs -n1 echo)"
@@ -581,7 +583,7 @@ def stage6_container_openstack_neutron():
 # Add the midonet section to the midonet plugin and the dhcp agent ini
 #
 for CONFIGFILE in "${MIDONET_PLUGIN}" "/etc/neutron/dhcp_agent.ini"; do
-    "${CONFIGHELPER}" set "${CONFIGFILE}" "MIDONET" "midonet_uri" "http://${MIDONET_API}:8080/midonet-api"
+    "${CONFIGHELPER}" set "${CONFIGFILE}" "MIDONET" "midonet_uri" "http://${MIDONET_API}:${MIDONET_API_URL}"
     "${CONFIGHELPER}" set "${CONFIGFILE}" "MIDONET" "username" "midonet"
     "${CONFIGHELPER}" set "${CONFIGFILE}" "MIDONET" "password" "${MIDONET_PASS}"
     "${CONFIGHELPER}" set "${CONFIGFILE}" "MIDONET" "project_id" "service"
@@ -610,7 +612,8 @@ def stage6_container_openstack_neutron():
         metadata.containers[metadata.roles["container_openstack_rabbitmq"][0]]["ip"],
         metadata.containers[metadata.roles["container_openstack_controller"][0]]["ip"],
         metadata.config["region"],
-        metadata.containers[metadata.roles["container_midonet_api"][0]]["ip"]
+        metadata.containers[metadata.roles["container_midonet_api"][0]]["ip"],
+        metadata.services["midonet"]["internalurl"]
     ))
 
     puts(green("running neutron-db-manage"))
@@ -956,6 +959,8 @@ def stage6_openstack_compute_nova_compute(compute_ip, compute_vpn_ip):
 
 COMPUTE_VPN_IP="%s"
 
+MIDONET_API_URL="%s"
+
 #
 # nova compute
 #
@@ -1005,7 +1010,7 @@ def stage6_openstack_compute_nova_compute(compute_ip, compute_vpn_ip):
     # used for midonet
     "${CONFIGHELPER}" set "${CONFIGFILE}" "DEFAULT" "libvirt_vif_driver" "nova.virt.libvirt.vif.LibvirtGenericVIFDriver"
 
-    "${CONFIGHELPER}" set "${CONFIGFILE}" "MIDONET" "midonet_uri" "http://${MIDONET_API}:8080/midonet-api"
+    "${CONFIGHELPER}" set "${CONFIGFILE}" "MIDONET" "midonet_uri" "http://${MIDONET_API}:${MIDONET_API_URL}"
     "${CONFIGHELPER}" set "${CONFIGFILE}" "MIDONET" "username" "midonet"
     "${CONFIGHELPER}" set "${CONFIGFILE}" "MIDONET" "password" "${MIDONET_PASS}"
     "${CONFIGHELPER}" set "${CONFIGFILE}" "MIDONET" "project_id" "service"
@@ -1084,6 +1089,7 @@ def stage6_openstack_compute_nova_compute(compute_ip, compute_vpn_ip):
         metadata.containers[metadata.roles["container_midonet_api"][0]]["ip"],
         metadata.containers[metadata.roles["container_openstack_neutron"][0]]["ip"],
         compute_vpn_ip,
+        metadata.services["midonet"]["internalurl"],
         service.upper()
     ))
 
@@ -1272,7 +1278,10 @@ def stage6_container_openstack_keystone_create_service_entity_api_endpoints():
             if service == 'midonet':
                 service_ip = metadata.containers[metadata.roles["container_midonet_api"][0]]["ip"]
             else:
-                service_ip = metadata.containers[metadata.roles["container_openstack_%s" % service_alias][0]]["ip"]
+                if service == 'swift':
+                    service_ip = metadata.containers[metadata.roles["container_openstack_controller"][0]]["ip"]
+                else:
+                    service_ip = metadata.containers[metadata.roles["container_openstack_%s" % service_alias][0]]["ip"]
 
             run("""
 if [[ "%s" == "True" ]] ; then set -x; fi
@@ -1700,12 +1709,23 @@ def stage6_container_openstack_mysql():
 
 EOF
 
+DEBIAN_FRONTEND=noninteractive apt-get -q --yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install mariadb-server || true
+
+mkdir -pv /var/lib/mysql
+touch /var/lib/mysql/debian-x.flag
+
+dpkg --configure -a
+
+apt-get -f install
+
+DEBIAN_FRONTEND=noninteractive apt-get -q --yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install mariadb-server
+
 """ % (
         metadata.config["debug"],
         open(os.environ["PASSWORDCACHE"]).read()
     ))
 
-    cuisine.package_ensure(["mariadb-server", "python-mysqldb"])
+    cuisine.package_ensure("python-mysqldb")
 
     configfile = "/etc/mysql/my.cnf"