From 6160260b208aa89a98732795264b6e8e5d3eb362 Mon Sep 17 00:00:00 2001
From: Ekaterina Chernova <efedorova@mirantis.com>
Date: Mon, 17 Sep 2018 11:00:09 +0300
Subject: [PATCH] Improve openstack-kubespray engine

* Flaoting IP is used for asscess by ssh
* Ansible host is internal IP
* Get rid of adding FIP to loopback function
* Set extra variables for lbaass
* Store redefined variables in k8s-cluster.yml, since it has more priority
---
 Dockerfile                            |  2 +-
 kqueen/config/demo_mirantis.py        |  3 ++
 kqueen/engines/openstack_kubespray.py | 52 +++++++++++----------------
 3 files changed, 24 insertions(+), 33 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 2c9864ed..bba63a69 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -20,7 +20,7 @@ RUN pip install ./kqueen
 
 # Avoid Ssh issues with docker overlayfs and sockets
 ENV ANSIBLE_SSH_CONTROL_PATH /dev/shm/cp%%h-%%p-%%r
-ENV ANSIBLE_SSH_RETRIES 3
+ENV ANSIBLE_SSH_RETRIES 5
 ENV ANSIBLE_TIMEOUT 25
 ENV KQUEEN_KS_KUBESPRAY_PATH /code/kubespray
 ENV KQUEEN_KS_ANSIBLE_CMD /usr/local/bin/ansible
diff --git a/kqueen/config/demo_mirantis.py b/kqueen/config/demo_mirantis.py
index 94198ee8..c5be7967 100644
--- a/kqueen/config/demo_mirantis.py
+++ b/kqueen/config/demo_mirantis.py
@@ -48,3 +48,6 @@ class Config(BaseConfig):
     POD_INFRA_IMAGE_TAG = 'v1.10.4-4'
 
     KS_DEFAULT_NAMESERVERS = "172.18.176.6"
+    KS_NO_PROXY = "127.0.0.1,localhost,docker-prod-local.docker.mirantis.net,172.16.48.254," \
+                  "cloud-cz.bud.mirantis.net,172.17.45.80"
+    KS_OS_BLOCKSTORAGE_VERSION = "v2"
diff --git a/kqueen/engines/openstack_kubespray.py b/kqueen/engines/openstack_kubespray.py
index 3fe61f9d..614b70f7 100644
--- a/kqueen/engines/openstack_kubespray.py
+++ b/kqueen/engines/openstack_kubespray.py
@@ -436,33 +436,9 @@ def deploy(self, cluster_metadata):
         self._save_inventory(inventory, "hosts.json")
         self._create_group_vars(cluster_metadata)
         self._wait_for_ping()
-        self._add_fip_to_lo(resources)
         self._run_ansible()
         return self._get_kubeconfig(resources["masters"][0]["fip"])
 
-    def _add_fip_to_lo(self, resources):
-        """Add floating ip to loopback interface.
-
-        This is workaround for buggy deployments where node can't reach
-        itself by floating ip.
-        """
-        cmd_fmt = (
-            "sudo /bin/sh -c 'cat > /etc/rc.local <<EOF\n"
-            "/sbin/ip addr add %s/32 scope host dev lo\n"
-            "EOF'"
-        )
-        for master in resources["masters"]:
-            ip = master["fip"]
-            host = "@".join((self.ssh_username, ip))
-            ssh_cmd = ("ssh", host) + self.ssh_common_args
-            try:
-                subprocess.check_call(ssh_cmd + (cmd_fmt % ip, ))
-                subprocess.check_call(ssh_cmd + ("sudo /bin/sh /etc/rc.local", ))
-            except subprocess.CalledProcessError as e:
-                raise RuntimeError("Enable to add a loopback "
-                                   "to make localhost accessible by floating IP. "
-                                   "The reason is: {}".format(e))
-
     def scale(self, resources):
         inventory = self._generate_inventory(resources)
         self._save_inventory(inventory, "hosts.json")
@@ -494,12 +470,22 @@ def _create_group_vars(self, metadata):
         kubespray_vars = {"persistent_volumes_enabled": True} if persistent_volumes else {}
 
         kubespray_vars["cloud_provider"] = "openstack"
-        kubespray_vars["openstack_blockstorage_version"] = "v2"
+        kubespray_vars["openstack_blockstorage_version"] = config.get("KS_OS_BLOCKSTORAGE_VERSION") or "v2"
+        kubespray_vars["calico_endpoint_to_host_action"] = "ACCEPT"
+        if config.get("KS_NO_PROXY"):
+            kubespray_vars["no_proxy"] = config.KS_NO_PROXY
+
+        kubespray_vars["openstack_lbaas_subnet_id"] = metadata["resources"]["subnet_id"]
+        kubespray_vars["openstack_lbaas_floating_network_id"] = metadata["resources"]["ext_net_id"]
+        # See https://github.com/kubernetes-incubator/kubespray/issues/2141
+        # Set this variable to true to get rid of this issue```
+        kubespray_vars["volume_cross_zone_attachment"] = True
+
         image_var_names = [var_name for var_name in dir(config) if var_name.endswith(('_IMAGE_REPO', '_IMAGE_TAG'))]
         image_variables = {k.lower(): getattr(config, k) for k in image_var_names}
         kubespray_vars.update(image_variables)
-        with open(os.path.join(dst, "all.yml"), "a") as all_yaml:
-            yaml.dump(kubespray_vars, all_yaml, default_flow_style=False)
+        with open(os.path.join(dst, "k8s-cluster.yml"), "a") as k8s_yaml:
+            yaml.dump(kubespray_vars, k8s_yaml, default_flow_style=False)
 
     def _make_files_dir(self):
         os.makedirs(self._get_cluster_path(), exist_ok=True)
@@ -548,7 +534,7 @@ def _generate_inventory(self, resources, keep_slaves=None):
         }
         for master in resources["masters"]:
             conf["all"]["hosts"][master["hostname"]] = {
-                "access_ip": master["fip"],
+                "access_ip": master["ip"],
                 "ansible_host": master["fip"],
                 "ansible_user": self.ssh_username,
                 "ansible_become": True,
@@ -577,7 +563,7 @@ def _generate_inventory(self, resources, keep_slaves=None):
     def _get_cluster_path(self, *args):
         return os.path.join(self.clusters_path, self.cluster_id, *args)
 
-    def _wait_for_ping(self, retries=30, sleep=10):
+    def _wait_for_ping(self, retries=15, sleep=10):
         args = [config.KS_ANSIBLE_CMD, "-m",
                 "ping", "all", "-i", "hosts.json"]
         while retries:
@@ -608,8 +594,9 @@ def _run_ansible(self, inventory="hosts.json", playbook="cluster.yml"):
         args = [
             config.KS_ANSIBLE_PLAYBOOK_CMD, "-b", "-i",
             inventory, playbook,
-            "--extra-vars", "delete_nodes_confirmation=yes",
-            "--extra-vars", "docker_dns_servers_strict=no",
+            "-e", "delete_nodes_confirmation=yes",
+            "-e", "docker_dns_servers_strict=no",
+            "-e", "ansible_python_interpreter=/usr/bin/python3"
         ]
         env = self._construct_env()
         self.ansible_log = os.path.join(self._get_cluster_path(), "ansible_log_for_{0}_playbook.txt".format(playbook))
@@ -717,8 +704,9 @@ def provision(self):
                                       subnet_name=self.stack_name,
                                       dns_nameservers=self.meta['dns'])
         router = self.c.create_router(name=self.stack_name,
-                                      ext_gateway_net_id=self.meta['ext_net'].id)
+                                      ext_gateway_net_id=self.meta["ext_net"].id)
         self.c.add_router_interface(router, subnet["id"])
+        resources["ext_net_id"] = self.meta["ext_net"].id
         resources["router_id"] = router["id"]
         resources["network_id"] = network["id"]
         resources["subnet_id"] = subnet["id"]