From f11ee839cdba1154c91afdbe608266c77975cfc2 Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Wed, 22 Jan 2025 11:27:55 -0800 Subject: [PATCH 01/16] add terraform --- terraform/.gitignore | 37 ++++++++++++++++++++ terraform/main.tf | 76 ++++++++++++++++++++++++++++++++++++++++++ terraform/output.tf | 14 ++++++++ terraform/providers.tf | 14 ++++++++ terraform/start_up.sh | 56 +++++++++++++++++++++++++++++++ terraform/variables.tf | 11 ++++++ 6 files changed, 208 insertions(+) create mode 100644 terraform/.gitignore create mode 100644 terraform/main.tf create mode 100644 terraform/output.tf create mode 100644 terraform/providers.tf create mode 100644 terraform/start_up.sh create mode 100644 terraform/variables.tf diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 0000000..21e6d3c --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,37 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Ignore transient lock info files created by terraform apply +.terraform.tfstate.lock.info + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc \ No newline at end of file diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..475560b --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,76 @@ +locals { + num_instances = 10 + project = var.project_id + service_name = "cosyne-service" + database_name = "test-database-users" +} + +# Creates a new instance +# resource "google_compute_instance" "test_instance" { +# count = local.num_instances +# name = "test-instance-${count.index}" +# machine_type = "e2-medium" +# zone = "us-west1-b" +# can_ip_forward = false + + +# boot_disk { +# initialize_params { +# image = "linux-pilot-sleap-gui-crd-pubsub-database-inuse-5" +# } +# } + +# network_interface { +# network = "default" +# access_config {} +# } + +# # for each instance, create a startup script +# metadata_startup_script = file("${path.module}/start_up.sh") + +# service_account { +# scopes = ["https://www.googleapis.com/auth/spanner.data", "https://www.googleapis.com/auth/devstorage.read_only", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/monitoring.write", "https://www.googleapis.com/auth/trace.append", "https://www.googleapis.com/auth/service.management.readonly", "https://www.googleapis.com/auth/servicecontrol"] +# } +# } + +# resource "google_pubsub_topic" "main" { +# name = var.instance_name + +# } + +# Create Spanner Database in the specific instance +resource "google_spanner_database" "default" { + name = local.database_name + instance = "vmassign-dev" + ddl = [ + "CREATE TABLE Users (Hostname STRING(1024) NOT NULL, Pin STRING(1024), CrdCmd STRING(1024), UserEmail STRING(1024), inUse BOOL,) PRIMARY KEY (Hostname)" + ] + deletion_protection = false +} + +resource "google_cloud_run_v2_service" "deployment" { + name = local.service_name + location = "us-central1" + project = var.project_id + deletion_protection = false + template { + containers { + image = "gcr.io/${var.project_id}/cosyne:latest" + } + } +} + +# traffic { +# percent = 100 +# latest_revision = true +# } +# } + +# data "google_iam_policy" "noauth" { +# binding { +# role = "roles/run.invoker" +# members = [ +# "allUsers", +# ] +# } +# } diff --git a/terraform/output.tf b/terraform/output.tf new file mode 100644 index 0000000..3c3f352 --- /dev/null +++ b/terraform/output.tf @@ -0,0 +1,14 @@ +# output "instance_id" { +# description = "Instance ID" +# value = [for instance in google_compute_instance.test_instance : instance.id] +# } + +# output "instance_name" { +# description = "Instance Name" +# value = [for instance in google_compute_instance.test_instance : instance.name] +# } + +output "service_url" { + description = "Service URL" + value = google_cloud_run_v2_service.deployment.uri +} diff --git a/terraform/providers.tf b/terraform/providers.tf new file mode 100644 index 0000000..9e66a1c --- /dev/null +++ b/terraform/providers.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "6.5.0" + } + } +} + +provider "google" { + project = "vmassign-dev" + region = "us-west1" + credentials = "./vmassign-dev-2818ad83c3ff.json" +} diff --git a/terraform/start_up.sh b/terraform/start_up.sh new file mode 100644 index 0000000..fd1437f --- /dev/null +++ b/terraform/start_up.sh @@ -0,0 +1,56 @@ +#!/bin/bash +PROJECT_ID="vmassign-dev" +USER_NAME="tutorial" +DB_INSTANCE="vmassign-dev" +DB_NAME="users" +DB_TABLE="Users" +UPDATE_COSYNE_VM_REPO="False" +TUTORIAL_REPO_TO_CLONE="" + +# Appending start_up_suffix.sh here... +# This is only the suffix of the start_up.sh file +# The prefix and entirety of the script is dynamically generated by create_instances.py. +# The entire script is called start_up.sh + +echo "USER_NAME: $USER_NAME" +echo "PROJECT_ID: $PROJECT_ID" +echo "DB_INSTANCE: $DB_INSTANCE" +echo "DB_NAME: $DB_NAME" +echo "DB_TABLE: $DB_TABLE" +echo "UPDATE_COSYNE_VM_REPO: $UPDATE_COSYNE_VM_REPO" +echo "TUTORIAL_REPO_TO_CLONE: $TUTORIAL_REPO_TO_CLONE" +echo + +# Clone specified repository if provided +if [ "$TUTORIAL_REPO_TO_CLONE" != "" ]; then + # Change to the user's Desktop directory + cd /home/$USER_NAME/Desktop + echo "Cloning the specified repository..." + # Clone the repository as the git user so user can git pull if needed + sudo -u $USER_NAME git clone $TUTORIAL_REPO_TO_CLONE +fi + +# Change to the directory where the Python script is located +cd /home/$USER_NAME/cosyne_vm + +# Check if the scripts need to be updated +if [ "$UPDATE_COSYNE_VM_REPO" = "True" ]; then + echo "Updating the cosyne_vm repository..." + git pull + /opt/conda/bin/python3 -m pip install -r requirements.txt +fi + +# We need to run the Python script with sudo -u $USER_NAME for the CRD command to work +# Listen for messages on pubsub topic hostname (need to use correct Python installation) +sudo -u $USER_NAME /opt/conda/bin/python3 subscribe_instance.py \ + --project_id $PROJECT_ID \ + --db_instance_id $DB_INSTANCE \ + --db_id $DB_NAME \ + --db_table_name $DB_TABLE & + +# Update the inuse status of the VM whenever sleap opens/closes +/opt/conda/bin/python3 update_inuse_status.py \ + --project_id $PROJECT_ID \ + --db_instance_id $DB_INSTANCE \ + --db_id $DB_NAME \ + --db_table_name $DB_TABLE & \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..c8647ad --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,11 @@ +variable "instance_name" { + description = "Name of the instance" + type = string + default = "web-server" +} + +variable "project_id" { + description = "Target Project ID" + type = string + default = "vmassign-dev" +} From 83297d002a8aa1260221ddc31851d19693218b4a Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Wed, 22 Jan 2025 11:37:53 -0800 Subject: [PATCH 02/16] fix terraform ignore --- terraform/.gitignore | 37 ----------------------------------- terraform/.terraform.lock.hcl | 22 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 37 deletions(-) delete mode 100644 terraform/.gitignore create mode 100644 terraform/.terraform.lock.hcl diff --git a/terraform/.gitignore b/terraform/.gitignore deleted file mode 100644 index 21e6d3c..0000000 --- a/terraform/.gitignore +++ /dev/null @@ -1,37 +0,0 @@ -# Local .terraform directories -**/.terraform/* - -# .tfstate files -*.tfstate -*.tfstate.* - -# Crash log files -crash.log -crash.*.log - -# Exclude all .tfvars files, which are likely to contain sensitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject -# to change depending on the environment. -*.tfvars -*.tfvars.json - -# Ignore override files as they are usually used to override resources locally and so -# are not checked in -override.tf -override.tf.json -*_override.tf -*_override.tf.json - -# Ignore transient lock info files created by terraform apply -.terraform.tfstate.lock.info - -# Include override files you do wish to add to version control using negated pattern -# !example_override.tf - -# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan -# example: *tfplan* - -# Ignore CLI configuration files -.terraformrc -terraform.rc \ No newline at end of file diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..b889018 --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "6.5.0" + constraints = "6.5.0" + hashes = [ + "h1:qKHzN9B+9uOjHDXxanQJanBdsd0bzCP3egUjIcyYxSY=", + "zh:14101a38e880d4a1ef14c0a24476b008a3b577540a260a246a471bcfb5c31f4c", + "zh:478a7b0810956d39843e785262ab8162324a7412c3f6cf1ceb43977e2c05f22e", + "zh:6c9b583abcbaa2093b1b55494ac08851bd3364919fe86850a9c3e8f6c46851d4", + "zh:7c400eb5488221ba7ea48725ab43db1464cefd96cb29a24e63fe1950666b465f", + "zh:82931b2c186403753356a73878d36efc209c9e5ae46d0b609bb7ca38aece931d", + "zh:87e7966ef7067de3684f658251cdede057be419bbfeaaad935ab6f501024046a", + "zh:a2f4aaa3b9260732a53f78c8053eb2cbcee2abf11d3d245c58f3065423ad30ab", + "zh:bbc4c3ca9d51287e77130fc95880792007dd919b9b5396433f9eed737119c6c3", + "zh:edcda54d37be1b8d4cbe029e30df6a228e0be3887831b892c11536502d87e840", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f95d92ca2ac527442f6767a217b8a557ba6b2d1915c39efba412382e986e4c3e", + "zh:f96148c3742126219b810a687551284460a8d78aa66efbfd2c58880f48049dda", + ] +} From 22f257fe3e593dc750d3cc6fc2ac27d0215f1186 Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Wed, 22 Jan 2025 13:05:32 -0800 Subject: [PATCH 03/16] remove ds_store --- .gitignore | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.gitignore b/.gitignore index 15201ac..2bf4750 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,47 @@ cython_debug/ # PyPI configuration file .pypirc + +# Local .terraform directories +terraform/.terraform + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +terraform/override.tf +terraform/override.tf.json +terraform/*_override.tf +terraform/*_override.tf.json + +# Ignore transient lock info files created by terraform apply +terraform/.terraform.tfstate.lock.info + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +terraform/.terraformrc +terraform/terraform.rc + +# Ignore credentials file (any json file that starts with "vmassign-dev-") +terraform/vmassign-dev-*.json + +# ignore DS_Store files +.DS_Store \ No newline at end of file From f78827bea3b8449d244cdb2bada848d40ec48142 Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Wed, 22 Jan 2025 13:56:48 -0800 Subject: [PATCH 04/16] add doc --- terraform/README.md | 54 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 terraform/README.md diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..1b1d5e9 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,54 @@ +# Terraform in Lablink + +## Overview +This directory contains the Terraform configuration files for the Lablink project. The Terraform configuration files are used to create the infrastructure for the Lablink project. The infrastructure creates the following resources: + +- A virtual machine (VM) instance in Google Cloud Platform (GCP) with SLEAP installed. +- A database instance in Google Spanner for VM assignment. +- A Cloud Run service for the Lablink API. + +## Prerequisites + +Before you can use the Terraform configuration files, running Terraform commands, you need to install Terraform. You can install Terraform by following the instructions in the [Terraform documentation](https://learn.hashicorp.com/tutorials/terraform/install-cli). + +Also, you need to have a Google Cloud Platform (GCP) account and a project in GCP. You can create a GCP account and a project by following the instructions in the [GCP documentation](https://cloud.google.com/gcp). + +After creating a GCP account and a project, you need to create a service account in GCP and download the service account key. You can create a service account by following the instructions in the [GCP Service Accounts documentation](https://cloud.google.com/iam/docs/creating-managing-service-accounts). After creating the service account, you need to download the service account key in JSON format. You can download the service account key by following the instructions in the [GCP Keys documentation](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). + +## Configuration + +The Terraform configuration files are located in the `terraform` directory. The configuration files are organized as follows: + +- `main.tf`: The main Terraform configuration file that defines the resources to create. +- `variables.tf`: The Terraform variables file that defines the input variables for the Terraform configuration files. +- `outputs.tf`: The Terraform outputs file that defines the output variables for the Terraform configuration files. +- `provider.tf`: The Terraform provider file that defines the provider for the Terraform configuration files. + +## Installation + +To install the Terraform configuration files, clone the repository: + +```bash +git clone https://github.com/talmolab/lablink.git +``` + +## Usage + +To use the Terraform configuration files, you need to run this command to initialize the Terraform configuration files: + +```bash +cd terraform +terraform init +``` + +After initializing the Terraform configuration files, you can run the following command to create the infrastructure: + +```bash +terraform apply +``` + +To destroy the infrastructure, you can run the following command: + +```bash +terraform destroy +``` From 1c007bfe9a9676325bebb7d7257e8099c26eb6ce Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Wed, 22 Jan 2025 14:01:19 -0800 Subject: [PATCH 05/16] edit doc --- terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/README.md b/terraform/README.md index 1b1d5e9..7afa391 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -11,7 +11,7 @@ This directory contains the Terraform configuration files for the Lablink projec Before you can use the Terraform configuration files, running Terraform commands, you need to install Terraform. You can install Terraform by following the instructions in the [Terraform documentation](https://learn.hashicorp.com/tutorials/terraform/install-cli). -Also, you need to have a Google Cloud Platform (GCP) account and a project in GCP. You can create a GCP account and a project by following the instructions in the [GCP documentation](https://cloud.google.com/gcp). +Also, you need to have a Google Cloud Platform (GCP) account and a project in GCP. You can create a GCP account and a project in the [Google Cloud Platform Official Website](https://cloud.google.com/gcp). After creating a GCP account and a project, you need to create a service account in GCP and download the service account key. You can create a service account by following the instructions in the [GCP Service Accounts documentation](https://cloud.google.com/iam/docs/creating-managing-service-accounts). After creating the service account, you need to download the service account key in JSON format. You can download the service account key by following the instructions in the [GCP Keys documentation](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). From c581c054d60452e8e8b5e0772a935128bdd612da Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Thu, 23 Jan 2025 14:42:53 -0800 Subject: [PATCH 06/16] terraform --- terraform/main.tf | 63 ++++++++++++------------------------------- terraform/start_up.sh | 56 -------------------------------------- 2 files changed, 17 insertions(+), 102 deletions(-) delete mode 100644 terraform/start_up.sh diff --git a/terraform/main.tf b/terraform/main.tf index 475560b..5b95509 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,42 +1,18 @@ +variable "resource_suffic" { + description = "Suffix to append to all resources" + type = string + default = "prod" +} + locals { - num_instances = 10 project = var.project_id service_name = "cosyne-service" database_name = "test-database-users" } -# Creates a new instance -# resource "google_compute_instance" "test_instance" { -# count = local.num_instances -# name = "test-instance-${count.index}" -# machine_type = "e2-medium" -# zone = "us-west1-b" -# can_ip_forward = false - - -# boot_disk { -# initialize_params { -# image = "linux-pilot-sleap-gui-crd-pubsub-database-inuse-5" -# } -# } - -# network_interface { -# network = "default" -# access_config {} -# } - -# # for each instance, create a startup script -# metadata_startup_script = file("${path.module}/start_up.sh") +# TODO: Add resources for Terraform to manage -# service_account { -# scopes = ["https://www.googleapis.com/auth/spanner.data", "https://www.googleapis.com/auth/devstorage.read_only", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/monitoring.write", "https://www.googleapis.com/auth/trace.append", "https://www.googleapis.com/auth/service.management.readonly", "https://www.googleapis.com/auth/servicecontrol"] -# } -# } - -# resource "google_pubsub_topic" "main" { -# name = var.instance_name - -# } +# TODO: Parameterize testing/development values # Create Spanner Database in the specific instance resource "google_spanner_database" "default" { @@ -58,19 +34,14 @@ resource "google_cloud_run_v2_service" "deployment" { image = "gcr.io/${var.project_id}/cosyne:latest" } } + traffic { + percent = 100 + } } -# traffic { -# percent = 100 -# latest_revision = true -# } -# } - -# data "google_iam_policy" "noauth" { -# binding { -# role = "roles/run.invoker" -# members = [ -# "allUsers", -# ] -# } -# } +data "google_iam_policy" "noauth" { + binding { + role = "roles/run.invoker" + members = ["allUsers"] + } +} diff --git a/terraform/start_up.sh b/terraform/start_up.sh deleted file mode 100644 index fd1437f..0000000 --- a/terraform/start_up.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -PROJECT_ID="vmassign-dev" -USER_NAME="tutorial" -DB_INSTANCE="vmassign-dev" -DB_NAME="users" -DB_TABLE="Users" -UPDATE_COSYNE_VM_REPO="False" -TUTORIAL_REPO_TO_CLONE="" - -# Appending start_up_suffix.sh here... -# This is only the suffix of the start_up.sh file -# The prefix and entirety of the script is dynamically generated by create_instances.py. -# The entire script is called start_up.sh - -echo "USER_NAME: $USER_NAME" -echo "PROJECT_ID: $PROJECT_ID" -echo "DB_INSTANCE: $DB_INSTANCE" -echo "DB_NAME: $DB_NAME" -echo "DB_TABLE: $DB_TABLE" -echo "UPDATE_COSYNE_VM_REPO: $UPDATE_COSYNE_VM_REPO" -echo "TUTORIAL_REPO_TO_CLONE: $TUTORIAL_REPO_TO_CLONE" -echo - -# Clone specified repository if provided -if [ "$TUTORIAL_REPO_TO_CLONE" != "" ]; then - # Change to the user's Desktop directory - cd /home/$USER_NAME/Desktop - echo "Cloning the specified repository..." - # Clone the repository as the git user so user can git pull if needed - sudo -u $USER_NAME git clone $TUTORIAL_REPO_TO_CLONE -fi - -# Change to the directory where the Python script is located -cd /home/$USER_NAME/cosyne_vm - -# Check if the scripts need to be updated -if [ "$UPDATE_COSYNE_VM_REPO" = "True" ]; then - echo "Updating the cosyne_vm repository..." - git pull - /opt/conda/bin/python3 -m pip install -r requirements.txt -fi - -# We need to run the Python script with sudo -u $USER_NAME for the CRD command to work -# Listen for messages on pubsub topic hostname (need to use correct Python installation) -sudo -u $USER_NAME /opt/conda/bin/python3 subscribe_instance.py \ - --project_id $PROJECT_ID \ - --db_instance_id $DB_INSTANCE \ - --db_id $DB_NAME \ - --db_table_name $DB_TABLE & - -# Update the inuse status of the VM whenever sleap opens/closes -/opt/conda/bin/python3 update_inuse_status.py \ - --project_id $PROJECT_ID \ - --db_instance_id $DB_INSTANCE \ - --db_id $DB_NAME \ - --db_table_name $DB_TABLE & \ No newline at end of file From 742137ca4ca3eb7a1639b912218832a339c5f469 Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Thu, 23 Jan 2025 14:58:49 -0800 Subject: [PATCH 07/16] add new variable --- terraform/variables.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/terraform/variables.tf b/terraform/variables.tf index c8647ad..dd332a1 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -9,3 +9,9 @@ variable "project_id" { type = string default = "vmassign-dev" } + +variable "resource_suffix" { + description = "Suffix to append to all resources" + type = string + default = "prod" +} From 76f798d459c12259dae293c87f8ffffe48993182 Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Tue, 28 Jan 2025 12:57:39 -0800 Subject: [PATCH 08/16] add test files --- Dockerfile | 20 ++++++++++++++++ app/app.py | 17 ++++++++++++++ app/requirements.txt | 1 + terraform/.terraform.lock.hcl | 19 ++++++++++++++++ terraform/main.tf | 43 +++++++++++++++-------------------- terraform/output.tf | 13 ----------- 6 files changed, 75 insertions(+), 38 deletions(-) create mode 100644 Dockerfile create mode 100644 app/app.py create mode 100644 app/requirements.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ede1787 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +# Use a lightweight Python base image +FROM python:3.9-slim + +# Set the working directory in the container +WORKDIR /app + +# Copy the requirements file +COPY app/requirements.txt requirements.txt + +# Install Python dependencies +RUN pip install -r requirements.txt + +# Copy the app code into the container +COPY ./app . + +# Expose the port Flask will run on +EXPOSE 8080 + +# Command to run the Flask app +CMD ["python", "app.py"] diff --git a/app/app.py b/app/app.py new file mode 100644 index 0000000..c240475 --- /dev/null +++ b/app/app.py @@ -0,0 +1,17 @@ +from flask import Flask + +app = Flask(__name__) + + +@app.route("/") +def home(): + return "Hello, World! This is a test Flask app." + + +@app.route("/health") +def health(): + return {"status": "healthy"}, 200 + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8080) diff --git a/app/requirements.txt b/app/requirements.txt new file mode 100644 index 0000000..0f4e7db --- /dev/null +++ b/app/requirements.txt @@ -0,0 +1 @@ +flask==2.2.5 diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index b889018..4b39e37 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -20,3 +20,22 @@ provider "registry.terraform.io/hashicorp/google" { "zh:f96148c3742126219b810a687551284460a8d78aa66efbfd2c58880f48049dda", ] } + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:I0Um8UkrMUb81Fxq/dxbr3HLP2cecTH2WMJiwKSrwQY=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + ] +} diff --git a/terraform/main.tf b/terraform/main.tf index 5b95509..6120037 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,13 +1,7 @@ -variable "resource_suffic" { - description = "Suffix to append to all resources" - type = string - default = "prod" -} - locals { project = var.project_id - service_name = "cosyne-service" - database_name = "test-database-users" + service_name = "${var.resource_suffix}-cosyne-service" + database_name = "users" } # TODO: Add resources for Terraform to manage @@ -24,24 +18,23 @@ resource "google_spanner_database" "default" { deletion_protection = false } -resource "google_cloud_run_v2_service" "deployment" { - name = local.service_name - location = "us-central1" - project = var.project_id - deletion_protection = false - template { - containers { - image = "gcr.io/${var.project_id}/cosyne:latest" - } - } - traffic { - percent = 100 - } +# Push an image to Google Container Registry +# NOTE: We can automate the process if we have the vmassign repo combined with this repo +# Google Artifact Registry +resource "google_artifact_registry_repository" "repo" { + format = "DOCKER" + location = "us-central1" + repository_id = "flask-app-repo" } -data "google_iam_policy" "noauth" { - binding { - role = "roles/run.invoker" - members = ["allUsers"] +resource "null_resource" "docker_build_and_push" { + provisioner "local-exec" { + command = < Date: Tue, 28 Jan 2025 13:57:07 -0800 Subject: [PATCH 09/16] add deployment test script --- terraform/main.tf | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 6120037..d146980 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -27,14 +27,45 @@ resource "google_artifact_registry_repository" "repo" { repository_id = "flask-app-repo" } -resource "null_resource" "docker_build_and_push" { - provisioner "local-exec" { - command = < Date: Thu, 30 Jan 2025 10:55:38 -0800 Subject: [PATCH 10/16] add a bash script that generates a service account --- create-service-account.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 create-service-account.sh diff --git a/create-service-account.sh b/create-service-account.sh new file mode 100644 index 0000000..98bf2b6 --- /dev/null +++ b/create-service-account.sh @@ -0,0 +1,23 @@ +# Project ID as an argument +PROJECT_ID=$1 + +# Create the service account +gcloud iam service-accounts create service-account-admin \ + --description="Service account to create and manage other service accounts" \ + --display-name="Service Account Admin" \ + --project="${PROJECT_ID}" + +# Grant roles to allow service account management +gcloud projects add-iam-policy-binding ${PROJECT_ID} \ + --member="serviceAccount:service-account-admin@${PROJECT_ID}.iam.gserviceaccount.com" \ + --role="roles/iam.serviceAccountAdmin" + +# Grant roles to allow assignment of roles to other service accounts +gcloud projects add-iam-policy-binding ${PROJECT_ID} \ + --member="serviceAccount:service-account-admin@${PROJECT_ID}.iam.gserviceaccount.com" \ + --role="roles/iam.roleAdmin" + +# Generate a key for the service account +gcloud iam service-accounts keys create service-account-admin-key.json \ + --iam-account="service-account-admin@${PROJECT_ID}.iam.gserviceaccount.com" \ + --project="${PROJECT_ID}" \ No newline at end of file From a4e8ec79dfd0031ef8c694dc4846bf31d6e2388a Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Thu, 30 Jan 2025 10:56:57 -0800 Subject: [PATCH 11/16] fix gitignore and fix bash --- .gitignore | 1 + create-service-account.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2bf4750..18cbeac 100644 --- a/.gitignore +++ b/.gitignore @@ -210,6 +210,7 @@ terraform/terraform.rc # Ignore credentials file (any json file that starts with "vmassign-dev-") terraform/vmassign-dev-*.json +terraform/service-account-*.json # ignore DS_Store files .DS_Store \ No newline at end of file diff --git a/create-service-account.sh b/create-service-account.sh index 98bf2b6..5335efc 100644 --- a/create-service-account.sh +++ b/create-service-account.sh @@ -18,6 +18,6 @@ gcloud projects add-iam-policy-binding ${PROJECT_ID} \ --role="roles/iam.roleAdmin" # Generate a key for the service account -gcloud iam service-accounts keys create service-account-admin-key.json \ +gcloud iam service-accounts keys create terraform/service-account-admin-key.json \ --iam-account="service-account-admin@${PROJECT_ID}.iam.gserviceaccount.com" \ --project="${PROJECT_ID}" \ No newline at end of file From 75cf1efd0989c893eea7acd4686a460598f740e0 Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Thu, 30 Jan 2025 12:41:32 -0800 Subject: [PATCH 12/16] add more permissions --- create-service-account.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/create-service-account.sh b/create-service-account.sh index 5335efc..ee7d865 100644 --- a/create-service-account.sh +++ b/create-service-account.sh @@ -17,6 +17,16 @@ gcloud projects add-iam-policy-binding ${PROJECT_ID} \ --member="serviceAccount:service-account-admin@${PROJECT_ID}.iam.gserviceaccount.com" \ --role="roles/iam.roleAdmin" +# Grant database admin roles +gcloud projects add-iam-policy-binding ${PROJECT_ID} \ + --member="serviceAccount:service-account-admin@${PROJECT_ID}.iam.gserviceaccount.com" \ + --role="roles/spanner.admin" + +# Grant cloud run deployment roles +gcloud projects add-iam-policy-binding ${PROJECT_ID} \ + --member="serviceAccount:service-account-admin@${PROJECT_ID}.iam.gserviceaccount.com" \ + --role="roles/run.developer" + # Generate a key for the service account gcloud iam service-accounts keys create terraform/service-account-admin-key.json \ --iam-account="service-account-admin@${PROJECT_ID}.iam.gserviceaccount.com" \ From da59c3e429017470cdcb81fef9d7619434da8fd9 Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Tue, 4 Feb 2025 12:25:15 -0800 Subject: [PATCH 13/16] add service account for each infastructure' --- terraform/main.tf | 70 ++++++++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index d146980..12ae281 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,51 +1,61 @@ locals { - project = var.project_id service_name = "${var.resource_suffix}-cosyne-service" - database_name = "users" + database_name = "users-${var.resource_suffix}" + is_test = var.resource_suffix == "test" + is_staging = var.resource_suffix == "staging" + is_production = var.resource_suffix == "prod" } -# TODO: Add resources for Terraform to manage +# Service Account Creation for spanner instance +resource "google_service_account" "spanner_admin_account" { + account_id = "spanner-admin" + display_name = "Spanner Admin Service Account" +} + +# Instance Creation +resource "google_spanner_instance" "database_instance" { + name = "vmassign-${var.resource_suffix}" + display_name = "Assign Instance ${var.resource_suffix}" + config = "regional-us-west1" + processing_units = 1000 +} -# TODO: Parameterize testing/development values -# Create Spanner Database in the specific instance +# Database Creation resource "google_spanner_database" "default" { - name = local.database_name - instance = "vmassign-dev" + name = "users" + instance = google_spanner_instance.database_instance.name ddl = [ "CREATE TABLE Users (Hostname STRING(1024) NOT NULL, Pin STRING(1024), CrdCmd STRING(1024), UserEmail STRING(1024), inUse BOOL,) PRIMARY KEY (Hostname)" ] deletion_protection = false } -# Push an image to Google Container Registry -# NOTE: We can automate the process if we have the vmassign repo combined with this repo -# Google Artifact Registry -resource "google_artifact_registry_repository" "repo" { - format = "DOCKER" - location = "us-central1" - repository_id = "flask-app-repo" +# Grant permissions to the new service account for Spanner +resource "google_spanner_database_iam_member" "spanner_permissions" { + project = var.project_id + instance = google_spanner_instance.database_instance.name + database = google_spanner_database.default.name + role = "roles/spanner.databaseAdmin" + member = "serviceAccount:${google_service_account.spanner_admin_account.email}" } -# resource "null_resource" "docker_build_and_push" { -# provisioner "local-exec" { -# command = < Date: Wed, 5 Feb 2025 16:23:07 -0800 Subject: [PATCH 14/16] implement delete service account --- delete-service-account.sh | 52 +++++++++++++++++++++++++++++++++++++++ terraform/providers.tf | 2 +- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 delete-service-account.sh diff --git a/delete-service-account.sh b/delete-service-account.sh new file mode 100644 index 0000000..53a627c --- /dev/null +++ b/delete-service-account.sh @@ -0,0 +1,52 @@ +#!/bin/bash +SHA=$1 + +# Exit on any error +set -e + +# Variables +PROJECT_ID="vmassign-dev" +SERVICE_ACCOUNT_NAME="serviceAccount:service-account-admin" +SERVICE_ACCOUNT_EMAIL="${SERVICE_ACCOUNT_NAME}@${PROJECT_ID}.iam.gserviceaccount.com" + +echo "Deleting IAM bindings for service account: $SERVICE_ACCOUNT_EMAIL" + +gcloud projects get-iam-policy $PROJECT_ID --flatten="bindings[].members" --format="table(bindings.role,bindings.members)" --filter="bindings.members:$SERVICE_ACCOUNT_EMAIL" + +# Get the IAM roles assigned to the service account +IAM_ROLES=$(gcloud projects get-iam-policy $PROJECT_ID --format=json | jq -r \ + '.bindings[] | select(.members[]? == "serviceAccount:'"$SHA"'") | .role') + +# Revoke IAM roles assigned to the service account +if [ -z "$IAM_ROLES" ]; then + echo "No active IAM roles found for $SERVICE_ACCOUNT_EMAIL." +else + # Remove IAM roles only for the active service account + for ROLE in $IAM_ROLES; do + echo "Removing IAM role: $ROLE" + gcloud projects remove-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SERVICE_ACCOUNT_EMAIL" \ + --role="$ROLE" \ + --quiet || echo "Failed to remove role $ROLE, it may have already been removed." + done +fi + + +# Delete the service account +echo "Deleting service account: $SERVICE_ACCOUNT_EMAIL" +gcloud iam service-accounts delete $SHA --quiet + + +# Check if the service account still exists before deleting +if gcloud iam service-accounts describe $SERVICE_ACCOUNT_EMAIL &>/dev/null; then + echo "Deleting service account: $SERVICE_ACCOUNT_EMAIL" + gcloud projects set-iam-policy $PROJECT_ID <(gcloud projects get-iam-policy $PROJECT_ID --format=json | jq 'del(.bindings[] | select(.members[] | contains("serviceAccount:'"$SERVICE_ACCOUNT_EMAIL"'")))') +else + echo "Service account $SERVICE_ACCOUNT_EMAIL does not exist." +fi + +# Delete the key file +echo "Deleting key file: terraform/service-account-admin-key.json" +rm -f terraform/service-account-admin-key.json + +echo "Service account $SERVICE_ACCOUNT_EMAIL deleted successfully!" diff --git a/terraform/providers.tf b/terraform/providers.tf index 9e66a1c..bbd09f6 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -10,5 +10,5 @@ terraform { provider "google" { project = "vmassign-dev" region = "us-west1" - credentials = "./vmassign-dev-2818ad83c3ff.json" + credentials = "./service-account-admin-key.json" } From 285cc8a8db9b35606b2efb0fa7db425614613004 Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Thu, 6 Feb 2025 10:48:29 -0800 Subject: [PATCH 15/16] handle empty argument --- create-service-account.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/create-service-account.sh b/create-service-account.sh index ee7d865..320b846 100644 --- a/create-service-account.sh +++ b/create-service-account.sh @@ -1,6 +1,11 @@ # Project ID as an argument PROJECT_ID=$1 +if [ -z "$PROJECT_ID" ]; then + echo "Project ID is required." + exit 1 +fi + # Create the service account gcloud iam service-accounts create service-account-admin \ --description="Service account to create and manage other service accounts" \ From 40b80910641f097935a912bd7017c706162ce509 Mon Sep 17 00:00:00 2001 From: 7174Andy Date: Thu, 6 Feb 2025 12:50:06 -0800 Subject: [PATCH 16/16] delete service account and its iam-policy-binding --- delete-service-account.sh | 25 +++++++++++++++++-------- kubectl.sha256 | 1 + 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 kubectl.sha256 diff --git a/delete-service-account.sh b/delete-service-account.sh index 53a627c..c1acbc5 100644 --- a/delete-service-account.sh +++ b/delete-service-account.sh @@ -1,21 +1,31 @@ #!/bin/bash -SHA=$1 # Exit on any error set -e # Variables PROJECT_ID="vmassign-dev" -SERVICE_ACCOUNT_NAME="serviceAccount:service-account-admin" +SERVICE_ACCOUNT_NAME="service-account-admin" SERVICE_ACCOUNT_EMAIL="${SERVICE_ACCOUNT_NAME}@${PROJECT_ID}.iam.gserviceaccount.com" echo "Deleting IAM bindings for service account: $SERVICE_ACCOUNT_EMAIL" -gcloud projects get-iam-policy $PROJECT_ID --flatten="bindings[].members" --format="table(bindings.role,bindings.members)" --filter="bindings.members:$SERVICE_ACCOUNT_EMAIL" +gcloud projects get-iam-policy $PROJECT_ID \ + --flatten="bindings[].members" \ + --format="table(bindings.role)" \ + --filter="bindings.members:serviceAccount:$SERVICE_ACCOUNT_EMAIL" # Get the IAM roles assigned to the service account -IAM_ROLES=$(gcloud projects get-iam-policy $PROJECT_ID --format=json | jq -r \ - '.bindings[] | select(.members[]? == "serviceAccount:'"$SHA"'") | .role') +IAM_ROLES=$(gcloud projects get-iam-policy $PROJECT_ID \ + --flatten="bindings[].members" \ + --format="table(bindings.role)" \ + --filter="bindings.members:$SERVICE_ACCOUNT_EMAIL") + +echo "IAM roles assigned to $SERVICE_ACCOUNT_EMAIL: $IAM_ROLES" + +# Remove ROLE column header from the output +IAM_ROLES=$(echo "$IAM_ROLES" | sed '1d') + # Revoke IAM roles assigned to the service account if [ -z "$IAM_ROLES" ]; then @@ -26,15 +36,14 @@ else echo "Removing IAM role: $ROLE" gcloud projects remove-iam-policy-binding $PROJECT_ID \ --member="serviceAccount:$SERVICE_ACCOUNT_EMAIL" \ - --role="$ROLE" \ - --quiet || echo "Failed to remove role $ROLE, it may have already been removed." + --role="$ROLE" done fi # Delete the service account echo "Deleting service account: $SERVICE_ACCOUNT_EMAIL" -gcloud iam service-accounts delete $SHA --quiet +gcloud iam service-accounts delete $SERVICE_ACCOUNT_EMAIL --quiet # Check if the service account still exists before deleting diff --git a/kubectl.sha256 b/kubectl.sha256 new file mode 100644 index 0000000..ad4b76a --- /dev/null +++ b/kubectl.sha256 @@ -0,0 +1 @@ +5b89f9598e2e7da04cc0b5dd6e8daca01d23855fd00c8ea259fd2aab993114db \ No newline at end of file