Merge pull request #126 from fabric-testbed/knit8

Knit8
fabric-testbed · Apr 24, 2024 · 2a5ad7e · 2a5ad7e
2 parents 01009cb + 357d30f
commit 2a5ad7e
Show file tree

Hide file tree

Showing 73 changed files with 3,474 additions and 384 deletions.
diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml
@@ -10,14 +10,26 @@ on:
   workflow_dispatch:
 
 env:
-  fabfed_image: fabfed:cicd
   FABRIC_TOKEN: ${{ secrets.FABRIC_TOKEN }}
   FABRIC_BASTION_KEY: ${{ secrets.FABRIC_BASTION_KEY }}
   FABRIC_SLIVER_KEY: ${{ secrets.FABRIC_SLIVER_KEY }}
   FABRIC_SLIVER_PUBKEY: ${{ secrets.FABRIC_SLIVER_PUBKEY }}
   FABRIC_PROJECT: ${{ secrets.FABRIC_PROJECT }}
   FABRIC_USER: ${{ secrets.FABRIC_USER }}
 
+  SENSE_USER: ${{ secrets.SENSE_USER }}
+  SENSE_PASSWORD: ${{ secrets.SENSE_PASSWORD }}
+  SENSE_SECRET: ${{ secrets.SENSE_SECRET }}
+  SENSE_SLIVER_KEY: ${{ secrets.SENSE_SLIVER_KEY }}
+
+  CLAB_USER: ${{ secrets.CLAB_USER }}
+  CLAB_SLIVER_KEY: ${{ secrets.CLAB_SLIVER_KEY }}
+  CLAB_PEM_FILE: ${{ secrets.CLAB_PEM_FILE }}
+
+  RUN_FABRIC_AWS_SENSE: ${{ vars.RUN_FABRIC_AWS_SENSE }}
+  RUN_CLAB: ${{ vars.RUN_CLAB }}
+  RUN_FABRIC_AWS_WITH_NODES:  ${{ vars.RUN_FABRIC_AWS_WITH_NODES }}
+
 jobs:
   UbuntuTest:
     runs-on: ubuntu-latest
@@ -43,6 +55,7 @@ jobs:
           pip list -v | grep neo4j
           pip install --no-cache-dir --ignore-requires-python -r requirements.txt
           python3 -m pip install --no-cache-dir .
+          pip install --no-cache-dir git+https://gitlab.flux.utah.edu/stoller/portal-tools.git
 
       - name: Run Unit Tests
         run: |
@@ -51,6 +64,9 @@ jobs:
       - name: Test Stitch Policy
         run: |
           fabfed stitch-policy -providers "chi,fabric"
+          fabfed stitch-policy -providers "cloudlab,fabric"
+          fabfed stitch-policy -providers "aws,fabric"
+          fabfed stitch-policy -providers "gcp,fabric"
 
       - name: Show Sessions 
         run: |
@@ -64,13 +80,44 @@ jobs:
           echo ${{ env.FABRIC_BASTION_KEY }} | base64 --decode > ${{ github.workspace }}/creds/bastion
           echo ${{ env.FABRIC_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver
           echo ${{ env.FABRIC_SLIVER_PUBKEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver.pub
+          echo ${{ env.SENSE_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/sense
+          echo ${{ env.CLAB_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/cloudlab
+          echo ${{ env.CLAB_PEM_FILE }} | base64 --decode > ${{ github.workspace }}/creds/cloudlab.pem
+
+      - name: Test Cloudlab
+        if: ${{ env.RUN_CLAB == 'true' }}
+        run: |
+          session=cicd-clab
+          echo "site: CLEM" > $session-varfile.yml
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/cloudlab $session $session-varfile.yml
 
       - name: Test Fabric FacilityPort
+        if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} 
         run: |
-          session=fabric-facility-port
+          session=cicd-fabric-facility-port
           echo "vlan: 3102" > $session-varfile.yml
           ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml
 
       - name: Test L2VPN with nodes.
+        if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} 
+        run: |
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_l2_vpn cicd-fabric-l2-vpn
+
+      - name: Test Fabric AWS (Fabric Only With No Nodes)
+        run: |
+          session=aes-aws-native-no-nodes
+          echo "vlan: 4" > $session-varfile.yml
+          echo "node_count: 0" >> $session-varfile.yml
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_native_aws $session $session-varfile.yml
+
+      - name: Test Fabric AWS (Fabric Only With Nodes)
+        if: ${{ env.RUN_FABRIC_AWS_WITH_NODES == 'true' || env.RUN_FABRIC_AWS_WITH_NODES == true }} 
+        run: |
+          session=aes-aws-native-with-nodes
+          echo "vlan: 4" > $session-varfile.yml
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_native_aws $session $session-varfile.yml
+
+      - name: Test SENSE/AWS.
+        if: ${{ env.RUN_FABRIC_AWS_SENSE == 'true' || env.RUN_FABRIC_AWS_SENSE == true  }} 
         run: |
-          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_l2_vpn fabric-l2-vpn
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_sense_aws cicd-sense-aws 
diff --git a/.github/workflows/ubuntu_22_and_test_gcp.yml b/.github/workflows/ubuntu_22_and_test_gcp.yml
@@ -0,0 +1,70 @@
+name: GCP-Ubuntu22.04 Test 
+
+on:
+  workflow_dispatch:
+
+env:
+  FABRIC_TOKEN: ${{ secrets.FABRIC_TOKEN }}
+  FABRIC_BASTION_KEY: ${{ secrets.FABRIC_BASTION_KEY }}
+  FABRIC_SLIVER_KEY: ${{ secrets.FABRIC_SLIVER_KEY }}
+  FABRIC_SLIVER_PUBKEY: ${{ secrets.FABRIC_SLIVER_PUBKEY }}
+  FABRIC_PROJECT: ${{ secrets.FABRIC_PROJECT }}
+  FABRIC_USER: ${{ secrets.FABRIC_USER }}
+
+  GCP_SERVICE_KEY_PATH: ${{ secrets.GCP_SERVICE_KEY_PATH }}
+
+  RUN_GCP: ${{ vars.RUN_GCP }}
+
+jobs:
+  GcpOnUbuntuTest:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Print Versions 
+        run: |
+          python3 --version
+          pip --version
+          pwd
+          whoami
+          echo ${{ github.workspace }}
+          echo $HOME
+
+      - name: Install Fabfed And Requirements 
+        run: |
+          pip install --upgrade pip 
+          pip install setuptools --upgrade
+          pip install --no-cache-dir --ignore-requires-python neo4j==5.18.0
+          pip list -v | grep neo4j
+          pip install --no-cache-dir --ignore-requires-python -r requirements.txt
+          python3 -m pip install --no-cache-dir .
+
+      - name: Run Unit Tests
+        run: |
+          pytest tests
+
+      - name: Test Stitch Policy
+        run: |
+          fabfed stitch-policy -providers "gcp,fabric"
+
+      - name: Show Sessions 
+        run: |
+          fabfed sessions -show 
+          fabfed sessions -show -json
+
+      - name: Save Credentials
+        run: |
+          mkdir -p ${{ github.workspace }}/creds
+          echo ${{ env.FABRIC_TOKEN }} | base64 --decode > ${{ github.workspace }}/creds/token.json
+          echo ${{ env.FABRIC_BASTION_KEY }} | base64 --decode > ${{ github.workspace }}/creds/bastion
+          echo ${{ env.FABRIC_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver
+          echo ${{ env.FABRIC_SLIVER_PUBKEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver.pub
+          echo ${{ env.GCP_SERVICE_KEY_PATH }} | base64 --decode > ${{ github.workspace }}/creds/gcp.json
+
+      - name: Test GCP
+        if: ${{ env.RUN_GCP == 'true' }}
+        run: |
+          session=cicd-gcp
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/gcp $session
diff --git a/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml b/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml
@@ -0,0 +1,89 @@
+name: Ubuntu22.04 Modify Test 
+
+on:
+  push:
+     branches:
+      - knit8 
+  workflow_dispatch:
+
+env:
+  FABRIC_TOKEN: ${{ secrets.FABRIC_TOKEN }}
+  FABRIC_BASTION_KEY: ${{ secrets.FABRIC_BASTION_KEY }}
+  FABRIC_SLIVER_KEY: ${{ secrets.FABRIC_SLIVER_KEY }}
+  FABRIC_SLIVER_PUBKEY: ${{ secrets.FABRIC_SLIVER_PUBKEY }}
+  FABRIC_PROJECT: ${{ secrets.FABRIC_PROJECT }}
+  FABRIC_USER: ${{ secrets.FABRIC_USER }}
+
+jobs:
+  UbuntuModifyTest:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Fabfed And Requirements 
+        run: |
+          pip install --upgrade pip 
+          pip install setuptools --upgrade
+          pip install --no-cache-dir --ignore-requires-python neo4j==5.18.0
+          pip list -v | grep neo4j
+          pip install --no-cache-dir --ignore-requires-python -r requirements.txt
+          python3 -m pip install --no-cache-dir .
+          pip install --no-cache-dir git+https://gitlab.flux.utah.edu/stoller/portal-tools.git
+
+      - name: Save Credentials
+        run: |
+          mkdir -p ${{ github.workspace }}/creds
+          echo ${{ env.FABRIC_TOKEN }} | base64 --decode > ${{ github.workspace }}/creds/token.json
+          echo ${{ env.FABRIC_BASTION_KEY }} | base64 --decode > ${{ github.workspace }}/creds/bastion
+          echo ${{ env.FABRIC_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver
+          echo ${{ env.FABRIC_SLIVER_PUBKEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver.pub
+
+      - name: Test Fabric Modify with FacilityPort 0 nodes
+        run: |
+          export DO_NOT_DESTROY=1
+          session=cicd-fabric-facility-port
+          echo "vlan: 3103" > $session-varfile.yml
+          echo "node_count: 0" >> $session-varfile.yml
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml
+
+      - name: Test Fabric Modify with FacilityPort Add One Node
+        run: |
+          export DO_NOT_DESTROY=1
+          session=cicd-fabric-facility-port
+          echo "vlan: 3103" > $session-varfile.yml
+          echo "node_count: 1" >> $session-varfile.yml
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml
+
+      - name: Test Fabric Modify with FacilityPort Add Another Node
+        run: |
+          export DO_NOT_DESTROY=1
+          session=cicd-fabric-facility-port
+          echo "vlan: 3103" > $session-varfile.yml
+          echo "node_count: 2" >> $session-varfile.yml
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml
+
+      - name: Test Fabric Modify with FacilityPort Remove One Node
+        run: |
+          export DO_NOT_DESTROY=1
+          session=cicd-fabric-facility-port
+          echo "vlan: 3103" > $session-varfile.yml
+          echo "node_count: 1" >> $session-varfile.yml
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml
+
+      - name: Test Fabric Modify with FacilityPort Remove One Node
+        run: |
+          export DO_NOT_DESTROY=1
+          session=cicd-fabric-facility-port
+          echo "vlan: 3103" > $session-varfile.yml
+          echo "node_count: 0" >> $session-varfile.yml
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml
+
+      - name: Test Fabric Modify with FacilityPort Destroy 
+        run: |
+          export DO_NOT_DESTROY=0
+          session=cicd-fabric-facility-port
+          echo "vlan: 3103" > $session-varfile.yml
+          echo "node_count: 0" >> $session-varfile.yml
+          ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml
diff --git a/.gitignore b/.gitignore
@@ -153,3 +153,6 @@ venv
 
 # emacs auto-save
 *.*~
+
+# jupyter artifacts
+.ipynb_checkpoints
diff --git a/README.md b/README.md
@@ -64,9 +64,9 @@ the <i>--config-dir</i>.  If this option is not present, the current directory i
 # Example to view stitch policy from cloudlab to fabric
 fabfed stitch-policy -providers "fabric,cloudlab"
 
+# Validation
 fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -validate
-
-fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -init [-summary] [-json]
+fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -stitch-info [-summary] [-json]
 
 fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -plan [-summary] [-json]
 

diff --git a/cicd/docker/Dockerfile.jupyter b/cicd/docker/Dockerfile.jupyter
@@ -0,0 +1,17 @@
+ARG base_image=ubuntu:24.04
+
+FROM ${base_image}
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get -u -y install lsb-release python3 binutils python3-venv python3-pip wget git bash-completion vim yq jq \
+    && apt-get -qq purge && apt-get -qq clean && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --break-system-packages jupyterlab
+COPY /requirements.txt /tmp/
+RUN pip install --no-cache-dir --break-system-packages --ignore-requires-python -r /tmp/requirements.txt
+
+COPY ./setup.py ./tools ./README.md ./MANIFEST.in /requirements.txt /
+COPY ./fabfed /fabfed
+COPY ./tools /tools
+RUN python3 -m pip install --break-system-packages --no-cache-dir .
diff --git a/cicd/fabfed_credentials.yml.cicd b/cicd/fabfed_credentials.yml.cicd
@@ -11,5 +11,21 @@ aws:
    secret_key: AWS_SECRET_KEY
 
 gcp:
-   service_key_path: GCP_SERVICE_KEY_PATH
-   project: GCP_PROJECT 
+   service_key_path: creds/gcp.json
+   project: fabfed
+
+cloudlab:
+  project: fabfed
+  certificate: creds/cloudlab.pem
+  user: CLAB_USER
+  slice-private-key-location: creds/cloudlab
+
+sense:
+   auth_endpoint: https://sense-o.es.net:8543/auth/realms/StackV/protocol/openid-connect/token
+   api_endpoint: https://sense-o-dev.es.net:8443/StackV-web/restapi
+   client_id: StackV
+   username: SENSE_USER
+   password: SENSE_PASSWORD
+   secret: SENSE_SECRET
+   verify: False
+   slice-private-key-location: creds/sense
diff --git a/cicd/run-fabfed.sh b/cicd/run-fabfed.sh
@@ -17,17 +17,65 @@ cp $script_dir/fabfed_credentials.yml.cicd ~/.fabfed/
 sed -i "s/FABRIC_PROJECT/$FABRIC_PROJECT/" ~/.fabfed/fabfed_credentials.yml.cicd
 sed -i "s/FABRIC_USER/$FABRIC_USER/" ~/.fabfed/fabfed_credentials.yml.cicd
 
+# SENSE
+sed -i "s/SENSE_USER/$SENSE_USER/" ~/.fabfed/fabfed_credentials.yml.cicd
+sed -i "s/SENSE_PASSWORD/$SENSE_PASSWORD/" ~/.fabfed/fabfed_credentials.yml.cicd
+sed -i "s/SENSE_SECRET/$SENSE_SECRET/" ~/.fabfed/fabfed_credentials.yml.cicd
+
+# CLAB
+sed -i "s/CLAB_USER/$CLAB_USER/" ~/.fabfed/fabfed_credentials.yml.cicd
+
 if [ -n "$var_file" ]
 then
   options="-v $3"
 fi
 
+echo "***************** APPLYING  ****************"
 echo fabfed workflow -c $conf_dir $options -s $session -apply
 fabfed workflow -c $conf_dir $options -s $session -apply
-fabfed workflow -c $conf_dir $options -s $session -show
+ret1=$?
+
+echo "***************** APPLY SUMMARY  ****************"
+echo fabfed workflow -c $conf_dir $options -s $session -show -summary
+fabfed workflow -c $conf_dir $options -s $session -show -summary > $session-apply-state.yaml
+fabfed workflow -c $conf_dir $options -s $session -show -summary
 fabfed sessions -show
+
+DO_NOT_DESTROY="${DO_NOT_DESTROY:=0}"
+
+if [ $DO_NOT_DESTROY -ne 0 ]; then
+   echo "NOT_DESTROYING:APPLY RESULTS:"
+   cat $session-apply-state.yaml
+   exit $ret1
+fi
+
 echo "***************** DESTOYING ****************"
 echo fabfed workflow -c $conf_dir $options -s $session -destroy
 fabfed workflow -c $conf_dir $options -s $session -destroy
+ret2=$? 
+
+echo "***************** DESTROY SUMMARY  ****************"
+fabfed workflow -c $conf_dir $options -s $session -show -summary
 fabfed sessions -show
-exit $? 
+
+echo "***************** APPLY RESULTS ****************"
+echo "APPLY RESULTS:"
+cat $session-apply-state.yaml
+
+if [[ ! $ret1 -eq 0 ]]; then
+     echo "Apply failed ....."
+fi
+
+if [[ ! $ret2 -eq 0 ]]; then
+     echo "Destroy failed ....."
+fi
+
+if [[ ! $ret1 -eq 0 ]]; then
+     exit 1
+fi
+
+if [[ ! $ret2 -eq 0 ]]; then
+     exit 2
+fi
+
+exit 0