Merge pull request #110 from SimonBaeumer/add-ssh-executor

Create exectuor interface and LocalExecutor for interchangable executors

.travis.yml

@@ -15,6 +15,9 @@ go:
 sudo: required
 dist: trusty
 
+services:
+  - docker
+
 before_install:
   - go get -u golang.org/x/lint/golint
   - curl -L https://github.com/SimonBaeumer/commander/releases/download/v1.2.1/commander-linux-amd64 -o ~/bin/commander
@@ -36,7 +39,7 @@ jobs:
     script:
       - curl -L https://github.com/SimonBaeumer/commander/releases/download/v0.3.0/commander-darwin-amd64 -o ~/bin/commander
       - chmod +x ~/bin/commander
-      - make integration
+      - make integration-unix
 
   - name: windows Unit
     os: windows
@@ -60,12 +63,12 @@ jobs:
       - chmod +x test-reporter
       - ./test-reporter before-build
     script:
-      - make test-coverage
+      - make test-coverage-all
     after_script:
       - ./test-reporter after-build -t gocov --exit-code $TRAVIS_TEST_RESULT
 
   - name: Integration test
-    script: make integration
+    script: make integration-linux
 
   - stage: deploy
     name: "Deployment"

CHANGELOG.md

@@ -1,3 +1,9 @@
+# v2.0.0
+
+  - Added `nodes` which allow remote execution of tests
+  - Added `SSHExecutor` and `LocalExecutor`
+  - Removed `concurrent` argument from `test` command
+
 # v1.3.0
 
  - Added `xml` assertion to `stdout` and `stderr`

Makefile

@@ -1,6 +1,7 @@
 exe = cmd/commander/*
 cmd = commander
 TRAVIS_TAG ?= "0.0.0"
+CWD = $(shell pwd)
 
 .PHONY: deps lint test integration integration-windows git-hooks init
 
@@ -30,9 +31,27 @@ test-coverage:
 	$(info INFO: Starting build $@)
 	go test -coverprofile c.out ./...
 
-integration: build
+
+test-coverage-all: export COMMANDER_SSH_TEST = 1
+test-coverage-all: export COMMANDER_TEST_SSH_HOST = localhost:2222
+test-coverage-all: export COMMANDER_TEST_SSH_USER = root
+test-coverage-all: export COMMANDER_TEST_SSH_IDENTITY_FILE = $(CWD)/integration/containers/ssh/id_rsa
+test-coverage-all:
+	$(info INFO: Starting build $@)
+	./integration/setup_unix.sh
+	go test -coverprofile c.out ./...
+	./integration/teardown_unix.sh
+
+integration-unix: build
+	$(info INFO: Starting build $@)
+	commander test commander_unix.yaml
+
+integration-linux: build
 	$(info INFO: Starting build $@)
+	./integration/setup_unix.sh
 	commander test commander_unix.yaml
+	commander test commander_linux.yaml
+	./integration/teardown_unix.sh
 
 integration-windows: build
 	$(info INFO: Starting build $@)

README.md

@@ -10,6 +10,7 @@
 Define language independent tests for your command line scripts and programs in simple `yaml` files.
 
  - It runs on `windows`, `osx` and `linux` 
+ - It can validate local machines, ssh hosts and docker containers
  - It is a self-contained binary - no need to install a heavy lib or language
  - It is easy and fast to write
 
@@ -47,6 +48,10 @@ For more information take a look at the [quick start](#quick-start), the [exampl
     - [interval](#interval)
     - [retries](#retries)
     - [timeout](#timeout)
+    - [nodes](#nodes)
+  + [Nodes](#nodes)
+    - [local](#local)
+    - [ssh](#ssh)
   + [Development](#development)
 * [Misc](#misc)
 
@@ -108,12 +113,32 @@ Count: 1, Failed: 0
 Here you can see an example with all features for a quick reference
 
 ```yaml
+nodes:
+  ssh-host1:
+    type: ssh
+    addr: 192.168.0.1:22
+    user: root
+    pass: pass
+  ssh-host2:
+    type: ssh
+    addr: 192.168.0.1:22
+    user: root
+    identity-file: /home/user/id_rsa.pub
+  docker-host1:
+    type: docker
+    image: alpine:2.4
+  docker-host2:
+    type: docker
+    instance: alpine_instance_1
+
 config: # Config for all executed tests
     dir: /tmp #Set working directory
     env: # Environment variables
         KEY: global
     timeout: 50s # Define a timeout for a command under test
     retries: 2 # Define retries for each test
+    nodes:
+    - ssh-host1 # define default hosts
 
 tests:
     echo hello: # Define command as title
@@ -141,7 +166,7 @@ tests:
         command: echo hello
         stdout:
             contains: 
-                - hello #See test "it should fail"
+              - hello #See test "it should fail"
             exactly: hello
             line-count: 1
         config:
@@ -152,6 +177,9 @@ tests:
                 ANOTHER: yeah # Add another env variable
             timeout: 1s # Overwrite timeout
             retries: 5
+            nodes: # overwrite default nodes
+              - docker-host1
+              - docker-host2
         exit-code: 0
 ```
 
@@ -607,6 +635,82 @@ If a tests exceeds the given `timeout` the test will fail.
 timeout: 600s
 ```
 
+### Nodes
+
+`Commander` has the option to execute tests against other hosts, i.e. via ssh.
+
+Available node types are currently:
+
+ - `local`, execute tests locally
+ - `ssh`, execute tests viá ssh
+
+```yaml
+nodes: # define nodes in the node section
+  ssh-host:
+    type: ssh # define the type of the connection 
+    user: root # set the user which is used by the connection
+    pass: password # set password for authentication
+    addr: 192.168.0.100:2222 # target host address
+    identity-file: ~/.ssh/id_rsa # auth with private key
+tests:
+  echo hello:
+    config:
+      nodes: # define on which host the test should be executed
+        - ssh-host
+    stdout: hello
+    exit-code: 0
+```
+
+You can identify on which node a test failed by inspecting the test output.
+The `[local]` and `[ssh-host]` represent the node name on which the test were executed.
+
+```
+✗ [local] it should test ssh host
+✗ [ssh-host] it should fail if env could not be set
+```
+
+#### local
+
+The `local` node is the default execution and will be applied if nothing else was configured.
+It is always pre-configured and available, i.e. if you want to execute tests on a node and locally.
+
+```yaml
+nodes:
+  ssh-host:
+    addr: 192.168.1.100
+    user: ...
+tests:
+  echo hello:
+    config:
+      nodes: # will be executed on local and ssh-host
+        - ssh-host
+        - local
+    exit-code: 0
+```
+
+#### ssh
+
+The `ssh` will execute tests against a configured node using ssh.
+
+**Limitations:** The `inhereit-env` config is disabled for ssh hosts, nevertheless it is possible to set env variables
+
+```yaml
+nodes: # define nodes in the node section
+  ssh-host:
+    type: ssh # define the type of the connection 
+    user: root # set the user which is used by the connection
+    pass: password # set password for authentication
+    addr: 192.168.0.100:2222 # target host address
+    identity-file: ~/.ssh/id_rsa # auth with private key
+tests:
+  echo hello:
+    config:
+      nodes: # define on which host the test should be executed
+        - ssh-host
+    stdout: hello
+    exit-code: 0
+```
+
 ### Development
 
 ```
@@ -622,13 +726,35 @@ $ make test
 # Coverage
 $ make test-coverage
 
-# Integration tests
-$ make integration
+# Coverage with more complex tests like ssh execution
+$ make test-coverage-all
+
+# Integration tests for linux and macos
+$ make integration-unix
+
+# Integration on linux
+$ make integration-linux
+
+# Integration windows
+$ make integration-windows
 
 # Add depdencies to vendor
 $ make deps
 ```
 
+### Unit tests
+
+Enables ssh tests in unit test suite and sets the credentials for the target host.
+`COMMANDER_SSH_TEST` must be set to `1` to enable ssh tests.
+
+```
+export COMMANDER_TEST_SSH=1
+export COMMANDER_TEST_SSH_HOST=localhost:2222
+export COMMANDER_TEST_SSH_PASS=pass
+export COMMANDER_TEST_SSH_USER=root
+export COMMANDER_TEST_SSH_IDENTITY_FILE=integration/containers/ssh/.ssh/id_rsa
+```
+
 ## Misc
 
 Heavily inspired by [goss](https://github.com/aelsabbahy/goss).

cmd/commander/commander.go

@@ -48,11 +48,6 @@ func createTestCommand() cli.Command {
 		Usage:     "Execute the test suite, by default it will use the commander.yaml from your current directory",
 		ArgsUsage: "[file] [title]",
 		Flags: []cli.Flag{
-			cli.IntFlag{
-				Name:   "concurrent",
-				EnvVar: "COMMANDER_CONCURRENT",
-				Usage:  "Set the max amount of tests which should run concurrently",
-			},
 			cli.BoolFlag{
 				Name:   "no-color",
 				EnvVar: "COMMANDER_NO_COLOR",

commander_linux.yaml

@@ -0,0 +1,12 @@
+tests:
+  test nodes:
+    command: ./commander test integration/linux/nodes.yaml
+    stdout:
+      contains:
+        - ✓ [ssh-host] it should test ssh host
+        - ✓ [ssh-host] it should set env variable
+        - ✓ [ssh-host-default] it should be executed on ssh-host-default
+        - ✓ [ssh-host] it should test multiple hosts
+        - ✓ [ssh-host-default] it should test multiple hosts
+        - ✓ [local] it should test multiple hosts
+    exit-code: 0

commander_unix.yaml

@@ -14,16 +14,16 @@ tests:
     command: ./commander test ./integration/unix/commander_test.yaml
     stdout:
       contains:
-        - ✓ it should exit with error code
+        - ✓ [local] it should exit with error code
       line-count: 16
     exit-code: 0
 
   it should assert that commander will fail:
     command: ./commander test ./integration/unix/failing_suite.yaml
     stdout:
       contains:
-        - ✗ 'it will fail', on property 'ExitCode'
-        - ✗ 'test timeout' could not be executed with error message
+        - ✗ [local] 'it will fail', on property 'ExitCode'
+        - ✗ [local] 'test timeout' could not be executed with error message
         - Command timed out after 10ms
         - "Count: 2, Failed: 2"
     exit-code: 1
@@ -32,7 +32,7 @@ tests:
     command: ./commander test ./integration/unix/test_big_output.yaml
     stdout:
       contains:
-        - ✓ cat ./integration/unix/_fixtures/big_out.txt
+        - ✓ [local] cat ./integration/unix/_fixtures/big_out.txt
         - "Count: 1, Failed: 0"
     exit-code: 0
 
@@ -43,9 +43,9 @@ tests:
         COMMANDER_FROM_SHELL: from_shell
     stdout:
       contains:
-        - ✓ should print global env value
-        - ✓ should print local env value
-        - ✓ should print env var from shell
+        - ✓ [local] should print global env value
+        - ✓ [local] should print local env value
+        - ✓ [local] should print env var from shell
     exit-code: 0
 
   test add command:
@@ -61,7 +61,7 @@ tests:
     command: ./commander test integration/unix/retries.yaml
     stdout:
       contains:
-        - ✗ echo hello, retries 3
-        - ✓ it should retry failed commands, retries 2
-        - ✗ it should retry failed commands with an interval, retries 2
-    exit-code: 1
+        - ✗ [local] echo hello, retries 3
+        - ✓ [local] it should retry failed commands, retries 2
+        - ✗ [local] it should retry failed commands with an interval, retries 2
+    exit-code: 1

commander_windows.yaml

@@ -21,16 +21,16 @@ tests:
     command: commander.exe test ./integration/windows/failing_suite.yaml
     stdout:
       contains:
-        - ✗ 'it will fail', on property 'ExitCode'
-        - ✗ 'test timeout' could not be executed with error message
+        - ✗ [local] 'it will fail', on property 'ExitCode'
+        - ✗ [local] 'test timeout' could not be executed with error message
         - "Count: 2, Failed: 2"
     exit-code: 1
 
   it should validate a big output:
     command: commander.exe test ./integration/windows/test_big_output.yaml
     stdout:
       contains:
-        - ✓ type integration\windows\_fixtures\big_out.txt
+        - ✓ [local] type integration\windows\_fixtures\big_out.txt
         - "Count: 1, Failed: 0"
     exit-code: 0
 
@@ -41,10 +41,10 @@ tests:
         COMMANDER_FROM_SHELL: from_shell
     stdout:
       contains:
-        - ✓ should print global
-        - ✓ should print local
-        - ✓ should execute in given dir
-        - ✓ should work with timeout
+        - ✓ [local] should print global
+        - ✓ [local] should print local
+        - ✓ [local] should execute in given dir
+        - ✓ [local] should work with timeout
     exit-code: 0
 
   test add command:
@@ -60,5 +60,5 @@ tests:
     command: commander.exe test integration/windows/retries.yaml
     stdout:
       contains:
-        - ✗ echo hello, retries 3
+        - ✗ [local] echo hello, retries 3
     exit-code: 1