Merge pull request #118 from brianhlin/create-io-images-dag

Move tests to the new host

README.md

@@ -163,9 +163,10 @@ Unfortunately, VM Universe jobs don't have the ssh\_to\_job capacity that's avai
 1.  Grab the make-interactive-image from git:
 
         git clone https://github.com/opensciencegrid/vm-test-runs.git
-2.  Run `make-interactive-image` using the flavor and version of Linux you need, VMU images are in `/mnt/gluster/chtc/VMs/` (NOTE: the output image needs to be in a directory that's readable by the `qemu` user):
+2.  Run `make-interactive-image` using the flavor and version of Linux you need, VMU images are in `/staging/osg-images/` (NOTE: the output image needs to be in a directory that's readable by the `qemu` user):
+
+        vm-test-runs/make-interactive-image /staging/osg-images/<INPUT IMAGE> <OUTPUT IMAGE>
 
-        vm-test-runs/make-interactive-image /mnt/gluster/chtc/VMs/<INPUT IMAGE> <OUTPUT IMAGE>
 3.  Make a copy of `libvirt-template.xml` and edit the `@DOMAIN@` and `@IMAGEFILE@` to a name that will be used by virsh and the path to the image file you created in the previous step
 4.  Define and start the VM with your copy of the xml file:
 
@@ -216,9 +217,9 @@ information will be printed for the listed packages across all output-NNN
 subdirectories for that set of osg test runs.
 
 If a VMU-RESULTS-URL is provided, the corresponding run dir will be used.
-Eg: "http://vdt.cs.wisc.edu/tests/20180604-1516/005/osg-test-20180604.log"
+Eg: "https://osg-sw-submit.chtc.wisc.edu/tests/20180604-1516/005/osg-test-20180604.log"
 for an individual output job (005),
-or: "http://vdt.cs.wisc.edu/tests/20180604-1516/packages.html"
+or: "https://osg-sw-submit.chtc.wisc.edu/tests/20180604-1516/packages.html"
 for a summary of all jobs for the run.
 
 Options:

bin/clean-job-output

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+serial=$1
+
+# cleanup test and debug files upon success
+unlink $serial.jobid
+
+for imagetype in result input; do
+    unlink $imagetype-image-$serial.qcow2
+done
+
+for suffix in err out; do
+    unlink create-io-image-$serial.$suffix
+    unlink process-job-output-$serial.$suffix
+done

bin/create-io-images → bin/create-io-image

@@ -73,7 +73,7 @@ def create_image(config_filename):
     f.close()
 
     serial_number = re.search(r'(\d+)', config_filename).group(1)
-    image_filename = 'input-image-%s.dsk' % (serial_number)
+    image_filename = 'input-image-%s.qcow2' % (serial_number)
 
     if contents_to_filename.has_key(config_contents):
         print 'Linking from "%s" -> "%s"' % (contents_to_filename[config_contents], image_filename)
@@ -109,7 +109,7 @@ def create_image(config_filename):
         os.mkdir(os.path.join(image_directory, 'output'))
 
         print 'Making "%s" from "%s"' % (image_filename, image_directory)
-        os.system('virt-make-fs --size=10M --format=raw %s %s' % (image_directory, image_filename))
+        os.system('virt-make-fs --size=10M --format=qcow2 %s %s' % (image_directory, image_filename))
         shutil.rmtree(image_directory)
 
         contents_to_filename[config_contents] = image_filename
@@ -122,11 +122,12 @@ if __name__ == '__main__':
     script_name = os.path.basename(sys.argv[0])
 
     # Process command-line arguments
-    if len(sys.argv) != 1:
-        vmu.die('usage: %s' % (script_name))
+    if len(sys.argv) != 2:
+        vmu.die('usage: %s <TEST CONFIG FILE>' % (script_name))
+
+    config_filename = sys.argv[1]
 
     # Write files
     contents_to_filename = {}
-    for config_filename in sorted(glob.glob('osg-test-*.conf')):
-        create_image(config_filename)
-        os.remove(config_filename)
+    create_image(config_filename)
+    os.remove(config_filename)

bin/email-analysis

@@ -50,8 +50,8 @@ my_directory = os.path.basename(vmu.RUN_DIR)
 my_timestamp = re.sub(r'^run-', '', my_directory)
 
 message = '''OSG VMU tests: %s
-http://vdt.cs.wisc.edu/tests/%s/results.html
-http://vdt.cs.wisc.edu/tests/%s/packages.html
+https://osg-sw-submit.chtc.wisc.edu/tests/%s/results.html
+https://osg-sw-submit.chtc.wisc.edu/tests/%s/packages.html
 https://jira.opensciencegrid.org/browse/SOFTWARE-1996\n\n''' % (RUN_LABEL, my_timestamp, my_timestamp)
 
 if not os.path.exists(ANALYSIS_FILENAME):

bin/extract-job-output

@@ -63,6 +63,4 @@ for image_path in paths:
             # print "  - extracting"
         g.download(full_image_path, os.path.join(output_dir, image_path))
 
-os.remove(image_filename)
-
 print "ok"

bin/generate-dag

@@ -8,23 +8,46 @@ import time
 
 import vmu
 
-BASE_IMAGE_PATH="/mnt/gluster/chtc/VMs"
+BASE_IMAGE_PATH="/staging/osg-images"
+
+DAG_FRAGMENT = '''
+#########################
+# Nodes {serial}
+# Platform: {platform}
+# Sources: {sources}
+# Packages: {package_set}
+#########################
+JOB CreateImage{serial} create-io-image.sub
+VARS CreateImage{serial} serial="{serial}"
+
+JOB TestRun{serial} single-test-run.sub
+VARS TestRun{serial} serial="{serial}" platform="{platform}" jobpriority="{priority}"
+PRIORITY TestRun{serial} {priority}
+SCRIPT POST TestRun{serial} ../bin/write-job-id {serial} $JOBID
+
+JOB ProcessResult{serial} process-job-output.sub
+VARS ProcessResult{serial} serial="{serial}"
+
+JOB CleanOutput{serial} clean-job-output.sub
+VARS CleanOutput{serial} serial="{serial}"
+
+PARENT CreateImage{serial} CHILD TestRun{serial}
+PARENT TestRun{serial} CHILD ProcessResult{serial}
+PARENT ProcessResult{serial} CHILD CleanOutput{serial}
+'''
 
 def generate_dag_fragment(serial, combo):
     platform, sources, package_set = combo
-    node_name = 'TestRun' + serial
 
     priority = 0
     if 'osg-tested-internal' in package_set.packages:
         priority = 1
 
-    contents = '\n'
-    contents += '# Node %s: %s - %s - %s\n' % (serial, platform, sources, ', '.join(package_set.packages))
-    contents += 'JOB %s single-test-run.sub\n' % (node_name)
-    contents += 'VARS %s serial="%s" platform="%s" jobpriority="%d"\n' % (node_name, serial, platform, priority)
-    contents += 'SCRIPT POST %s ../bin/process-job-output %s $JOBID\n' % (node_name, serial)
-    contents += 'PRIORITY %s %d\n' % (node_name, priority)
-    return contents
+    return DAG_FRAGMENT.format(serial=serial,
+                               platform=platform,
+                               sources=sources,
+                               priority=priority,
+                               package_set=', '.join(package_set.packages))
 
 def write_osg_test_configuration(serial, combo, directory, nightly=False):
     _, sources, package_set = combo
@@ -90,6 +113,7 @@ if __name__ == '__main__':
     # Start DAG file
     dag_contents = '# osg-test run generated %s\n' % (time.strftime('%Y-%m-%d %H:%M'))
     dag_contents += 'CONFIG inner-dag.config\n'
+    dag_contents += 'VARS ALL_NODES run_dir="%s"' % os.path.dirname(test_run_directory)
 
     # bail if there aren't corresponding VM images before writing any config
     for param_file in run_params:

bin/list-rpm-versions

@@ -23,9 +23,9 @@ information will be printed for the listed packages across all output-NNN
 subdirectories for that set of osg test runs.
 
 If a VMU-RESULTS-URL is provided, the corresponding run dir will be used.
-Eg: "http://vdt.cs.wisc.edu/tests/20180604-1516/005/osg-test-20180604.log"
+Eg: "https://osg-sw-submit.chtc.wisc.edu/tests/20180604-1516/005/osg-test-20180604.log"
 for an individual output job (005),
-or: "http://vdt.cs.wisc.edu/tests/20180604-1516/packages.html"
+or: "https://osg-sw-submit.chtc.wisc.edu/tests/20180604-1516/packages.html"
 for a summary of all jobs for the run.
 
 Options: