diff --git a/.github/workflows/check_tpv_config.yml b/.github/workflows/check_tpv_config.yml index a94e5320a..fe78f44b9 100644 --- a/.github/workflows/check_tpv_config.yml +++ b/.github/workflows/check_tpv_config.yml @@ -31,9 +31,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install total-perspective-vortex==$TPV_VERSION pyyaml galaxy-app==22.1.1 + pip install total-perspective-vortex==$TPV_VERSION pyyaml galaxy-app==23.1.1 env: - TPV_VERSION: '2.3.0' + TPV_VERSION: '2.3.2' - name: Check TPV files and job conf env: SETTING: github diff --git a/files/galaxy/config/local_tool_conf_dev.xml b/files/galaxy/config/local_tool_conf_dev.xml index 328bb1801..1736e4705 100644 --- a/files/galaxy/config/local_tool_conf_dev.xml +++ b/files/galaxy/config/local_tool_conf_dev.xml @@ -19,6 +19,6 @@
- +
diff --git a/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/production_tools.yml b/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/production_tools.yml new file mode 120000 index 000000000..8c87a0031 --- /dev/null +++ b/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/production_tools.yml @@ -0,0 +1 @@ +../../production/total_perspective_vortex/tools.yml \ No newline at end of file diff --git a/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_config.yml b/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_config.yml index 03a4a2bb5..5a5ca9801 100644 --- a/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_config.yml +++ b/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_config.yml @@ -35,7 +35,7 @@ tools: toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/.*: cores: 2 scheduling: - prefer: + require: - pulsar rules: - if: 0.01 <= input_size < 2 @@ -129,38 +129,3 @@ tools: - if: 0.05 <= input_size cores: 8 mem: 30.7 - -users: - jenkins_bot@usegalaxy.org.au: - cores: 1 - mem: cores * 3.8 - params: - nativeSpecification: "--nodes=1 --ntasks={cores} --ntasks-per-node={cores} --mem={round(mem*1024)} --partition=main" - scheduling: - require: - - slurm - pulsar_paw_user@usegalaxy.org.au: - rules: - - if: tool.id.startswith('toolshed') or tool.id.startswith('testtoolshed') - scheduling: - accept: - - pulsar - - offline - require: - - pulsar-paw - pqld@usegalaxy.org.au: - rules: - - if: tool.id.startswith('toolshed') or tool.id.startswith('testtoolshed') - scheduling: - accept: - - pulsar - - offline - require: - - pulsar-QLD - pulsar_nci_test_user@usegalaxy.org.au: - rules: - - if: tool.id.startswith('toolshed') or tool.id.startswith('testtoolshed') - scheduling: - require: - - pulsar - - pulsar-nci-test diff --git a/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_default_tool.yml b/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_default_tool.yml new file mode 100644 index 000000000..56dee75ef --- /dev/null +++ b/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_default_tool.yml @@ -0,0 +1,18 @@ +global: + default_inherits: default + +tools: + default: + context: + partition: main + test_cores: 1 + cores: 1 + mem: cores * 3.8 + params: {} + env: {} + scheduling: + reject: + - offline + rules: [] + rank: | + helpers.weighted_random_sampling(candidate_destinations) diff --git a/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_destinations.yml.j2 b/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_destinations.yml.j2 index b6cd3e738..310ea099e 100644 --- a/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_destinations.yml.j2 +++ b/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_destinations.yml.j2 @@ -61,8 +61,8 @@ destinations: slurm: inherits: _slurm_destination runner: slurm - max_accepted_cores: 32 - max_accepted_mem: 122.86 + max_cores: 32 + max_mem: 122.86 tags: {{ job_conf_limits.environments['slurm'].tags }} scheduling: accept: @@ -110,8 +110,8 @@ destinations: pulsar-QLD: inherits: _pulsar_destination runner: pulsar-QLD_runner - max_accepted_cores: 16 - max_accepted_mem: 62.72 + max_cores: 16 + max_mem: 62.4 tags: {{ job_conf_limits.environments['pulsar-QLD'].tags }} params: jobs_directory: /mnt/tmp/files/staging @@ -119,6 +119,7 @@ destinations: scheduling: accept: - pulsar-QLD + - pulsar-blast require: - pulsar diff --git a/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_users.yml b/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_users.yml new file mode 100644 index 000000000..4851db0cd --- /dev/null +++ b/files/galaxy/dynamic_job_rules/load-testing/total_perspective_vortex/testing_tpv_users.yml @@ -0,0 +1,52 @@ +users: + test_user: + rules: + - id: singularity_test_user_rule + if: | + enable_singularity = False + if tool.id.startswith('toolshed') or tool.id.startswith('testtoolshed'): + try: + enable_singularity = 'singularity' in [hta.user_tname for hta in job.history.tags] + except: + pass + enable_singularity + params: + singularity_enabled: True + dependency_resolution: none + jenkins_bot@usegalaxy.org.au: + inherits: test_user + cores: test_cores + mem: cores * 3.8 + params: + nativeSpecification: "--nodes=1 --ntasks={cores} --ntasks-per-node={cores} --mem={round(mem*1024)} --partition=main" + scheduling: + require: + - slurm + pulsar_paw_user@usegalaxy.org.au: + inherits: test_user + rules: + - if: tool.id.startswith('toolshed') or tool.id.startswith('testtoolshed') + scheduling: + accept: + - pulsar + - offline + require: + - pulsar-paw + pqld@usegalaxy.org.au: + inherits: test_user + rules: + - if: tool.id.startswith('toolshed') or tool.id.startswith('testtoolshed') + scheduling: + accept: + - pulsar + - offline + require: + - pulsar-QLD + pulsar_nci_test_user@usegalaxy.org.au: + inherits: test_user + rules: + - if: tool.id.startswith('toolshed') or tool.id.startswith('testtoolshed') + scheduling: + require: + - pulsar + - pulsar-nci-test diff --git a/files/galaxy/dynamic_job_rules/production/total_perspective_vortex/destinations.yml.j2 b/files/galaxy/dynamic_job_rules/production/total_perspective_vortex/destinations.yml.j2 index a3df198b2..b1a6800cb 100644 --- a/files/galaxy/dynamic_job_rules/production/total_perspective_vortex/destinations.yml.j2 +++ b/files/galaxy/dynamic_job_rules/production/total_perspective_vortex/destinations.yml.j2 @@ -120,7 +120,6 @@ destinations: require: - pulsar-paw - pulsar - - offline # on loan to etca pulsar-mel3: inherits: _pulsar_destination runner: pulsar-mel3_runner @@ -159,6 +158,7 @@ destinations: require: - pulsar - pulsar-quick # tag given to tools where jobs are bound to complete in less than a day + - offline pulsar-qld-high-mem0: inherits: _pulsar_destination runner: pulsar-qld-high-mem0_runner @@ -236,7 +236,7 @@ destinations: require: - pulsar - pulsar-quick # tag given to tools where jobs are bound to complete in less than a day - - offline # etca testing + - offline # volume migration pulsar-azure: inherits: _pulsar_destination runner: pulsar_azure_0_runner diff --git a/files/galaxy/dynamic_job_rules/production/total_perspective_vortex/tools.yml b/files/galaxy/dynamic_job_rules/production/total_perspective_vortex/tools.yml index 0f3a237f6..94b100036 100644 --- a/files/galaxy/dynamic_job_rules/production/total_perspective_vortex/tools.yml +++ b/files/galaxy/dynamic_job_rules/production/total_perspective_vortex/tools.yml @@ -1,17 +1,4 @@ tools: - ### Tools with resource updates being tested due to OOM errors ### - toolshed.g2.bx.psu.edu/repos/iuc/biom_convert/biom_convert/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/rnateam/mafft/rbc_mafft_add/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/iuc/dada2_dada/dada2_dada/.*: - cores: 2 - mem: 10 - toolshed.g2.bx.psu.edu/repos/iuc/mlst/mlst/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/iuc/macs2/macs2_bdgcmp/.*: - mem: 12 - ### ### testtoolshed.g2.bx.psu.edu/repos/fubar/htseq_bams_to_count_matrix/htseqsams2mxlocal/.*: cores: 3 mem: 11.5 @@ -29,6 +16,8 @@ tools: testtoolshed.g2.bx.psu.edu/repos/simon-gladman/phyloseq_ordination_plot/phyloseq_ordinate/.*: cores: 16 mem: 61.4 + toolshed.g2.bx.psu.edu/repos/artbio/cap3/cap3/.*: + mem: 8 toolshed.g2.bx.psu.edu/repos/artbio/repenrich/repenrich/.*: cores: 8 mem: 30.7 @@ -71,8 +60,6 @@ tools: cores: 2 mem: 7.6 toolshed.g2.bx.psu.edu/repos/bgruening/autodock_vina/docking/.*: - cores: 1 - mem: 3.8 params: singularity_enabled: true toolshed.g2.bx.psu.edu/repos/bgruening/bionano_scaffold/bionano_scaffold/.*: @@ -80,7 +67,7 @@ tools: mem: 61.4 params: docker_enabled: true - toolshed.g2.bx.psu.edu/repos/bgruening/blobtoolkit/blobtoolkit/.*: # + toolshed.g2.bx.psu.edu/repos/bgruening/blobtoolkit/blobtoolkit/.*: params: singularity_enabled: true toolshed.g2.bx.psu.edu/repos/bgruening/canu/canu/.*: @@ -90,9 +77,6 @@ tools: accept: - pulsar - pulsar-training-large - toolshed.g2.bx.psu.edu/repos/bgruening/cpat/cpat/.*: - params: - singularity_enabled: true toolshed.g2.bx.psu.edu/repos/bgruening/chemfp/ctb_chemfp_butina_clustering/.*: cores: 1 mem: 3.8 @@ -113,6 +97,12 @@ tools: mem: 3.8 params: singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/bgruening/circexplorer/circexplorer/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/bgruening/cpat/cpat/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_bam_compare/deeptools_bam_compare/.*: cores: 8 mem: 30.7 @@ -132,8 +122,8 @@ tools: - pulsar - pulsar-quick toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_compute_matrix/deeptools_compute_matrix/.*: - cores: 10 - mem: 90 # TODO: delete this and get cores/mem from shared db if they are adjusted to 10c90g + cores: 10 + mem: 90 params: singularity_enabled: true scheduling: @@ -162,11 +152,18 @@ tools: toolshed.g2.bx.psu.edu/repos/bgruening/diamond/bg_diamond_view/.*: params: singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/bgruening/diffbind/diffbind/.*: + mem: 8 + toolshed.g2.bx.psu.edu/repos/bgruening/enumerate_charges/enumerate_charges/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/bgruening/flye/flye/.*: context: test_cores: 2 cores: 120 mem: 1922 + params: + singularity_enabled: true scheduling: accept: - pulsar @@ -186,9 +183,13 @@ tools: toolshed.g2.bx.psu.edu/repos/bgruening/hicexplorer_hicpca/hicexplorer_hicpca/.*: cores: 10 mem: 38.3 + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/bgruening/hifiasm/hifiasm/.*: cores: 120 mem: 1922 + params: + singularity_enabled: true scheduling: accept: - pulsar @@ -242,23 +243,29 @@ tools: scheduling: accept: - pulsar - toolshed.g2.bx.psu.edu/repos/bgruening/salmon/salmon/.*: - cores: 5 - mem: 19.1 toolshed.g2.bx.psu.edu/repos/bgruening/sailfish/sailfish/.*: cores: 4 mem: 15.3 + scheduling: + accept: + - pulsar rules: - id: sailfish_small_input_rule if: input_size < 0.2 cores: 1 mem: 3.8 - scheduling: - accept: - - pulsar + toolshed.g2.bx.psu.edu/repos/bgruening/salmon/salmon/.*: + cores: 5 + mem: 19.1 + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/bgruening/sdf_to_tab/sdf_to_tab/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/bgruening/text_processing/tp_sort_header_tool/.*: + mem: 10 params: - tmp_dir: true + tmp_dir: true toolshed.g2.bx.psu.edu/repos/bgruening/text_processing/tp_sorted_uniq/.*: cores: 1 mem: 3.8 @@ -318,6 +325,8 @@ tools: if: input_size >= 0.03 fail: Too much data, Prokka is designed to annotate bacterial/virus genomes only. Prokka will not annotate eukaryotic genomes nor metagenomes. + toolshed.g2.bx.psu.edu/repos/crs4/taxonomy_krona_chart/taxonomy_krona_chart/.*: + mem: 12 toolshed.g2.bx.psu.edu/repos/csbl/repeatmodeler/repeatmodeler/.*: cores: 8 mem: 30.7 @@ -451,13 +460,11 @@ tools: scheduling: accept: - pulsar + toolshed.g2.bx.psu.edu/repos/devteam/emboss_5/EMBOSS.*: + mem: 10 toolshed.g2.bx.psu.edu/repos/devteam/fastq_groomer/fastq_groomer/.*: cores: 2 mem: 7.6 - # scheduling: # 9/9/23: removing from pulsar due to large output files compared to inputs - # accept: - # - pulsar - # - pulsar-quick toolshed.g2.bx.psu.edu/repos/devteam/fastq_paired_end_deinterlacer/fastq_paired_end_deinterlacer/.*: cores: 8 mem: 30.7 @@ -501,8 +508,8 @@ tools: cores: 8 mem: 30.7 env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G SINGULARITYENV__JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G + _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G scheduling: accept: - pulsar @@ -521,6 +528,9 @@ tools: if: 0.01 <= input_size < 2 cores: 4 mem: 15.3 + toolshed.g2.bx.psu.edu/repos/devteam/flanking_features/flanking_features_1/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/devteam/freebayes/bamleftalign/.*: params: singularity_enabled: true @@ -649,8 +659,8 @@ tools: cores: 3 mem: 11.5 env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G SINGULARITYENV__JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G + _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G scheduling: accept: - pulsar @@ -668,6 +678,8 @@ tools: toolshed.g2.bx.psu.edu/repos/devteam/samtool_filter2/samtool_filter2/.*: cores: 3 mem: 11.5 + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/devteam/samtools.*: params: singularity_enabled: true @@ -692,6 +704,9 @@ tools: if: input_size < 0.5 cores: 1 mem: 3.8 + toolshed.g2.bx.psu.edu/repos/devteam/sicer/peakcalling_sicer/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/devteam/tophat2/tophat2/.*: cores: 8 mem: 30.7 @@ -703,9 +718,25 @@ tools: if: input_size < 0.5 cores: 1 mem: 3.8 + toolshed.g2.bx.psu.edu/repos/ebi-gxa/anndata_ops/anndata_ops/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/ebi-gxa/scanpy.*: - cores: 1 - mem: 3.8 + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/ecology/pampa_communitymetrics/pampa_communitymetrics/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/ecology/pampa_glmcomm/pampa_glmcomm/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/ecology/pampa_glmsp/pampa_glmsp/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/ecology/pampa_plotglm/pampa_plotglm/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/ecology/pampa_presabs/pampa_presabs/.*: params: singularity_enabled: true toolshed.g2.bx.psu.edu/repos/galaxy-australia/alphafold2/alphafold/.*: @@ -713,9 +744,10 @@ tools: cores: 8 mem: 69 params: - tmp_dir: true docker_enabled: true - docker_volumes: $job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/data/alphafold_databases:/data:ro + docker_volumes: + $job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/data/alphafold_databases:/data:ro + tmp_dir: true rules: - if: | user and 'Alphafold' in [role.name for role in user.all_roles() if not role.deleted] and not ( @@ -744,39 +776,43 @@ tools: )]) fail: | This tool is currently being beta-tested on GPUs and your account has not been given access. Contact help@genome.edu.au if you think this is in error. - toolshed.g2.bx.psu.edu/repos/galaxy-australia/alphafold2/alphafold/2.1.*: + inherits: toolshed.g2.bx.psu.edu/repos/galaxy-australia/alphafold2/alphafold/.* params: - docker_run_extra_arguments: --gpus all --env ALPHAFOLD_AA_LENGTH_MIN=16 --env + docker_run_extra_arguments: --gpus all --env ALPHAFOLD_AA_LENGTH_MIN=5 --env ALPHAFOLD_AA_LENGTH_MAX=3000 --env ALPHAFOLD_DB=/data/v2.1 - inherits: toolshed.g2.bx.psu.edu/repos/galaxy-australia/alphafold2/alphafold/.* toolshed.g2.bx.psu.edu/repos/galaxy-australia/alphafold2/alphafold/2.3.*: + inherits: toolshed.g2.bx.psu.edu/repos/galaxy-australia/alphafold2/alphafold/.* params: - docker_run_extra_arguments: --gpus all --env ALPHAFOLD_AA_LENGTH_MIN=16 --env + docker_run_extra_arguments: --gpus all --env ALPHAFOLD_AA_LENGTH_MIN=5 --env ALPHAFOLD_AA_LENGTH_MAX=3000 --env ALPHAFOLD_DB=/data/v2.3 rules: - - if: | - min_version = '2.3.1+galaxy2' - helpers.tool_version_gte(tool, min_version) - params: - docker_run_extra_arguments: --gpus all --env ALPHAFOLD_AA_LENGTH_MIN=16 --env ALPHAFOLD_AA_LENGTH_MAX=3000 --env ALPHAFOLD_DB=/data - inherits: toolshed.g2.bx.psu.edu/repos/galaxy-australia/alphafold2/alphafold/.* - + - if: | + min_version = '2.3.1+galaxy2' + helpers.tool_version_gte(tool, min_version) + params: + docker_run_extra_arguments: --gpus all --env ALPHAFOLD_AA_LENGTH_MIN=5 --env + ALPHAFOLD_AA_LENGTH_MAX=3000 --env ALPHAFOLD_DB=/data toolshed.g2.bx.psu.edu/repos/galaxy-australia/cactus_cactus/cactus_cactus/.*: context: test_cores: 10 - cores: 60 - mem: 961 + cores: 24 + mem: 200 params: singularity_enabled: true scheduling: accept: - pulsar toolshed.g2.bx.psu.edu/repos/galaxy-australia/cactus_export/cactus_export/.*: - cores: 1 - mem: 3.8 + context: + test_cores: 10 + cores: 24 + mem: 200 params: singularity_enabled: true + scheduling: + accept: + - pulsar toolshed.g2.bx.psu.edu/repos/galaxy-australia/hifiadapterfilt/hifiadapterfilt/.*: cores: 12 mem: 46.0 @@ -885,6 +921,18 @@ tools: scheduling: accept: - pulsar + toolshed.g2.bx.psu.edu/repos/galaxyp/rawtools/rawtools/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/genouest/braker3/braker3/.*: + cores: 6 + mem: 4 + params: + singularity_container_id_override: docker://teambraker/braker3:v.1.0.4 + singularity_enabled: true + scheduling: + require: + - pulsar toolshed.g2.bx.psu.edu/repos/iracooke/protk_proteogenomics/.*: cores: 1 mem: 3.8 @@ -917,7 +965,7 @@ tools: mem: 3.8 scheduling: prefer: - - slurm + - slurm - id: abricate_medium_input_rule if: 0.005 <= input_size < 0.02 cores: 4 @@ -977,10 +1025,14 @@ tools: if: input_size < 0.5 cores: 1 mem: 3.8 + toolshed.g2.bx.psu.edu/repos/iuc/bedtools/bedtools_coveragebed/.*: + mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/bedtools/bedtools_intersectbed/.*: scheduling: accept: - pulsar + toolshed.g2.bx.psu.edu/repos/iuc/biom_convert/biom_convert/.*: + mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/busco/busco/3.*: cores: 2 mem: 7.6 @@ -1073,6 +1125,14 @@ tools: if: 0.0002 <= input_size < 0.01 cores: 2 mem: 7.6 + toolshed.g2.bx.psu.edu/repos/iuc/dada2_assigntaxonomyaddspecies/dada2_assignTaxonomyAddspecies/.*: + cores: 3 + mem: 11.5 + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/iuc/dada2_dada/dada2_dada/.*: + cores: 2 + mem: 10 toolshed.g2.bx.psu.edu/repos/iuc/datamash_ops/datamash_ops/.*: cores: 1 mem: 3.8 @@ -1094,15 +1154,27 @@ tools: mem: 3.8 params: singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/iuc/dropletutils/dropletutils/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/iuc/edger/edger/.*: cores: 5 mem: 19.1 + toolshed.g2.bx.psu.edu/repos/iuc/egsea/egsea/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/iuc/ena_upload/ena_upload/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/iuc/exonerate/exonerate/.*: cores: 30 mem: 480.5 scheduling: accept: - pulsar + toolshed.g2.bx.psu.edu/repos/iuc/export2graphlan/export2graphlan/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/iuc/fasta_stats/fasta-stats/.*: cores: 3 mem: 11.5 @@ -1140,13 +1212,18 @@ tools: params: singularity_enabled: true scheduling: - prefer: - - pulsar require: - - gtdbtk_database + - gtdbtk_database + prefer: + - pulsar + toolshed.g2.bx.psu.edu/repos/iuc/gtftobed12/gtftobed12/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/iuc/gubbins/gubbins/.*: cores: 16 mem: 61.4 + params: + singularity_enabled: true scheduling: accept: - pulsar @@ -1179,12 +1256,6 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/humann/humann/.*: cores: 32 mem: 122.8 - # scheduling: - # accept: - # - pulsar - # reject: - # - pulsar-mel2 - # - pulsar-QLD rules: - id: humann_small_input_rule if: input_size < 0.01 @@ -1257,6 +1328,9 @@ tools: accept: - pulsar - pulsar-quick + toolshed.g2.bx.psu.edu/repos/iuc/iwtomics_loadandplot/iwtomics_loadandplot/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/iuc/jbrowse/jbrowse/.*: cores: 3 mem: 11.5 @@ -1275,6 +1349,8 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/kallisto_quant/kallisto_quant/.*: cores: 5 mem: 19.1 + params: + singularity_enabled: true rules: - id: kallisto_quant_small_input_rule if: input_size < 0.1 @@ -1314,12 +1390,23 @@ tools: execute: | from galaxy.jobs.mapper import JobNotReadyException raise JobNotReadyException() + toolshed.g2.bx.psu.edu/repos/iuc/kraken_biom/kraken_biom/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/iuc/krakentools_extract_kraken_reads/krakentools_extract_kraken_reads/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/iuc/krakentools_kreport2krona/krakentools_kreport2krona/.*: params: singularity_enabled: true toolshed.g2.bx.psu.edu/repos/iuc/limma_voom/limma_voom/.*: cores: 7 mem: 26.8 + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/iuc/links/links/.*: + cores: 4 + mem: 15.3 toolshed.g2.bx.psu.edu/repos/iuc/lofreq_call/lofreq_call/.*: cores: 5 mem: 19.1 @@ -1344,6 +1431,8 @@ tools: accept: - pulsar - pulsar-quick + toolshed.g2.bx.psu.edu/repos/iuc/macs2/macs2_bdgcmp/.*: + mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/macs2/macs2_bdgdiff/.*: scheduling: accept: @@ -1398,6 +1487,8 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/megahit/megahit/.*: cores: 16 mem: 61.4 + params: + singularity_enabled: true scheduling: accept: - pulsar @@ -1418,6 +1509,8 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/meryl/meryl/.*: cores: 60 mem: 961 + params: + singularity_enabled: true scheduling: accept: - pulsar @@ -1460,7 +1553,8 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/mothur.*: params: docker_enabled: true - docker_run_extra_arguments: --pids-limit 10000 --ulimit fsize=1000000000 --env TERM=vt100 + docker_run_extra_arguments: --pids-limit 10000 --ulimit fsize=1000000000 --env + TERM=vt100 toolshed.g2.bx.psu.edu/repos/iuc/mothur_align_seqs/mothur_align_seqs/.*: cores: 7 mem: 26.8 @@ -1672,6 +1766,12 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/multiqc/multiqc/.*: cores: 3 mem: 11.5 + toolshed.g2.bx.psu.edu/repos/iuc/mummer.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/iuc/mummer_mummer/mummer_mummer/.*: + cores: 2 + mem: 10 toolshed.g2.bx.psu.edu/repos/iuc/mummer_nucmer/mummer_nucmer/.*: cores: 5 mem: 19.1 @@ -1686,6 +1786,9 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/nanoplot/nanoplot/.*: cores: 3 mem: 11.5 + toolshed.g2.bx.psu.edu/repos/iuc/nextclade/nextclade/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/iuc/pathview/pathview/.*: params: singularity_enabled: true @@ -1704,54 +1807,54 @@ tools: if: 0.015 <= input_size < 1 cores: 5 mem: 19.1 - toolshed.g2.bx.psu.edu/repos/iuc/picrust_categorize/picrust_categorize/.*: + toolshed.g2.bx.psu.edu/repos/iuc/phyml/phyml/.*: + cores: 120 + mem: 1922 + scheduling: + accept: + - pulsar + - pulsar-training-large + toolshed.g2.bx.psu.edu/repos/iuc/picrust2_add_descriptions/picrust2_add_descriptions/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust_compare_biom/picrust_compare_biom/.*: + toolshed.g2.bx.psu.edu/repos/iuc/picrust2_hsp/picrust2_hsp/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust_format_tree_and_trait_table/picrust_format_tree_and_trait_table/.*: + toolshed.g2.bx.psu.edu/repos/iuc/picrust2_metagenome_pipeline/picrust2_metagenome_pipeline/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust_metagenome_contributions/picrust_metagenome_contributions/.*: + toolshed.g2.bx.psu.edu/repos/iuc/picrust2_pathway_pipeline/picrust2_pathway_pipeline/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust_normalize_by_copy_number/picrust_normalize_by_copy_number/.*: + toolshed.g2.bx.psu.edu/repos/iuc/picrust2_pipeline/picrust2_pipeline/.*: + cores: 7 + mem: 26.8 params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust_predict_metagenomes/picrust_predict_metagenomes/.*: + toolshed.g2.bx.psu.edu/repos/iuc/picrust2_place_seqs/picrust2_place_seqs/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust2_add_descriptions/picrust2_add_descriptions/.*: + toolshed.g2.bx.psu.edu/repos/iuc/picrust2_shuffle_predictions/picrust2_shuffle_predictions/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust2_hsp/picrust2_hsp/.*: + toolshed.g2.bx.psu.edu/repos/iuc/picrust_categorize/picrust_categorize/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust2_metagenome_pipeline/picrust2_metagenome_pipeline/.*: + toolshed.g2.bx.psu.edu/repos/iuc/picrust_compare_biom/picrust_compare_biom/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust2_pathway_pipeline/picrust2_pathway_pipeline/.*: + toolshed.g2.bx.psu.edu/repos/iuc/picrust_format_tree_and_trait_table/picrust_format_tree_and_trait_table/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust2_pipeline/picrust2_pipeline/.*: - cores: 4 - mem: 15.3 + toolshed.g2.bx.psu.edu/repos/iuc/picrust_metagenome_contributions/picrust_metagenome_contributions/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust2_place_seqs/picrust2_place_seqs/.*: + toolshed.g2.bx.psu.edu/repos/iuc/picrust_normalize_by_copy_number/picrust_normalize_by_copy_number/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/picrust2_shuffle_predictions/picrust2_shuffle_predictions/.*: + toolshed.g2.bx.psu.edu/repos/iuc/picrust_predict_metagenomes/picrust_predict_metagenomes/.*: params: singularity_enabled: true - toolshed.g2.bx.psu.edu/repos/iuc/phyml/phyml/.*: - cores: 120 - mem: 1922 - scheduling: - accept: - - pulsar - - pulsar-training-large toolshed.g2.bx.psu.edu/repos/iuc/pilon/pilon/.*: cores: 16 mem: 61.4 @@ -1833,6 +1936,8 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/qualimap_bamqc/qualimap_bamqc/.*: cores: 9 mem: 34.5 + params: + singularity_enabled: true rules: - id: qualimap_bamqc_small_input_rule if: input_size < 0.2 @@ -1842,14 +1947,20 @@ tools: if: 0.2 <= input_size < 30 cores: 5 mem: 19.1 + toolshed.g2.bx.psu.edu/repos/iuc/qualimap_multi_bamqc/qualimap_multi_bamqc/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/iuc/qualimap_rnaseq/qualimap_rnaseq/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/iuc/quast/quast/.*: cores: 16 mem: 61.4 rules: - id: quast_small_input_rule if: input_size < 1 - cores: 1 - mem: 3.8 + cores: 2 + mem: 10 toolshed.g2.bx.psu.edu/repos/iuc/raven/raven/.*: cores: 20 mem: 200 @@ -1906,6 +2017,8 @@ tools: cores: 9 mem: 34.5 toolshed.g2.bx.psu.edu/repos/iuc/rnaspades/rnaspades/.*: + params: + singularity_enabled: true scheduling: accept: - pulsar @@ -1984,6 +2097,18 @@ tools: if: 0.05 <= input_size < 0.5 cores: 5 mem: 19.1 + toolshed.g2.bx.psu.edu/repos/iuc/scater_create_qcmetric_ready_sce/scater_create_qcmetric_ready_sce/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/iuc/scater_filter/scater_filter/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/iuc/scater_plot_dist_scatter/scater_plot_dist_scatter/.*: + params: + singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/iuc/scater_plot_pca/scater_plot_pca/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/iuc/seqtk/seqtk_seq/.*: cores: 20 mem: 200 @@ -2060,18 +2185,114 @@ tools: cores: 3 mem: 11.5 toolshed.g2.bx.psu.edu/repos/iuc/spades_biosyntheticspades/spades_biosyntheticspades/.*: - inherits: toolshed.g2.bx.psu.edu/repos/nml/spades/spades/.* + cores: 20 + mem: 500 + params: + singularity_enabled: true + scheduling: + accept: + - pulsar + - pulsar-training-large + rules: + - id: spades_biosyntheticspades_small_input_rule + if: input_size < 0.005 + cores: 2 + mem: 7.6 + - id: spades_biosyntheticspades_medium_input_rule + if: 0.005 <= input_size < 2 + cores: 8 + mem: 30.7 + - id: spades_biosyntheticspades_fail_rule + if: input_size >= 60 + fail: Too much data, please don't use Spades for this toolshed.g2.bx.psu.edu/repos/iuc/spades_coronaspades/spades_coronaspades/.*: + cores: 20 + mem: 500 scheduling: accept: - pulsar-paw - inherits: toolshed.g2.bx.psu.edu/repos/nml/spades/spades/.* + - pulsar + - pulsar-training-large + rules: + - if: | + minimum_singularity_version = '3.15.4+galaxy0' + helpers.tool_version_gte(tool, minimum_singularity_version) + params: + singularity_enabled: true + - id: spades_coronaspades_small_input_rule + if: input_size < 0.005 + cores: 2 + mem: 7.6 + - id: spades_coronaspades_medium_input_rule + if: 0.005 <= input_size < 2 + cores: 8 + mem: 30.7 + - id: spades_coronaspades_fail_rule + if: input_size >= 60 + fail: Too much data, please don't use Spades for this toolshed.g2.bx.psu.edu/repos/iuc/spades_metaplasmidspades/spades_metaplasmidspades/.*: - inherits: toolshed.g2.bx.psu.edu/repos/nml/spades/spades/.* + cores: 20 + mem: 500 + params: + singularity_enabled: true + scheduling: + accept: + - pulsar + - pulsar-training-large + rules: + - id: spades_metaplasmidspades_small_input_rule + if: input_size < 0.005 + cores: 2 + mem: 7.6 + - id: spades_metaplasmidspades_medium_input_rule + if: 0.005 <= input_size < 2 + cores: 8 + mem: 30.7 + - id: spades_metaplasmidspades_fail_rule + if: input_size >= 60 + fail: Too much data, please don't use Spades for this toolshed.g2.bx.psu.edu/repos/iuc/spades_metaviralspades/spades_metaviralspades/.*: - inherits: toolshed.g2.bx.psu.edu/repos/nml/spades/spades/.* + cores: 20 + mem: 500 + params: + singularity_enabled: true + scheduling: + accept: + - pulsar + - pulsar-training-large + rules: + - id: spades_metaviralspades_small_input_rule + if: input_size < 0.005 + cores: 2 + mem: 7.6 + - id: spades_metaviralspades_medium_input_rule + if: 0.005 <= input_size < 2 + cores: 8 + mem: 30.7 + - id: spades_metaviralspades_fail_rule + if: input_size >= 60 + fail: Too much data, please don't use Spades for this toolshed.g2.bx.psu.edu/repos/iuc/spades_plasmidspades/spades_plasmidspades/.*: - inherits: toolshed.g2.bx.psu.edu/repos/nml/spades/spades/.* + cores: 20 + mem: 500 + params: + singularity_enabled: true + scheduling: + accept: + - pulsar + - pulsar-training-large + rules: + - id: spades_plasmidspades_small_input_rule + if: input_size < 0.005 + cores: 2 + mem: 7.6 + - id: spades_plasmidspades_medium_input_rule + if: 0.005 <= input_size < 2 + cores: 8 + mem: 30.7 + - id: spades_plasmidspades_fail_rule + if: input_size >= 60 + fail: Too much data, please don't use Spades for this toolshed.g2.bx.psu.edu/repos/iuc/spades_rnaviralspades/spades_rnaviralspades/.*: cores: 10 mem: 250 @@ -2080,6 +2301,11 @@ tools: - pulsar - pulsar-training-large rules: + - if: | + minimum_singularity_version = '3.15.4+galaxy1' + helpers.tool_version_gte(tool, minimum_singularity_version) + params: + singularity_enabled: true - id: spades_rnaviralspades_small_input_rule if: input_size < 0.005 cores: 2 @@ -2129,18 +2355,10 @@ tools: mem: 61.4 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_ustacks/stacks2_ustacks/.*: cores: 16 - mem: 61.4 + mem: 100 scheduling: accept: - pulsar - - pulsar-quick - rules: - - id: rule_stacks2_ustacks_limit_concurrency - if: | - helpers.concurrent_job_count_for_tool(app, tool, user) >= 3 # concurrent jobs per user - execute: | - from galaxy.jobs.mapper import JobNotReadyException - raise JobNotReadyException() toolshed.g2.bx.psu.edu/repos/iuc/stringtie/stringtie/.*: cores: 5 mem: 19.1 @@ -2308,7 +2526,7 @@ tools: cores: 1 mem: 3.8 - id: unicycler_medium_input_rule - if: 0.05 <= input_size < 2 + if: 0.05 <= input_size < 0.3 cores: 8 mem: 30.7 toolshed.g2.bx.psu.edu/repos/iuc/vcf2maf/vcf2maf/.*: @@ -2376,6 +2594,8 @@ tools: toolshed.g2.bx.psu.edu/repos/nml/metaspades/metaspades/.*: cores: 16 mem: 61.4 + params: + singularity_enabled: true scheduling: accept: - pulsar @@ -2395,6 +2615,8 @@ tools: toolshed.g2.bx.psu.edu/repos/nml/spades/spades/.*: cores: 20 mem: 500 + params: + singularity_enabled: true scheduling: accept: - pulsar @@ -2408,12 +2630,14 @@ tools: if: 0.005 <= input_size < 2 cores: 8 mem: 30.7 - - id: spades_fail_rule # TODO: Ask IM whether we still need the spades fail rule - if: input_size >= 20 + - id: spades_fail_rule + if: input_size >= 60 fail: Too much data, please don't use Spades for this toolshed.g2.bx.psu.edu/repos/peterjc/sample_seqs/sample_seqs/.*: params: singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/peterjc/seq_select_by_id/seq_select_by_id/.*: + mem: 12 toolshed.g2.bx.psu.edu/repos/pjbriggs/macs21/macs2_1_peakcalling/.*: cores: 9 mem: 34.5 @@ -2421,8 +2645,8 @@ tools: cores: 3 mem: 11.5 env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G SINGULARITYENV__JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G + _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G scheduling: accept: - pulsar @@ -2447,6 +2671,9 @@ tools: toolshed.g2.bx.psu.edu/repos/recetox/.*: params: singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/rnateam/chipseeker/chipseeker/.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/rnateam/mafft/rbc_mafft/.*: cores: 8 mem: 30.7 @@ -2466,18 +2693,6 @@ tools: toolshed.g2.bx.psu.edu/repos/rnateam/sortmerna/bg_sortmerna/.*: cores: 8 mem: 30.7 - toolshed.g2.bx.psu.edu/repos/saskia-hiltemann/krona_text/krona-text/.*: - cores: 7 - mem: 26.8 - rules: - - id: krona-text_small_input_rule - if: input_size < 0.02 - cores: 1 - mem: 3.8 - - id: krona-text_medium_input_rule - if: 0.02 <= input_size < 2 - cores: 5 - mem: 19.1 toolshed.g2.bx.psu.edu/repos/simon-gladman/velvetoptimiser/velvetoptimiser/.*: cores: 4 mem: 15.3 @@ -2498,6 +2713,9 @@ tools: mem: 3.8 params: singularity_enabled: true + toolshed.g2.bx.psu.edu/repos/wolma/mimodd.*: + params: + singularity_enabled: true toolshed.g2.bx.psu.edu/repos/wolma/mimodd_aln/mimodd_align/.*: cores: 5 mem: 19.1 @@ -2506,9 +2724,6 @@ tools: if: input_size < 0.01 cores: 1 mem: 3.8 - toolshed.g2.bx.psu.edu/repos/iuc/dada2_assigntaxonomyaddspecies/dada2_assignTaxonomyAddspecies/.*: - cores: 3 - mem: 11.5 .*bowtie2_index_builder_data_manager.*: cores: 16 mem: 61.4 @@ -2601,6 +2816,26 @@ tools: require: - pulsar - pulsar-azure-gpu + cellranger: + cores: 6 + mem: 20 + params: + singularity_enabled: true + rules: + - if: | + not user or not 'cellranger' in [role.name for role in user.all_roles() if not role.deleted] + fail: Your account has not been given access to cellranger. Contact help@genome.edu.au + if you think this is in error. + fgenesh: + cores: 24 + mem: 12 + params: + singularity_enabled: true + rules: + - if: | + not user or not 'Fgenesh' in [role.name for role in user.all_roles() if not role.deleted] + fail: Your account has not been given access to fgenesh. Contact help@genome.edu.au + if you think this is in error. gtdbtk_classify_wf_test: cores: 16 mem: 60 @@ -2644,33 +2879,7 @@ tools: cores: 1 mem: 3.8 wig_to_bigWig: - cores: 5 mem: 19.1 - cellranger: - cores: 6 - mem: 20 - params: - singularity_enabled: true - rules: - - if: | - not user or not 'cellranger' in [role.name for role in user.all_roles() if not role.deleted] - fail: Your account has not been given access to cellranger. Contact help@genome.edu.au if you think this is in error. - fgenesh: - cores: 24 - mem: 12 - params: - singularity_enabled: true - rules: - - if: | - not user or not 'Fgenesh' in [role.name for role in user.all_roles() if not role.deleted] - fail: Your account has not been given access to fgenesh. Contact help@genome.edu.au if you think this is in error. - - toolshed.g2.bx.psu.edu/repos/genouest/braker3/braker3/.*: - mem: 4 - cores: 6 - params: - singularity_container_id_override: docker://teambraker/braker3:v.1.0.4 - singularity_enabled: true scheduling: - require: - - pulsar + accept: + - training-exempt # these jobs are exempt from training rules that would reduce allocated memory diff --git a/files/keys/internal_hop_key.pub b/files/keys/internal_hop_key.pub new file mode 100644 index 000000000..98812490b --- /dev/null +++ b/files/keys/internal_hop_key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+HmTcHhYpvhy0J+h5/XUrWJ/S6/94Ju48B9NF+4cqdcIGtUFFK8mM5bDPGc36a7+NkH0JL+DoG1X7hbjX13B2kQTVVB5tvSIzsSpx79zTwxMSSQJIcZ0Z99GXclzt81KLQ0zCYnN8X42+Cq4mU9jaXUnnmFlcmjTP2XmB311VGqv3ujCdYjaoyqB9C8kP2/dtuMHXCDZ9urg9pGlkb/3354tehahbFgA9XazCe8sMGApXloa9xMHdiCkTZ0YXSI9bvyEQIuWmgZL5ZNjlF41hnYk/b+u/22xFlvetot5OMYOONuQN6UMlVwua2lakmO7Q8zWRLunGIBxfmuJceheT internal_hop_key diff --git a/galaxy-backup_playbook.yml b/galaxy-backup_playbook.yml index 1252c045c..5a335a58c 100644 --- a/galaxy-backup_playbook.yml +++ b/galaxy-backup_playbook.yml @@ -6,6 +6,7 @@ - group_vars/VAULT - host_vars/galaxy-backup.yml - secret_group_vars/stats_server_vault + - secret_group_vars/ubuntu_maintenance_key pre_tasks: - name: Attach volume to instance include_role: @@ -16,7 +17,7 @@ roles: - common - geerlingguy.pip - # - slg.db-backup # Do not include this role until galaxy production is in Sydney + - slg.db-backup - dj-wasabi.telegraf post_tasks: - name: Ensure desired version of ansible is installed diff --git a/galaxy-db_playbook.yml b/galaxy-db_playbook.yml index 5961263e7..7cf3c4663 100644 --- a/galaxy-db_playbook.yml +++ b/galaxy-db_playbook.yml @@ -7,6 +7,7 @@ - group_vars/dbservers.yml - host_vars/galaxy-db.usegalaxy.org.au.yml - secret_group_vars/stats_server_vault + - secret_group_vars/ubuntu_maintenance_key pre_tasks: - name: Attach volume to instance include_role: diff --git a/galaxy-misc-nfs_playbook.yml b/galaxy-misc-nfs_playbook.yml index bef1ecd49..0258cd261 100644 --- a/galaxy-misc-nfs_playbook.yml +++ b/galaxy-misc-nfs_playbook.yml @@ -6,6 +6,7 @@ - group_vars/VAULT - group_vars/nfs_servers.yml - host_vars/galaxy-misc-nfs.usegalaxy.org.au.yml + - secret_group_vars/ubuntu_maintenance_key - secret_group_vars/stats_server_vault pre_tasks: - name: Create nfs directories diff --git a/galaxy-queue_playbook.yml b/galaxy-queue_playbook.yml index 00be7bff4..78f01f2af 100644 --- a/galaxy-queue_playbook.yml +++ b/galaxy-queue_playbook.yml @@ -9,6 +9,7 @@ - group_vars/galaxy_etca_slurm.yml - secret_group_vars/stats_server_vault - secret_group_vars/etca_vault.yml + - secret_group_vars/ubuntu_maintenance_key roles: - common - insspb.hostname diff --git a/galaxy-user-nfs_playbook.yml b/galaxy-user-nfs_playbook.yml index 25313e55f..ba99c1112 100644 --- a/galaxy-user-nfs_playbook.yml +++ b/galaxy-user-nfs_playbook.yml @@ -7,10 +7,11 @@ - group_vars/nfs_servers.yml - host_vars/galaxy-user-nfs.usegalaxy.org.au.yml - secret_group_vars/stats_server_vault + - secret_group_vars/ubuntu_maintenance_key pre_tasks: - - name: Attach volumes to instance - include_role: - name: attached-volumes + # - name: Attach volumes to instance # handle user nfs mounting manually + # include_role: + # name: attached-volumes - name: Ensure galaxy nfs user data directories exist file: path: "{{ item }}" diff --git a/galaxy-workers_playbook.yml b/galaxy-workers_playbook.yml index 95e77dee7..4f6c16dbc 100644 --- a/galaxy-workers_playbook.yml +++ b/galaxy-workers_playbook.yml @@ -6,6 +6,7 @@ - group_vars/VAULT - group_vars/galaxy_etca_slurm.yml - group_vars/galaxy_workers.yml + - secret_group_vars/ubuntu_maintenance_key # TODO: when branches are merged, rename this file to internal_keys - secret_group_vars/stats_server_vault pre_tasks: - name: Attach volume to instance diff --git a/galaxy_playbook.yml b/galaxy_playbook.yml index 765903df2..1de09aa7f 100644 --- a/galaxy_playbook.yml +++ b/galaxy_playbook.yml @@ -33,9 +33,11 @@ - mounts - geerlingguy.pip - galaxyproject.galaxy - - role: galaxyproject.miniconda - become: true - become_user: galaxy + # - role: galaxyproject.miniconda # this role is needed for a fresh installation, but for subsequent playbook runs can take far too much time. TODO: fix this somehow + # become: true + # become_user: galaxy + - usegalaxy_eu.galaxy_subdomains + - webhooks - nginx-upload-module - galaxyproject.nginx - galaxyproject.tusd @@ -48,9 +50,10 @@ - geerlingguy.nfs - galaxyproject.gxadmin - pg-post-tasks - # - remote-pulsar-cron # commented out until galaxy etca is production galaxy + - postfix-mail-relay + - remote-pulsar-cron # - delete-tmp-jwds-cron # commented out until galaxy etca is production galaxy, also needs to be enabled - # - slg.galaxy_stats # commented out until galaxy etca is production galaxy + - slg.galaxy_stats - galaxy-pg-cleanup - galaxyproject.tiaas2 post_tasks: diff --git a/galaxy_update_configs_playbook.yml b/galaxy_update_configs_playbook.yml new file mode 100644 index 000000000..c205afe4c --- /dev/null +++ b/galaxy_update_configs_playbook.yml @@ -0,0 +1,59 @@ +- hosts: galaxy-handlers + become: true + vars_files: + - group_vars/all.yml + - group_vars/galaxy_etca.yml + - group_vars/VAULT + - group_vars/galaxyservers.yml + - group_vars/galaxy_etca_slurm.yml + - host_vars/galaxy.usegalaxy.org.au.yml + - host_vars/galaxy-handlers.usegalaxy.org.au.yml + - secret_group_vars/stats_server_vault + handlers: + - include: handlers/galaxy_handlers.yml + tasks: + - name: copy job_conf file + template: + src: "{{ galaxy_config_template_src_dir }}/config/galaxy_job_conf.yml.j2" + dest: "{{ galaxy_config_dir }}/job_conf.yml" + notify: restart handlers + # TODO: better linting of job conf, then it is probably entirely acceptable to let the jenkins_bot trigger a graceful restart of handlers + when: not ansible_user == 'jenkins_bot' # do not let the automatic process update the job conf + # Previous process would update object store, toolbox filters. TODO: Find another place to do this + # - name: template object store conf file + # template: + # src: "{{ galaxy_config_template_src_dir }}/config/galaxy_object_store_conf.xml.j2" + # dest: "{{ galaxy_config_dir }}/object_store_conf.xml" + # notify: restart galaxy + # when: not ansible_user == 'jenkins_bot' # do not let the automatic process object store or restart galaxy + # - name: template toolbox filters + # template: + # src: "{{ galaxy_config_template_src_dir }}/toolbox/filters/ga_filters.py.j2" + # dest: "{{ galaxy_server_dir }}/lib/galaxy/tools/toolbox/filters/ga_filters.py" + # notify: restart galaxy + # when: not ansible_user == 'jenkins_bot' # do not let the automatic process restart galaxy + - name: Copy local tool conf file + copy: + src: "{{ galaxy_config_file_src_dir }}/config/{{ item }}" + dest: "{{ galaxy_config_dir }}/{{ item }}" + with_items: + - local_tool_conf.xml + when: not ansible_user == 'jenkins_bot' + - name: Install dynamic job rules (static) # dynamic job rules tasks copied from https://github.com/galaxyproject/ansible-galaxy/blob/main/tasks/static_setup.yml + copy: + src: "{{ galaxy_dynamic_job_rules_src_dir }}/{{ item }}" + dest: "{{ galaxy_dynamic_job_rules_dir }}/{{ item }}" + mode: 0644 + with_items: "{{ galaxy_dynamic_job_rules }}" + when: not item.endswith(".j2") + notify: restart handlers + - name: Install dynamic job rules (template) + template: + src: "{{ galaxy_dynamic_job_rules_src_dir }}/{{ item }}" + dest: "{{ galaxy_dynamic_job_rules_dir }}/{{ item | regex_replace(regex) }}" + mode: 0644 + vars: + regex: '\.j2$' + with_items: "{{ galaxy_dynamic_job_rules }}" + when: item.endswith(".j2") + notify: restart handlers diff --git a/galaxy_update_slurm_playbook.yml b/galaxy_update_slurm_playbook.yml index a056d7cd2..2c56d86cd 100644 --- a/galaxy_update_slurm_playbook.yml +++ b/galaxy_update_slurm_playbook.yml @@ -1,4 +1,4 @@ -- hosts: galaxy, galaxy-queue, galaxy_workers +- hosts: galaxy, galaxy-handlers, galaxy-queue, galaxy_workers become: true vars_files: - group_vars/all.yml diff --git a/group_vars/VAULT b/group_vars/VAULT index d33d5fddb..bae822b44 100644 --- a/group_vars/VAULT +++ b/group_vars/VAULT @@ -1,430 +1,441 @@ $ANSIBLE_VAULT;1.1;AES256 -66343736633237643664383138363633346638363363623930656366393638646239666632356161 -3130366233323238623036313334353561643464393164630a663962356665626231393534363332 -32383733653463366165323834633031373938303366646238623538623533376138306338313831 -6533353636363834630a663665623462636137376331373861653364653063643637613561623363 -36363665386365363737386139356263643134306336616131623063666533356634336562346239 -33646465353464636431373165636531303062306563343165656235613036393966383433316463 -32653165343933313132373438656432616564383833386263623734363462623463333632643238 -65383939376639383432646530633437656132666536313537656532376633303966623463366431 -30346539633334663366343934646334366131346363346331313039663530653133636233626633 -35383665663335343238343564633130633230353932366563663337373433346530666331623663 -36623735386339303633346237353431666130383731313164313361393135333366663230663934 -39323031393339356636396362666234303330366331623239393266306137323264323231336262 -36626161316131373535663961643762623330306632653831663134343632313938656332376435 -31333862353065653338656631386565346338306666366639393631653031623231353732313135 -30646338666530393433313835653738613164316361303365663965633139616635383661623366 -66623038356436386430333337373634633161343765316566393739663165313639626532646366 -65653539656365323465343661306630366639396139303635366564343561653166383631313532 -37656361363630613264346162386363633264346235343439666233333830653131613961303362 -39313237633734373430376232383436316133343936386333363365313034363633383563393630 -66353562663766363732386365623966323738656663383861323835613430333065353538663132 -39343036383932643461643062316566653836396436343365316539666138383130393535323734 -30623239303266303862613437386162353930613266626163643035336332333462313364626565 -32336461363937643264356438313034623066323334636337646661343232323236393032343564 -66343364383034636337323066353463383331623638326538356463343739623863396139623861 -32376338613832623331363937613561356537303762383639336232613334326666626636313162 -32666236303163396139323733343462613636646336353761616239383563653461383032343931 -65326634346538616466313835623562626132643737613838646233653831393836386262353830 -35613164363864633133663864616161663039303436653033396538343930666335353265393832 -64373334316465386661613435313465653839643830646335393763333639623866306466643635 -39623061393335643135633766323439373165326332333537633137313837633838643861373832 -30313732323538383361336464616265316632636562633031633333636539383663636663353462 -62663137623861633364393033626637353265353464613038643261643666303634633366613835 -34306236613665643138373937646435623934323837636634323665323331636139303166653761 -33363034336230346338663638383031383763313338336562356433646134333865376136656532 -39323038633431323562336235623538616237623264343830353763323863343932353737623737 -36383032366339313534626137316438663565663130316538643234653737316439333339626665 -32396330316235626463326162666630636637306166646435336435616132346361643038303539 -64363864633639646538363237636166316430626330636132666237323735386565396162363166 -36393565343533623233373764626331363735653835633736626635393736613761323032663361 -32613335376634626363306263336637643562613262393734646635656634643533336665343837 -38383031643536346636363634376139666435383838366166303966373963366535366530653034 -66306234376530613962656462623565393866303965363962633066633733363235333134396239 -61326236616461333732303664326136386334633662336163643936393464383565373635616438 -62336232333562393365346433363763396666373536356137386464323836643266386665353030 -61623235336135613335623634666534383965333038633864633636633036653838656230653566 -36366436653233336635643463666364323361316466643938373835383564626337623033356266 -31316539623065623730666433366230376364633564363266633664653530333333343339303034 -39623665363230386132323961323237653738666634666133653135666638306533366234326434 -32353130346165386232383338393832313763396632663161343139313833323137646462376136 -35373431303064396136353136376263633438356230313934366131396631616566633038626564 -66643935376234353765623634343361313464303738303736343238656464326362313738323964 -38626633383331623037633264653635383636356333366666616630393139363962643935343466 -37623432663261353630373130383765343764313166363261643764626161303135376165626536 -65386234623634663066363666663839353361373733393539613266643262366564663431343639 -63653566333863393531343661643664643130343530313230646432333464633137346532333831 -63343539633962313830366437306564656263623134333864336432336666393663383465643030 -35363563633361323737653462313165346539663030386161303135363039613131663234353834 -33346266633233613135666363323635626666303032313530626635653763613266323134666366 -32313863616162663236626336313966356664306239323166326364646131383066376463613366 -38393366303939313361333636393634326265366136396132653036303731336531326538356463 -31653533383866653135323765326464306364363630363562336233336531323965313962393333 -37613863616330656336623332346364323364326539626534396464376236663830656431613236 -31333531363438326162623537343934313961623830653335636266316130663261613463373431 -64393465323430393766656433386637333030646236646339613431636137643733363931336537 -39646464633434656366626636386361353833623734323462396635313765343136666661663639 -34656664633062656163303733646333373530636335633130636465313731313364393264333365 -39363636633936303963373737346234666634636638626634356461633965653261633230393936 -64393630626161313336343133626135636134343863353665633132653231353539383132373134 -34336536326234373438373334316338643564646236343561353839363461333862316563663530 -31613265636232643665653531313730373030373131316233346663656262363731666264373835 -62643461323861313037656632343533613931663437656433336230633664393235663034653362 -38666339366636386434653530373533633537623065393934346664313533653933333834626565 -61643936643433663232636465396130356138383531306263336530306239386236363034666264 -30646632393164373239313434396537353730643965373635366663666634653163613565643064 -38316539356433313263363561343332646231366237356337323839313163373766336535376635 -65306433623434656237303931623737383737343432383265323330373963383365326639333137 -37313835626564316438636531616462333265663662626633643266316139653466343535363231 -66333266363237646333383335323166386139633432636566393961653737653135623433356532 -39613932346163336264633836373864303835353934326533646531636532373435363362396561 -63333462646362383961313332643233633837373036376532303533373866636438333931376132 -65626538333736633331336337316538343230326537333066393363356633653561306631613238 -35343466626134326239393065616266353065343932363737323565383435313566623636346535 -30303437633162376362326133383239396665613331336537613965393866633332376539383437 -62626336663535353931373231346365623236636561303933656136333266323662376630666264 -38316330643432616332316130313466663064356136613237656437616165663961376562613137 -31356532363164626365613838313762626230636364363036643031363663333431646534643038 -30356465303561376439313366643266383166336365306530326465656435303836663830386436 -34636164323635323236333130343964313431343236383630383836333762316165393533306335 -61356165303231376230343139663930353430313834366334616131346565326465353239306330 -31653763666537646431633966336138363135353735346231373733313966663437343463376633 -30366331646263336137303633313565313033343162663137656538343765363734356535326136 -31646561626539643532393533303531326532343963646135313131626639653763346530663931 -63323366316433646563393838653837393035646631333034383630393632396633666437643566 -36356432323365343935326631653039393234633463616665643930346234653034656361346631 -63356634623030663838333561323338626539383331346131653137646535646163386664663965 -37643232633264353032383030323335633637616364313430623137326330666261366436363630 -30316264343536616537313464393437396464376664313639353061306666346631623336666632 -64646462333964396337316337613732633430373763306234666664333437616534386131623939 -62323136396531376362306339306235303939653633356338303139336264623266643239343936 -37316266306535313432623365303432373136643466313164383833303562383235323864363635 -34636231633837303165653538633336663934313830633038373334646339303865656131346230 -63626336653064363237636436303237356265383130363534636438613666653232336437303139 -38386438386237363233653931316461393231363331396163376339663435613862356565313833 -32626633366237343861343331346434626239326136623534303561373134393739663631303139 -63643662356236653136313330613261386162373031626334366437386266313031393561623661 -34663764666364356231306363626361303330626639633939663335613331616163343266666162 -33336236353862626531623736313630363931396139616534346534653538666665613936366635 -62666437643935396635656534653265323635343336303932653565376263306364366564663734 -66316436393865323039613865376330336339313266386538346564376263633436643435653962 -38366636363535626566653631383265653338353138333139323832373565626164623538656531 -30333030303131303536613833643330333731393365323264306234356265353435353531326565 -35336133306161353363383762623536626164313830616561666439666630363531373038613764 -35346631396434646432663236663733663039303535336533646533313563393534336661326563 -34636566313639396434636236666463316639303566653533633662346335373638333665343065 -64313035313437303238323334616662643138623963383232306230656665373430643130353233 -34316535363137373235626631323934643738383136653232373361333164623737303730653135 -38376132633133366634323030306361623461373838316131653235323034326132393837643966 -35373064666638333134343862356239313035373863333236656436393331386436386463383561 -37666266663932653533626362633333663239653330343335343736663661373439663063333863 -30626662666564353537346438363839656333666336313461303264656464313539336563346536 -64323233633038323837363364356431633538636132656232363730336339306138626331636230 -33383561626130646561303234393830666537323565363764623238323365653332656565336561 -30306365653134373765313534336434336135313166373939613765303863663763353865396331 -30353734323065613331313938633331393531393134326136313236393638653435353838383234 -63376438363031626231393739333130323936393336636663333233613264373837646333373735 -63636337313539383165636362616366323463383332336232633532383431376635373839326461 -37363362386638663530353933383938336231346538383966303133303130626564653636313465 -33346164303835356566353865323962363636613834356461363834613434373834336332313766 -62323532323365333132616161316334353837313864366331376139383464323334636263393539 -63646165663136363138376436323734316531326131366363626138343966646433316230666563 -38356462316432313261643037636161623361303639346334623630336261623736386136363638 -34356633383062656139626635616133376561626263323630336633303861616634623466343331 -34336633386566313664323465346431393733616434623533363063333032643464613737333936 -36356532663232373436373762313561663364336237353762303831326230393234343931646437 -31386232333035306332306134613636333038343239353462666261383033323665326330383638 -32636665613933626665303966313361376134653264383732663837313330616464636634303562 -66623838646337646261633031366135373730393865373261656134393636386565393565303362 -36316631393163396337326639323961326230316336313965636130396530353764656263633538 -35623937346336613665323562393161343831616336333337613237636163613962373661313734 -62366431663563623630633036613539396230303837346262366633633533326266333765376662 -31386365636264356435656262333533323264373934393831383539636534326566333339656234 -33383366396165343234373764343939383564353265393431333832386232653463656232656334 -37303661656538383536656338613963353938306535653761306266643562333133363166613933 -61303731306361393432373934666531663165343963646462316435343165373363623162666631 -62396637383165616563626331323836343334653763323738616266336562346335393863316536 -37313434396161646231643636633364393131353831636431386361636165613464666261356261 -38333033383962383033343462643430323430636633353735373530396234353937383836386634 -38313062316639366463613738643261343035646437663632323764343538626461336537666334 -38303237386261633431353261646336616635383666343264306334353166363631313561396535 -61363935336166353536646562343838653232316262346434613466666330616565343265326461 -31626261373461343964323236313966383035383366663231666465656232343365363438376462 -39306236633631376330316166316663366634366632643638363666333933316430346335356139 -30663233663538363334346564373664326538303361393166643266353233396537366364396664 -30653739626162663066633835323865643634323531616566316465356264343061313334316137 -39393637666161663565306331343033333436353934373336623331666562353563393265656661 -32363766656538343631653465656436306332613963626134383434653064383761373465336330 -30613035663362383962366234626264323561613936656164303536353039643364316461626139 -36396365326434366632353564373032326236356230333136356366636462653932306564326663 -65613736666464663263333563663432393564383937373661653264326563373936643661616433 -37396561656161336439313433366536313432633132323666656466373132633833623936333163 -38346537633830653863303061373034333061383761666161663434663131313938303030623235 -31663665616266346136313336343238653736363832616439376663646539333132363431343130 -63376264313234633735643336626264353966333439373465643364393262353165303137376531 -66626461303463623463323261376232616532613039333730366261666565633133356431396431 -66366265346630663862373234383564663861333830616262373162666130386461363134313863 -34393438653065613738343965393266663134316666623330333339666433653839626165376461 -66393037646336666337643536336231666262643931346566346238323962613835316337613532 -32663435396462373330663333366230666234633965656663633135636434386466383830653530 -30616661663063643336663933623835653233656135396462616664336463303661383461656666 -33613032313662366630626162393330376231303862626536633434343633343933386539313739 -31323937653632333636323135613063303163613034633036346431366333333432643139333836 -30646131323330383866613732363039303065363436666365643530633137646433323435333131 -37643263646331656337333835346633363332383164393034313938343565356130363738376132 -39613366323861633338653961353533643339643361333866363730656466363662343631306562 -35643333623235383864656630316363663365613331373262303731333837343535653333643263 -63363732653330333335383561373537316361333963646636346330626462393266386134613430 -61616531366537623438396433356566303032303366626564393832623865613863343234336137 -31363161656239616163636264373739396264613634646332393265633734616338663636633661 -34663836393164353638326336323564346539313562666339333164383437616338383536303464 -33376235386164363738366661383764613835363631363839363839316632353936383330303461 -64383236353064313338393335363135663164363937626339633939363136633362363630323364 -66343665333063616230366538343963656261656631306131636163396234313433333062386466 -32333538653832326433363135613461363230346163626432343836303034643539363836323732 -64353830646636363736343337313030353261643063343065643462373836376461303036646665 -32326235346464393166333437336238653563663635326532656265363466376130353366396336 -38656432363433363030366239313439626337363737373432646439346135616635336639346334 -61616162343138363564333336383265373535623763386563663865623466386239353766386436 -31336162393364313830376566363238313833326230316632313466643838386339383034333930 -39343266383032323162353735306333376662373162356339316331336432666439383565386633 -38663137333839636264653637313530643336316437663832353066653732333434363234346134 -35633866626562343738613238376464366563333137383132386637626537326231363533626233 -65313932353739313237666331303865383965353166666162336336376266326239656137666361 -35643739643064643738316638303737386234336130373262356563393363376635613632666463 -64373634373935383561666365383533616465653062343637306639366237366130336535373465 -38646334343036363263666238393062643132393137393864316362353135313136366138626536 -30303736653130313437643966363362363436336161313734373937323063313664333931323733 -37313631653037363265356231393439373666353430343835376133323962643833656533303832 -64613938336233333938353639356265306362613666626234326361616533653030623464666564 -31366431616235313636356462366164353436336662303432386435386462653430303434333836 -64326261666230393031393438323862323238346133326161643135613133653732623166363235 -63643832353230663961343863616661313862633933316262393731383431356463633163323366 -63303766353462366437326434393837633136393265396438313962393565393238356530313831 -35313033633065343230336434623531643332363230313532303161356664393437383063373066 -61386663316631653731353165613631666530326339613136653164396661333431306364366131 -62646466656338666231616237623633616232383062613266356565383463653531633736396131 -63336164303361356137336164613331393934656463343837393933396438306665646261653362 -38336637373134626266363734323462633838633132616237353439343361666138373436303630 -31613637323330643435636664386435326463626239633438636436303338636363343062363963 -31386239303939373861343237356637363234363665623337313434336636313433633062636366 -65343131636363333962313065336563636535656438353133393863346133633130303532626331 -61306331323265323764333138643032303439346164356632366361353762383162613632306564 -35653433656232303166353265653961653636303839396631663335353333323832393635656636 -36616663343430643233663665343866666631646337333264396662616561376566666536323431 -33353333393366323865363562363162666332393537333638373732623866326639356639333032 -66393234613463306533366666636262613832333463313565623536393866326664366338346664 -61326263316462323066666665636531633235613766663633303638346631363333303563633463 -63353734363533386331333235363565353164326139396565323264353465363663656362336531 -61633866323936633736643061353836346237326432666639633033616162313032343639383430 -39613066663939633131623331363533646230636432363037303832663435393463613330316365 -38666562343664386330623139363264363339396164353638623363613033326562653665616266 -38333438383661663336376335316136393766386436306334373536346332333331303932343930 -36323739363639643638393339616261363363363031346564643862636164613730343035626363 -63376433633364313231663137646330376361646664303231326466656565363032393934313135 -32633134653732656138396364313762656465623162363236333862376230633932653661393333 -35316239323664643930366630336163643330646165623530323531396564663766303938386431 -30643936653333316238626437653035396631376134633162656366393638383136636334636263 -65653030353231376133663466393832633166333163306163313564396139343333383732366266 -30343533373939383433616466373431376434653135323632383031333166646431323864616537 -33393632653438656565323635323262313832353864363139643162333062663963303734323139 -35653139326661333766346331303366356665303537343364356133663930393530633865313634 -61653363303765303739316639346135636466363062613538666130366430393133616436613039 -61666265366361383536343933643239633538316162363631666365363763623938626663626162 -63373462323136373938646531336264373565366466303036336433323538393434353732303838 -31316462636530613734623030646630346166626430636236326635616337643438613238646239 -33343933656365353561633365633632663830643532636662623534323665373935326431623537 -32623534366462353234633535666535373634333762613433316633376665333337623830663734 -36316465373862636162303736643738633962616138306464653062646230356134313334316665 -61356466646630336338376566363538653832633363356539633036656330656130336463613265 -37343332613265373239383433313430663232643564376135333739666165393836656133316662 -38663037653732376332383564656335386234663534313536633936373066393330386335343139 -64616330346639303061653436613431663732383030316533343237316533306332326663313132 -30616239366233356235316638633030353431383134663761643364646361306432666161346430 -62343066356339633033373135626438303730613935353433633835326339613737343334396137 -33373132383466646463363038613634353134343839663465383330353733356463333039333265 -38333932623765353237656330623239653565663562346137343036656530366533363138376638 -63653962656338346439303966316632373435613938366339343661616233633933636634353062 -34373738383130313861653762616166323132366264353833343930326363383331383366343833 -37356436383438316230386437373863383061626664626132383837383439303032303437366535 -38336536323164373334336430393839666433613734636235303634326266323230643463393464 -65616161613236613263376430343038383766383162383335363636353263383461373862646236 -62353931386331363165333434356664333937303966373163353138613135356434343361346334 -36643532373232383464363130306438616162613631613564353665336164323734643534396233 -33643031643566383262633339313933313161623566333735396163323835373735613538613635 -31653836616239356266323837663136643465396261646463393832336131303834373161383636 -30373030373332353332373963663536626439303135653362393539616236313337353865646435 -65386632373037613139323036373163663865356161333138383661616230663532313232356662 -31396163353436656265316265303263313564623266343963656233333537383432343337386135 -64653034323166326530343563346130313063613563336235383961323265613561333066353862 -36353638373334373234323566333339303234333539363761666562646638326662373261383261 -37626437373964393166326438653366393230326533393030633833316239656238313062323938 -37396337303932323438346366653334626361656437383236376235643331633531326666336239 -30616631336365343166363364333038386330613266613934363236626633636166666330336364 -36323135663837333165313936303734643265643035656462356566306364333738386337326633 -33306538366337316131323466356662636639323732333836326439306433316434393830333231 -64353237613363323439383135353634316436336338663333363836346538633736353262643538 -34373134313164323638373161643132303933313364323830623365633435656430616332363739 -37326363396264373762626535396539643964376137616338353838313539613931663664366362 -66323230316431626466626631626532383131396363633734616533393663383364616231363364 -63346335636238396335323163633366366663356634656265636666323163623037303138383663 -33353434663633643433613833636265363363373565346265376231633431626236333439383565 -64393032636662363966376532396263616466636464383136383538326231393434386430323865 -35343239333732353732383765346334376164383638363137633136356233373534313233383239 -32633338653033383562356137663931633833313132343531323665613261376237386265623466 -37303034643431383166313238333563393364353832613063623939666231616165356131653364 -64643532316236303561393933386565326631646561333734616333343139313035333762323135 -39323063633666313434623961613936386337343734613964643261653066353235303465313762 -61613133303430353064303936626664613231373239336630646261353336663733353838316535 -64363033663232653461643635303133333765303639393630306438666437623362306532353239 -36343335303063643532353734353865363936653466323635653533313530623163383231323764 -64313835373733663436336562343539643630306431333666343065303537363062643034383162 -37373134383363623133653131323062653332636132666434323233386661343861323132643534 -38343863376434376232376430343464643737386234363533396663353066383062386462396463 -36373132366565653361383138636633616333633265616534333936333234623264653934393861 -32663631323635633565663237353630316565323366653631626539393336316664363763326239 -32303631643436343530353635646263363735613836623563373661303533623566353231393138 -66386263343334653530366339616538326239616462393334633564386165616530316639623264 -64613961323263366538653266393431646138376462393861396431373133323036336538313634 -36336630613163313464353230306432613562636535376635373730366561363034636332633834 -34633733633938396365396132303339636661656563363538656266323132643166376463316631 -34383738303835313432636263653439343232353436663031636337386336623431333239343061 -30663131323632663263633432383538313765666365656436613135646566326135346464383261 -36646137386238396237313966616237323366323463373163346539376266666432666464666639 -31316537613539396337346536356363396131333835663733386662393338336239316237303436 -61396139643434303336666539303830376230613435313939313731333961313633343736613437 -36653963393333623531613037373838626564633262343530393166346332326363376266626130 -65623530626239396662656563646139663461666364383965383531643732336431353630323436 -65646362303335643864323166353961316265623161666335396163373330373932613236626133 -66316664363239343838623835343466666231613433643338376464653035333133323130653731 -38663035633135353533313730316166343163393931376632393431653332333538393934343565 -32363238653136623337663338326461386335623538326134396439646136626364623739626631 -62633961373039386564653536393866343366643830326436393536626433653832336265363066 -64633132393530343630313764326230343136373136623036633837313262393937626135323630 -39333031643364303361636433613563396331646135633834643337633934333061306362303532 -63316364656231663134343035663164323962326363383164376437623636396464313061386531 -31336161633864343463663136643234373038656163393336393130343838386532363865633736 -34386430383234343262303231303933623539326537353430653465376464343333356434623136 -61633062386264376434316632313264633239666362636264333739653034306666633031653263 -37356164626164346531666434363137636665366435663261353365393432663938396661646330 -36383163613763353938306638636433356263356365333631333437643536303036333036373264 -65323637623063643363613832313866396662646335313338313837333635386331393232346339 -37663431383230393266303930303038356334336662363562343430636163383265373437356439 -63656234313961306433646132333462366134326534613935616635643761613036393637666532 -62656562373935373864626531303630383466396435633139323035343331643035376665613932 -35323939333632653832616138393336336164613266366334613435353237653335666364326436 -66313365323334316239313262393139626535646265353062616231356339383366653561363737 -31306339343430613963383763613461316466346163373935636464633439393336366562323563 -37353264666265643936666662646139366564303066653139306232313562326639393036396562 -61623662336531616663316539643863393130393335326666313831393033646436333734366466 -35346433363965653261613332396539376433663066646435633633616262386565306666303633 -36376630333463663336656439373230643937343162363731353738626430653730393662616435 -65333636363565613261363937363531333331376364343530346637636336363130343462653764 -37366666613436323939663531363731396462373531313166353562333439336230373338366663 -63636636346636613738623938633264626531363664613734343563366361396539666561323464 -33666266333761326239343265316161636639386364323962326162303761643762653164663830 -35623032653634653763303132646537633366656630666234323539373235303736626666356434 -30613364336137316464373431323232353565396435313630366633643538336365643737373264 -36356538666661333862613732383535303062346466336230623764393231323031636264316366 -31396165636565663537636139633032313666396462633266643334393962303339346362306236 -66653165393538656365333634383931303134633539613335636331306638333336646533666538 -36653233616339613863376536366262396238383034633962353464353865373731353562363632 -61336635386563356330346337323939373663633362383533326434323563396139376430373639 -65313537356137323230623265383533306462623636646432653362623332346566663034336537 -65356636656332333061346538393064623033623162656561333734376331643862356239623564 -62393166343138353365653763313762393866303730653934336432636135643963626532623365 -36343639313934313439633935323032656161643764316639616539363566356338363338633437 -66366330346632393833383738616165383130343464386535643630336238346237316661353631 -39613331616233363138636563313932643230646539363761613835366637353238383332393031 -65313766653265333162666135653037313932373561613333336361356564326362313230666562 -39333261386330636663366537366465613331636234306534353861316562383963623738336165 -65623138616635306431383064643162336664373536316261643066343932333338656134376134 -38663036393534346537333534656564646465316639373536303363643137373134393266646334 -64373938663563326438643630366261653834323464373865356666326631313366323032656332 -66386532363634363533323133396434643437653936633264653831326136303038366565336231 -31663464313061396138623130356366646539636239306431326535376136653235353135383333 -34626133653866643064303635336334396533393434373063633831313062386438373465303930 -39323962343362643333333334303533666233376239623139666263383234373063643161393231 -62303436323339666330666230663238363530316638313562366565613638393636663265383439 -65663364366332396465636136303135313036353238653663666266613435666238313330613839 -39623535633438636430326230623136633635393730303861373038373632343936316131333532 -38313563623033343865613433653065333765633432666437393562623131343439393666643137 -35656164666437663538633534653532613133336539353261393236333830666131646139616530 -35373262656130366166613930396465623465323234653864656235656433646365303832343234 -61363030633065643631353962356166643837363434663061333366633562653263623063643238 -31333031366363653130373331316337343065333138383239666537636230303262353338383763 -31633566326361313734383738313930346633633437373363356339373762326662383062316465 -36313365663038303866666264656163336262316435613938613831356565303830646363316132 -33343365633839623062646135366565633038346662393961373863666162306665653734313732 -31353936353238613930363339346431366333626265393663303336383362663266626430393832 -63323234626163663639396637336133656230326632656137373132633666356636313564363263 -33636438633232383462663534343733376137616338303939343832623631373161353665623662 -63373363636564663866303165643861383535666336633638353730376432376333393962323663 -66666562636431383839326633616164303439646531306136353136303532656364333038653163 -65386161613230386134306333323233636663376430633436663561666430633534313633373730 -62653236323965316431356565336236323330316164326662633932626566363263643232303764 -30653030323561383032616334643962613931346563636361383334363930616530663763363238 -61323265643633336166393537343031653730323432383435356462366638313937616433313837 -39336663346237616664363737383138656533376361383365303534376139356363356363356461 -33353638646561663262626331656431376662343038356634643138343136663134663231386663 -66633664353063313464663131316438313137643238346666666461383932376230663761396131 -36363966303937336637313333326333656663383535303539386638333634376337333431336434 -31373338396333363431613233613837623733316434363263346135383732663565323439323366 -31373930653638346433313561633730643063393966343632303561353332303336326632363030 -66323864313439396364613635376665646630643033366266373039346630313530393463313433 -35313336613465383636656531373833373539653036653764306231383539303732356333383930 -39656638616135646362666365383638636234333537353738356330613662313362373339616532 -30636230616334376661363637333939356461366161613130393161363138343936613434356161 -34653962613536343732653434316362353339353739396632363139323163333739663437626163 -39363035383838623932633439333730646438633234373634633730333361383837326533336232 -61616439316639326535623831346231373265316533343636663238613831333166343265393163 -64353238386232666533613165646563666262633638396134646339656139626132343561396335 -37333862363235323435363034343365653534376161663236353965376566626230323662366662 -33356237373539643232653332333964626164303436373136646264616230646433653964666535 -37303861386635323931343834343236656239616639626166333032656637383362336633366632 -63666630386532366232303238653962353362376534306365643031653661643539633733626537 -31303831646361663862363636343736666130333136333431633362613630316430376431313836 -39383733623164623465636463396430646465333238356263363032633665376564323736653834 -31346162396163363731646435333065366638663661323931323238333563366436356139656133 -66363561646132376562323466633765353833393463343761636136636531626530303337333734 -66653164666261376166313961343963303063653464653962393063323532623539613165613633 -62323339653666343034636230623338343765663362353435336539353231316364353631343762 -35366233353565393530373936326135323931386565643235643662353666663835343533633732 -63643935333961353130336465313237643765643335666533376438306264663838363038613732 -34663633636130613830373666336662383761633738616263636461653666313831383164653965 -31663733316534633631383439633462333233666239306535656562396435336533386430343230 -33663332366537636635376465386437356634326238333631663862376331616631303263653136 -38636633366434626532376331376532666138346163656662303239623862366165393434653065 -63633737396331343364653032633231353435666131353762303432623262333662353465363963 -62383566353264323765396134396165633030306462663961333861376436333164383465626162 -63656435646562623166376138666434323237373864613965633030316232633130323433346632 -32666231336263373334643435346132636432633761653966626130633863643234633931333138 -66316531383838636666653830616461343166643361316265353965383962633034386266376635 -35633265616339353735306365346338356161303535306337323236653061613632363863643963 -36376632616666313230363431653563646139316332333763383730343530623865653232306632 -30326363393963383661643038353464653636353939383835383665333630623436313434303533 -30626239326233663264666139616337346132636239343031663861623837636164316336396130 -64363337633435376363376565313530626635633431306265663662623361343464343138386363 -30363638663965363631616663306439303963646439366461616165373662353735643831373537 -31633962376339613665386262653734346636396239336561303765373837623834633238616661 -65366564313765633132643165623134663031353161613931656130613130306635653838383831 -35373966643663363830303161323739636565386237626364303164326134363337386431333334 -36363235383734356164396238336465313033373236323039363733353862636532626636323732 -31633334346138316635353538343336396335313233383938323866616537333563613738323938 -31646632646366663463363638646533653162616634356633386536303739366664373464356332 -31323938396561326138373666343532386135653463653738386366666138393735623334626236 -64366361316264386464386231383333376561656365613337386136376566643737653336343163 -66633862366436636431386261316336386539373931363832646137376130623433656262643433 -33336530356430646330323334623332333134626338636361396336613764616535343039626432 -66396462356666636565646135653261353961633435356136313635353138636265653366313364 -35343364646137353634633836363663656438656334373636356361633465613762653962303939 -66343962356261366162333764643636643336343539643139366234373764393162303737663762 -35393463393533316531303934353837623938316466623961663637303663336538616563656437 -33653033333866623231616235616265303436393639383461623230646464636534333134393231 -36623161633837363263653964313664373039663937623732666365323165623065656235393536 -31343135666264313966383232363531353539313861653936323662396635396661613632613832 -30336561323231643834393033373261393139393434303962643332376533363731616537643838 -64333434356434666336653532396266626438326636643963626638363237653633636661333930 -34396536666536313439376161666232303634353266353162613930373339613739306533336433 -38616365383639656230363037313264373163643761373661313134643336646162653135323264 -6562 +31376564643938383138656638663930346137336363333637343162366534396438303831333832 +3136626431613632666332336537303364313763316361620a646265666331376565613961636466 +62636239633530363237393831656536316336353132313666613863383866643465366462663864 +3261663436663939330a343364613039623739383532623333353337386430303666393537653238 +35393361626463643464383435626266373930373631623237623662613361326532356435613332 +38623062666362333839373938336564653739613435373537616365326337333561623937313961 +38356336653036353139326439303535323835353630383132356131396363393064303538666236 +61313936396334373337663461333438663030326638303034316436643736613264323765343730 +39666136623930656363373233646264646630356637363338623265363238623932366166346265 +33633661343636666163656465386632376530383561383865383964643039386634363239396363 +39363830316464643564666230393065396131626464386461383237663632646533613331303165 +31333836366665366238633237386366373665383534623862363335316533646335653364643737 +30633539656133303039623837326136303464353264336237633936333334653237396665616436 +30326463666432633864623230333663373536316665343530346638396165356135613532663431 +66353437396162656133326138363361383037373036306662323237306431636263303763623466 +35353335383261366638636535323634303764333466323730363562336664313263653634626632 +61313864626139623638373864336563613734333832393832346438623662653330633036623131 +32383665333130383839333964616331333031343236333830663934646464326463386562353235 +33333265383935356430653964333335383435623437386338323735376465653263376162616135 +62646164396564383139643163613831656332366564383466346235386230636665613336613331 +33613533666236636334633366336537376163336633346666313661626462623832333531386361 +35323562313437643538396565323031333166353535376566613739653839383064623634323230 +34383065373037383631303036366132346131386437353663383663316237633534333534383430 +39666238353932353365363132306637393638633463323561353535343731626462303236373034 +65343365383937396466316461363232616430653063333334633532666235316534333037623764 +37613432396439663931373661333430373665383038326364366433633464343736633530353837 +36303161666131386135346432656439336535653563643637633566396332353835633561323535 +37396437613138663631653861336265376362373766383934326439643563346334363661373733 +33643430623865376337346534326238613361373936303433653336336137303961376463313361 +62356630353239363834363164643261313964313938613133306162326464626264383363326163 +34343330643833613262376631306338303439656230376238346664323264643364373131373133 +38643639643635343131626163613430643965633534336530356631343738623139393031653336 +62666361366531666166376338323264633837326438343662383061616163666166313038366434 +38383230336435386434643565643037336562386439313239303532363236396164376435643661 +66346366323563623436373237383965363262613961386165613538653838353032633632303539 +32396337303961313334346464303630353139343730313035353039383734343130326664646362 +35633839386233393165623766316134396639646135386435396561633365663732396363633933 +64633136616434353631393330316636353432393139663963393265643338393537323739623936 +36313939326533626236643638306136363062343565393465623135666565373061373664373466 +64613536646161313736633830636336363863323830636136666534663364376636613764643630 +64353034313934383533646236333835663863316534343465653666393634616236373935306666 +36313665356161356238643035373334643837616161303233633839396530653161333437613430 +37343563306161646238376338393635366235623434616436643665633465366563613365333132 +37353263326636336233313032323534363838623066363963663463313538383665313336303033 +32393865323266663634626466376461663730316635316462356530623639623135303930313737 +34363262636564393763616535353134323130666265373837313139373462393263353861306536 +31306462313339666332353961333833336166636236373434656630336366666338643536353135 +65393438616338303364353230363432343332623532626661663466333962326164303132333566 +35356639633432396536376134333735653264333733356462353137613666636165356234623334 +39643633383463633132333963653330306636646232396636373538656463653530653936333439 +66626531633534613435363366323633323462616537386230353336386435663061623361653938 +62656139623266656531316261363464333464623761663463386230386463643333623730653866 +36633563336430633263326463643334366331616365336463653536633366393930356563636565 +36626166313839323066306561613031323335323362353538303937396630663566303630346133 +30376235373039303138386334393034643034383333313835376564383637653139306530343732 +37323030306161323932366562653630613832333566343232626133346631326237626338313636 +36636631343038393831636261356463373765313732633731623862646238633266383334343731 +32623130356434396162353536376361303232626439356563396638613530336461333330326239 +39643434323963356633663131386663373034373261373435613537626266396531323763336539 +33316662366233303964623434363536323164373161383663303230623531326433316238663633 +64366533303364376662663836376334383264646533333135376338353663313637323535313734 +61343564383531373831336637393531633064396533383534663637363938333362343561646133 +30343665346130663430393932303737333162373632313831623037316139363161616135303634 +37646561663537633465326361376534363966353033306164376638353861353465306431333535 +33306264623132346639626237366662663232663664663831336137393838623939316438323033 +39373966336366303635373939366161363866363232396330396133316463343462633162326564 +30313330376536353137363262343738323234633861613561663037616338333166393862396265 +33666264323264643430326230393738623138356634396432356331303861333762653766346337 +32653839656635303930663665626132366464303164396665636337373666353936623338363137 +66616131383231306461643833663731343439626335623737383637613561373138633435653234 +39316265626265626230616239343432653166663963646539633830636564663034613236353961 +34326536666332313161393066633565323931383532333432316533356566313066613230313266 +34316331656532383666623933356236353831356239323463663461656132376335363634636462 +33616132613035343163393430633466666361373563306437343733383836353264346230643562 +36626233613734393639323233613833363237333232343531643130643431343533373633383033 +37373738303239313063363362636536386533623963623163633737653762386261373065326338 +66393838383532396264663362613836316238636633353263383465346637356432613863363332 +39663562343438656336363337303931663239653064393739363938643934383138376465333735 +63336633613637643538393534663231633763616364363132326133663431323363646638326238 +33613736653430386366653163623733383130376137333563326334636163353461656266326261 +39366330353238333732656236333236656636386234343966326162636637623035363866333730 +31623064323564356333663934356465383039613964323930393538363830623130663931333064 +34306235643964613139376139656661316666306466626465333261653766306364346363366638 +64626138623965646362626166646132313661383961393761633862393335353664306464646435 +37653265613061616237346235316431636665363561613363336432343235373238333631373932 +37363633356233643662383766663833316337396236306131336332616431646164353635653830 +65316535363763393434353666636563623631613436666566313233636263363561633630356633 +39393332623439383732326630393666376531653161626662326235313934376435353732613431 +61623935633264613762373464313333656266356461303135326237633039323133636261393063 +30663330313633333935376166613437373837386436363861656239353133656331656230343837 +38653363333566636663323966333635323236336331386235303963373034643939636339623065 +38323261376434393533643732366239653663613865323135616362366264336334623066393862 +33663161616231653439366331333539356631333937343261393435616634356531623434663535 +63383130623566623332316138363464643762623432623130376561663130396339363137623339 +39373830316333616233383633376636666565313433333239306336366538333835653863353230 +66363861623939376264326165323334656661396535393334373230656566386132666138376132 +63386463333764363165636631376536376334643234343963346464643834383039653564323834 +35383733663066656462633962303134653066646664396465326535393537363139306533663234 +31343161653863386566636131393239656430303464313462336337306539333239376335643362 +62616337663230653630346162353134353832343363343338346565636363303438383030373037 +31633530333762633737303136346238616437303830383466396261363231643831653539323563 +35333336303762393066303264333062666262373864663934623361383632316134366638366566 +66623838376464323831363566383530393861613133353561366565633262373239623863656264 +36616233303237313239356662373461653566383364346534326136663832303836663934643930 +30626630663530356364316261633332306465323831623230656134643639383064643465333663 +64353335366339613036363162303264386531663133663538386332316637353632643466316661 +61356664383738613037393834643066656463343239623330323266343136313130393564383761 +39623661623064643935333530333365323838306537333335643635323131656366666134366566 +64386438346432323861366139646364336161303938396532636166643234353231666138336633 +63346663393731343963363931313937643563303631623738646636643661323930326635396334 +61346164636238666438343064323061326234343364333836303233623637633438383461366437 +62633463393336353539373433653435633863336439663961646633373839306334303062346531 +31353135643230646466363861646534383064306530656432353139323232623336646161333030 +39633530633337383337633538323363663766343238336238353636386239333939393966643133 +39393562393666343961306238653463633431313466376139356136376433613036653230663166 +36366637333662366132653464663831653063663836353038326364633065373930343632653638 +66386263363765616131383964316662656238613935663136636136343332313837376663646466 +36363933653063633433613834333964613136383438303138353635666234336366336539353064 +32323033336262666338373564613263336663663865303839663734313636316561666432393134 +31656439653237343633373262343837646533626332373435303339623166393537353737386539 +39323861306561613430363534613434376133636662306565386366626661336463333437623261 +31313138313738643361373366333533633463326436323739613932623039353766386236626366 +34653161663263363466396265643662303937363966363730663535393831393536333737623939 +33623362616339653538346331386639333662646365363538373638383131643434663839393639 +65626561613765353163316663343938643837396235613930366635326362393236326466393733 +61613164613832666362343732653530643033336334313165383564633563303830343137363237 +34616638303564356263663034356236346235646564333033303162363761313234323935386133 +62623766353431303237613536373064363661366331396537306431336563303736376531646364 +63333335613230363366656433353066343866313230613834343634386132323637346462316332 +64303035643030643861363435333732313464656135376534373064663762623261316231633530 +39656439323338363563333732396265303961333761383566383238616364613134666331656636 +35626163393438633434663831306533383234353937643933393837303539346463303838316464 +62383134663137306435663261386261656666376236383837323231666436316332376535643163 +35336262346531343539343138373339383265393838616365333038353834663561636435633636 +63643065346365613266656633386464393631323561393237623530626463326233623233313436 +37366232396239646436316233363031373936373462616635323434623233663733663465366462 +33623231616435326337643766666639313536356564376364656631366537653532313031663536 +37613434663733353630393930343166393864343462316138616130343235303765383239363639 +62363634356663383736316437643434613338623362386562313937396365323630643061393864 +32336432343132336136383137393932396631336666343062396335313561656662636435323832 +39646535323538353035636634643039323162343932373563356233393561633737613937616139 +30303433366162616565383561306430393037383862373335386431333465303238633464633238 +65383237353061653461643939383862343035336262336133356366333130616261633636303533 +37343762306265663363613132383961363866366339376234313731616135666430643833613237 +66383833356637633334346339303664653662646635653035616132633938336237653239366636 +64393262613531323335636637656639656634316336636536633138303630666264366662653562 +38373335616438313937393962366662626139616632373133653135393231623364386133336565 +62363130646633326561623963336533323636613363346363613462383933343039303865646533 +38326637643839396461626132323864303562666261356663303065346538303038373562613236 +39363732303465323136366335333435653139316534653430353937653365356663343134366463 +30613964333662346134356664636163613366323762346463633463636566376463613330313937 +33306439353336393436346234333235663435663265643936353366343233663134343364623364 +61616662373139656430643037336537353137303666333434643330663937333631353430626639 +39336636633163396235333539656531313437653961363936623664323639633239636533666161 +36393535336338663362363361363533383237333538613933303134303831373663633461633738 +31386132393762386138623838363663396637303737633165323263643836303238306661303866 +35313831336666646631343431353939353733396462343966333439646666663633393434663665 +35373335636333333966383233353930643533623932643431333161636662313031313838333134 +63363534303966633564383537356666313630613738636265386637316130646635303836636466 +62366365646135393666326534646466363932636464636538636335363566353936626339633530 +35383065626139663864376438643435623635633237323763383635643236643636343763666266 +34313066633036613065656233363563636264613234623965373434633765343935366132313962 +37373036316362366433663362643738626133356366336166383135353664313563323030326664 +35636533336133623539613662326434663565643165356539643137393235333530363637623263 +34663832333130653465613831323936343165663561626461626163393030626535356437386461 +38616331393130313930323765623930313333316132323861313135613163316239343734656432 +34643166663436383834623963623636636638383664346565313462323362393137613566303034 +64623864343939623739633064626564613334633966386433366631303862663533333437313634 +32346531636530373964386238376630313733366133323766626661333065613336343831393866 +38663432346434646237663732366437643265346634333666333538393664666532336635366632 +34326633373165393465396135343734393035383466393937313936653131656265303661386633 +31306561396562646433613330393463366338303563336637303562313332346531663734313130 +62353166653161613435633930316263373235353961343638366632626665626535656638366265 +63353639326232316434323639313234366333613864323935326461366230303730656638636239 +63333933353264633233303839333138353234323636616237373131313066623232323130616230 +61313238653632613034376565366362633266386261393561313061333032313538393333653062 +64326335393862643461643537316433313866616133663339313361386562623762616161346530 +65626539616336363964663638666661343461623737666433366536613736636461663564376533 +37636661636634613138386338303432326634643861313962666138373065636165633865303235 +35376139666534643237356631633632336138353461373830393461383737393164366565666431 +37323663653537383862343731346462303138366237623935633130373633373433663663623138 +39656565373932646235393933613539303538343435663534353365666263313964623965333739 +38356133353961316666643832306332653235316639646330386138623837373332356333643839 +32643564383135333762356235386434626166316439623736353662303337656637333661393936 +34636164633337326636376530383064333130383634346231336434666232623062643065356263 +35313232343339623939323435653738386262633264663433646365373837363536623833643430 +64363630626535666539393666363861373662383434346230616366386562313738643837623862 +64353730666265613238646631633065303036333732653939343266666133623532363136633536 +66303862653139376439343537323939326536303330326666636434633364626535353831323836 +35323464353162306338396464303638666664663130336634616162353331316531636231663937 +37653666393736393063393063623139343238313735316130653230653062333139373464313566 +63303934613737386631623365353936396162666530343331386333326564616166303438653065 +35376234316637366535363337356630623235653764643530306362303665366663353131623630 +63366138633733613039666164336630396463643261616537323466363162313331316365356239 +31383564663237333735306664633131646233383764643564316562303764346430376631363235 +32376461386132313062323161346664373434633536633666376333383534303237656230376530 +36303039656337663538656635653762656532363834333438623665643631666663643930343837 +64393138303231646130663536373938336539663337396639343732373366633939383535653262 +37653234373639626362623033336330376563643030653537653637383865366633633336316532 +36653632333035633731313261373237646536323737613232656432663364363338646439346432 +30346361396462333132383339386566636537376635313438333966613332306332383466623637 +64373430313131346664623865383562383238383337396235623530316539646136316431326264 +64356263323865356166373137363637323930363035313666363232626633306434383036316137 +30333837313239313465386462343933396134333330623564353735313432646464363633333539 +35306264643161333966633735663462343434326535306530333137666535306333396531666639 +37336539363132373666366464636137383736396364666630616130313939366139613263353430 +37633739393237323765356635643966633464656666306534353764373764623764383832343632 +38636538326162613236636462313038396235386231633236333237333165613338363531643134 +30376339306639626238363964366664396363666636653536333033336637653637613831383761 +30316534613962333634356361383639356437663966323764323536346631306137626234376231 +38656361646132396138303432333633356431643164656439346462363539613132663330616165 +32306633383965363164343966323435366264383736303730656638356339663331343235666566 +61303962326133666331643065323838343238646138343132646236373766326430346561316261 +61653937663364383934326432623463663434613161653865346362613232323265363931336262 +64323133623064386231353534363636303131663433353433343764373462626363663531663933 +37666263383461616635663261646237353133323533373536616261626230666266376332346662 +63653232613264323231393632353539333436353663663633643130656434666135323033356561 +63333538616438363434346435626235373036636563396463313536363830663663353334316433 +66356332616533653532656265363963613665646465333363666333356333363361303938306262 +37313638333035343661623563303035346333343965336563613263663231643035623963303432 +61323534386534373663646266353234613938313262393437333562666362616438393236313365 +64353538326561366130366438643336373065353735643963636538373332633563616638386431 +38326161663265656562306535393861643630323565666338383263363363353166316331363866 +36346633636366613138373966663134386338633966623565626631376133353233386232306339 +62366136393561383537303234373465353163663137343538373630323935636638306365616334 +64366261313537616338663265356433636462623866616236313730613035323733366265313561 +31303266666334363335386137336161383536356337336161666239343231656533323939636664 +64643331356136316465363935643265333934376461663738643063646261363033306439313065 +64316437363265366334373166383639393966343534393333303638626432363865323931373463 +66363466633263313361333262336162623964333534366335363135316236633634303232626463 +37643936333963326466313234643663363866383865356436346235626637353566393836343439 +30613638646439663037306634316132663938643062333830633136316635663336333837633435 +30383365363663396530643465613661353332343033653263396334383838626631643961353338 +37383233363333616365636330393938633632306365303863643666383730313661383133383530 +38353065376238623339316437653363646631643461396633333765663232616464363938306462 +37646663323939393463653736333366613634653239363961343837316563363938623363666130 +35356431303466323566653732336635336435656537643833643931613963626634643962633339 +61383461303262623439396138303137643934613531313730636361326434653439656532376464 +30376563383262613335306231396334613938393461633761613166663562643331333530323737 +30316235326430653932323963653738323365323163613230303263346463653262386130613436 +39336666656566326134363337353035313536303230343230353965636139633630623334373038 +63343063376366343631386631343638323437646233393861386461373332636533313330376532 +65383432313765623534376136323463386431326631343530633437373933633938326437663736 +32386664663963313337616130626239306335343735303138333637346361623934643030363638 +65326433363331653137333265313265653830663431333836333363376266653763653061613537 +62333830663464396337653165383661393864316632393135656332653134616564383535333432 +62316231366635386131326639326437396438316437336661623566303635313762633032363533 +61653531333436303931396337663835666136663865373838613433653435353536633335666564 +35393233396337303536313163396135326134313961336163356134386530353834313466346431 +63383666626537386464306137633537643265323931316637333337663365313034323339396166 +37616532646436656264333834666434393966643963643637386635333463323338643262373939 +61636334353262323033666632636532313032646235353562356535373861666438363537393035 +33623234396266336234376161373761653838393938623337633034633930373465303961346466 +39316633646636326564356235336135306139653765363834623063393462366566323334336237 +35623534623462343832396532633330623231363865623861626561653663383761383539653330 +61383334326233636230373838396163323032623231646335343832346633383638663832386264 +37353430636232303033626232396166633565393135343561636533336664623633636433393132 +37366664386364373063333962383962653062663434623430313737326465626666393461633330 +65343365396131386565343930333733643062393333666331663631353439616564396534383532 +66643935643530333531393261323163373730616233353164333361323432306338663232326665 +38373363386134646235383065323065366664306663613038643264653336303735316237373730 +63376437623130396564323836656631353731366463623732393664336139643666626633363038 +30333262313561373366643163663065613564633866323563646163383334643861393230386165 +37616235653866376566393266383763333666396134333732616364363630653532616663353366 +30613036316639613134633665333662363961653365313839323564633137336333306266356137 +37613633313834643738646530303938353566316138393361623931666365306534366464366362 +36646364336337653132343439313466613362626131643836626461326466373364613535306231 +31616637663339313030376233633038636162616363646162356537373961333436653965316132 +39643634636639633865393035643832343063663638663334353736636163623332623832613334 +38353133363838613134643937633134643030386165323639663332333236376162646663303461 +36313264373132386332613131663463343162396236386536646130376461613533613730373864 +32306332613538336435323662356536633030386136396130383433643531663662326664343438 +34366639633835353235653837333363623939303461336532636437383762363334636533653734 +62316662666135613234663864346234313537393836363862356661323038333030656532326466 +31626133376165623734663638653236666661383861323438306265383931623235343138393732 +31616361613631653135613837616136343039303431646332646332616432643938633134363134 +66313330636431643438346364303232336230343337363132316239343262646636633665383534 +65313931613331306636643736353063386264383235326461343332666363343230333862353332 +33343661383566333336386264313639346439323664653136373832343266303630346234666662 +36656634323433623035643739643764396565646338303935363230363332393665623436373038 +61346639366433656664623633306537663633373430396431616133656438313433653565323165 +31323234626265353466343163663536653565646632343337663864303735366266333839316266 +38643563366438306430336539633163346262386331343064303466373561303638363738303164 +31303965383063653031623033646339636563393738306264303432363933653862376661376235 +64383937366563653233376437656134656537363137383233303435373361656635333433346430 +65363863626161396532653331373862303162646634653766376538303165623964333161646566 +36393962623261353737363761393139326638343463613635343131373936363864346466636538 +34326234383537663038393663613465646162383636626133626561653138333166633163333439 +35393636353561326163383037323332633563383030393966333835326139323234656630643063 +33343532663161623132616630343166396664373033663236316665393432363134636234383963 +62646461363536613830353834346134313331363933663737316137353139306266363833303863 +33376534363133316266366133616236303465663038373639326665333235326165373733336633 +64636539363038653666383365393431313362613530643131306431653535613432336334383264 +61366361616238393436393839383062643731396630373534383862613130356265633338666664 +38373830363538376564663638373138396634386231356361653466643063656635653136376131 +63346434613331333765346162646463663664666337333832356239643838623562313562326566 +65303238313639616334383531396462356130613832333663613666393939343462356239653138 +61323832616366383664313864303738376635616339373162353566343162343233656430666631 +31326665653631346264633431386366376434363932316434623836356138393838303566326335 +30666134663139636264333134636138333335656662336666303236306362396331303665316461 +61363165366164626533306166323032343665363034346164356564313333323939393564356637 +33626662623836353736643736303433646361363839323662353838616438313064363763646433 +30663333313839323036636633326532343830333661346236656138643664316238323563303434 +30633262323461316232393361353165393938653031656130306530623033396134333366633432 +62363336626265613566333964303733386234303132313265393465616664613165313932383830 +36643637376238316262333639316436636234636230363035313231336366646330616465653963 +61313632393062323338313337336363623430326364363832623531623762663363303061643266 +39646630616461623331323136313564373638663666366564313865653836656434636330373230 +63333536313663303338363234653337633762303338613639373661613365353961613864326364 +32646464376136313466303664303366623562333832376636343963353631353634313466666636 +39326431353632396264386330323065623238333961346665336238386136323132333762383362 +35346137383762663464643039326362386430333531353630653465626539323036616136383937 +33666366343038326435613362656236313030663732333836633434666663353162363436313138 +32353839376138643735653832623038343138373866333666366161306363333463366139346436 +39613066636230323066653765383865343032313464306465386130373432623030303836303031 +64653465373732366461383462366338323934613931643637386538653137356463353530366336 +32623137303937613637323938666364656233363839353362376265393032623933356230363863 +30326232356336666461373039656633643239316438616431363866613934663035366233366265 +35646361333237643566343562656632393866383166303461616163616437386232646239383132 +37656331303366313065366133333330623137353138383934653934383532663663393035313131 +62346638386462363965336139396265343462663039343939346434396338393266306565636461 +35643937346565343435653632616633386338366166376363633965363931313462343630333230 +33633835383465333833653934313633613335626230626534666436626566373430613735613730 +62313831326233616337323065303832386462386535643832343862303536353638636536366138 +34616338313134393636383933626466336534663134373237316663316465326434656332343937 +64346630663265643766393563616635393933306664336436633163636238616363353465653262 +62616636633237323732633565633631643031303930623937376161666162333639393739373562 +62623639373931376535386463376364653735396335303839383262663635303339316430346230 +61333730373539646162333233333138373337373761643334613334623737393338353032316433 +31626263333762313333323238323435653132333566306239653831636265353133616236643036 +63666466323138313634373565386138396166386638373636346230343864336330663564333432 +34343236396363616462316231346135343363383339653864323265393931363565353566313332 +37623934323734663564343536626365326538646233303130346434383139653063333564623437 +61623063353561613635303261333762636561613736653436333737303830313438323536313564 +39643934396136343431303534373639346134336136303438363833313438623337353138353638 +66656230643762396433343333643832623263333761303561663262623662363830613839623736 +30663338353035356636346439313139623230663533363837383139656231346633623735666638 +35373432626534316637386264343862653931623466386364393439613339373331373837663361 +62346164303565616439353238343534613231656662626665663937373465616565643461616361 +62666661393037363730663931373363386330656563373339343835366537643331306137633361 +35306662323766663361633564336563373435363866653961653733323230343232333161363039 +35633863316534643032343736646131393831353938323866356434643732313061353061613166 +30323634343730343930393737656138633134343862643834616230626338343838366366646636 +35626263623362666634663566326361396633383039623562326636303036626331633830343234 +65656330646530646566613965303962626530643935646333326438616261326666643036666564 +66373830646238393163356235353033656662663932326239343834653364613635366163633733 +64306633623533326364663364666365396332653262636365336562613166363233363666396432 +63626462313866356337633164323737343265656565363838643866363336313934303331383730 +65303436356335366232396132386431316330393430353337636365313365623565383231386335 +36623065323639633033376238333664366333663333626666306233366561666133666461633861 +31353538633838346462383837663465663631626533323535646365613439633963383464393133 +35616638336131643865336530316165333931613032323263306165343235623466393530646663 +37663561393734636563336531623935646666383463303734353766613438326133373931633261 +34633639336638333863386237616661336136393434333464626463663730386562353533333366 +61623139386532636636306336646331363938393935353563316436316464353764623130393639 +30316561393836616164346162363064616631323561613037613138353863316335383765326238 +37353064353265313966303165353938333235646563643731613436656566376365343239326338 +64386634366630303463366237313038333031313632366563306638366663363564653736333765 +63323033363562376636633832383937353665653530353365633131343365376635376134323935 +35313434613235636534313561306539626139363834303239343331343132636464383366333034 +65326533646637623831663361643764356438333463363930613132396264326461306532616230 +30636164393135373063643231376433386330363362666539663933613261373030396637643632 +37643633323364653362393138326466353339366130663834303861373434333131393330336630 +63333335306133313562616362343161316637646434393735303566373462333266643166313231 +66303334623738323366303665393062326635333230653466636563653432383532666137663335 +61366465376336646437333865613163393663653538626234313963393464373031333966636261 +65386365623430313536653662643738353264336462363239613733386339343561653835373132 +64383263313631393963633139333662633636303961623935633930643739616436663233313636 +39646238376462313963316163346333663339333333356334383566303838663761366463353032 +38393335666530316132303464653734633738333032393834323838663462343331313237326434 +39396338643730653364353431376335643333343865313762633435623832303638316363323534 +37346137613434333861633966626361313535616137363634323230383433393261646166363337 +35313038313362303531376430626138663531373561326262663061373465333962646633343036 +30333432353936316334323163633761346231616262633239653666383762313163663838643366 +66646634366164346464636639333762363037386433396662333838373132363164373434376638 +62346332623437356262333362373763636661306637616262366431656465343934333639363433 +35663837616339366330323362616630623236336138336232303531316562373235653262636134 +30376261613234356465333039383932393466333061333634313034363038633566626434626632 +63333563323365323330363438316337333835666635643537633132306433393939633038333763 +39333234623937316130646138346238316133653465313635313633663463643435376533396637 +30333366653232666230343963316563623865326266376332383836326134313064363465376161 +37303334343835393038396135373236316331363063653436353863306134623737396237373434 +64383461353238663733613565313238376431643131633232663636346132396631643862326233 +39656162366663353436343935303062623933656230313261646136643065613635353336363664 +65356234333734383432383533316438316536323637356539356139613363643931643866633936 +36623233336161313861656264313937646135626465336236353635616439303161613464633138 +33303633363463633836333232383863666134353435613333373437396636356464343465383135 +31343563643337663739613338626632653333373234383663663939663638643637313935353361 +32643135373163376132303831653537393561376637393466646261303066383637666431333565 +64363639646666323733333563666661343737303966303262656133333035653766623638336638 +31633663353031316631623232343033346462633338623132353033663830313830383139303638 +66656434663366366431633266666437323732316435346430636564326432663335653133643361 +32393332643633323266363536663836346162313535373764323034333264343730346433303137 +63326166316538323239623964363435313465343038613633363034316265356435396462306230 +34653232646331646535313466373736393961306439393930653234626137373037316333666466 +38643565643766643062643462333435616433326538616662333933343633616638326135313839 +35636235353135613130376137363163646435333361376333613436363364383361323638623564 +38346238393339653635333135666635663435333833343733306536613036643430353232373262 +30323831666264653233316461396433633235336339393239656335306137666439346439666436 +30326633623533313034373538326532323464353434616631373639353934316531623563333364 +33346362653933313837633265303839353337313732356466303135373237343662366234316534 +36343732306264366137333834333339383433666238646130376465663863376336643331643863 +39306666366266356139313466346163306164366234373739303634386331366561333966383165 +39343265346635303837306539633233373066343262643237323564333635626330643139646530 +33313437623938663032663739333933616533613866326565613135383131626435383063613161 +38316363393436303531613630653461336363613462666638653265316463306139396462343535 +61383066303832653237656637646631343630623464326434393139653334363036626238323832 +32373762616131376437316562643865626464613161363533393632313333633732616561313631 +39643461366338636435323139353237313533376235663731653138313963333364353733663563 +37663932343131323165313762313262353463306535393739373436396561363261313438316437 +31653863313563646434626166366464626236373939353966663333386635656436623130323939 +64393563613839303236303232306533303732323230383664393930373661643638623337616462 +63303330643233356463323661623337623063346332376166353934656262393131633338373033 +39663564383966333433653830303162373434343632346137333639366239346265383063376461 +37313230396639346261306465313236346637636163343738396237643139633762333264383930 +36656536633064393863323235326231393262343537343134643030313730353665393562353230 +33386161633565366535323538373130396239656230663538373433616130333234313030333437 +36633765323863386166316538613434316338653765373138616664333036363661353337316466 +35616262343636366634353234633261386562646564383435356463646366316630626135356635 +34643431653535393535313730646330623362623239313036343039343634613662663865643039 +31616262323835323861643962663033393733313034396263663463376662356138663639313965 +30313136386530346234613333343434626633396265316232656532386330333465656232363037 +65636632643761303962356562323831313364653331616565633138616336303965383031646364 +33333138316362613862616261343265313363626539393535306563383531343631383034396434 +63623934333335326635396164393736316161333062663531666631643132613163393538363435 +38356234353239383535306361646563353433356139336339643935333662346165303036383737 +64333335613864336163363233633163306561356238666630353438366665386138636162313535 +66623563366661303362393039346666653730313431333364373938363662333333313632373330 +65386138366138393464386537633264343865323634626535643361386231396261663332353463 +37376330653761393661383661356238393839643166343464623132333631633961653631626131 +32366234333838346463363732613636316236333065636234373436306531353666316231636531 +30333564623430663733343338343637326636633564656435363532373031616636626431393262 +34373662633239353331326266313239356664646565383662633937663230316163643737353439 +62393037666535363463343763653130343964643463613638363661343464323531356635636534 +65393535386633386261353032656135643236313763353164316234363161303963333836383266 +31613033373138356233333537623831353631656662656237373061303266303838313333303733 +30393636653166616631366336643839313662313333313761636366626535623936356364636562 +63613237646162313533313561343666303662356265623261333963663865306431353530373933 +61626163313633616265323534626239383037646336353362346333366433643765626438623639 +32373330313036636331 diff --git a/group_vars/all.yml b/group_vars/all.yml index 3aa891ae2..08560b01e 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -156,6 +156,8 @@ common_packages: - jq - glances - fail2ban + - software-properties-common + - pigz #Set pip to be pip3 by default - see roles/geerlingguy.pip/defaults/main.yml pip_package: python3-pip @@ -226,6 +228,10 @@ internal_ssh_keys: private_key_content: "{{ vault_ubuntu_maintenance_key }}" public_key_src: files/keys/ubuntu_maintenance_key.pub user: ubuntu + internal_hop_key: + private_key_content: "{{ vault_internal_hop_key }}" + public_key_src: files/keys/internal_hop_key.pub + user: ubuntu # Golang golang_version: '1.15.6' # default for role version 2.6.3 diff --git a/group_vars/galaxy_etca.yml b/group_vars/galaxy_etca.yml index 92f5f7252..e07bc0f12 100644 --- a/group_vars/galaxy_etca.yml +++ b/group_vars/galaxy_etca.yml @@ -6,6 +6,8 @@ use_internal_ips: true influx_url: stats.usegalaxy.org.au grafana_server_url: "https://{{ influx_url }}:8086" +install_python_311: True + # Monitoring telegraf_agent_output: - type: influxdb @@ -23,59 +25,63 @@ galaxy_db_tiaasadmin_password: "{{ vault_galaxy_db_tiaasadmin_password }}" galaxy_db_tiaas_password: "{{ vault_galaxy_db_tiaas_password }}" galaxy_server_and_worker_shared_mounts: # Everything mounted on galaxy, galaxy_handlers and workers + # galaxy-misc-nfs - path: /mnt/tools src: "{{ hostvars['galaxy-misc-nfs']['internal_ip'] }}:/mnt/tools" fstype: nfs state: mounted - - path: /mnt/user-data-test # TEMPORARY - src: "{{ hostvars['galaxy-user-nfs']['internal_ip'] }}:/mnt/volB/user-data-test" - fstype: nfs - state: mounted - opts: 'actimeo=2,defaults' - path: /mnt/custom-indices src: "{{ hostvars['galaxy-misc-nfs']['internal_ip'] }}:/mnt/custom-indices" fstype: nfs state: mounted -# - path: /mnt/user-data -# src: "pawsey-user-nfs.usegalaxy.org.au:/mnt/user-data" -# fstype: nfs -# state: mounted -# - path: /mnt/user-data-2 -# src: "pawsey-user-nfs.usegalaxy.org.au:/user-data-2" -# fstype: nfs -# state: mounted -# - path: /mnt/user-data-3 -# src: "pawsey-user-nfs.usegalaxy.org.au:/user-data-3" -# fstype: nfs -# state: mounted -# - path: /mnt/user-data-4 -# src: "{{ hostvars['galaxy-user-nfs']['internal_ip'] }}:/mnt/user-data4" -# fstype: nfs -# state: mounted -# - path: /mnt/user-data-5 -# src: "{{ hostvars['galaxy-user-nfs']['internal_ip'] }}:/mnt/user-data5" -# fstype: nfs -# state: mounted -# - path: /mnt/user-data-6 -# src: "{{ hostvars['galaxy-user-nfs']['internal_ip'] }}:/user-data6" -# fstype: nfs -# state: mounted -# - path: /mnt/user-data-7 -# src: "{{ hostvars['galaxy-user-nfs']['internal_ip'] }}:/mnt/user-data7" -# fstype: nfs -# state: mounted + # galaxy-job-nfs - path: /mnt/tmp src: "{{ hostvars['galaxy-job-nfs']['internal_ip'] }}:/mnt/tmp" fstype: nfs state: mounted -# - path: /mnt/files -# src: "galaxy-aust-exports.genome.edu.au:/Q0028_files" -# fstype: nfs -# state: mounted -# - path: /mnt/files2 -# src: "galaxy-aust-exports.genome.edu.au:/mnt/files2" -# fstype: nfs -# state: mounted + opts: 'actimeo=0,defaults' + # galaxy-user-nfs + - path: /mnt/user-data-volA # 150T volume + src: "{{ hostvars['galaxy-user-nfs']['internal_ip'] }}:/mnt/volA" + fstype: nfs + opts: 'actimeo=0,defaults' + state: mounted + - path: /mnt/user-data-volB # 50T volume + src: "{{ hostvars['galaxy-user-nfs']['internal_ip'] }}:/mnt/volB" + fstype: nfs + opts: 'actimeo=0,defaults' + state: mounted + - path: /mnt/user-data-volC # 42T volume + src: "{{ hostvars['galaxy-user-nfs']['internal_ip'] }}:/mnt/volC" + fstype: nfs + opts: 'actimeo=0,defaults' + state: mounted + # pawsey data volumes + - path: /mnt/user-data + src: "pawsey-user-nfs.usegalaxy.org.au:/mnt/user-data" + fstype: nfs + state: mounted + - path: /mnt/user-data-2 + src: "pawsey-user-nfs.usegalaxy.org.au:/user-data-2" + fstype: nfs + state: mounted + - path: /mnt/user-data-3 + src: "pawsey-user-nfs.usegalaxy.org.au:/user-data-3" + fstype: nfs + state: mounted + - path: /mnt/user-data-4 + src: "pawsey-user-nfs.usegalaxy.org.au:/user-data-4" + fstype: nfs + state: mounted + # QLD data volumes + - path: /mnt/files + src: "galaxy-aust-exports.genome.edu.au:/Q0028_files" + fstype: nfs + state: mounted + - path: /mnt/files2 + src: "galaxy-aust-exports.genome.edu.au:/mnt/files2" + fstype: nfs + state: mounted galaxy_worker_mounts: - path: /mnt/galaxy diff --git a/group_vars/galaxy_etca_slurm.yml b/group_vars/galaxy_etca_slurm.yml index 5a5ea297e..365100783 100644 --- a/group_vars/galaxy_etca_slurm.yml +++ b/group_vars/galaxy_etca_slurm.yml @@ -12,51 +12,61 @@ slurm_nodes: NodeAddr: "{{ hostvars['galaxy-w1']['internal_ip'] }}" CPUs: 16 RealMemory: 64000 + Weight: 10 State: UNKNOWN - name: galaxy-w2 NodeAddr: "{{ hostvars['galaxy-w2']['internal_ip'] }}" CPUs: 16 RealMemory: 64000 + Weight: 10 State: UNKNOWN - name: galaxy-w3 NodeAddr: "{{ hostvars['galaxy-w3']['internal_ip'] }}" CPUs: 16 RealMemory: 64000 + Weight: 10 State: UNKNOWN - name: galaxy-w4 NodeAddr: "{{ hostvars['galaxy-w4']['internal_ip'] }}" CPUs: 16 RealMemory: 64000 + Weight: 10 State: UNKNOWN - name: galaxy-w5 NodeAddr: "{{ hostvars['galaxy-w5']['internal_ip'] }}" CPUs: 16 RealMemory: 64000 + Weight: 10 State: UNKNOWN - name: galaxy-w6 NodeAddr: "{{ hostvars['galaxy-w6']['internal_ip'] }}" CPUs: 16 RealMemory: 64000 + Weight: 10 State: UNKNOWN - name: galaxy-w7 NodeAddr: "{{ hostvars['galaxy-w7']['internal_ip'] }}" CPUs: 16 RealMemory: 64000 + Weight: 10 State: UNKNOWN - name: galaxy-w8 NodeAddr: "{{ hostvars['galaxy-w8']['internal_ip'] }}" CPUs: 32 RealMemory: 128000 + Weight: 100 State: UNKNOWN - name: galaxy-w9 NodeAddr: "{{ hostvars['galaxy-w9']['internal_ip'] }}" CPUs: 32 RealMemory: 128000 + Weight: 100 State: UNKNOWN - name: galaxy-w10 NodeAddr: "{{ hostvars['galaxy-w10']['internal_ip'] }}" CPUs: 32 RealMemory: 128000 + Weight: 100 State: UNKNOWN - name: galaxy-queue NodeAddr: "{{ hostvars['galaxy-queue']['internal_ip'] }}" @@ -69,11 +79,11 @@ slurm_nodes: RealMemory: 64000 State: UNKNOWN ##& Include galaxy-handlers in slurm config for 2-VM galaxy server. Comment out when using a single VM for galaxy server - # - name: galaxy-handlers - # NodeAddr: "{{ hostvars['galaxy-handlers']['internal_ip'] }}" - # CPUs: 16 - # RealMemory: 64000 - # State: UNKNOWN + - name: galaxy-handlers + NodeAddr: "{{ hostvars['galaxy-handlers']['internal_ip'] }}" + CPUs: 16 + RealMemory: 64000 + State: UNKNOWN slurm_partitions: diff --git a/group_vars/galaxy_workers.yml b/group_vars/galaxy_workers.yml index 12288b933..749b091f4 100644 --- a/group_vars/galaxy_workers.yml +++ b/group_vars/galaxy_workers.yml @@ -28,3 +28,9 @@ docker_users: - "{{ galaxy_user.name }}" docker_daemon_options: # TODO: Uncomment as soon as there is a volume data-root: /mnt/docker-data + +# internal ssh keys +extra_keys: + - id: internal_hop_key + type: public + from: "{{ hostvars['galaxy']['internal_ip'] }},{{ hostvars['galaxy-backup']['internal_ip'] }},{{ hostvars['galaxy-queue']['internal_ip'] }},{{ hostvars['galaxy-handlers']['internal_ip'] }}" diff --git a/group_vars/galaxyservers.yml b/group_vars/galaxyservers.yml index bc2e77b81..da759fbef 100644 --- a/group_vars/galaxyservers.yml +++ b/group_vars/galaxyservers.yml @@ -86,6 +86,8 @@ nginx_upload_module_url: https://swift.rc.nectar.org.au/v1/AUTH_377/galaxy_infra # miniconda miniconda_prefix: "{{ galaxy_conda_prefix }}" +num_gunicorn_processes: 1 + # singularity cache of galaxy user (this defaults to /home/galaxy/.singularity). Not to be confused with cache directory for built sif files in galaxy_config.galaxy.container_resolvers_conf galaxy_user_singularity_cachedir: /mnt/singularity_data galaxy_user_singularity_tmpdir: "{{ galaxy_user_singularity_cachedir }}/tmp" @@ -122,6 +124,8 @@ galaxy_config_dir: "{{ galaxy_root }}/config" galaxy_conda_prefix: "{{ galaxy_tools_indices_dir }}/tool_dependencies/_conda" galaxy_conda_exec: mamba +galaxy_toolbox_filters_dir: "{{ galaxy_server_dir }}/lib/galaxy/tools/toolbox/filters" + galaxy_config_file: "{{ galaxy_config_dir }}/galaxy.yml" tpv_version: "2.3.2" @@ -156,7 +160,6 @@ group_galaxy_config: use_interactive: false expose_dataset_path: true enable_unique_workflow_defaults: true - database_engine_option_server_side_cursors: true track_jobs_in_database: true use_heartbeat: true allow_user_dataset_purge: true @@ -204,6 +207,7 @@ group_galaxy_config: themes_config_file: "{{ galaxy_config_dir }}/themes_conf.yml" library_import_dir: "{{ galaxy_tmp_dir }}/library_import_dir" cluster_files_directory: "{{ galaxy_tmp_dir }}/pbs" + short_term_storage_dir: "{{ galaxy_tmp_dir }}/short_term_web_storage" templates_dir: "{{ galaxy_config_dir }}" diff --git a/group_vars/pulsarservers.yml b/group_vars/pulsarservers.yml index 71bedcea2..f73921647 100644 --- a/group_vars/pulsarservers.yml +++ b/group_vars/pulsarservers.yml @@ -68,6 +68,7 @@ pulsar_optional_dependencies: - pycurl - drmaa - kombu>=5.2.0 + - sentry-sdk install_drmaa: true pulsar_drmaa_library_path: /usr/lib/slurm-drmaa/lib/libdrmaa.so.1 @@ -107,6 +108,7 @@ pulsar_yaml_config: amqp_publish_retry_interval_start: 10 amqp_publish_retry_interval_step: 10 amqp_publish_retry_interval_max: 60 + sentry_dsn: "{{ vault_sentry_url_pulsar_production if ansible_hostname in groups['production_pulsars'] else omit }}" pulsar_conda_prefix: "{{ pulsar_dependencies_dir }}/_conda" miniconda_prefix: "{{ pulsar_conda_prefix }}" diff --git a/host_vars/aarnet.usegalaxy.org.au.yml b/host_vars/aarnet.usegalaxy.org.au.yml index ac4e06381..60d2b4896 100644 --- a/host_vars/aarnet.usegalaxy.org.au.yml +++ b/host_vars/aarnet.usegalaxy.org.au.yml @@ -140,7 +140,7 @@ host_galaxy_config_templates: - src: "{{ galaxy_config_template_src_dir }}/config/aarnet_job_conf.yml.j2" dest: "{{ galaxy_config_dir }}/job_conf.yml" - src: "{{ galaxy_config_template_src_dir }}/toolbox/filters/ga_filters.py.j2" ## File cannot be created at dest on first run of playbook - dest: "{{ galaxy_server_dir }}/lib/galaxy/tool_util/toolbox/filters/ga_filters.py" + dest: "{{ galaxy_toolbox_filters_dir }}/ga_filters.py" host_galaxy_config: # renamed from __galaxy_config gravity: diff --git a/host_vars/dev.gvl.org.au.yml b/host_vars/dev.gvl.org.au.yml index f73bf9b02..04b5606b7 100644 --- a/host_vars/dev.gvl.org.au.yml +++ b/host_vars/dev.gvl.org.au.yml @@ -81,7 +81,7 @@ host_galaxy_config_templates: - src: "{{ galaxy_config_template_src_dir }}/config/dev_job_conf.yml.j2" dest: "{{ galaxy_config_dir}}/job_conf.yml" - src: "{{ galaxy_config_template_src_dir }}/toolbox/filters/ga_filters.py.j2" - dest: "{{ galaxy_server_dir }}/lib/galaxy/tool_util/toolbox/filters/ga_filters.py" + dest: "{{ galaxy_toolbox_filters_dir }}/ga_filters.py" - src: "{{ galaxy_config_file_src_dir }}/config/web-gravity.yml" dest: /opt/galaxy/web-gravity.yml diff --git a/host_vars/galaxy-backup.yml b/host_vars/galaxy-backup.yml index 17bbe7f3c..303963718 100644 --- a/host_vars/galaxy-backup.yml +++ b/host_vars/galaxy-backup.yml @@ -1,40 +1,51 @@ --- # volume stuff -# attached_volumes: # TODO: Probably the same. check this. Uncomment when this VM has a volume -# - device: /dev/vdb -# path: /mnt -# fstype: ext4 -# partition: 1 +attached_volumes: + - device: /dev/vdb + path: /mnt + fstype: ext4 + partition: 1 #paths script_path: /home/ubuntu/galaxy_backup backup_path: /mnt/galaxy-backups -# Comment out galaxy db backup variables until galaxy Sydney is production galaxy -# #swift for backups -# use_swift: true - -# swift_backup_container: 'usegalaxy-au-db' -# swift_cred_file: "{{ script_path }}/GenomicsVL-openrc.sh" # IMPORTANT: this file needs to be added by a user with credentials -# venv_location: "{{ script_path }}/venv" - -# #slack integration -# use_slack: true -# slack_galaxy_log_webhook: "{{ vault_slack_galaxy_log_webhook }}" - -# #The name of the database to backup -# psql_db: 'galaxy' - -# #Database connection vars -# db_user: galaxy -# db_password: "{{ vault_galaxy_db_user_password }}" -# db_server: "{{ hostvars['galaxy-db']['internal_ip'] }}" -# db_port: 5432 - -# # Retention time variables -# monthly_backup_day: 1 #Monthly backups will run on this day of the month -# weekly_backup_day: 6 #Weekly backups will run on this day of the week -# retention_day: 6 #Keep daily backups for this many days (6 days) -# retention_week: 21 #Keep weekly backups for this many days (21 days = 3 weeks) -# retention_month: 92 #Keep monthly backups for this many days (92 days ~ 3 months) \ No newline at end of file +#swift for backups +use_swift: true + +swift_backup_container: 'usegalaxy-au-db' +swift_cred_file: "{{ script_path }}/GenomicsVL-openrc.sh" # IMPORTANT: this file needs to be added by a user with credentials +venv_location: "{{ script_path }}/venv" + +#slack integration +use_slack: true +slack_galaxy_log_webhook: "{{ vault_slack_galaxy_log_webhook }}" + +#The name of the database to backup +psql_db: 'galaxy' + +#Database connection vars +db_user: galaxy +db_password: "{{ vault_galaxy_db_user_password }}" +db_server: "{{ hostvars['galaxy-db']['internal_ip'] }}" +db_port: 5432 + +# Retention time variables +monthly_backup_day: 1 #Monthly backups will run on this day of the month +weekly_backup_day: 6 #Weekly backups will run on this day of the week +retention_day: 6 #Keep daily backups for this many days (6 days) +retention_week: 21 #Keep weekly backups for this many days (21 days = 3 weeks) +retention_month: 61 #Keep monthly backups for this many days (61 days ~ 2 months) + +extra_keys: + - id: internal_hop_key + type: private + - id: internal_hop_key + type: public + from: "{{ hostvars['galaxy']['internal_ip'] }},{{ hostvars['galaxy-queue']['internal_ip'] }},{{ hostvars['galaxy-handlers']['internal_ip'] }}" + +# ssh config, only for ubuntu +ssh_config_id_file: "/home/{{ ssh_config_user }}/.ssh/internal_hop_key" +ssh_config_user: ubuntu +ssh_config_hosts: "{{ groups['galaxy_group'] }}" diff --git a/host_vars/galaxy-db.usegalaxy.org.au.yml b/host_vars/galaxy-db.usegalaxy.org.au.yml index 950adac53..9c088ef58 100644 --- a/host_vars/galaxy-db.usegalaxy.org.au.yml +++ b/host_vars/galaxy-db.usegalaxy.org.au.yml @@ -17,7 +17,7 @@ postgresql_conf: # CPUs num: 8 # Connections num: 100 # Data Storage: san - - max_connections: 200 + - max_connections: 300 - listen_addresses: "'*'" # Allow remote connections - data_directory: "'/data/production'" - shared_buffers: 8GB @@ -46,3 +46,8 @@ postgresql_pg_hba_conf: - host all all "{{ hostvars['galaxy']['internal_ip'] }}/32" md5 - host all all "{{ hostvars['galaxy-handlers']['internal_ip'] }}/32" md5 - host all all "{{ hostvars['galaxy-backup']['internal_ip'] }}/32" md5 + +extra_keys: + - id: internal_hop_key + type: public + from: "{{ hostvars['galaxy']['internal_ip'] }},{{ hostvars['galaxy-backup']['internal_ip'] }},{{ hostvars['galaxy-handlers']['internal_ip'] }},{{ hostvars['galaxy-queue']['internal_ip'] }}" diff --git a/host_vars/galaxy-handlers.usegalaxy.org.au.yml b/host_vars/galaxy-handlers.usegalaxy.org.au.yml index d072b8799..9af7a3d60 100644 --- a/host_vars/galaxy-handlers.usegalaxy.org.au.yml +++ b/host_vars/galaxy-handlers.usegalaxy.org.au.yml @@ -72,3 +72,14 @@ flower_galaxy_conf: "{{ galaxy_config_file }}" # #flower_broker_api: "https://flower:{{ rabbitmq_password_flower }}@{{ rabbitmq_url }}/api/" flower_broker_url: "pyamqp://flower:{{ vault_rabbitmq_password_galaxy_prod }}@{{ hostvars['galaxy-queue']['internal_ip'] }}:5671//galaxy/galaxy_queues?ssl=1" + +extra_keys: + - id: internal_hop_key + type: private + - id: internal_hop_key + type: public + from: "{{ hostvars['galaxy-backup']['internal_ip'] }},{{ hostvars['galaxy']['internal_ip'] }},{{ hostvars['galaxy-queue']['internal_ip'] }}" + +# override docker-data location +docker_daemon_options: + data-root: /pvol/docker-data diff --git a/host_vars/galaxy-job-nfs.usegalaxy.org.au.yml b/host_vars/galaxy-job-nfs.usegalaxy.org.au.yml index 24ff7bed8..b2b461c62 100644 --- a/host_vars/galaxy-job-nfs.usegalaxy.org.au.yml +++ b/host_vars/galaxy-job-nfs.usegalaxy.org.au.yml @@ -13,3 +13,6 @@ extra_keys: - id: ubuntu_maintenance_key type: public from: "{{ hostvars['galaxy']['internal_ip'] }}" + - id: internal_hop_key + type: public + from: "{{ hostvars['galaxy']['internal_ip'] }},{{ hostvars['galaxy-backup']['internal_ip'] }},{{ hostvars['galaxy-queue']['internal_ip'] }},{{ hostvars['galaxy-handlers']['internal_ip'] }}" diff --git a/host_vars/galaxy-misc-nfs.usegalaxy.org.au.yml b/host_vars/galaxy-misc-nfs.usegalaxy.org.au.yml index d0e92e83c..9a56226b9 100644 --- a/host_vars/galaxy-misc-nfs.usegalaxy.org.au.yml +++ b/host_vars/galaxy-misc-nfs.usegalaxy.org.au.yml @@ -25,3 +25,9 @@ nfs_exports: - "{{ nfs_app_dir }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" - "{{ nfs_tools_dir }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" - "{{ nfs_custom_indices_dir }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" + +# internal ssh keys +extra_keys: + - id: internal_hop_key + type: public + from: "{{ hostvars['galaxy']['internal_ip'] }},{{ hostvars['galaxy-backup']['internal_ip'] }},{{ hostvars['galaxy-queue']['internal_ip'] }},{{ hostvars['galaxy-handlers']['internal_ip'] }}" diff --git a/host_vars/galaxy-queue.usegalaxy.org.au.yml b/host_vars/galaxy-queue.usegalaxy.org.au.yml index b79ba8606..d963c2277 100644 --- a/host_vars/galaxy-queue.usegalaxy.org.au.yml +++ b/host_vars/galaxy-queue.usegalaxy.org.au.yml @@ -215,4 +215,16 @@ pip_install_packages: nginx_check_conf: False redis_requirepass: "{{ vault_redis_requirepass }}" -redis_bind_interface: 0.0.0.0 \ No newline at end of file +redis_bind_interface: 0.0.0.0 + +extra_keys: + - id: internal_hop_key + type: private + - id: internal_hop_key + type: public + from: "{{ hostvars['galaxy-backup']['internal_ip'] }},{{ hostvars['galaxy']['internal_ip'] }},{{ hostvars['galaxy-handlers']['internal_ip'] }}" + +# ssh config, only for ubuntu +ssh_config_id_file: "/home/{{ ssh_config_user }}/.ssh/internal_hop_key" +ssh_config_user: ubuntu +ssh_config_hosts: "{{ groups['galaxy_group'] }}" diff --git a/host_vars/galaxy-user-nfs.usegalaxy.org.au.yml b/host_vars/galaxy-user-nfs.usegalaxy.org.au.yml index 3f229dc7e..be9f83634 100644 --- a/host_vars/galaxy-user-nfs.usegalaxy.org.au.yml +++ b/host_vars/galaxy-user-nfs.usegalaxy.org.au.yml @@ -1,32 +1,44 @@ -nfs_user_data_test_dir: /mnt/volB/user-data-test +volA_path: /mnt/volA +volB_path: /mnt/volB +volC_path: /mnt/volC + +nfs_user_data_5_dir: "{{ volA_path }}/user-data-5" +nfs_user_data_7_dir: "{{ volA_path }}/user-data-7" +nfs_data08_dir: "{{ volA_path }}/data08" + +nfs_user_data_6_dir: "{{ volB_path }}/user-data-6" +nfs_data09_dir: "{{ volB_path }}/data09" +nfs_user_data10_dir: "{{ volC_path }}/data10" attached_volumes: - device: /dev/vdb - path: /mnt/volA - fstype: ext4 partition: 1 + path: "{{ volA_path }}" # contains user-data-5, user-data-7, data08 + fstype: ext4 - device: /dev/vdc partition: 1 - path: /mnt/volB + path: "{{ volB_path }}" # contains user-data-6, data09 + fstype: ext4 + - device: /dev/vdd + partition: 1 + path: "{{ volC_path }}" # contains data10 fstype: ext4 nfs_dirs: - - "{{ nfs_user_data_test_dir }}" - -# nfs_user_data_dir: /mnt/user-data -# nfs_user_data_2_dir: /mnt/user-data2 -# nfs_user_data_3_dir: /mnt/user-data3 -# nfs_user_data_4_dir: /mnt/user-data4 -# nfs_user_data_5_dir: /mnt/user-data5 -# nfs_user_data_6_dir: /user-data6 -# nfs_user_data_7_dir: /mnt/user-data7 + - "{{ nfs_user_data_5_dir }}" + - "{{ nfs_user_data_7_dir }}" + - "{{ nfs_data08_dir }}" + - "{{ nfs_user_data_6_dir }}" + - "{{ nfs_data09_dir }}" + - "{{ nfs_user_data10_dir }}" nfs_exports: - - "{{ nfs_user_data_test_dir }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" -# - "{{ nfs_user_data_dir }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" -# - "{{ nfs_user_data_2_dir }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" -# - "{{ nfs_user_data_3_dir }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" -# - "{{ nfs_user_data_4_dir }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" -# - "{{ nfs_user_data_5_dir }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" -# - "{{ nfs_user_data_6_dir }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" -# - "{{ nfs_user_data_7_dir }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" + - "{{ volA_path }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" + - "{{ volB_path }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" + - "{{ volC_path }} {{ hostvars['galaxy'].internal_ip.split('.')[:-1] | join('.') }}.0/24(rw,async,no_root_squash,no_subtree_check)" + +# internal ssh keys +extra_keys: + - id: internal_hop_key + type: public + from: "{{ hostvars['galaxy']['internal_ip'] }},{{ hostvars['galaxy-backup']['internal_ip'] }},{{ hostvars['galaxy-queue']['internal_ip'] }},{{ hostvars['galaxy-handlers']['internal_ip'] }}" diff --git a/host_vars/galaxy.usegalaxy.org.au.yml b/host_vars/galaxy.usegalaxy.org.au.yml index 929ef6d49..b3d63c8fa 100644 --- a/host_vars/galaxy.usegalaxy.org.au.yml +++ b/host_vars/galaxy.usegalaxy.org.au.yml @@ -3,8 +3,8 @@ galaxy_config_file: /opt/galaxy/galaxy.yml # legacy_file_mounts_available: set to true if /mnt/files, /mnt/files2 should be in the object store -legacy_file_mounts_available: False # assume true unless set to false -pawsey_file_mounts_available: False # assume true unless set to false +legacy_file_mounts_available: True # assume true unless set to false +pawsey_file_mounts_available: True # assume true unless set to false # variables for attaching mounted volume to application server attached_volumes: # TODO: check this @@ -15,10 +15,11 @@ attached_volumes: # TODO: check this certbot_domains: - "{{ hostname }}" # galaxy.usegalaxy.org.au -# - "usegalaxy.org.au" -# - "www.usegalaxy.org.au" -# - "*.interactivetoolentrypoint.interactivetool.usegalaxy.org.au" - - "*.interactivetoolentrypoint.interactivetool.galaxy.usegalaxy.org.au" + - "usegalaxy.org.au" + - "www.usegalaxy.org.au" + - "*.interactivetoolentrypoint.interactivetool.usegalaxy.org.au" + - "genome.usegalaxy.org.au" + - "proteomics.usegalaxy.org.au" certbot_dns_provider: cloudflare certbot_dns_credentials: api_token: "{{ vault_dns_cloudflare_api_token }}" @@ -28,31 +29,26 @@ nginx_ssl_servers: - galaxy - galaxy-gie-proxy ## we still do this? +num_gunicorn_processes: 2 + # gie proxy hostname -interactive_tools_server_name: "galaxy.usegalaxy.org.au" +interactive_tools_server_name: "usegalaxy.org.au" galaxy_repo: https://github.com/galaxyproject/galaxy.git -galaxy_commit_id: release_23.0 +galaxy_commit_id: release_23.1 # TODO: check list concatenation with versions of ansible > 2.12 shared_mounts: "{{ galaxy_server_and_worker_shared_mounts + galaxy_web_server_mounts }}" # sourced from galaxy_etca.yml # ansible-galaxy -# galaxy_dynamic_job_rules_src_dir: files/galaxy/dynamic_job_rules/production -# galaxy_dynamic_job_rules_dir: "{{ galaxy_root }}/dynamic_job_rules" -# galaxy_dynamic_job_rules: -# - total_perspective_vortex/tools.yml -# - total_perspective_vortex/destinations.yml.j2 -# - total_perspective_vortex/users.yml -# - total_perspective_vortex/roles.yml -# - total_perspective_vortex/default_tool.yml.j2 -# - readme.txt - -galaxy_dynamic_job_rules_src_dir: files/galaxy/dynamic_job_rules/load-testing +galaxy_dynamic_job_rules_src_dir: files/galaxy/dynamic_job_rules/production galaxy_dynamic_job_rules_dir: "{{ galaxy_root }}/dynamic_job_rules" galaxy_dynamic_job_rules: - - total_perspective_vortex/testing_tpv_config.yml - - total_perspective_vortex/testing_tpv_destinations.yml.j2 + - total_perspective_vortex/tools.yml + - total_perspective_vortex/destinations.yml.j2 + - total_perspective_vortex/users.yml + - total_perspective_vortex/roles.yml + - total_perspective_vortex/default_tool.yml.j2 - readme.txt galaxy_systemd_mode: gravity @@ -64,25 +60,33 @@ galaxy_custom_indices_dir: /mnt/custom-indices galaxy_tmp_dir: /mnt/tmp galaxy_tus_upload_store: "{{ galaxy_tmp_dir }}/tus" -galaxy_infrastructure_url: https://galaxy.usegalaxy.org.au +galaxy_infrastructure_url: https://usegalaxy.org.au -galaxy_file_path: /mnt/user-data-test -nginx_upload_store_base_dir: "{{ galaxy_file_path }}/upload_store" +upload_store_volume_dir: /mnt/user-data-volA +nginx_upload_store_base_dir: "{{ upload_store_volume_dir }}/upload_store" nginx_upload_store_dir: "{{ nginx_upload_store_base_dir }}/uploads" nginx_upload_job_files_store_dir: "{{ nginx_upload_store_base_dir }}/job_files" nginx_upload_store_set_cleanup_cron_job: true +# galaxy_virtualenv_python: python3.11 # deferred until release 23.2 probably + host_galaxy_config_files: - src: "{{ galaxy_config_file_src_dir }}/config/trs_servers_conf.yml" dest: "{{ galaxy_config_dir }}/trs_servers_conf.yml" + - src: "{{ galaxy_config_file_src_dir }}/config/tool_data_table_conf.xml" # TODO: etca will need this too + dest: "{{ galaxy_config_dir }}/tool_data_table_conf.xml" + - src: "{{ galaxy_config_file_src_dir }}/config/activation-email.html" + dest: "{{ galaxy_config_dir }}/mail/activation-email.html" + - src: "{{ galaxy_config_file_src_dir }}/config/activation-email.txt" + dest: "{{ galaxy_config_dir }}/mail/activation-email.txt" host_galaxy_config_templates: - - src: "{{ galaxy_config_template_src_dir }}/config/galaxy_object_store_conf.xml.j2" # TODO: create me + - src: "{{ galaxy_config_template_src_dir }}/config/galaxy_object_store_conf.xml.j2" dest: "{{ galaxy_config_dir }}/object_store_conf.xml" - - src: "{{ galaxy_config_template_src_dir }}/config/galaxy_job_conf.yml.j2" # TODO: create me + - src: "{{ galaxy_config_template_src_dir }}/config/galaxy_job_conf.yml.j2" dest: "{{ galaxy_config_dir }}/job_conf.yml" -# - src: "{{ galaxy_config_template_src_dir }}/toolbox/filters/ga_filters.py.j2" ## File cannot be created at dest on first run of playbook -# dest: "{{ galaxy_server_dir }}/lib/galaxy/tool_util/toolbox/filters/ga_filters.py" + - src: "{{ galaxy_config_template_src_dir }}/toolbox/filters/ga_filters.py.j2" ## File cannot be created at dest on first run of playbook + dest: "{{ galaxy_toolbox_filters_dir }}/ga_filters.py" host_galaxy_config_gravity: @@ -92,18 +96,30 @@ host_galaxy_config_gravity: app_server: gunicorn virtualenv: "{{ galaxy_venv_dir }}" gunicorn: - # listening options - bind: "unix:{{ galaxy_mutable_config_dir }}/gunicorn.sock" - # performance options - workers: 4 - # Other options that will be passed to gunicorn - extra_args: '--forwarded-allow-ips="*"' - preload: true - environment: - VIRTUAL_ENV: "{{ galaxy_venv_dir }}" - DRMAA_LIBRARY_PATH: "/usr/lib/slurm-drmaa/lib/libdrmaa.so.1" - SINGULARITY_CACHEDIR: "{{ galaxy_user_singularity_cachedir }}" - SINGULARITY_TMPDIR: "{{ galaxy_user_singularity_tmpdir }}" + - bind: unix:{{ galaxy_mutable_config_dir }}/gunicorn1.sock + workers: 2 + # Other options that will be passed to gunicorn + extra_args: '--forwarded-allow-ips="*"' + timeout: 600 + restart_timeout: 600 + preload: true + environment: + VIRTUAL_ENV: "{{ galaxy_venv_dir }}" + DRMAA_LIBRARY_PATH: "/usr/lib/slurm-drmaa/lib/libdrmaa.so.1" + SINGULARITY_CACHEDIR: "{{ galaxy_user_singularity_cachedir }}" + SINGULARITY_TMPDIR: "{{ galaxy_user_singularity_tmpdir }}" + - bind: unix:{{ galaxy_mutable_config_dir }}/gunicorn2.sock + workers: 2 + # Other options that will be passed to gunicorn + extra_args: '--forwarded-allow-ips="*"' + timeout: 600 + restart_timeout: 600 + preload: true + environment: + VIRTUAL_ENV: "{{ galaxy_venv_dir }}" + DRMAA_LIBRARY_PATH: "/usr/lib/slurm-drmaa/lib/libdrmaa.so.1" + SINGULARITY_CACHEDIR: "{{ galaxy_user_singularity_cachedir }}" + SINGULARITY_TMPDIR: "{{ galaxy_user_singularity_tmpdir }}" tusd: enable: true tusd_path: /usr/local/sbin/tusd @@ -121,39 +137,39 @@ host_galaxy_config_gravity: bind: "unix:{{ galaxy_mutable_config_dir }}/reports.sock" config_file: "{{ galaxy_config_dir }}/reports.yml" ##& handlers and celery are on galaxy-handlers VM with 2-VM galaxy server. Uncomment these entries when using a single VM - celery: - concurrency: 2 - loglevel: DEBUG - handlers: - handler: - environment: - DRMAA_LIBRARY_PATH: "/usr/lib/slurm-drmaa/lib/libdrmaa.so.1" - SINGULARITY_CACHEDIR: "{{ galaxy_user_singularity_cachedir }}" - SINGULARITY_TMPDIR: "{{ galaxy_user_singularity_tmpdir }}" - processes: 5 - pools: - - job-handlers - - workflow-schedulers + # celery: + # concurrency: 2 + # loglevel: DEBUG + # handlers: + # handler: + # environment: + # DRMAA_LIBRARY_PATH: "/usr/lib/slurm-drmaa/lib/libdrmaa.so.1" + # SINGULARITY_CACHEDIR: "{{ galaxy_user_singularity_cachedir }}" + # SINGULARITY_TMPDIR: "{{ galaxy_user_singularity_tmpdir }}" + # processes: 5 + # pools: + # - job-handlers + # - workflow-schedulers host_galaxy_config: # renamed from __galaxy_config gravity: "{{ host_galaxy_config_gravity }}" galaxy: ##& amqp_internal_connection for 2-VM galaxy server. Comment this out when out when using a single VM - # amqp_internal_connection: "pyamqp://galaxy_queues:{{ vault_rabbitmq_password_galaxy_prod }}@{{ hostvars['galaxy-queue']['internal_ip'] }}:5671//galaxy/galaxy_queues?ssl=1" + amqp_internal_connection: "pyamqp://galaxy_queues:{{ vault_rabbitmq_password_galaxy_prod }}@{{ hostvars['galaxy-queue']['internal_ip'] }}:5671//galaxy/galaxy_queues?ssl=1" admin_users: "{{ machine_users | selectattr('email', 'defined') | selectattr('roles', 'contains', 'galaxy_admin') | map(attribute='email') | join(',') }},{{ bpa_email }}" - brand: "E T C A" + brand: "Australia" database_connection: "postgresql://galaxy:{{ galaxy_db_user_password }}@{{ hostvars['galaxy-db']['internal_ip'] }}:5432/galaxy" - id_secret: "{{ vault_aarnet_id_secret }}" # TODO: this need to stay the same wherever production galaxy is runninig, but the name of the vault variable is misleading - file_path: "{{ galaxy_file_path }}" + id_secret: "{{ vault_aarnet_id_secret }}" # TODO: this need to stay the same wherever production galaxy is running, but the name of the vault variable is misleading object_store_config_file: "{{ galaxy_config_dir }}/object_store_conf.xml" smtp_server: localhost - # ga_code: "{{ vault_prod_ga_code }}" ## swich off ga_code while testing. TODO: uncomment this when we go live + ga_code: "{{ vault_prod_ga_code }}" ## swich off ga_code while testing. TODO: uncomment this when we go live interactivetools_enable: true interactivetools_map: "{{ gie_proxy_sessions_path }}" - # enable_oidc: true # TODO: Can we switch this on before galaxy-etca is production galaxy or should we wait until moving day? - # oidc_config_file: "{{ galaxy_config_dir }}/oidc_config.xml" - # oidc_backends_config_file: "{{ galaxy_config_dir }}/oidc_backends_config.xml" + enable_oidc: true + oidc_config_file: "{{ galaxy_config_dir }}/oidc_config.xml" + oidc_backends_config_file: "{{ galaxy_config_dir }}/oidc_backends_config.xml" + tool_data_table_config_path: "{{ galaxy_config_dir }}/tool_data_table_conf.xml,{{ galaxy_mutable_config_dir }}/shed_tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/managed/location/tool_data_table_conf.xml" # nginx upload module nginx_upload_store: "{{ nginx_upload_store_dir }}" nginx_upload_path: "/_upload" @@ -170,23 +186,25 @@ host_galaxy_config: # renamed from __galaxy_config tus_upload_store: "{{ galaxy_tus_upload_store }}" # Allow cross-subdomain cookie sharing: - # cookie_domain: usegalaxy.org.au # TODO: switch from null to usegalaxy.org.au when etca is production site - cookie_domain: null - - ##& comment out celery_conf and enable_celery_tasks when using single VM - # celery_conf: - # result_backend: "redis://:{{ vault_redis_requirepass }}@{{ hostvars['galaxy-queue']['internal_ip'] }}:6379/0" - # # result_backend: "{{ redis_connection_string }}" - # task_routes: - # galaxy.fetch_data: disabled - # # galaxy.fetch_data: galaxy.external - # galaxy.set_job_metadata: galaxy.external - - # # Offload long-running tasks to a Celery task queue. Activate this - # # only if you have setup a Celery worker for Galaxy. For details, see - # # https://docs.galaxyproject.org/en/master/admin/production.html - # enable_celery_tasks: true + cookie_domain: usegalaxy.org.au + + user_activation_on: true + activation_email: + #sentry_dsn: "{{ vault_sentry_url_galaxy_production }}" # commented out, waiting for change in 23.1 to reduce severity of startup logs + + celery_conf: + result_backend: "redis://:{{ vault_redis_requirepass }}@{{ hostvars['galaxy-queue']['internal_ip'] }}:6379/0" + # result_backend: "{{ redis_connection_string }}" + task_routes: + galaxy.fetch_data: disabled + # galaxy.fetch_data: galaxy.external + galaxy.set_job_metadata: galaxy.external + + # Offload long-running tasks to a Celery task queue. Activate this + # only if you have setup a Celery worker for Galaxy. For details, see + # https://docs.galaxyproject.org/en/master/admin/production.html + enable_celery_tasks: true # cvmfs cvmfs_cache_base: /mnt/var/lib/cvmfs @@ -213,8 +231,14 @@ add_monthly_stats: true add_utilisation_info: true add_queue_info: true +# host-specific settings for postfix +postfix_host_domain: "usegalaxy.org.au" +postfix_hostname: "galaxy" +smtp_login: "{{ vault_smtp_login_prod }}" +smtp_password: "{{ vault_smtp_password_prod }}" + #Vars for TIaaS -tiaas_galaxy_db_host: "galaxy-db.usegalaxy.org.au" +tiaas_galaxy_db_host: "{{ hostvars['galaxy-db']['internal_ip'] }}" tiaas_galaxy_db_port: "5432" tiaas_galaxy_db_user: "tiaas" tiaas_galaxy_db_pass: "{{ galaxy_db_tiaas_password }}" @@ -222,7 +246,7 @@ tiaas_info: owner: "Galaxy Australia" owner_email: help@genome.edu.au owner_site: "https://site.usegalaxy.org.au" - domain: galaxy.usegalaxy.org.au # TODO: update when galaxy-etca is production galaxy + domain: usegalaxy.org.au tiaas_other_config: | EMAIL_HOST="localhost" EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' @@ -346,6 +370,11 @@ rpc_pulsar_machines: extra_keys: - id: ubuntu_maintenance_key type: private + - id: internal_hop_key + type: private + - id: internal_hop_key + type: public + from: "{{ hostvars['galaxy-backup']['internal_ip'] }},{{ hostvars['galaxy-handlers']['internal_ip'] }},{{ hostvars['galaxy-queue']['internal_ip'] }}" # # grt-sender role # TODO: what of GRT?? # grt_sender_dir: /mnt/var/galactic_radio_telescope @@ -511,19 +540,16 @@ slurm_singularity_volumes_list_restricted: - $job_directory:rw - $galaxy_root:ro - $tool_directory:ro - - /mnt/user-data-test # TEMPORARY -# - /mnt/user-data-7:ro -# - /mnt/user-data-6:ro -# - /mnt/user-data-5:ro -# - /mnt/user-data-4:ro -# - /mnt/user-data:ro + - /mnt/user-data-volA:ro + - /mnt/user-data-volB:ro + - /mnt/user-data-volC:ro - /mnt/custom-indices:ro - /cvmfs/data.galaxyproject.org:ro - /tmp:rw legacy_singularity_volumes_list: "{{ ['/mnt/files:ro', '/mnt/files2:ro'] if legacy_file_mounts_available|d(True) else [] }}" -pawsey_singularity_volumes_list: "{{ ['/mnt/user-data-2:ro', '/mnt/user-data-3:ro'] if pawsey_file_mounts_available|d(True) else [] }}" +pawsey_singularity_volumes_list: "{{ ['/mnt/user-data:ro', '/mnt/user-data-2:ro', '/mnt/user-data-3:ro', '/mnt/user-data-4:ro'] if pawsey_file_mounts_available|d(True) else [] }}" slurm_singularity_volumes_list: "{{ slurm_singularity_volumes_list_restricted + legacy_singularity_volumes_list + pawsey_singularity_volumes_list }}" @@ -555,4 +581,59 @@ dt_connection_string: "postgres://reader:{{ galaxy_db_reader_password }}@{{ host # NFS stuff (for exporting so galaxy-handlers can mount) nfs_exports: - - "{{ galaxy_root }} *(rw,async,no_root_squash,no_subtree_check)" \ No newline at end of file + - "{{ galaxy_root }} *(rw,async,no_root_squash,no_subtree_check)" + +# Subdomains +galaxy_subsite_base_domain: "usegalaxy.org.au" +galaxy_subsite_default_welcome: "https://site.usegalaxy.org.au" +galaxy_subsite_dir: /mnt/galaxy/subsites +galaxy_subsites: + - name: genome + brand: Genome Lab + iframe: "https://site.usegalaxy.org.au/landing/genome" + wallpaper: false # If wallpaper is true, then files/subsites/{name}.png will be copied to /static/dist/{name}.png, and can be used exactly like that in the CSS. + tool_sections: [] + custom_css: | + #masthead .navbar-nav>li.active { + background: #2c3067 !important; + } + #masthead { + background: #3a3e87 !important; + } + #masthead a.navbar-brand::after { + content: "Australia - Genome Lab"; + } + #masthead .navbar-text { + display: none; + } + - name: proteomics + hidden: true + brand: Proteomics lab + iframe: "https://site.usegalaxy.org.au/landing/proteomics" + wallpaper: false # If wallpaper is true, then files/subsites/{name}.png will be copied to /static/dist/{name}.png, and can be used exactly like that in the CSS. + tool_sections: [] + custom_css: | + #masthead .navbar-nav>li.active { + background: #402668 !important; + } + #masthead { + background: #542a95 !important; + } + #masthead a.navbar-brand::after { + content: "Australia - Proteomics Lab"; + } + #masthead .navbar-text { + display: none; + } + +webhook_plugins: + - subdomain_switcher + # These ones aren't in the playbook as they are included in the galaxy repo: + - demo + - gtn + - news + +# ssh config, only for ubuntu +ssh_config_id_file: "/home/{{ ssh_config_user }}/.ssh/internal_hop_key" +ssh_config_user: ubuntu +ssh_config_hosts: "{{ groups['galaxy_group'] }}" diff --git a/host_vars/pawsey-user-nfs.usegalaxy.org.au.yml b/host_vars/pawsey-user-nfs.usegalaxy.org.au.yml index 2f909a5e1..43284a91f 100644 --- a/host_vars/pawsey-user-nfs.usegalaxy.org.au.yml +++ b/host_vars/pawsey-user-nfs.usegalaxy.org.au.yml @@ -20,7 +20,7 @@ galaxy_user_data_3_dir: /user-data-3 galaxy_user_data_4_dir: /user-data-4 nfs_exports: - - "{{ galaxy_user_data_dir }} 192.168.0.0/24(rw,async,no_root_squash,no_subtree_check) 138.44.80.40/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.80.112/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check)" - - "{{ galaxy_user_data_2_dir }} 192.168.0.0/24(rw,async,no_root_squash,no_subtree_check) 138.44.80.40/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.80.112/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check)" - - "{{ galaxy_user_data_3_dir }} 192.168.0.0/24(rw,async,no_root_squash,no_subtree_check) 138.44.80.40/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.80.112/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check)" - - "{{ galaxy_user_data_4_dir }} 192.168.0.0/24(rw,async,no_root_squash,no_subtree_check) 138.44.80.40/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.80.112/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check)" + - "{{ galaxy_user_data_dir }} 192.168.0.0/24(rw,async,no_root_squash,no_subtree_check) 138.44.80.40/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.80.112/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.153/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.157/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.152/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check)" + - "{{ galaxy_user_data_2_dir }} 192.168.0.0/24(rw,async,no_root_squash,no_subtree_check) 138.44.80.40/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.80.112/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.153/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.157/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.152/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check)" + - "{{ galaxy_user_data_3_dir }} 192.168.0.0/24(rw,async,no_root_squash,no_subtree_check) 138.44.80.40/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.80.112/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.153/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.157/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.152/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check)" + - "{{ galaxy_user_data_4_dir }} 192.168.0.0/24(rw,async,no_root_squash,no_subtree_check) 138.44.80.40/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.80.112/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.153/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.157/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check) 138.44.7.152/32(rw,async,all_squash,anonuid=10010,anongid=10010,no_subtree_check)" diff --git a/host_vars/pulsar-QLD/pulsar-QLD-job-nfs.genome.edu.au.yml b/host_vars/pulsar-QLD/pulsar-QLD-job-nfs.genome.edu.au.yml index 99a9b3fff..dabcc6344 100644 --- a/host_vars/pulsar-QLD/pulsar-QLD-job-nfs.genome.edu.au.yml +++ b/host_vars/pulsar-QLD/pulsar-QLD-job-nfs.genome.edu.au.yml @@ -9,8 +9,3 @@ galaxy_job_dir: /mnt/tmp nfs_exports: #- "{{ pulsar_custom_indices_dir }} *(rw,async,no_root_squash,no_subtree_check)" - "{{ galaxy_job_dir }} 203.101.224.0/19(rw,async,no_root_squash,no_subtree_check)" - -extra_keys: - - id: ubuntu_maintenance_key - type: public - from: "{{ hostvars['pawsey']['ansible_ssh_host'] }}" diff --git a/host_vars/pulsar-QLD/pulsar-QLD.genome.edu.au.yml b/host_vars/pulsar-QLD/pulsar-QLD.genome.edu.au.yml index deadd9e7a..43f80ecd6 100644 --- a/host_vars/pulsar-QLD/pulsar-QLD.genome.edu.au.yml +++ b/host_vars/pulsar-QLD/pulsar-QLD.genome.edu.au.yml @@ -49,4 +49,5 @@ pulsar_conda_exec: "mamba" # Use mamba as replacement for conda extra_keys: - id: ubuntu_maintenance_key type: public - from: "{{ hostvars['aarnet']['ansible_ssh_host'] }}" + from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}" + diff --git a/host_vars/pulsar-high-mem1/pulsar-high-mem1.yml b/host_vars/pulsar-high-mem1/pulsar-high-mem1.yml index bf0416971..59b0fdcae 100644 --- a/host_vars/pulsar-high-mem1/pulsar-high-mem1.yml +++ b/host_vars/pulsar-high-mem1/pulsar-high-mem1.yml @@ -13,7 +13,7 @@ ssl_email: "help@genome.edu.au" #host specific pulsar settings rabbitmq_password_galaxy_au: "{{ vault_rabbitmq_password_galaxy_hm1_prod }}" -pulsar_queue_url: "aarnet-queue.usegalaxy.org.au" +pulsar_queue_url: "galaxy-queue.usegalaxy.org.au" pulsar_rabbit_username: "galaxy_hm1" pulsar_rabbit_vhost: "/pulsar/galaxy_hm1" @@ -70,7 +70,7 @@ slurm_munge_key: files/keys/munge.key extra_keys: - id: ubuntu_maintenance_key type: public - from: "{{ hostvars['aarnet']['ansible_ssh_host'] }}" + from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}" # docker docker_daemon_options: diff --git a/host_vars/pulsar-high-mem2/pulsar-high-mem2.yml b/host_vars/pulsar-high-mem2/pulsar-high-mem2.yml index 74d6245c5..6230bcd7e 100644 --- a/host_vars/pulsar-high-mem2/pulsar-high-mem2.yml +++ b/host_vars/pulsar-high-mem2/pulsar-high-mem2.yml @@ -13,7 +13,7 @@ ssl_email: "help@genome.edu.au" #host specific pulsar settings rabbitmq_password_galaxy_au: "{{ vault_rabbitmq_password_galaxy_hm2_prod }}" -pulsar_queue_url: "aarnet-queue.usegalaxy.org.au" +pulsar_queue_url: "galaxy-queue.usegalaxy.org.au" pulsar_rabbit_username: "galaxy_hm2" pulsar_rabbit_vhost: "/pulsar/galaxy_hm2" @@ -69,7 +69,7 @@ slurm_munge_key: files/keys/munge.key extra_keys: - id: ubuntu_maintenance_key type: public - from: "{{ hostvars['aarnet']['ansible_ssh_host'] }}" + from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}" # docker docker_daemon_options: diff --git a/host_vars/pulsar-mel2/pulsar-mel2.genome.edu.au.yml b/host_vars/pulsar-mel2/pulsar-mel2.genome.edu.au.yml index e23f4a5fc..5ef6a214e 100644 --- a/host_vars/pulsar-mel2/pulsar-mel2.genome.edu.au.yml +++ b/host_vars/pulsar-mel2/pulsar-mel2.genome.edu.au.yml @@ -29,7 +29,7 @@ ssl_email: "help@genome.edu.au" #host specific pulsar settings rabbitmq_password_galaxy_au: "{{ vault_rabbitmq_password_galaxy_mel2_prod }}" -pulsar_queue_url: "aarnet-queue.usegalaxy.org.au" +pulsar_queue_url: "galaxy-queue.usegalaxy.org.au" pulsar_rabbit_username: "galaxy_mel2" pulsar_rabbit_vhost: "/pulsar/galaxy_mel2" @@ -38,7 +38,7 @@ pulsar_conda_exec: "mamba" # Use mamba as replacement for conda extra_keys: - id: ubuntu_maintenance_key type: public - from: "{{ hostvars['aarnet']['ansible_ssh_host'] }}" + from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}" # docker docker_daemon_options: diff --git a/host_vars/pulsar-mel3/pulsar-mel3.genome.edu.au.yml b/host_vars/pulsar-mel3/pulsar-mel3.genome.edu.au.yml index 6d7f82454..fd9990772 100644 --- a/host_vars/pulsar-mel3/pulsar-mel3.genome.edu.au.yml +++ b/host_vars/pulsar-mel3/pulsar-mel3.genome.edu.au.yml @@ -20,7 +20,7 @@ ssl_email: "help@genome.edu.au" #host specific pulsar settings rabbitmq_password_galaxy_au: "{{ vault_rabbitmq_password_galaxy_mel3_prod }}" -pulsar_queue_url: "aarnet-queue.usegalaxy.org.au" +pulsar_queue_url: "galaxy-queue.usegalaxy.org.au" pulsar_rabbit_username: "galaxy_mel3" pulsar_rabbit_vhost: "/pulsar/galaxy_mel3" @@ -38,7 +38,7 @@ attached_volumes: extra_keys: - id: ubuntu_maintenance_key type: public - from: "{{ hostvars['aarnet']['ansible_ssh_host'] }}" + from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}" # docker docker_daemon_options: diff --git a/host_vars/pulsar-nci-training/pulsar-nci-training.usegalaxy.org.au.yml b/host_vars/pulsar-nci-training/pulsar-nci-training.usegalaxy.org.au.yml index 1a31e8485..ad6b7df43 100644 --- a/host_vars/pulsar-nci-training/pulsar-nci-training.usegalaxy.org.au.yml +++ b/host_vars/pulsar-nci-training/pulsar-nci-training.usegalaxy.org.au.yml @@ -20,14 +20,15 @@ add_hosts_workers: yes #host specific pulsar settings rabbitmq_password_galaxy_au: "{{ vault_rabbitmq_password_galaxy_nci_training_prod }}" -pulsar_queue_url: "aarnet-queue.usegalaxy.org.au" +pulsar_queue_url: "galaxy-queue.usegalaxy.org.au" pulsar_rabbit_username: "galaxy_nci_training" pulsar_rabbit_vhost: "/pulsar/galaxy_nci_training" extra_keys: - id: ubuntu_maintenance_key type: public - from: "{{ hostvars['aarnet']['ansible_ssh_host'] }}" + from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}" + # docker docker_daemon_options: diff --git a/host_vars/pulsar-paw/pulsar-paw.genome.edu.au.yml b/host_vars/pulsar-paw/pulsar-paw.genome.edu.au.yml index 3e8c30700..99e347bbb 100644 --- a/host_vars/pulsar-paw/pulsar-paw.genome.edu.au.yml +++ b/host_vars/pulsar-paw/pulsar-paw.genome.edu.au.yml @@ -27,7 +27,7 @@ attached_volumes: extra_keys: - id: ubuntu_maintenance_key type: public - from: "{{ hostvars['aarnet']['ansible_ssh_host'] }}" + from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}" # docker docker_daemon_options: diff --git a/host_vars/pulsar-qld-blast.yml b/host_vars/pulsar-qld-blast.yml index 5ab2fcf0a..5d2abb815 100644 --- a/host_vars/pulsar-qld-blast.yml +++ b/host_vars/pulsar-qld-blast.yml @@ -13,7 +13,7 @@ ssl_email: "help@genome.edu.au" #host specific pulsar settings rabbitmq_password_galaxy_au: "{{ vault_rabbitmq_password_galaxy_qld_blast_prod }}" -pulsar_queue_url: "aarnet-queue.usegalaxy.org.au" +pulsar_queue_url: "galaxy-queue.usegalaxy.org.au" pulsar_rabbit_username: "galaxy_qld_blast" pulsar_rabbit_vhost: "/pulsar/galaxy_qld_blast" @@ -68,7 +68,7 @@ slurm_munge_key: files/keys/munge.key extra_keys: - id: ubuntu_maintenance_key type: public - from: "{{ hostvars['aarnet']['ansible_ssh_host'] }}" + from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}" # docker docker_daemon_options: diff --git a/host_vars/qld-pulsar-himem-0.yml b/host_vars/qld-pulsar-himem-0.yml index f68292f8a..f4f83c5c8 100644 --- a/host_vars/qld-pulsar-himem-0.yml +++ b/host_vars/qld-pulsar-himem-0.yml @@ -13,7 +13,7 @@ ssl_email: "help@genome.edu.au" #host specific pulsar settings rabbitmq_password_galaxy_au: "{{ vault_rabbitmq_password_galaxy_qld_hm0_prod }}" -pulsar_queue_url: "aarnet-queue.usegalaxy.org.au" +pulsar_queue_url: "galaxy-queue.usegalaxy.org.au" pulsar_rabbit_username: "galaxy_qld_hm0" pulsar_rabbit_vhost: "/pulsar/galaxy_qld_hm0" @@ -74,7 +74,7 @@ slurm_munge_key: files/keys/munge.key extra_keys: - id: ubuntu_maintenance_key type: public - from: "{{ hostvars['aarnet']['ansible_ssh_host'] }}" + from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}" # docker docker_daemon_options: diff --git a/host_vars/qld-pulsar-himem-1.yml b/host_vars/qld-pulsar-himem-1.yml index 45d689158..7cc2a4eba 100644 --- a/host_vars/qld-pulsar-himem-1.yml +++ b/host_vars/qld-pulsar-himem-1.yml @@ -13,7 +13,7 @@ ssl_email: "help@genome.edu.au" #host specific pulsar settings rabbitmq_password_galaxy_au: "{{ vault_rabbitmq_password_galaxy_qld_hm1_prod }}" -pulsar_queue_url: "aarnet-queue.usegalaxy.org.au" +pulsar_queue_url: "galaxy-queue.usegalaxy.org.au" pulsar_rabbit_username: "galaxy_qld_hm1" pulsar_rabbit_vhost: "/pulsar/galaxy_qld_hm1" @@ -74,7 +74,7 @@ slurm_munge_key: files/keys/munge.key extra_keys: - id: ubuntu_maintenance_key type: public - from: "{{ hostvars['aarnet']['ansible_ssh_host'] }}" + from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}" # docker docker_daemon_options: diff --git a/host_vars/qld-pulsar-himem-2.yml b/host_vars/qld-pulsar-himem-2.yml index 7d98a3e86..8615ecf75 100644 --- a/host_vars/qld-pulsar-himem-2.yml +++ b/host_vars/qld-pulsar-himem-2.yml @@ -13,7 +13,7 @@ ssl_email: "help@genome.edu.au" #host specific pulsar settings rabbitmq_password_galaxy_au: "{{ vault_rabbitmq_password_galaxy_qld_hm2_prod }}" -pulsar_queue_url: "aarnet-queue.usegalaxy.org.au" +pulsar_queue_url: "galaxy-queue.usegalaxy.org.au" pulsar_rabbit_username: "galaxy_qld_hm2" pulsar_rabbit_vhost: "/pulsar/galaxy_qld_hm2" @@ -74,7 +74,7 @@ slurm_munge_key: files/keys/munge.key extra_keys: - id: ubuntu_maintenance_key type: public - from: "{{ hostvars['aarnet']['ansible_ssh_host'] }}" + from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}" # docker docker_daemon_options: diff --git a/host_vars/staging.gvl.org.au.yml b/host_vars/staging.gvl.org.au.yml index 7d250ca1c..d08c71783 100644 --- a/host_vars/staging.gvl.org.au.yml +++ b/host_vars/staging.gvl.org.au.yml @@ -65,7 +65,7 @@ host_galaxy_config_templates: - src: "{{ galaxy_config_template_src_dir }}/config/staging_job_conf.yml.j2" dest: "{{ galaxy_config_dir}}/job_conf.yml" - src: "{{ galaxy_config_template_src_dir }}/toolbox/filters/ga_filters.py.j2" - dest: "{{ galaxy_server_dir }}/lib/galaxy/tool_util/toolbox/filters/ga_filters.py" + dest: "{{ galaxy_toolbox_filters_dir }}/ga_filters.py" host_galaxy_config_files: - src: "{{ galaxy_config_file_src_dir }}/config/oidc_config.xml" diff --git a/hosts b/hosts index 827ae507b..b7f8ce694 100644 --- a/hosts +++ b/hosts @@ -172,6 +172,18 @@ pulsar-nci-training-w9 ansible_ssh_host=10.0.2.254 internal_ip=10.0.2.254 pulsar-nci-training-w10 ansible_ssh_host=10.0.1.110 internal_ip=10.0.1.110 pulsar-nci-training-w11 ansible_ssh_host=10.0.2.82 internal_ip=10.0.2.82 +[production_pulsars:children] +pulsar_mel3_head +pulsar_mel2_head +pulsar_paw_head +pulsar_QLD_head +pulsar_qld_himems +pulsar_high_mem2 +pulsar_high_mem1 +pulsar_qld_blast +pulsar_nci_training_head +pulsar_azure_0_head + [nvme] qld-nvme ansible_ssh_host=203.101.231.166 diff --git a/jenkins/config_backup/config_backup_playbook.yml b/jenkins/config_backup/config_backup_playbook.yml index 819779fbb..940d687d4 100644 --- a/jenkins/config_backup/config_backup_playbook.yml +++ b/jenkins/config_backup/config_backup_playbook.yml @@ -1,5 +1,5 @@ - hosts: - - aarnet + - galaxy - staging - dev become: true @@ -26,7 +26,8 @@ dest: "{{ backup_dir }}/{{ ansible_hostname }}/config/" with_items: - build_sites.yml - - galaxy.yml + # - galaxy.yml # TODO: galaxy.yml is now at /opt/galaxy/galaxy.yml on both galaxy and galaxy-handlers and these are different files. + # ansible saving backups of these in /mnt/galaxy/config when changes are made - these will have to do for now. - job_conf.yml - object_store_conf.xml - nagios_tool_conf.xml diff --git a/jenkins/update_labels/update_tool_labels.sh b/jenkins/update_labels/update_tool_labels.sh index 11a32905e..f634860ed 100644 --- a/jenkins/update_labels/update_tool_labels.sh +++ b/jenkins/update_labels/update_tool_labels.sh @@ -5,6 +5,6 @@ source jenkins/utils.sh activate_virtualenv echo "secret" > .vault_pass.txt # ansible needs .vault_pass.txt to exist -ansible-playbook -i hosts jenkins/update_labels/update_tool_labels_playbook.yml --extra-vars "ansible_user=jenkins_bot update_labels_hostname=staging_galaxy_server" +ansible-playbook -i hosts jenkins/update_labels/update_tool_labels_playbook.yml --extra-vars "ansible_user=jenkins_bot update_labels_hostname=staging" -ansible-playbook -i hosts jenkins/update_labels/update_tool_labels_playbook.yml --extra-vars "ansible_user=jenkins_bot update_labels_hostname=aarnet_galaxy_server" \ No newline at end of file +ansible-playbook -i hosts jenkins/update_labels/update_tool_labels_playbook.yml --extra-vars "ansible_user=jenkins_bot update_labels_hostname=galaxy" \ No newline at end of file diff --git a/jenkins/update_tool_destinations.sh b/jenkins/update_tool_destinations.sh index 4b57b146b..89eb9e453 100644 --- a/jenkins/update_tool_destinations.sh +++ b/jenkins/update_tool_destinations.sh @@ -31,7 +31,7 @@ echo "helloworld" > .vault_pass.txt # ansible needs .vault_pass.txt to exist if [ "$PROD_TPV_UPDATED" ]; then echo -e "\nUpdating tool destinations file on Galaxy production instance\n" - ansible-playbook -i hosts aarnet_update_configs_playbook.yml --extra-vars "ansible_user=jenkins_bot" --vault-password-file "$VAULT_PASS" + ansible-playbook -i hosts galaxy_update_configs_playbook.yml --extra-vars "ansible_user=jenkins_bot" --vault-password-file "$VAULT_PASS" fi if [ "$DEV_RELEVANT_FILES_UPDATED" ]; then diff --git a/one-offs/migrate-tools/templates/tool_migration_rsync_script.sh.j2 b/one-offs/migrate-tools/templates/tool_migration_rsync_script.sh.j2 index a186eb27e..307714499 100644 --- a/one-offs/migrate-tools/templates/tool_migration_rsync_script.sh.j2 +++ b/one-offs/migrate-tools/templates/tool_migration_rsync_script.sh.j2 @@ -42,5 +42,5 @@ rsync -e "ssh -i {{ ssh_key }} -o StrictHostKeyChecking=no" -av \ {% if migrate_custom_indices %} rsync -e "ssh -i {{ ssh_key }} -o StrictHostKeyChecking=no" -av \ galaxy@{{ source_galaxy_ip }}:{{ source_custom_indices_dir }} \ - {{ dest_custom_indices_dir }} -{% endif %} \ No newline at end of file + {{ dest_custom_indices_dir | dirname }} +{% endif %} diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index b8a0e14af..f19d75d10 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -9,7 +9,6 @@ ansible.builtin.apt_key: url: https://repos.influxdata.com/influxdata-archive_compat.key state: present - when: ansible_distribution_major_version is version('22', '<') - name: Update the apt repos and base OS apt: @@ -38,6 +37,20 @@ vars: hosts_filename: "/etc/hosts" +- name: Install python 3.11 + block: + - name: Add ppa + apt_repository: + repo: ppa:deadsnakes/ppa + state: present + - name: Install python 3.11 + apt: + name: + - python3.11 + - python3.11-venv + state: present + when: install_python_311|d(False) + - name: Add Galaxy group to relevant machines become: yes become_user: root @@ -77,6 +90,10 @@ loop_control: loop_var: extra_key +- name: Add ssh config + import_tasks: ssh_config.yml + when: ssh_config_hosts is defined + - name: Make sure cloud configured hostname control is not present file: path: /etc/cloud/cloud.cfg.d/07_hostnamefix.cfg diff --git a/roles/common/tasks/ssh_config.yml b/roles/common/tasks/ssh_config.yml new file mode 100644 index 000000000..da7389cf9 --- /dev/null +++ b/roles/common/tasks/ssh_config.yml @@ -0,0 +1,12 @@ +- name: Add entries to ssh config + blockinfile: + marker: "# {mark} ANSIBLE MANAGED BLOCK (common role: add ssh config entries)" + dest: "/home/{{ ssh_config_user }}/.ssh/config" + create: true + block: | + {% for item in ssh_config_hosts %} + Host {{ item }} + HostName {{ item }} + User {{ ssh_config_user }} + IdentityFile {{ ssh_config_id_file }} + {% endfor %} \ No newline at end of file diff --git a/roles/install-tpv/tasks/main.yml b/roles/install-tpv/tasks/main.yml index 5c1b21f78..cea5d1d5f 100644 --- a/roles/install-tpv/tasks/main.yml +++ b/roles/install-tpv/tasks/main.yml @@ -3,10 +3,12 @@ pip: name: "total-perspective-vortex=={{ tpv_version }}" virtualenv: "{{ galaxy_venv_dir }}" + virtualenv_python: "{{ galaxy_virtualenv_python | default(omit) }}" when: tpv_version is defined - name: Install Total Perspective Vortex (from git commit) pip: name: "git+{{ tpv_repo }}{% if tpv_commit_id|d('') %}@{{ tpv_commit_id }}{% endif %}" virtualenv: "{{ galaxy_venv_dir }}" + virtualenv_python: "{{ galaxy_virtualenv_python | default(omit) }}" state: forcereinstall when: tpv_version is not defined and tpv_repo is defined \ No newline at end of file diff --git a/secret_group_vars/ubuntu_maintenance_key b/secret_group_vars/ubuntu_maintenance_key index 1aef0914a..06976344b 100644 --- a/secret_group_vars/ubuntu_maintenance_key +++ b/secret_group_vars/ubuntu_maintenance_key @@ -1,182 +1,279 @@ $ANSIBLE_VAULT;1.1;AES256 -34646232633761396232623538313065613037323535313639336538643031393065373234386265 -6236646365623530653133623438383766666538303738330a376638383034613537656637323037 -61616665396161353137383062633138643933643365623334353434633366396537383937333632 -6336663436323765380a633633336466326466613537626536373232323532303637343761313237 -66376439646161346161363639656564653039643336396662333465623432336464376437323032 -33333631353039356330383236313536356538356361643532666135333137343265663766666362 -33383830386633383064356637303364326635636263306666303731346532346638303136343937 -33636236636133366162396538363431653864633161336430383734646661633136666133346138 -34373530313630363632653764363830383966613762663762326534633233343035383737336362 -62326433393430643831353633653738656266356235636630333564396264643932636165396663 -39666163646331376665626333333131353362383563333339633632643465316364366634643865 -31363534333864643435316639616435666363393032653131653630376535306534333134643963 -34633434383737383463303438353863343964313365656235376132313531353861376536373231 -64366564663162623735663165656464316136316437373364363935383330646630336562616562 -36336232366135393832313639303132663365366433386538636437306663303263616364316561 -64393039303164376438626435316638613631626436653835346365353331393536303538303038 -37383730613564383665393536346433363363633261353162396563616138376636343634383734 -66613566396662353338663763326664323637376266636637656635666534616434323434353933 -34333231643436393265353935393738306336373764636565653265386130663066376535363538 -63373566306430343461663031313666303363613232633464663665633632393138613636353731 -66306631633161333233323331663638616266306366393363383864623034613038653938306632 -63366361366163346564643336653461653261653666386666653535623863666166376230643261 -38343166306133383563393731613236326664623135363634383334343238323733353631623432 -38663331336338643566323139326662376435323433633363326666313564633430303536613666 -66386639396561356234353262313538343936396166626561323063393362633065316334356464 -33396163396430666363313539346533366466383930616361376263613335643366353461356666 -39666637343830323863396165376165366139643638303230356536643635636638336161323735 -32633338363939363865353864326430343037333237306664373231633863313561366635373565 -37326233633661373735363263373433613034336334383437633165313164653735353131656539 -36646263396139663565393634616364386565343066336439623861336366666365623730326364 -63353638313635653636313365383435663632373230356438343030393062383538306664636630 -37613636346664323337323232306635356435336236656539326135626138613934383763633933 -30323335613866373163663538323131323566653266613361323239356665623361376136356662 -66366135396364353139313831643630376463626339663963663232326336613839376162356235 -38643130636563616633306434663465316366353465383431333661323265363361656233623833 -38666633656139386533343730306561666364333836616230316263383037653061353965346638 -39363764323734653836613330383662396139396634626235393764353163663935626438636331 -30343937353465653366363464346465626630383064383065353030623061653738346365393732 -32363162333463333963393332623339303037383537383031373966616264656562373066376463 -64656236376632636332326231343232353966643066666461613139656438306261643538633033 -39613937323830613065646466373865306130396135323662363832326363356362353830646266 -62343762633238376532353735386166643438613132363065383862306532383362623933393334 -32316165633961646265613839656166353532373335653365316130643236666331663531343363 -62613563363437303864376332303764383031376566363639323036353638623035353534646461 -36353139386236373533303033306164663863306362626463386161336630656365356636613333 -34383933316462643836373637633439373032653065376162663735333937393662663333663537 -65343464393531613731643438313766353234633635653535373533383263643365383431333531 -61363239666236623566613062336632343466643735386230386566383238376366313963396539 -65336436333235656262646231376361363965613536386462663263393335663064323538616236 -65656663613461626166646164626564393233373035386263393039343238623938626332336232 -30303032626232636530343261653861353535336137303537626562383937396463393531653163 -62646161656332623561663534663832373730666539646131346331316437353765666437326637 -39346562313165303537363465303236623531313133383466323366343262313130346135353063 -35373133386632316532323734636565313864373339323434376166333630623936643564353635 -33333437626137653033396632383138626165343037306239663531663264636661303639303561 -61666531663930616630613761373265383537363562333433623736386164373734346536376231 -31623335346132623165656130346332646431646235303561373531616461363837626633326438 -66323338663764646336303964333661343838386234373831326635336438646562633864626232 -61353030343466373065366663343163313830646133356665333432643665636232373131616366 -62653066313435613932656334653335323039666534336631636365646262303966366636356439 -63623430383037373034353064396330636137646435323534343865353934303132613839626166 -64643938613664643732393062313661666634646430623530353863376438633136383436653066 -30303864353637323664306365383734363830393564613666336466663663316163346665343530 -36616563376434366530666362353361363835383661323235363834303536646536376339323465 -63326331616562333861666365646166613331366664313564383365306533333235323363373465 -64346463326666653838323262653937316236306463333734366236656138306434353638306463 -36653461326461383937393562356539643564383233303032333434633338316432666533643838 -32326439336530633732313231306537333865316635636161643964313433396562333336653139 -64623031386132616432623731373839363035333037353531336439333635303536373361623966 -35653134623062643165303238373361333438323731356362383739613765323539353864366565 -39373634343164366235336266386138343533623835393235393634366534313564626565396130 -62353331376139666533346534383439326564323564626439363730626231346334666332653863 -32636434323239616164366132653766616266326166653564303333663766373666643361623732 -62613861346238666361333465313831303838636536326530386366363262313637636338366334 -64346661616131623430666437383761623936616134373234326461643565333961393366333439 -32666466613737343832343664643231333261323864626539653165316235643864383430663934 -39376639326562316534393861386239353433303333316430613530623833336233366365313937 -62393762613464656333323535313731613638376137333361376239306532396364653630613639 -63326436346437633261613134616539643130653939363663363863616162326331323634343638 -64663238323465383531343664663466613430643463306431343435333662363061613039653964 -32626138323230303638373939396339396535386537666232346238393530336333663361646133 -33613566623936646432396565353033346531356663626261393430663431393161653936306565 -35613832363064653332363664396235333839646132303035353235366430383133393261663363 -34623566613132613534653135643563366636383064646435303665396231643862383137383932 -33623133646537613035616534653535656433643134663261313532623235356464646332393864 -33306365393663373838616666363062646138326132633239363237626635326664386362306435 -30353764633531313865653261376135376465646165383132383236316233373934663032303065 -37356161306632353066303230613339653761316131356338646536343837343338633039653962 -65623134643334616131393936363061366338666238616136373239653436626135316463386636 -32323563336334653538343261396531653461323030383636646439343165623831306533633139 -30313034643835646137333861363337303561623136613932356532346335343462616365363166 -66343032386133356466373363353034656664303334653865666137326532373064366161313736 -33646365653633333666353136313336663161383661333963633638326264613035613539613164 -33356132636439303434343033393736376439613038353135643366376230643537303663643462 -35376262393836313638396161623034656665326263303763366237323164623630353236633466 -62386130366432633465333032373634326562346365393037656530663465383438663838663539 -36383431366632613334623734656634653431633965646362356661326464386233373464323833 -38393139666562643932663561623564386636656231656665373164306537326236383465303832 -61396261626235363766636439333362633836343033653835616136633632343262393231396530 -38636334336661393564643865633932356563643066633665356539306537376266656561616433 -66373630333165653630633539653862333436663431313064393530626639643265316537616631 -30663836633637393566333865303566613965643736623430333832663938633966373365346138 -64633531303039396232613963393735663865333435333366333966373962313030646338336134 -31343631313035613534303433306635316437343539663466333765656539326435313330383761 -33613964373739303261373831316436323634626362626166383361626665663638353364663931 -66356232636532396636643834633461346538616530666462363736373238333839333263333633 -35343632386132353262326331313963646532383436373962663130643837373664306630353633 -30326664303831323439643339663962343866303636363637396436393566633035373063353263 -32373234323837386132623336663466356133633864623334363465356435633834613461386166 -36643033616137366661633331663061393963653831306434333930396263393933303363643532 -62386362626533373430666366643163366264323062633164316561616536343735643063313563 -30343637633337643035336662303537343765326566326462613434386637333735343739636338 -64386236356362376136356638353137366136313031353033616536643836336463353030393832 -34653464633538616631336232666634646439336139616338373866626637303334316332363030 -61653066326266353332366439636562396437303232616134306533386231383737306463353130 -65323733396362626563613732376465313864656161353863653966643639323733653761376133 -31613261663439323834323634333266366431306463623337356365626635653439626336366162 -35333730653862623733333530326666653534303230373762383836623565623234316133326438 -30366663653835663336343333376465653736326238613532346662623466656234643035383937 -31343136323862323461616334356233333463366436643334633736386136316338303131333532 -65363465336338366665333934343565383061303833373438353536653462363161643831306561 -30323064353632383633623838623663613963663133343135623931613665303566623165326534 -66353561656264386335313838393538633531353364346335376165343131383961656239353034 -35363232343431306135373933656539383064633034666433303463346165363236633537666534 -36643734303532396133633564663564396335643437643637626361356133396364656436383836 -37393762333739663739366531633763633164396564373165363033646537366639383235653839 -64356533373566366235383165663133623930323364643937376331313865353537353838323565 -64623239656262346564613632363838343563326138366337663539383037396435663932626561 -38306631393661623461356230613833653334626663363236633533323535653030613662393433 -30323636623762373564343932633136626332653365613236643239303535393865643736356539 -63646663363762363435313037316637356166663032626532383236383930343337613139646132 -33303832353864363235313564636538353331653361336463386131303735353334343633386563 -31333562386461373161326161353937353038326637353237326431333439346433386637316435 -61336137336363643037356166326437396665653334653930393566653462623764643264366536 -62356637303038336130313665363534353031313334373735623634666133643634623630656563 -39306438653861653161373935393233346336393735653733373831313438653661353539373837 -63653236356531383237373733363739393534383265643137383634643936623135323462633963 -34636166353966633130306132363561646238383164346334613433386162376466303365356635 -61383663323937623664626333663466643737313336353139323961333238343363326563666330 -38643130303666343930633663306437373161653631346131643830353563303563643563623066 -35393336366163656237386537313937326265653165346631653832316663623538316330613263 -32363932303966353965626665336332363535356562353234656231643932636130656463656639 -36636137373863323132646164666430313466383366343561616532616161383138363430316635 -31313166363564313433383630623061316135363463316361616166666131633964353664323039 -63393763306437393738353664663166396538633430653461386261633738303936313836366435 -35333365646134643863373334303331613133343935663038356465313064666532626662386130 -36353334373961643434316163353936616639383137633462636165646562343164626461663463 -33353765346563326536346531336362363130383234366365353331636432346535373062303663 -30653139623937646361383761616265653861303534393334646333363436373666313339373638 -37646566656134336564626632653730643336363433356137306562353939383765303439373530 -65643836373838386565333835613032613139626465623030363664326161666133346139346636 -37636666626362306632643038626364383362396663323562396561306234653363653664363763 -64383864363637333338646566303131653832613030646562613732383233626132323864656438 -35643730653162343737653236643136633538303364653638373736623963316631623331623633 -61343436303737326330656362663261646363303263646635666337393664313335316538313964 -37383065366130393835623761333766323637396633366538333639623330653530303538653131 -33353062316532333862346632353661633339333737356563393531633032306433656463386662 -37366135666336333734383266363330333330616433636536366132636263316232643562346131 -32626262323431383636653933653666316137383464306162653438366134666265663436383734 -38656230306437383335343263663364333163336363333430383265323139623964636436613166 -61383165323631636361363064336566383432616461343662633436333037396434306330663263 -33306166633632363065373738626331323133623863306238313535336462343162663033313230 -62313066336263636435373135383735626665636334643830316136383163383032356133343031 -33373162646131353564613830333832653331616131646163353231313239383339346664396333 -35663864656466626661643637386262303365353937336132626336646238633433626464393465 -37633836666638333039656461303732396238306263346435336463353432363136636461313137 -30633937313265373365383134623366393336333733653061313862366632313363646133623032 -38666663373831633331636232346439653138333734613439386430376264323438353161623464 -32353566383337323464386539663237316433386431623663653763316664646237353161373633 -64636461346435303662316335386366666165623333363430316232313330643635353237316364 -63666635353930653230323365376461623566326638326337623334306430303265326461333439 -33323264643866373231386466303866376530396134636464636133663432313265353036613462 -62636463383766666262383965633630623337303233626265323035373364643730646433353161 -63623733356432623366323664623735336661616266633833373936313365303866373331373934 -30623939363837613732393638373662333637383166353962346638393963306131373561656266 -61386131653562646362356330653465396463373033303739623434636334343538623030363539 -66383561396237383461343061343736316136626234346338643335633265376535303963316533 -66316462623563646239396364613362356237366166306135663564666130323835326633633765 -34366231343966316664393338383538383035663636316138316465646665616635343736383035 -66666234346534353932386463366330353861626334663661353639303931333138616465323833 -3834 +32633233396138623732323265616436356436346331333966346364353537313939646636353466 +3831366530646633363739313637333630373861313330350a626565336430356639346130363135 +66356633333332336232323463313165343431333534393032326632666631633633303238613739 +3439616465393530630a376533653831316435656362393065313034643965376239356136353361 +35663635316434346263616366353534356431633333656462356135643362666333383961303035 +38653632663733346433393563396237643430613434316138326333613062323134373137333163 +31666362366161373633346362326465396665663231373962616334363034363837643363336339 +37333361356266383966383434363263366137303665613930643066393465386433623763646437 +64323864366637326434343930326134313632326561393237323562653035326437373831666565 +34653234313137636437356366396331336366383932653061373061666334386266626238643932 +30336136313166393237396664346638663361363936343438393636363931393265333131373334 +36616565373864623063373362356463646232343963353033616162313661383139343531363662 +63313861323537646330383237336330363633353962613331653266356566396365373030346230 +65316230383035623363376662373333666337393037353237613265396137383838653031343835 +32383331353736326633613032646463666630313639383237663464306537633338346461303633 +37396436343065626236653862653736326139656337333065656166623362343339366531316131 +61656336653437313464643036346539633565386565633936383865383161393464336235363032 +66663566303538373935643237396238626533356235363961363339353263643133373162383935 +33363636313362616135633966356663353130643831366365623439383034303966343266646534 +62373834336231383065646231646632653830316466396464323137323061363663363865643035 +39633834336236663739316136343230353734356436316264326339633233306333376336653461 +65363530333134396337343762326431363334653734323130316632313432623832613036313932 +62366337303830623733343530643864626166323932356536323831306435646139306536363032 +65626563613631656362333434386535373630666461336233386631373737383236353036623137 +30376331353865393531303864346666333835356162303662346139633131613435623630393733 +38663233323137643639316364613761653966393031313337333065653733316133363463373532 +66313961643135303935393835313363343233303238653933326264373739636130333337643735 +61323037666165383238653763323961663964613337333564373561663462346133323066633466 +37353364366334356463656332316164353935333664326134613765343339343037633961636536 +36323136326466666161303262306230646161663938356632343930653162326333383939633466 +31633630313933626366383233323338326461326131343534333137376636646331653562643335 +64386531326131663063646635636262333533613038363131636233303630356332616165613662 +33643131313837653732336330653565303835666534343763386362653862633363393833663162 +30376331383861636363326131386230326230646536323931313932333136363262613763333262 +37336338333037643735366566663139323737373363633636666433346361393663626463366366 +33356437656663366436393833303337363565626536636666613331316538373333633364316664 +30386664656563613736336634646165663737396664666636646161386636393738663064356232 +35316636626361663163316434393862376533623733613861383464323038656532363863316331 +35396339663337616433313961663164313038366338613630323735636164613234303633303239 +30396636343763623938333834333764663732363730386661613437343533633436373537653139 +63383661363361646131326662336537613630616661346562346164633131383266353835356638 +39313061623035666664656163323436306364393935323632303539616162666166386461623637 +32326234643638613139646236393661636462393332633333626563383836313037343136386561 +39346232323933663837376137303835616635363831623162353333323134653931316139616465 +35303735316639393938386132663437366466313766616639313733613237356638636362656136 +65636262373861356439343636663332646131356430626539376462643563643038653837363635 +62373132306266313938326230646538303335666666656535633461383537633334353934306463 +66383832323338636465313837363431373964353035313262656661356538383036633865636562 +37346166613534663236623232336132383262353836646232313630353239663036396263626632 +33666565383766656630636465653838353536303364643330396562386436333538623835386261 +65326561373962386130333133633233613432656239613562376363323331333961646664653761 +35316334386435646235303066333661313136376236313564333237613461663138623137393131 +32356437666137666639373763663966303535386464336539643766663935643665303939336331 +34643936303365623065653033306532613665343366363231663634623932623364386130393066 +61636532363964633565326136356236623361343361393239613862623339376566653237333235 +35363336303164663830373033393565383535383537323439623133366163333835623939616137 +62313561376130633361663963396639316332666339313138323664396564343133376465323266 +61383364383836613462373739353635656634373961333262666563353238363033346137323531 +65343564343134316339343035333764353132656233613837626365303563623465386663316239 +38326165346132636133373630323530643133306535303661373531663638373638366236313330 +30653961376638383261313334366661333733643130653761373065626265303639643331373839 +39326239383966336266346464363134343334316564653137666564326564663862613663323630 +62316637666437663133353438316565653264303031376331353864333563616132653164343362 +36626332643164636236353536396336653165646636613130396634396136373063383866633337 +32336230643932363366343965316330623635373861396437663331363038386165323461643466 +63633332663839333935336539616162316265626261386231326533643634363365333062393361 +66393565653139646366396232623462336531313030363566373731316432633965323531306235 +30636432356639346262623633663831333236613935356638373561306165356537343132313432 +64646132656438356431623765303063636664343931613633376635373530393962316366616239 +64346239336437356433616262396138663034313330373464653062663262643361336330366461 +64643732313563306363666636353638656633326139343564356365366536663461356466653434 +61383432393663316666366435356631346466373363323366633364343466623336653664316134 +32663936613039363934313938363135336461623034343139313565646231643564326233633062 +36393932323934346538393237633836373138363531616235363531303238383135346664363237 +30613838646661306635393331306236306663393966666662313537386661343239333432323761 +32303662663632356537313933393562363139343836353363633163303836636262646464386631 +32396565643533643832333964393262376630303138353331313561373064616139623733353466 +61333133616635343264343837633138303034663038316138323862623037663866373135616438 +66646236396330376237633234643532313235323861366262643936363932643565646539323538 +34313735616363393634343338383737373433633962333330303962303961323037616432643932 +32336163316534326365633431323030386233326130393830633232313636393362623939356465 +64393335626361373261643664316539303032346436333936356331383531333166616363653331 +39613039666363613131383465373634613365386466373461666634303936373534613033366136 +34653532636235303830653563653038303430386630653231653062323738633261386261333830 +31303638346265653163636266613662666431386631613334613130353635623838383330346435 +65396664386163633231376461306436383331333538306337353831646461353233663835396364 +38306239373164376430666438623134643361616339326530393631643037326166373434653834 +33643539663832306430653439613834663737633862626438336163663330373035653461643038 +35333131363532383266346531343137376561326266633938323331653333366266316366653633 +32323630613166656462303433633533613837353361663337356162333765396639323430383462 +61356165623130306164323731303636393163316333313461303330626661636331343138626661 +66373333613663333330663562393463353765303630353464303061373661663231363066353234 +66366461373130376637343363623839366465613161386364346138646233613231613562353232 +35353438323230653263653761303461336638366533306261666265333133303534636431386234 +37303063366237333064636532346163363861363030343432653039636235303834313335626330 +35346335663736636333306662616137623539643730313062323337356634346237386565396263 +34333338316431363462613235326266303565326532333665356661316630626530663837303664 +39356131313135653566306532663934373038373862316565393231363235633632316634613338 +31666233356132303838313630396366636634306630623738616264346230643831343439396162 +38343131323465373735393933306363626236336165323161326432373062663539393631333231 +36616236353636643162663539636537393332626462343661323536303566353632656438353738 +64666133386232663066636663623436393536663433326333616465383531333037326237393037 +34666639353235343832393630396234373238306664303033373466333366303132376463633163 +39643033376165366466623037653632353165343836393136393063633936363838613833376462 +65656537383438396234353439613632393937393666383539366330366632366436663062323334 +39656662626163616334326564323863636237323838643262303761393333613235626335396333 +37393439623066323030313732626663646636373737303361663636303861646530656564306562 +63653164643463663130383635313638316562616634303830636337663230623939613135376133 +63343632386537336331386633333335353865663631376361653164323030396432363061656332 +37396261626134346430303034333363363633633539396630343639316332303163313034373435 +32373964316336663764343230373263643538333261383065633737646636656531303337333338 +30393930643164636633393534623433383864633061623861653931643761383666633839316137 +61343833336161616335363730656465336235643362333139356632313432373539616663396530 +37343931636466373465383463313432623866393336326661323563306361356635626661646362 +38356661383933333831343066383962363931613733343964623161393863623136356533313339 +64396139643835343565633636366532376565376163643335613131313233393063386130333430 +32383839356661656231393261646636306435666631323535393032303736666434636431643437 +66373436633562376366346632316134613535623337646331313835633135393936303637323762 +66343962666635653030376162643035656263303963363862343434303337633238396138326431 +39363131376664336462636235303534386533393661663432303161633637623966663561326664 +38363566646539353265366636346339396434633065303739373334363332663932643061383665 +61356662333134393861386164383962326463353339333933323966636664316366613864643665 +65343366383636346435643339393338353636613431363932313363346131363530353337613435 +36303764393330363233323832633339633366613962633661343633326464653837623333623637 +33356535363863626232353637636166656531643064643435313531616235336666313733393631 +31346535316539643331653838323439366431343563616263386566366334353332376630363638 +62343761356338336562316463313437626562626263323939386634356264373766316635643336 +39356666656235633536326638666139666434316539643430303239353536356264346432383562 +36386638353362306333386235326339333163613864646635636438356162393663666237663064 +64636166343432306637323534343330656162636662633439366365656363656131636564646461 +37633830356266393165336564333230303632343265623565383831323366663564393966356236 +65366664656336626333303231366337663863643239333538373831353036366639663965633032 +30303231663663376634633066663832306633386235343962646234346533386138333762643435 +66633933636261383334313662383337636635363737663533643066366130303032313739616637 +31646230613730356435363063666437333533343365613135313734653131656364386364316536 +66343531663963616534373038646536353339316238323133373563303539633935636239373036 +63393566333065366133666161386436303661303234356136653662643464633338356239353638 +64666631323462613761383731356236373636646333383861613666323539323837616339653734 +62376334613839386332346336363462303332326365363339356236666165326631323338643061 +37326363346236346566366333613735373163336136653331356133326135343730646433366335 +64386334643936616334313064636562623264363366333639333531373737376664653361663931 +38366434393334326136646266353433323438346435313163616162346361383365353462356437 +64623634656237303762616362323862613732363639383831616434633661316436663230326130 +62366636366232376535343164633835393835376430656562393466643962626432653134646537 +34383238633239346162343963303434333637373964636237346138363634613631356164623663 +33663666343862376638626564343362393830646239663431383630663364663032643435653962 +31383465333836646634663261376232343965613937363263353365666338616638393334366133 +62366261663365336161663931333965653336636633373333346261613163366439663136363761 +32343837626161666430303265323039363334353833363162343335663365316564663535626531 +62383334626537383738613162373031373639656537343839663536663535383236373932313662 +66353765323265316362303963663433326634613262323732616133663734626634326161643137 +61383163336564333364653535353063303333356234303131306636666637623862616430303031 +62356531643039633138643439656537363165396531383639613634626536303033663939323632 +65616237306437393066366234303437633736343833663762333433623633346565303236313763 +36343336313039336334663464643065636232666633613236356231636662313933656166666436 +33646365323531343863663530393632666164633639383933633539343866393165336365363532 +63366538373834343566393165353335383535323666306564333837373934616337366166656163 +64353234323535653830666462393736653035626336373131653366393263666330633937623339 +34396361306632393137333463323538613964343436333039386662613766303261616139613662 +35386165666130376664653638363635666632383832306134346461353136353865336264303934 +38663035313364643035383066303563393934336464636136623061653435383161366234633630 +37366539323232373436626666303263663863626233656239613730613261306234653034383564 +32643639396233306439336137383139323765623965383565373064343963666336306166303732 +34663930613662396137343637663239333961383630646266643531643237313333313666303836 +36643362343966323564393364653463646263306635313534356432396235623066666564366637 +31633532666330303938363661626335326564376464373230373731373835656535633532663364 +64323236386165303466343137336133316638623165313036326237633830346363373137366635 +64386461383334643763653364306635343739346665343566653538336638663038656239353765 +37393437393735623263346531343937663139353334343434626537653461396338633261636231 +61656533306636653034336262343334396434656230383439633562396135343733393061396231 +34623330393163386663383930386362366165343066643833396161373263383433643565636536 +31643033386136636261346663623565316161316335333761626430363965306130386532326561 +39353334663664306335613165623864626138313138396435313633653034653533653764643363 +64353535623137663363623233643531316237336630333261396137636661666333666561616164 +65373936313234316466383461373637313933323366323136643336323665306234643966373636 +61386639356466646662313034363361613533353530343362306163646564633861386362323231 +62613331646266383435393938306634646663326664356238643363313131623839393932303262 +36363439356330623437646335383665343830376431386633663635373631373166313632323963 +63613839396364633632356334616262386430343634633733373266373135666434373932396237 +65646338363035643533313439633838656132663863626661393662383635666530663538353338 +61376132333663393034666232666537656361383261656530346139646230383137376531626539 +61636236636465656130396263656164613934653930343165623766626665643538666336653264 +61373765373633323031633765663139366235393231633431613562366664636238313538346165 +66383435363265656263363765656433613035373930396562376562613561383330656333333664 +66663335393838343131643864343632386434616334663030376338313034633263666130376461 +35333865616164393462353966306564663163396433306364303335343061646235633664643962 +62316230353739643532643936623933353037643938653533646239623366613666393632303533 +38613532613965336565643763663137333463303866633136376562646463346137363037636266 +65356332646635643435666534323962646436393362616232656163646632313637393631373061 +63613935343932386436643733313361646239333863643661653937623035363462336534333065 +35353031663536313830346264376132393234393235643836303665653539333238393733383964 +39346465346430343862383363333630323236316264396331353765383663393363366263303134 +30666639396433373362376133636338333066636333643034343132616239383663333132396632 +33626331333134393533633635613135363564636163383433336638616161376533646635643064 +34333162653636656334613234326562343163613465323531343237373133663230653466303036 +63326533353339323063376238386539343337326261623037366363333938623564336136373035 +35636239663535653537323563373866373561313635336562636461646335316365326133356634 +38336663616161396630393861323332666331643939656333313061336336366334633234626436 +31343131333362626539313539643932373263393034616535353062646231613961343533623938 +36323364303763333065336131366365653663313838396139366138396630323663646535643837 +61316234636562333237356361326231623064336537326432303938313531616534333436306230 +33303432623730356532623931616334333531336631613763616263366233346662316532653963 +30396235313664656562626338343063383066383137346639633235373762373764343439313136 +39653661376463323730363461653733313965306132383339653436323533666461323263626131 +34663362326136363233623231633730633762323063653062366433636133636439613533313665 +39313731636166303431366666626236353161633361643037303435633131353134653864303232 +64323739663661386530633365353932393666343537646361396533616231356634613966636230 +35633732306339613739396265333234653134656563386439346165396330396336633032396566 +62643735623365366339663734663139386363626239376633333664373131316561626566636666 +36636237336266363734316632393039343231373133353137316562653464313335616230396663 +64396439626533376238663765346466626138346533623566616364356430663339303938313666 +39613839653736393332623565333166306364363036623563613266353836636430323262666133 +36303336633161613663336165653766363534363730623338656234666634373365306237396537 +39316230613666336363316533373166383333353932616336376564353262633065616339623138 +37373431366631313761343833656265346266366536313464373733626530653163373564383664 +65636365396532393836666165623538303131646165323663376562666339383939363731646137 +64373530393536383935643965393732626565623461383735653265383065636665316232623232 +65663730396564343435303461393264313663653531343730303639663838646630646335306664 +62623134346533393937303835653463356332373938353066663037376530646331643863633430 +35376235323963353564643230356262363134343236336332646162666362653031303661313766 +31663437616433396535313531653865633336643232616530323361373234636666346235616264 +37346161333466393963343063636337353037656466366137613536613535313033333565383565 +38386262646538333236366263613531653161636130663161326439393963323365383862396462 +31343462656635396438336464303161376663326366396538643562343932656466373834653864 +64316632626635393637613732626362333033333739336363363630393132633263313331313330 +38373866353930653135363662623439623531663030393466383561633633636633623233353737 +35316138633266323639353863633862336434393734386362373266643961633234633330396537 +31333431303233333131616139653539323838393431643238353365663936346236613162313930 +39316234313334313661373739396131656530616234646234363832643836393366653031386162 +62333135383237393830643330336631613833396366633534626364653564656235363464343638 +31323265646332376230613230396361613938623836626162386434393132333866653436663031 +37613563636535326538313964643737336137623635643362306639626337373663373230376461 +38643365626137363535626663313533396230653230303664646131393535366561356239393566 +64623833623765316230306165303834326339363461346562653931386533383039643030623864 +61633563613431363233326633616462356235326531326639396261656235636231333762383032 +62643830653637356539373933653965663061373730383638643162336635373863643261613330 +64653634383633376336303564386332373366316632613238323663626532633861313461373932 +35353565623161303436396233383939656637633332363065646663373065396432363033306361 +39383932323636643133633832623438316332323064323333626131346335303465323334383964 +61666661656532613431613539643536386435396164633863303839383130373266333436663861 +61346439616562306537326364343631366139643666653362626164383337656533303836343831 +61643062353034383333353962633963316330336635323631393838643431306164343032343662 +30343639633834643963643632333839613964636665326134646131376632313763656362613433 +34333533653533333638393134386133376636316466613233363031616534663032626237353063 +31663733303036656165643034336639633432376432653961326131383236343835653338303130 +36663230633263626165373962613631313132393738646234383661313832363539393461343633 +33313237376435343631653864396134613762653961373661393037626139353762636532376638 +39353831656636373765613962336234383134363033633033353038306562353137653239333939 +35663030636230346363663039363436393661313161303461616466306663633162353761326565 +66333662383238396663653633346334343936313266303366323931623065613036353131363936 +66646131326233396232646366646634333738656631316463613133623437643063623566663939 +33373434393138383836646339613363316665323061343463623939643634666134353533623366 +64383434383564653030643861653566643633303035376166633664366261613931653931316465 +32393265373133313564366138646662393234393563313836323031383339396561346362313238 +65656163306237653132633639333332626665323831656364333434353631386661323331393764 +38373734663431373163633566363034356538313938663264666161353638643763383937303634 +38346538313931626664313863663234346233343466336537396666303336656165626537323461 +34336265646363633466646436636330346430613339613166643562363536356530613730373232 +35323331313062653764333562333264373531656564666536363638653162326532323339653837 +39643130613030306538363264393764636632646632366430373566313665386238643439613533 +65623230643766303964656265623164643539613535316236613064306430386332643536633239 +33636264653965363663346332333861306639386530646438643830353962653439623862626438 +30323338396639303034356661303131313032363138666365643165666131373166336661633337 +61653166353930353136323337336265636664313462366435376231323236326461663363356337 +31353834323362373631666661653931303934326634373836643063646139663564346463643039 +37643835336435626132613737356231373961383639613038316562623032353139383264633162 +39623064326533333030303336383639373331316130396664386138633439366133363437333435 +63313736393864373433653162656633613232353564396165396130386539356232633134313433 +31326531363639323939333462386362653138666233353762643231323534346565373336363133 +38363265373664383832653061326234396532376331316630616633623536336435386237346561 +35303435346331646630333038626637626436356463313138613838653230363764363931373130 +61363331613136643830636563373037373663383962366261306633346337303265643339336533 +64343930323264353765336332316161653031626336396235636131386465643837383964623165 +66393134323632326562393130376234323165376134363565373231633935373834303638306365 +32636433303039653963373464323662313361623835346566653437333634396366653530663464 +32333430356638303437313064333533313564313464363965613539333833653639373565626335 +31326261623364373662303464303439393938613266663538323266396430373331616439636262 +3238316533396635633638343262343635633636326231326138 diff --git a/templates/galaxy/config/galaxy_job_conf.yml.j2 b/templates/galaxy/config/galaxy_job_conf.yml.j2 index 3ea2ac1ba..6931fea4f 100644 --- a/templates/galaxy/config/galaxy_job_conf.yml.j2 +++ b/templates/galaxy/config/galaxy_job_conf.yml.j2 @@ -23,7 +23,7 @@ runners: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 amqp_url: "pyamqp://galaxy_mel2:{{ vault_rabbitmq_password_galaxy_mel2_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_mel2?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ pulsar-paw_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner @@ -33,7 +33,7 @@ runners: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 amqp_url: "pyamqp://galaxy_paw:{{ vault_rabbitmq_password_galaxy_paw_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_paw?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ pulsar-mel3_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner @@ -43,7 +43,7 @@ runners: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 amqp_url: "pyamqp://galaxy_mel3:{{ vault_rabbitmq_password_galaxy_mel3_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_mel3?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ pulsar-high-mem1_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner @@ -53,7 +53,7 @@ runners: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 amqp_url: "pyamqp://galaxy_hm1:{{ vault_rabbitmq_password_galaxy_hm1_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_hm1?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ pulsar-high-mem2_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner @@ -63,7 +63,7 @@ runners: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 amqp_url: "pyamqp://galaxy_hm2:{{ vault_rabbitmq_password_galaxy_hm2_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_hm2?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ pulsar-qld-high-mem0_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner @@ -73,7 +73,7 @@ runners: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 amqp_url: "pyamqp://galaxy_qld_hm0:{{ vault_rabbitmq_password_galaxy_qld_hm0_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_qld_hm0?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ pulsar-qld-high-mem1_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner @@ -83,7 +83,7 @@ runners: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 amqp_url: "pyamqp://galaxy_qld_hm1:{{ vault_rabbitmq_password_galaxy_qld_hm1_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_qld_hm1?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ pulsar-qld-high-mem2_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner @@ -93,7 +93,7 @@ runners: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 amqp_url: "pyamqp://galaxy_qld_hm2:{{ vault_rabbitmq_password_galaxy_qld_hm2_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_qld_hm2?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ pulsar-nci-training_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner @@ -103,7 +103,7 @@ runners: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 amqp_url: "pyamqp://galaxy_nci_training:{{ vault_rabbitmq_password_galaxy_nci_training_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_nci_training?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ pulsar-qld-blast_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner @@ -113,7 +113,7 @@ runners: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 amqp_url: "pyamqp://galaxy_qld_blast:{{ vault_rabbitmq_password_galaxy_qld_blast_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_qld_blast?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ pulsar-QLD_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner @@ -123,12 +123,12 @@ runners: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 amqp_url: "pyamqp://galaxy_QLD:{{ vault_rabbitmq_password_galaxy_QLD_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_QLD?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ pulsar_azure_0_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner amqp_url: "pyamqp://galaxy_azure_0:{{ vault_rabbitmq_password_galaxy_azure_0_prod }}@galaxy-queue.usegalaxy.org.au:5671//pulsar/galaxy_azure_0?ssl=1" - galaxy_url: "{{ galaxy_infrastructure_url }}" + galaxy_url: https://galaxy.usegalaxy.org.au manager: _default_ amqp_acknowledge: True amqp_ack_republish_time: 1200 @@ -149,14 +149,18 @@ execution: function: map_tool_to_destination rules_module: tpv.rules tpv_config_files: - # shared db commented out until rules are updated to accommodate it (rank function, high-mem tag) - # - https://raw.githubusercontent.com/galaxyproject/tpv-shared-database/main/tools.yml - # - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/default_tool.yml" - # - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/tools.yml" - # - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/users.yml" - # - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/roles.yml" - # - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/destinations.yml" - - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/testing_tpv_config.yml" - - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/testing_tpv_destinations.yml" -limits: {} # TODO - put job conf limits back here once one load testing is finished (saved in trello card) + - https://raw.githubusercontent.com/galaxyproject/tpv-shared-database/main/tools.yml + - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/default_tool.yml" + - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/tools.yml" + - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/users.yml" + - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/roles.yml" + - "{{ galaxy_dynamic_job_rules_dir }}/total_perspective_vortex/destinations.yml" +{% if job_conf_limits %} +{% for env in job_conf_limits.environments|d({}) %} + {{ env }}: + tags: {{ job_conf_limits.environments[env].tags }} +{% endfor %} +limits: +{{ job_conf_limits.limits | to_nice_yaml }} +{% endif %} diff --git a/templates/galaxy/config/galaxy_object_store_conf.xml.j2 b/templates/galaxy/config/galaxy_object_store_conf.xml.j2 index e0ec8a7c1..9056a8d76 100644 --- a/templates/galaxy/config/galaxy_object_store_conf.xml.j2 +++ b/templates/galaxy/config/galaxy_object_store_conf.xml.j2 @@ -1,9 +1,46 @@ - - + + + + + + + + + + + + + + + + + +{% if pawsey_file_mounts_available|d(True) %} + + + + + + + + + + + + +{% endif %} +{% if legacy_file_mounts_available|d(True) %} + + + + + + +{% endif %} diff --git a/templates/galaxy/toolbox/filters/test_ga_filters.py b/templates/galaxy/toolbox/filters/test_ga_filters.py index 27bd7ba19..9010f6843 100644 --- a/templates/galaxy/toolbox/filters/test_ga_filters.py +++ b/templates/galaxy/toolbox/filters/test_ga_filters.py @@ -24,6 +24,10 @@ def restrict_alphafold(context, tool): 'id': 'maxquant_test', 'role': 'maxquant_test', }, + { + 'id': 'cellranger', + 'role': 'cellranger', + }, ] def hide_test_tools(context, tool): @@ -84,7 +88,7 @@ def restrict_alphafold(context, tool): assert restrict_alphafold(context1, tool2) == True # any tool other than alphafold is visible -roleA = Role('maxquant_test', False) +roleA = Role('cellranger', False) userA = User(lambda: [roleA], 'amy@email.org') userB = User(lambda: [], 'amy@email.org') @@ -98,7 +102,7 @@ def restrict_alphafold(context, tool): contextB = context(transB) contextC = context(transC) -toolA = Tool('maxquant_test', ['test']) +toolA = Tool('cellranger', ['test']) toolB = Tool('something_else', ['test']) assert hide_test_tools(contextA, toolA) == True diff --git a/templates/nginx/galaxy.j2 b/templates/nginx/galaxy.j2 index 0658373bc..7e2fd34b3 100644 --- a/templates/nginx/galaxy.j2 +++ b/templates/nginx/galaxy.j2 @@ -1,3 +1,10 @@ +{% if num_gunicorn_processes == 2 %} +upstream galaxy { + server unix://{{ galaxy_mutable_config_dir }}/gunicorn1.sock; + server unix://{{ galaxy_mutable_config_dir }}/gunicorn2.sock; +} +{% endif %} + server { # Listen on port 443 listen *:443 ssl default_server; @@ -12,32 +19,9 @@ server { uwsgi_buffer_size 16k; uwsgi_busy_buffers_size 24k; -{% if galaxy_commit_id == "release_22.01" %} - - # The most important location block, by default all requests are sent to uWSGI - location / { - # This is the backend to send the requests to. - uwsgi_pass 127.0.0.1:8080; - uwsgi_param UWSGI_SCHEME $scheme; - include uwsgi_params; - - add_header X-Clacks-Overhead 'GNU James Taylor (@jxtx) Simon Gladman (@slugger70)'; - } - - location ~ ^/api/dataset_collections/([^/]+)/download/?$ { - proxy_buffering off; - uwsgi_pass 127.0.0.1:8080; - uwsgi_param UWSGI_SCHEME $scheme; - include uwsgi_params; - } - -{% endif %} - -{% if galaxy_commit_id != "release_22.01" %} - # The most important location block, by default all requests are sent to Gunicorn location / { - proxy_pass http://unix://{{ galaxy_mutable_config_dir }}/gunicorn.sock; + proxy_pass {% if num_gunicorn_processes == 2 %}http://galaxy{% else %}http://unix://{{ galaxy_mutable_config_dir }}/gunicorn.sock{% endif %}; proxy_redirect off; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -49,15 +33,13 @@ server { location ~ ^/api/dataset_collections/([^/]+)/download/?$ { proxy_buffering off; - proxy_pass http://unix://{{ galaxy_mutable_config_dir }}/gunicorn.sock; + proxy_pass {% if num_gunicorn_processes == 2 %}http://galaxy{% else %}http://unix://{{ galaxy_mutable_config_dir }}/gunicorn.sock{% endif %}; proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Upgrade $http_upgrade; } -{% endif %} - # Static files can be more efficiently served by Nginx. Why send the # request to uWSGI which should be spending its time doing more useful # things like serving Galaxy!