Merge pull request #170 from broadinstitute/ps-release-2024-sept

Release (september 2024)
broadinstitute · Oct 8, 2024 · 65bfb86 · 65bfb86
2 parents 4a0105a + 5ae402a
commit 65bfb86
Show file tree

Hide file tree

Showing 5 changed files with 6 additions and 6 deletions.
diff --git a/bioindex/src/main/resources/singleCellMetadata.py b/bioindex/src/main/resources/singleCellMetadata.py
@@ -13,9 +13,9 @@
 def download_data():
     out = []
     cmd = 'aws s3 cp s3://dig-analysis-data/single_cell/ ./data/ ' \
-          '--recursive --exclude="*" --include="*dataset_metadata"'
+          '--recursive --exclude="*" --include="*dataset_metadata.json"'
     subprocess.check_call(cmd, shell=True)
-    for file in glob.glob('data/*/dataset_metadata'):
+    for file in glob.glob('data/*/dataset_metadata.json'):
         with open(file, 'r') as f:
             out.append(json.load(f))
     return out

diff --git a/bioindex/src/main/scala/SingleCellMetadataStage.scala b/bioindex/src/main/scala/SingleCellMetadataStage.scala
@@ -7,7 +7,7 @@ import org.broadinstitute.dig.aws.emr._
 class SingleCellMetadataStage(implicit context: Context) extends Stage {
   import MemorySize.Implicits._
 
-  val metadata: Input.Source = Input.Source.Raw("single_cell/*/dataset_metadata")
+  val metadata: Input.Source = Input.Source.Raw("single_cell/*/dataset_metadata.json")
 
   override val cluster: ClusterDef = super.cluster.copy(
     instances = 1,

diff --git a/bottom-line/src/main/resources/runMinP.py b/bottom-line/src/main/resources/runMinP.py
@@ -38,7 +38,7 @@ def min_p(df):
 
 
 def num_dataset(phenotype, ancestry):
-    path = f'{s3dir}/out/metaanalysis/variants/{phenotype}/'
+    path = f'{s3_in}/out/metaanalysis/variants/{phenotype}/'
     files = subprocess.check_output(['aws', 's3', 'ls', path, '--recursive']).decode().strip().split('\n')
     datasets = set()
     for file in files:

diff --git a/bottom-line/src/main/resources/runNaive.py b/bottom-line/src/main/resources/runNaive.py
@@ -39,6 +39,7 @@ def naive(df):
     df = df \
         .withColumn('weight', 1.0 / df.stdErr / df.stdErr) \
         .withColumn('weighted_beta', df.beta / df.stdErr / df.stdErr)
+    df = df.filter(~df.weight.isNull())
     df = df \
         .groupBy(df.varId, df.chromosome, df.position, df.reference, df.alt, df.phenotype, df.ancestry) \
         .agg({'weight': 'sum', 'weighted_beta': 'sum', 'n': 'sum'})

diff --git a/bottom-line/src/main/scala/ClumpedAssociationsStage.scala b/bottom-line/src/main/scala/ClumpedAssociationsStage.scala
@@ -25,8 +25,7 @@ class ClumpedAssociationsStage(implicit context: Context) extends Stage {
 
   /** Simple cluster with more memory. */
   override val cluster: ClusterDef = super.cluster.copy(
-    masterInstanceType = Strategy.generalPurpose(mem = 64.gb),
-    instances = 1
+    instances = 5
   )
 
   /** Build the job. */