snpcacheload-3-5-1-1

Configuration.default

@@ -1,71 +1,109 @@
-#!/bin/csh
+#!/bin/sh
 
 #
 # Configuration.default
 #
-# Source this file from any csh script to set the env variables
+# Source this file from any bourne shell script to set the
+# env variables
 #
+#
+#  If the MGICONFIG environment variable does not have a local override,
+#  use the default "live" settings.
+#
+if [ "${MGICONFIG}" = "" ]
+then
+    MGICONFIG=/usr/local/mgi/live/mgiconfig
+    export MGICONFIG
+fi
 
-if ( ${?MGICONFIG} == 0 ) then
-	setenv MGICONFIG /usr/local/mgi/live/mgiconfig
-endif
-
-source ${MGICONFIG}/master.config.csh
+. ${MGICONFIG}/master.config.sh
 
 # install directory
-setenv CACHEINSTALLDIR		${DBUTILS}/snpcacheload
+CACHEINSTALLDIR=${DBUTILS}/snpcacheload
 
 # output dir
-setenv CACHEDIR 		/data/loads/mgi/snpcacheload
+CACHEDIR=/data/loads/mgi/snpcacheload
 
 # data directory
-setenv CACHEDATADIR             ${CACHEDIR}/output
+CACHEDATADIR=${CACHEDIR}/output
 
 # logs directory
-setenv CACHELOGSDIR             ${CACHEDIR}/logs
+CACHELOGSDIR=${CACHEDIR}/logs
 
-# full path to back-end snp database backup file. 
-# this directory is on the unix server on which the sybase server resides
-# e.g. lindon when backing up PROD1_MGI..snp
-setenv SNP_BACKUP	/extra1/sybase/snp.backup
+# directory in which to archive the contents of output and logs directories
+ARCHIVEDIR=${CACHEDIR}/archive
 
-# full path to SNP_BACKUP in order to load it from shire (weekly snpcacheload)
-setenv PRODSNP_BACKUP   /net/mtdoom/vol/lindon${SNP_BACKUP}
+export CACHEINSTALLDIR CACHEDIR CACHEDATADIR CACHELOGSDIR ARCHIVEDIR
+
+#
+# settings for snpmarker_weekly.sh 
+#
+# full path to PROD1_MGI..snp backup file on lindon
+SNP_BACKUP_LOCALPATH=/extra1/sybase/snp.backup
+
+export SNP_BACKUP_LOCALPATH
+
+# source this for file archiving and other utilities
+DLAJOBSTREAMFUNC=${DBUTILS}/dlautils/DLAJobStreamFunctions.sh
+
+export DLAJOBSTREAMFUNC
 
 # mgd refseq _LogicalDB_key
-setenv REFSEQ_LOGICALDB_KEY     27
+REFSEQ_LOGICALDB_KEY=27
 
 # mgd SNP_ConsensusSnp_Marker _MGIType_key
-setenv SNPMRKR_MGITYPE_KEY              32
+SNPMRKR_MGITYPE_KEY=32
 
 # mgd refSNP _LogicalDB_key
-setenv CS_LOGICALDB_KEY         73
+CS_LOGICALDB_KEY=73
 
 # mgd SNP_ConsensusSnp _LogicalDB_key
-setenv CS_MGITYPE_KEY           30
+CS_MGITYPE_KEY=30
+
+export REFSEQ_LOGICALDB_KEY SNPMRKR_MGITYPE_KEY CS_LOGICALDB_KEY CS_MGITYPE_KEY
 
 # mgd EntrezGene _LogicalDB_key
-setenv EG_LOGICALDB_KEY         55
+EG_LOGICALDB_KEY=55
 
 # mgd MRK_Marker _MGIType_key
-setenv MRKR_MGITYPE_KEY 2
+MRKR_MGITYPE_KEY=2
+
+# QTL MRK_Marker _Marker_Type_key
+MRKR_QTLTYPE_KEY=6
+
+export EG_LOGICALDB_KEY MRKR_MGITYPE_KEY MRKR_QTLTYPE_KEY
 
 # snp table and bcp file names 
-setenv ACC_TABLE	SNP_Accession
-setenv ACC_FILE		${ACC_TABLE}.bcp
-setenv TMP_FXN_TABLE	TMP_SNP_Marker_Fxn
-setenv TMP_FXN_FILE	${TMP_FXN_TABLE}.bcp
-setenv SNP_MRK_TABLE	SNP_ConsensusSnp_Marker
-setenv SNP_MRK_FILE	${SNP_MRK_TABLE}.bcp
-setenv SNP_MRK_WITHIN_FILE	${SNP_MRK_TABLE}_Within.bcp
-setenv MRKLOC_CACHETABLE	MRK_Location_Cache
-setenv MRKLOC_CACHEFILE	${MRKLOC_CACHETABLE}.bcp
+ACC_TABLE=SNP_Accession
+ACC_FILE=${ACC_TABLE}.bcp
+
+export ACC_TABLE ACC_FILE
+
+TMP_FXN_TABLE=TMP_SNP_Marker_Fxn
+TMP_FXN_FILE=${TMP_FXN_TABLE}.bcp
+
+export TMP_FXN_TABLE TMP_FXN_FILE
+
+SNP_MRK_TABLE=SNP_ConsensusSnp_Marker
+SNP_MRK_FILE=${SNP_MRK_TABLE}.bcp
+SNP_MRK_WITHIN_FILE=${SNP_MRK_TABLE}_Within.bcp
+
+export SNP_MRK_TABLE SNP_MRK_FILE SNP_MRK_WITHIN_FILE
+
+MRKLOC_CACHETABLE=MRK_Location_Cache
+MRKLOC_CACHEFILE=${MRKLOC_CACHETABLE}.bcp
+
+export MRKLOC_CACHETABLE MRKLOC_CACHEFILE
 
 # max number of lines per bcp file to keep below 2G
-setenv MAX_BCP_LINES	25000000 
+MAX_BCP_LINES=25000000 
 
 # max number of ConsensusSnp keys in a snpmrkwithin.py query batch
-setenv MAX_QUERY_BATCH  100000
+MAX_QUERY_BATCH=100000
+
+export MAX_BCP_LINES MAX_QUERY_BATCH
 
 # Are dbSNP and MGI coordinates synchronized (same mouse genome build) ?
-setenv IN_SYNC	yes
+IN_SYNC=yes
+
+export IN_SYNC

HISTORY

@@ -1,3 +1,47 @@
+TAG: snpcacheload-3-5-1-1
+DATE: 09/06/2006
+STAFF: sc
+RELEASE: MGI3.51, build 36
+CHANGES: merge build36 branch to trunK (note this is a second merge
+         of this branch to the trunk
+
+TAG: snpcacheload-mgiconfig-7
+DATE: 09/06/2006
+STAFF: sc
+RELEASE: MGI3.51, build 36
+CHANGES:
+1) added system_docs
+
+TAG: snpcacheload-mgiconfig-6
+TAG: snpcacheload-mgiconfig-5
+DATE: 08/24/2006
+STAFF: sc
+RELEASE: MGI3.51, build 36
+CHANGES:
+1) corrected mgidbutilities config var name
+2) added sys.stdout.flush
+
+TAG: snpcacheload-mgiconfig-4
+DATE: 08/10/2006
+STAFF: sc
+RELEASE: MGI3.51, build 36
+CHANGES:
+1) remove snplocation.py
+2) Configuration - removed SNP_BACKUP_REMOTEPATH to mgiconfig
+                 - updated SNP_BACKUP to SNP_BACKUP_LOCALPATH
+		 - added MRKR_QTLTYPE_KEY
+3) snpmarker_weekly.sh - updated to use SNP_BACKUP_REMOTEPATH and 
+                 SNP_BACKUP_LOCALPATH
+4) snpmrkwithin.py - updated to not load distance relationships, only
+                   - update locus_region and determine 'within coordinates of'
+
+TAG: snpcacheload-mgiconfig-3
+DATE: 08/01/2006
+STAFF: sc
+RELEASE: MGI3.51, build 36
+CHANGES:
+1) converted to bourne shell
+
 TAG: snpcacheload-3-5-0-4
 DATE: 07/01/2006
 STAFF: lec

Install

@@ -1,4 +1,4 @@
-#!/bin/csh
+#!/bin/sh
 
 #
 # Install
@@ -9,19 +9,26 @@
 # 2. ln -s snpcacheload-#-#-# snpcacheload
 # 3. Copy Configuration.default Configuration
 # 4. Edit Configuration
-#
+# 5. Install
 
-cd `dirname $0` && source ./Configuration
+cd `dirname $0` && . ./Configuration
 
-if ( ! -d ${CACHEDIR} ) then
+if [ ! -d ${CACHEDIR} ]
+then
         mkdir -p ${CACHEDIR}
-endif
+fi
 
-if ( ! -d ${CACHEDATADIR} ) then
+if [ ! -d ${CACHEDATADIR} ] 
+then
         mkdir -p ${CACHEDATADIR}
-endif
+fi
 
-if ( ! -d ${CACHELOGSDIR} ) then
+if [ ! -d ${CACHELOGSDIR} ] 
+then
         mkdir -p ${CACHELOGSDIR}
-endif
+fi
 
+if [ ! -d ${ARCHIVEDIR} ] 
+then
+        mkdir -p ${ARCHIVEDIR}
+fi

snpmarker.py

@@ -16,9 +16,11 @@
 #    	   2) bcp files
 # History
 #
-# 08/17/2005	sc
-#	- SNP (TR 1560)
-# 03/16/2006 sc convert to snp database
+# lec   06/30/2006 - modified for mgiconfig
+#
+# sc	03/16/2006 - convert to snp database
+#
+# sc    08/17/2005 - SNP (TR 1560)
 #
 # lec   06/30/2006 - modified for mgiconfig
 #
@@ -89,6 +91,7 @@ def initialize():
     # Effects: queries a database
     # Throws:  db.error, db.connection_exc
     print 'connecting to mgd and loading markerLookup...%s' % NL
+    sys.stdout.flush()
     # set up connection to the mgd database
     db.useOneConnection(1)
     db.set_sqlLogin(user, password, mgdServer, mgdDB)
@@ -108,8 +111,10 @@ def initialize():
 	markerLookup[ r['egId'] ] = r['_Marker_key'] 
 
     print 'connecting to %s..%s ...%s' % (snpServer, snpDB, NL)
+    sys.stdout.flush()
     # set up connection the snp database
     db.useOneConnection(0)
+    db.useOneConnection(1)
     db.set_sqlLogin(user, password, snpServer, snpDB)
 
 def deleteAccessions():
@@ -119,6 +124,7 @@ def deleteAccessions():
     # Effects: queries a database, deletes records from a database
     # Throws:  db.error, db.connection_exc
     print 'deleting accessions ...%s' % NL
+    sys.stdout.flush()
 
     cmds = []
     cmds.append('select a._Accession_key ' + \
@@ -145,6 +151,7 @@ def getMaxAccessionKey():
     # current max(_Accession_key)
     global accKey
     print 'getting max snp accession key ...%s' % NL
+    sys.stdout.flush()
     cmds = []
     cmds.append('select max(_Accession_key) ' + \
             'from SNP_Accession')
@@ -162,6 +169,7 @@ def createBCP():
     print 'creating %s...%s' % (snpMrkrFile, mgi_utils.date())
     print 'and  %s...%s%s' % (accFile, mgi_utils.date(), NL)
     print 'querying ... %s' % NL
+    sys.stdout.flush()
 
     cmds = []
     # get set of DP_SNP_Marker attributes into a temp table
@@ -186,6 +194,7 @@ def createBCP():
     results = db.sql(cmds, 'auto')
 
     print 'writing bcp file ...%s' % NL
+    sys.stdout.flush()
     # current primary key
     primaryKey = 0
     for r in results[1]:
@@ -272,6 +281,7 @@ def finalize():
 #
 
 print 'snpmarker.py start: %s' % mgi_utils.date()
+sys.stdout.flush()
 try:
     initialize()
     getMaxAccessionKey()
@@ -288,4 +298,5 @@ def finalize():
     sys.exit(message)
 
 print 'snpmarker.py end: %s' % mgi_utils.date()
+sys.stdout.flush()