Replaced bucket region lookup process for all of the apps so this is not

dependent on the .aws/credentials file.

.classpath

@@ -9,6 +9,7 @@
 	<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="optional" value="true"/>
 		</attributes>
 	</classpathentry>
 	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
@@ -22,6 +23,7 @@
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
 			<attribute name="test" value="true"/>
+			<attribute name="optional" value="true"/>
 		</attributes>
 	</classpathentry>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">

.settings/org.eclipse.jdt.core.prefs

@@ -1,6 +1,8 @@
 eclipse.preferences.version=1
 org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
 org.eclipse.jdt.core.compiler.compliance=1.8
+org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
 org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
 org.eclipse.jdt.core.compiler.release=disabled
 org.eclipse.jdt.core.compiler.source=1.8

src/main/java/edu/utah/hci/aws/apps/copy/S3Copy.java

@@ -58,10 +58,8 @@ public class S3Copy {
 	private int iterations = 100;
 
 	//internal fields
-	private String defaultRegion = "us-east-1";
 	private CopyRequest[] copyRequests = null; 
 	private StringBuilder log = new StringBuilder();
-	private String region = null;
 	private boolean resultsCheckOK = true;
 	private int maxTries = 5;
 	private int secToWait = 15;
@@ -156,14 +154,12 @@ private HashMap<String, String> fetchBucketRegions() {
 
 		//for each one
 		HashMap<String, String> bucketNameRegion = new HashMap<String, String>();
-		AmazonS3 userS3 = fetchS3Client(region);
+
 		ArrayList<String> errorMessages = new ArrayList<String>();
 		for (String bn: bucketNames) {
 			try {
-				String bucketRegion = Util.fetchBucketRegion(userS3, bn);
+				String bucketRegion = Util.fetchBucketRegion(profile, bn);
 				 bucketNameRegion.put(bn, bucketRegion);
-				//if (regionAccess[1].equals("OK")) bucketNameRegion.put(bn, regionAccess[0]);
-				//else errorMessages.add("Cannot access bucket: "+bn);
 			} catch (Exception e) {
 				errorMessages.add(e.getMessage());
 			}
@@ -433,8 +429,6 @@ public void processArgs(String[] args) {
 
 			if (jobString == null) Util.printErrAndExit("\nERROR: please provide a file path or string with copy job information, see the help menu.\n");
 
-			region = Util.getRegionFromCredentials(profile, defaultRegion);		
-			if (region == null) Util.printErrAndExit("\nERROR: failed to find your profile in ~/.aws/credentials, "+profile);
 			credentials = new ProfileCredentialsProvider(profile);
 
 		} catch (Exception e) {
@@ -587,10 +581,6 @@ public boolean isDryRun() {
 		return dryRun;
 	}
 
-	public String getRegion() {
-		return region;
-	}
-
 	public int getNumberDaysToRestore() {
 		return numberDaysToRestore;
 	}

src/main/java/edu/utah/hci/aws/apps/copy/TestS3Copy.java

@@ -4,6 +4,7 @@
 
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.SdkClientException;
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.ListObjectsV2Result;
@@ -33,9 +34,10 @@ public class TestS3Copy {
 
 	/*Adjust these fields to match your testing environment	 */
 
-	/**Be sure these bucket exists and doesn't contain anything you care about. WARNING, they will be emptied!*/
+	/**Be sure these bucket exists, are in the same region, and don't contain anything you care about. WARNING, they will be emptied!*/
 	private static final String sourceBucketName = "hcibioinfo-nix-test";
 	private static final String destinationBucketName = "hcibioinfo-gsync-test"; 
+	private static final String profile = "default";
 
 	/**Full path to a small test file.*/
 	private static final String pathToSmallTestFile = "/Users/u0028003/Code/AwsApps/TestData/GSync/Bam/testShortIndex.bam.S3.txt";
@@ -53,9 +55,11 @@ public class TestS3Copy {
 	public void bucketToBucketNoRecursive() {
 		AmazonS3 s3 = null;
 		try {
-			//make a client
-			String region = Util.getRegionFromCredentials();
-			s3 = AmazonS3ClientBuilder.standard().withRegion(region).build();
+			//parse the credentials
+			ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile);
+			//make a client using the source
+			String regionSource = Util.fetchBucketRegion(credentials, sourceBucketName);
+			s3 = AmazonS3ClientBuilder.standard().withRegion(regionSource).build();
 
 			//empty the buckets
 			emptyS3Bucket(s3, sourceBucketName);
@@ -144,9 +148,13 @@ public void bucketToBucketNoRecursive() {
 	public void bucketToBucketRecursive() {
 		AmazonS3 s3 = null;
 		try {
+			//parse the credentials
+			ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile);
+			//make a client using the source
+			String regionSource = Util.fetchBucketRegion(credentials, sourceBucketName);
+
 			//make a client
-			String region = Util.getRegionFromCredentials();
-			s3 = AmazonS3ClientBuilder.standard().withRegion(region).build();
+			s3 = AmazonS3ClientBuilder.standard().withRegion(regionSource).build();
 
 			//empty the buckets
 			emptyS3Bucket(s3, sourceBucketName);
@@ -183,9 +191,13 @@ public void bucketToBucketRecursive() {
 	public void bucketToLocal() {
 		AmazonS3 s3 = null;
 		try {
+			//parse the credentials
+			ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile);
+			//make a client using the source
+			String regionSource = Util.fetchBucketRegion(credentials, sourceBucketName);
+
 			//make a client
-			String region = Util.getRegionFromCredentials();
-			s3 = AmazonS3ClientBuilder.standard().withRegion(region).build();
+			s3 = AmazonS3ClientBuilder.standard().withRegion(regionSource).build();
 
 			//empty the buckets
 			emptyS3Bucket(s3, sourceBucketName);
@@ -230,9 +242,13 @@ public void bucketToLocal() {
 	public void archiveRetrieval() {
 		AmazonS3 s3 = null;
 		try {
+			//parse the credentials
+			ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile);
+			//make a client using the source
+			String regionSource = Util.fetchBucketRegion(credentials, sourceBucketName);
+
 			//make a client
-			String region = Util.getRegionFromCredentials();
-			s3 = AmazonS3ClientBuilder.standard().withRegion(region).build();
+			s3 = AmazonS3ClientBuilder.standard().withRegion(regionSource).build();
 
 			//empty the buckets
 			emptyS3Bucket(s3, sourceBucketName);

src/main/java/edu/utah/hci/aws/apps/gsync/GSync.java

@@ -123,7 +123,7 @@ public GSync (String[] args){
 
 	void doWork() {
 		try {
-			region = Util.getRegionFromCredentials();
+			region = Util.fetchBucketRegion("default", bucketName);
 
 			initializeFields();
 
@@ -1147,6 +1147,9 @@ public void printDocs(){
 				"**************************************************************************************\n" +
 				"**                                   GSync : June 2020                              **\n" +
 				"**************************************************************************************\n" +
+				"This is depreciated and should not be used\n\n"+
+
+
 				"GSync pushes files with a particular extension that exceed a given size and age to \n" +
 				"Amazon's S3 object store. Associated genomic index files are also moved. Once \n"+
 				"correctly uploaded, GSync replaces the original file with a local txt placeholder file \n"+

src/main/java/edu/utah/hci/aws/apps/gsync/TestGSync.java

@@ -649,7 +649,7 @@ public void testMove() {
 	private HashMap<String, S3ObjectSummary> fetchS3Objects() {
 		AmazonS3 s3 = null;
 		try {
-			String region = Util.getRegionFromCredentials();
+			String region = Util.fetchBucketRegion("default", testS3BucketName);
 			s3 = AmazonS3ClientBuilder.standard().withRegion(region).build();
 
 			HashMap<String, S3ObjectSummary> kos = new HashMap<String, S3ObjectSummary>();
@@ -677,7 +677,7 @@ private HashMap<String, S3ObjectSummary> fetchS3Objects() {
 	private void deleteS3Object(String key) {
 		AmazonS3 s3 = null;
 		try {
-			String region = Util.getRegionFromCredentials();
+			String region = Util.fetchBucketRegion("default", testS3BucketName);
 			s3 = AmazonS3ClientBuilder.standard().withRegion(region).build();
 			s3.deleteObject(testS3BucketName, key);
 
@@ -695,7 +695,7 @@ private void deleteS3Object(String key) {
 	private void emptyS3Bucket() {
 		AmazonS3 s3 = null;
 		try {
-			String region = Util.getRegionFromCredentials();
+			String region = Util.fetchBucketRegion("default", testS3BucketName);
 			s3 = AmazonS3ClientBuilder.standard().withRegion(region).build();
 
 			ListObjectsV2Result result = s3.listObjectsV2(testS3BucketName);

src/main/java/edu/utah/hci/aws/apps/jobrunner/TestJobRunner.java

@@ -36,6 +36,7 @@ public class TestJobRunner {
 
 	/**Be sure this bucket exists, is private, and doesn't contain anything you care about. WARNING, it will be emptied!*/
 	private static final String s3BucketUri = "s3://hcibioinfo-jobrunner-test/";
+	private static final String s3BucketName = "hcibioinfo-jobrunner-test";
 
 	/**Directory in the AwsApps project containing the JobRunner test files. */
 	private static final File testDataDir = new File("/Users/u0028003/Code/AwsApps/TestData/JobRunner");
@@ -44,6 +45,7 @@ public class TestJobRunner {
 	private static final File awsTmpDir = new File("/Users/u0028003/TmpDelmeAWS");
 
 	private static final File awsCredentialsFile = new File("/Users/u0028003/.aws/credentials");
+	private static final String profile = "default";
 
 	private static final String awsPath="/usr/local/bin/aws";
 
@@ -340,7 +342,7 @@ public void setPaths() throws Exception {
 			envPropToAdd.put("PATH", "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin");
 		}
 
-		region = Util.getRegionFromCredentials();
+		region = Util.fetchBucketRegion(profile, s3BucketName);
 	}
 
 	public static void p(String s) {

src/main/java/edu/utah/hci/aws/apps/versions/VersionManager.java

@@ -219,15 +219,21 @@ public void processArgs(String[] args) throws Exception {
 			el("\nError: please provide the name of a versioned AWS S3 bucket.\n");
 			System.exit(1);
 		}
+		if (bucketName.contains("/")) {
+			el("\nError: please remove any s3:// or / from your bucket name.\n");
+			System.exit(1);
+		}
 
 		//set number of threads
 		int availThreads = Runtime.getRuntime().availableProcessors()-1;
 		if (maxThreads > availThreads) maxThreads = availThreads;
 
-		//fetch region from credentials
-		region = Util.getRegionFromCredentials(profile);		
-		if (region == null) Util.printErrAndExit("\nERROR: failed to find your region in ~/.aws/credentials for the profile ["+profile+"]");
+		//fetch region from credentials or use the default
+
+		//region = Util.getRegionFromCredentials(profile);		
+		//if (region == null) Util.printErrAndExit("\nERROR: failed to find your profile in ~/.aws/credentials for ["+profile+"]");
 		cred = ProfileCredentialsProvider.builder().profileName(profile).build();
+		region = Util.fetchBucketRegion(profile, bucketName);
 
 		printOptions();
 

src/main/java/edu/utah/hci/aws/util/Util.java

@@ -42,6 +42,8 @@
 import com.amazonaws.services.s3.transfer.Transfer.TransferState;
 import com.amazonaws.services.s3.transfer.TransferProgress;
 
+import software.amazon.awssdk.regions.Region;
+
 /**Static utility methods.*/
 public class Util {
 
@@ -154,7 +156,7 @@ public static HashMap<String, String> parseKeyValues (File f) throws IOException
 		return attributes;
 	}
 
-	/**Extracts the region from the ~/.aws/credentials file. Can only be one profile.*/
+	/**Extracts the region from the ~/.aws/credentials file. Can only be one profile.
 	public static String getRegionFromCredentials() throws IOException {
 		String path = System.getProperty("user.home")+"/.aws/credentials";
 		File cred = new File (path);
@@ -163,10 +165,10 @@ public static String getRegionFromCredentials() throws IOException {
 		String region = a.get("region");
 		if (region == null) throw new IOException("Failed to find a 'region' key in ~/.aws/credentials file");
 		return region;
-	}
+	}*/
 
 	/**Extracts the region from the ~/.aws/credentials file for a particular profile, returns null the profile wasn't found
-	 * and either the profile's region or the default.*/
+	 * and either the profile's region or the default.
 	public static String getRegionFromCredentials(String profile, String defaultRegion) throws IOException {
 		String path = System.getProperty("user.home")+"/.aws/credentials";
 		File cred = new File (path);
@@ -191,10 +193,10 @@ else if (lines[j].startsWith("region")) {
 		if (profileFound) return defaultRegion;
 		//Thus profile not found
 		return null;
-	}
+	}*/
 
 	/**Extracts the region from the ~/.aws/credentials file for a particular profile, returns null the profile wasn't found
-	 * and either the profile's region or the default.*/
+	 * and either the profile's region or the default.
 	public static String getRegionFromCredentials(String profile) throws IOException {
 		String path = System.getProperty("user.home")+"/.aws/credentials";
 		File cred = new File (path);
@@ -216,13 +218,13 @@ else if (lines[j].startsWith("region")) {
 			}
 		}
 		return null;
-	}
+	}*/
 
 	public static final Pattern REGION_MESSAGE = Pattern.compile(".+region: ([\\w-]+)\\..+");
 
 	/**Will fetch the region for any bucket.
 	 * @return bucket region
-	 * @throws IOException if the bucket doesn't exist.*/
+	 * @throws IOException if the bucket doesn't exist.
 	public static String fetchBucketRegion(AmazonS3 sourceS3, String bucketName) throws IOException{
 		String region = null;
 		try {
@@ -243,14 +245,76 @@ public static String fetchBucketRegion(AmazonS3 sourceS3, String bucketName) thr
 		return region;
 	}
 	
-	public static void main (String[] args) throws IOException {
-		ProfileCredentialsProvider credentials = new ProfileCredentialsProvider("kohli");
-		String region = "us-east-1";
-		AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(credentials).withRegion(region).build();
-		Util.pl(Util.fetchBucketRegion(s3Client, "hci-kohli-temp-transfers"));
+	public static String fetchBucketRegion(String profile, String defaultRegion, String bucketName) throws Exception{
 		
+		ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile);
+		AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(credentials).withRegion(defaultRegion).build();
+		String bucketRegion = Util.fetchBucketRegion(s3Client, bucketName);
+		s3Client.shutdown();
+		return bucketRegion;
+	}*/
+
+	public static final String[] awsBucketRegions = {"us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-east-1", "ap-south-2", "ap-southeast-3", "ap-southeast-4", "ap-south-1", "af-south-1"};
+
+	/**Ridiculous loop to find a bucket's region. WTH AWS!
+	 * @return null if it can't determine the region. */
+	public static String fetchBucketRegion(String profile, String bucketName) throws Exception{
 
+		ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile);
+		String bucketRegion = null;
+		AmazonS3 s3Client = null;
 
+		for (String abr: awsBucketRegions) {
+			s3Client = AmazonS3ClientBuilder.standard().withCredentials(credentials).withRegion(abr).build();
+
+			try {
+				//use a head request to attempt to find the region for the given bucket, this typically fails
+				HeadBucketRequest request = new HeadBucketRequest(bucketName);
+				HeadBucketResult  res = s3Client.headBucket(request);
+				bucketRegion  = res.getBucketRegion();
+			} catch (Exception e) {
+				//attempt to parse the actual bucket region from the error message, ugg!
+				//'The bucket is in this region: us-east-2. Please use this region...'
+				String message = e.getMessage();
+				if (message.contains("bucket is in this region:")) {
+					Matcher mat = REGION_MESSAGE.matcher(message);
+					if (mat.matches()) bucketRegion = mat.group(1);
+				}
+			}
+			s3Client.shutdown();
+			if (bucketRegion != null) return bucketRegion;
+		}
+		return null;
+	}
+
+	/**Ridiculous loop to find a bucket's region. WTH AWS!
+	 * @return null if it can't determine the region. */
+	public static String fetchBucketRegion(ProfileCredentialsProvider credentials, String bucketName) throws Exception{
+
+		String bucketRegion = null;
+		AmazonS3 s3Client = null;
+
+		for (String abr: awsBucketRegions) {
+			s3Client = AmazonS3ClientBuilder.standard().withCredentials(credentials).withRegion(abr).build();
+
+			try {
+				//use a head request to attempt to find the region for the given bucket, this typically fails
+				HeadBucketRequest request = new HeadBucketRequest(bucketName);
+				HeadBucketResult  res = s3Client.headBucket(request);
+				bucketRegion  = res.getBucketRegion();
+			} catch (Exception e) {
+				//attempt to parse the actual bucket region from the error message, ugg!
+				//'The bucket is in this region: us-east-2. Please use this region...'
+				String message = e.getMessage();
+				if (message.contains("bucket is in this region:")) {
+					Matcher mat = REGION_MESSAGE.matcher(message);
+					if (mat.matches()) bucketRegion = mat.group(1);
+				}
+			}
+			s3Client.shutdown();
+			if (bucketRegion != null) return bucketRegion;
+		}
+		return null;
 	}
 
 	/**Converts milliseconds to days.*/