From d5681992dbdacbf94bd1f0ba8e4d73563af5b090 Mon Sep 17 00:00:00 2001 From: u0028003 Date: Wed, 22 May 2024 09:05:06 -0600 Subject: [PATCH] Replaced bucket region lookup process for all of the apps so this is not dependent on the .aws/credentials file. --- .classpath | 2 + .settings/org.eclipse.jdt.core.prefs | 2 + .../edu/utah/hci/aws/apps/copy/S3Copy.java | 14 +-- .../utah/hci/aws/apps/copy/TestS3Copy.java | 36 +++++--- .../edu/utah/hci/aws/apps/gsync/GSync.java | 5 +- .../utah/hci/aws/apps/gsync/TestGSync.java | 6 +- .../hci/aws/apps/jobrunner/TestJobRunner.java | 4 +- .../hci/aws/apps/versions/VersionManager.java | 12 ++- src/main/java/edu/utah/hci/aws/util/Util.java | 88 ++++++++++++++++--- 9 files changed, 127 insertions(+), 42 deletions(-) diff --git a/.classpath b/.classpath index 002ad57..f7e4a1d 100644 --- a/.classpath +++ b/.classpath @@ -9,6 +9,7 @@ + @@ -22,6 +23,7 @@ + diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index b8947ec..2f5cc74 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -1,6 +1,8 @@ eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.release=disabled org.eclipse.jdt.core.compiler.source=1.8 diff --git a/src/main/java/edu/utah/hci/aws/apps/copy/S3Copy.java b/src/main/java/edu/utah/hci/aws/apps/copy/S3Copy.java index a4061ae..47de2a4 100644 --- a/src/main/java/edu/utah/hci/aws/apps/copy/S3Copy.java +++ b/src/main/java/edu/utah/hci/aws/apps/copy/S3Copy.java @@ -58,10 +58,8 @@ public class S3Copy { private int iterations = 100; //internal fields - private String defaultRegion = "us-east-1"; private CopyRequest[] copyRequests = null; private StringBuilder log = new StringBuilder(); - private String region = null; private boolean resultsCheckOK = true; private int maxTries = 5; private int secToWait = 15; @@ -156,14 +154,12 @@ private HashMap fetchBucketRegions() { //for each one HashMap bucketNameRegion = new HashMap(); - AmazonS3 userS3 = fetchS3Client(region); + ArrayList errorMessages = new ArrayList(); for (String bn: bucketNames) { try { - String bucketRegion = Util.fetchBucketRegion(userS3, bn); + String bucketRegion = Util.fetchBucketRegion(profile, bn); bucketNameRegion.put(bn, bucketRegion); - //if (regionAccess[1].equals("OK")) bucketNameRegion.put(bn, regionAccess[0]); - //else errorMessages.add("Cannot access bucket: "+bn); } catch (Exception e) { errorMessages.add(e.getMessage()); } @@ -433,8 +429,6 @@ public void processArgs(String[] args) { if (jobString == null) Util.printErrAndExit("\nERROR: please provide a file path or string with copy job information, see the help menu.\n"); - region = Util.getRegionFromCredentials(profile, defaultRegion); - if (region == null) Util.printErrAndExit("\nERROR: failed to find your profile in ~/.aws/credentials, "+profile); credentials = new ProfileCredentialsProvider(profile); } catch (Exception e) { @@ -587,10 +581,6 @@ public boolean isDryRun() { return dryRun; } - public String getRegion() { - return region; - } - public int getNumberDaysToRestore() { return numberDaysToRestore; } diff --git a/src/main/java/edu/utah/hci/aws/apps/copy/TestS3Copy.java b/src/main/java/edu/utah/hci/aws/apps/copy/TestS3Copy.java index 7fd1e89..2b880ad 100644 --- a/src/main/java/edu/utah/hci/aws/apps/copy/TestS3Copy.java +++ b/src/main/java/edu/utah/hci/aws/apps/copy/TestS3Copy.java @@ -4,6 +4,7 @@ import com.amazonaws.AmazonServiceException; import com.amazonaws.SdkClientException; +import com.amazonaws.auth.profile.ProfileCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ListObjectsV2Result; @@ -33,9 +34,10 @@ public class TestS3Copy { /*Adjust these fields to match your testing environment */ - /**Be sure these bucket exists and doesn't contain anything you care about. WARNING, they will be emptied!*/ + /**Be sure these bucket exists, are in the same region, and don't contain anything you care about. WARNING, they will be emptied!*/ private static final String sourceBucketName = "hcibioinfo-nix-test"; private static final String destinationBucketName = "hcibioinfo-gsync-test"; + private static final String profile = "default"; /**Full path to a small test file.*/ private static final String pathToSmallTestFile = "/Users/u0028003/Code/AwsApps/TestData/GSync/Bam/testShortIndex.bam.S3.txt"; @@ -53,9 +55,11 @@ public class TestS3Copy { public void bucketToBucketNoRecursive() { AmazonS3 s3 = null; try { - //make a client - String region = Util.getRegionFromCredentials(); - s3 = AmazonS3ClientBuilder.standard().withRegion(region).build(); + //parse the credentials + ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile); + //make a client using the source + String regionSource = Util.fetchBucketRegion(credentials, sourceBucketName); + s3 = AmazonS3ClientBuilder.standard().withRegion(regionSource).build(); //empty the buckets emptyS3Bucket(s3, sourceBucketName); @@ -144,9 +148,13 @@ public void bucketToBucketNoRecursive() { public void bucketToBucketRecursive() { AmazonS3 s3 = null; try { + //parse the credentials + ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile); + //make a client using the source + String regionSource = Util.fetchBucketRegion(credentials, sourceBucketName); + //make a client - String region = Util.getRegionFromCredentials(); - s3 = AmazonS3ClientBuilder.standard().withRegion(region).build(); + s3 = AmazonS3ClientBuilder.standard().withRegion(regionSource).build(); //empty the buckets emptyS3Bucket(s3, sourceBucketName); @@ -183,9 +191,13 @@ public void bucketToBucketRecursive() { public void bucketToLocal() { AmazonS3 s3 = null; try { + //parse the credentials + ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile); + //make a client using the source + String regionSource = Util.fetchBucketRegion(credentials, sourceBucketName); + //make a client - String region = Util.getRegionFromCredentials(); - s3 = AmazonS3ClientBuilder.standard().withRegion(region).build(); + s3 = AmazonS3ClientBuilder.standard().withRegion(regionSource).build(); //empty the buckets emptyS3Bucket(s3, sourceBucketName); @@ -230,9 +242,13 @@ public void bucketToLocal() { public void archiveRetrieval() { AmazonS3 s3 = null; try { + //parse the credentials + ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile); + //make a client using the source + String regionSource = Util.fetchBucketRegion(credentials, sourceBucketName); + //make a client - String region = Util.getRegionFromCredentials(); - s3 = AmazonS3ClientBuilder.standard().withRegion(region).build(); + s3 = AmazonS3ClientBuilder.standard().withRegion(regionSource).build(); //empty the buckets emptyS3Bucket(s3, sourceBucketName); diff --git a/src/main/java/edu/utah/hci/aws/apps/gsync/GSync.java b/src/main/java/edu/utah/hci/aws/apps/gsync/GSync.java index e0e1a47..933da40 100644 --- a/src/main/java/edu/utah/hci/aws/apps/gsync/GSync.java +++ b/src/main/java/edu/utah/hci/aws/apps/gsync/GSync.java @@ -123,7 +123,7 @@ public GSync (String[] args){ void doWork() { try { - region = Util.getRegionFromCredentials(); + region = Util.fetchBucketRegion("default", bucketName); initializeFields(); @@ -1147,6 +1147,9 @@ public void printDocs(){ "**************************************************************************************\n" + "** GSync : June 2020 **\n" + "**************************************************************************************\n" + + "This is depreciated and should not be used\n\n"+ + + "GSync pushes files with a particular extension that exceed a given size and age to \n" + "Amazon's S3 object store. Associated genomic index files are also moved. Once \n"+ "correctly uploaded, GSync replaces the original file with a local txt placeholder file \n"+ diff --git a/src/main/java/edu/utah/hci/aws/apps/gsync/TestGSync.java b/src/main/java/edu/utah/hci/aws/apps/gsync/TestGSync.java index 03b3f9e..d85729d 100644 --- a/src/main/java/edu/utah/hci/aws/apps/gsync/TestGSync.java +++ b/src/main/java/edu/utah/hci/aws/apps/gsync/TestGSync.java @@ -649,7 +649,7 @@ public void testMove() { private HashMap fetchS3Objects() { AmazonS3 s3 = null; try { - String region = Util.getRegionFromCredentials(); + String region = Util.fetchBucketRegion("default", testS3BucketName); s3 = AmazonS3ClientBuilder.standard().withRegion(region).build(); HashMap kos = new HashMap(); @@ -677,7 +677,7 @@ private HashMap fetchS3Objects() { private void deleteS3Object(String key) { AmazonS3 s3 = null; try { - String region = Util.getRegionFromCredentials(); + String region = Util.fetchBucketRegion("default", testS3BucketName); s3 = AmazonS3ClientBuilder.standard().withRegion(region).build(); s3.deleteObject(testS3BucketName, key); @@ -695,7 +695,7 @@ private void deleteS3Object(String key) { private void emptyS3Bucket() { AmazonS3 s3 = null; try { - String region = Util.getRegionFromCredentials(); + String region = Util.fetchBucketRegion("default", testS3BucketName); s3 = AmazonS3ClientBuilder.standard().withRegion(region).build(); ListObjectsV2Result result = s3.listObjectsV2(testS3BucketName); diff --git a/src/main/java/edu/utah/hci/aws/apps/jobrunner/TestJobRunner.java b/src/main/java/edu/utah/hci/aws/apps/jobrunner/TestJobRunner.java index b6164af..9d511d1 100644 --- a/src/main/java/edu/utah/hci/aws/apps/jobrunner/TestJobRunner.java +++ b/src/main/java/edu/utah/hci/aws/apps/jobrunner/TestJobRunner.java @@ -36,6 +36,7 @@ public class TestJobRunner { /**Be sure this bucket exists, is private, and doesn't contain anything you care about. WARNING, it will be emptied!*/ private static final String s3BucketUri = "s3://hcibioinfo-jobrunner-test/"; + private static final String s3BucketName = "hcibioinfo-jobrunner-test"; /**Directory in the AwsApps project containing the JobRunner test files. */ private static final File testDataDir = new File("/Users/u0028003/Code/AwsApps/TestData/JobRunner"); @@ -44,6 +45,7 @@ public class TestJobRunner { private static final File awsTmpDir = new File("/Users/u0028003/TmpDelmeAWS"); private static final File awsCredentialsFile = new File("/Users/u0028003/.aws/credentials"); + private static final String profile = "default"; private static final String awsPath="/usr/local/bin/aws"; @@ -340,7 +342,7 @@ public void setPaths() throws Exception { envPropToAdd.put("PATH", "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin"); } - region = Util.getRegionFromCredentials(); + region = Util.fetchBucketRegion(profile, s3BucketName); } public static void p(String s) { diff --git a/src/main/java/edu/utah/hci/aws/apps/versions/VersionManager.java b/src/main/java/edu/utah/hci/aws/apps/versions/VersionManager.java index e189ada..2021162 100644 --- a/src/main/java/edu/utah/hci/aws/apps/versions/VersionManager.java +++ b/src/main/java/edu/utah/hci/aws/apps/versions/VersionManager.java @@ -219,15 +219,21 @@ public void processArgs(String[] args) throws Exception { el("\nError: please provide the name of a versioned AWS S3 bucket.\n"); System.exit(1); } + if (bucketName.contains("/")) { + el("\nError: please remove any s3:// or / from your bucket name.\n"); + System.exit(1); + } //set number of threads int availThreads = Runtime.getRuntime().availableProcessors()-1; if (maxThreads > availThreads) maxThreads = availThreads; - //fetch region from credentials - region = Util.getRegionFromCredentials(profile); - if (region == null) Util.printErrAndExit("\nERROR: failed to find your region in ~/.aws/credentials for the profile ["+profile+"]"); + //fetch region from credentials or use the default + + //region = Util.getRegionFromCredentials(profile); + //if (region == null) Util.printErrAndExit("\nERROR: failed to find your profile in ~/.aws/credentials for ["+profile+"]"); cred = ProfileCredentialsProvider.builder().profileName(profile).build(); + region = Util.fetchBucketRegion(profile, bucketName); printOptions(); diff --git a/src/main/java/edu/utah/hci/aws/util/Util.java b/src/main/java/edu/utah/hci/aws/util/Util.java index 4d658b2..a788474 100644 --- a/src/main/java/edu/utah/hci/aws/util/Util.java +++ b/src/main/java/edu/utah/hci/aws/util/Util.java @@ -42,6 +42,8 @@ import com.amazonaws.services.s3.transfer.Transfer.TransferState; import com.amazonaws.services.s3.transfer.TransferProgress; +import software.amazon.awssdk.regions.Region; + /**Static utility methods.*/ public class Util { @@ -154,7 +156,7 @@ public static HashMap parseKeyValues (File f) throws IOException return attributes; } - /**Extracts the region from the ~/.aws/credentials file. Can only be one profile.*/ + /**Extracts the region from the ~/.aws/credentials file. Can only be one profile. public static String getRegionFromCredentials() throws IOException { String path = System.getProperty("user.home")+"/.aws/credentials"; File cred = new File (path); @@ -163,10 +165,10 @@ public static String getRegionFromCredentials() throws IOException { String region = a.get("region"); if (region == null) throw new IOException("Failed to find a 'region' key in ~/.aws/credentials file"); return region; - } + }*/ /**Extracts the region from the ~/.aws/credentials file for a particular profile, returns null the profile wasn't found - * and either the profile's region or the default.*/ + * and either the profile's region or the default. public static String getRegionFromCredentials(String profile, String defaultRegion) throws IOException { String path = System.getProperty("user.home")+"/.aws/credentials"; File cred = new File (path); @@ -191,10 +193,10 @@ else if (lines[j].startsWith("region")) { if (profileFound) return defaultRegion; //Thus profile not found return null; - } + }*/ /**Extracts the region from the ~/.aws/credentials file for a particular profile, returns null the profile wasn't found - * and either the profile's region or the default.*/ + * and either the profile's region or the default. public static String getRegionFromCredentials(String profile) throws IOException { String path = System.getProperty("user.home")+"/.aws/credentials"; File cred = new File (path); @@ -216,13 +218,13 @@ else if (lines[j].startsWith("region")) { } } return null; - } + }*/ public static final Pattern REGION_MESSAGE = Pattern.compile(".+region: ([\\w-]+)\\..+"); /**Will fetch the region for any bucket. * @return bucket region - * @throws IOException if the bucket doesn't exist.*/ + * @throws IOException if the bucket doesn't exist. public static String fetchBucketRegion(AmazonS3 sourceS3, String bucketName) throws IOException{ String region = null; try { @@ -243,14 +245,76 @@ public static String fetchBucketRegion(AmazonS3 sourceS3, String bucketName) thr return region; } - public static void main (String[] args) throws IOException { - ProfileCredentialsProvider credentials = new ProfileCredentialsProvider("kohli"); - String region = "us-east-1"; - AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(credentials).withRegion(region).build(); - Util.pl(Util.fetchBucketRegion(s3Client, "hci-kohli-temp-transfers")); + public static String fetchBucketRegion(String profile, String defaultRegion, String bucketName) throws Exception{ + ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile); + AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(credentials).withRegion(defaultRegion).build(); + String bucketRegion = Util.fetchBucketRegion(s3Client, bucketName); + s3Client.shutdown(); + return bucketRegion; + }*/ + + public static final String[] awsBucketRegions = {"us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-east-1", "ap-south-2", "ap-southeast-3", "ap-southeast-4", "ap-south-1", "af-south-1"}; + + /**Ridiculous loop to find a bucket's region. WTH AWS! + * @return null if it can't determine the region. */ + public static String fetchBucketRegion(String profile, String bucketName) throws Exception{ + ProfileCredentialsProvider credentials = new ProfileCredentialsProvider(profile); + String bucketRegion = null; + AmazonS3 s3Client = null; + for (String abr: awsBucketRegions) { + s3Client = AmazonS3ClientBuilder.standard().withCredentials(credentials).withRegion(abr).build(); + + try { + //use a head request to attempt to find the region for the given bucket, this typically fails + HeadBucketRequest request = new HeadBucketRequest(bucketName); + HeadBucketResult res = s3Client.headBucket(request); + bucketRegion = res.getBucketRegion(); + } catch (Exception e) { + //attempt to parse the actual bucket region from the error message, ugg! + //'The bucket is in this region: us-east-2. Please use this region...' + String message = e.getMessage(); + if (message.contains("bucket is in this region:")) { + Matcher mat = REGION_MESSAGE.matcher(message); + if (mat.matches()) bucketRegion = mat.group(1); + } + } + s3Client.shutdown(); + if (bucketRegion != null) return bucketRegion; + } + return null; + } + + /**Ridiculous loop to find a bucket's region. WTH AWS! + * @return null if it can't determine the region. */ + public static String fetchBucketRegion(ProfileCredentialsProvider credentials, String bucketName) throws Exception{ + + String bucketRegion = null; + AmazonS3 s3Client = null; + + for (String abr: awsBucketRegions) { + s3Client = AmazonS3ClientBuilder.standard().withCredentials(credentials).withRegion(abr).build(); + + try { + //use a head request to attempt to find the region for the given bucket, this typically fails + HeadBucketRequest request = new HeadBucketRequest(bucketName); + HeadBucketResult res = s3Client.headBucket(request); + bucketRegion = res.getBucketRegion(); + } catch (Exception e) { + //attempt to parse the actual bucket region from the error message, ugg! + //'The bucket is in this region: us-east-2. Please use this region...' + String message = e.getMessage(); + if (message.contains("bucket is in this region:")) { + Matcher mat = REGION_MESSAGE.matcher(message); + if (mat.matches()) bucketRegion = mat.group(1); + } + } + s3Client.shutdown(); + if (bucketRegion != null) return bucketRegion; + } + return null; } /**Converts milliseconds to days.*/