Skip to content

Commit

Permalink
Merge pull request #55 from lsst-dm/tickets/DM-44847
Browse files Browse the repository at this point in the history
DM-44847: Add ability to ingest with profile.
  • Loading branch information
ktlim authored Jun 18, 2024
2 parents 41deacf + 1b850f4 commit 518233b
Showing 1 changed file with 7 additions and 1 deletion.
8 changes: 7 additions & 1 deletion src/ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@
logger = setup_logging(__name__)
r = setup_redis()
bucket = os.environ["BUCKET"]
if (pos := bucket.find("@")) >= 0:
pos += 1
profile = bucket[:pos]
bucket = bucket[pos:]
else:
profile = ""
if bucket.startswith("rubin:"):
os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"
redis_queue = f"QUEUE:{bucket}"
Expand Down Expand Up @@ -199,7 +205,7 @@ def main():
# Process any entries on the worker queue.
if r.llen(worker_queue) > 0:
blobs = r.lrange(worker_queue, 0, -1)
resources = [ResourcePath(f"s3://{b.decode()}") for b in blobs]
resources = [ResourcePath(f"s3://{profile}{b.decode()}") for b in blobs]

# Ingest if we have resources
if resources:
Expand Down

0 comments on commit 518233b

Please sign in to comment.