From 69e14e0ebcdcc301a8d41c61bf3c86171b9427d5 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 16 Nov 2023 13:18:01 -0600 Subject: [PATCH 1/5] Initial Commit for project osggid assignment script --- .gitignore | 1 + group_identifier_assigner.py | 271 +++++++++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 .gitignore create mode 100644 group_identifier_assigner.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba698c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vscode/launch.json diff --git a/group_identifier_assigner.py b/group_identifier_assigner.py new file mode 100644 index 0000000..02f6677 --- /dev/null +++ b/group_identifier_assigner.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 + +import os +import re +import sys +import json +import getopt +import urllib.error +import urllib.request + +SCRIPT = os.path.basename(__file__) +ENDPOINT = "https://registry-test.cilogon.org/registry/" +OSG_CO_ID = 8 +MINTIMEOUT = 5 +MAXTIMEOUT = 625 +TIMEOUTMULTIPLE = 5 + +GET = "GET" +PUT = "PUT" +POST = "POST" +DELETE = "DELETE" + +_usage = f"""\ +usage: [PASS=...] {SCRIPT} [OPTIONS] + +OPTIONS: + -u USER[:PASS] specify USER and optionally PASS on command line + -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) + -d passfd specify open fd to read PASS + -f passfile specify path to file to open and read PASS + -e ENDPOINT specify REST endpoint + (default = {ENDPOINT}) + -o outfile specify output file (default: write to stdout) + -t minTimeout set minimum timeout, in seconds, for API call (default to {MINTIMEOUT}) + -T maxTimeout set maximum timeout, in seconds, for API call (default to {MAXTIMEOUT}) + -h display this help text + +PASS for USER is taken from the first of: + 1. -u USER:PASS + 2. -d passfd (read from fd) + 3. -f passfile (read from file) + 4. read from $PASS env var +""" + + +def usage(msg=None): + if msg: + print(msg + "\n", file=sys.stderr) + + print(_usage, file=sys.stderr) + sys.exit() + + +class Options: + endpoint = ENDPOINT + user = "co_8.william_test" + osg_co_id = OSG_CO_ID + outfile = None + authstr = None + min_timeout = MINTIMEOUT + max_timeout = MAXTIMEOUT + + +options = Options() + + +def getpw(user, passfd, passfile): + if ":" in user: + user, pw = user.split(":", 1) + elif passfd is not None: + pw = os.fdopen(passfd).readline().rstrip("\n") + elif passfile is not None: + pw = open(passfile).readline().rstrip("\n") + elif "PASS" in os.environ: + pw = os.environ["PASS"] + else: + usage("PASS required") + return user, pw + + +def mkauthstr(user, passwd): + from base64 import encodebytes + + raw_authstr = "%s:%s" % (user, passwd) + return encodebytes(raw_authstr.encode()).decode().replace("\n", "") + + +def mkrequest(method, target, data, **kw): + url = os.path.join(options.endpoint, target) + if kw: + url += "?" + "&".join("{}={}".format(k, v) for k, v in kw.items()) + req = urllib.request.Request(url, json.dumps(data).encode("utf-8")) + req.add_header("Authorization", "Basic %s" % options.authstr) + req.add_header("Content-Type", "application/json") + req.get_method = lambda: method + return req + + +def call_api(target, **kw): + return call_api2(GET, target, **kw) + + +def call_api2(method, target, **kw): + return call_api3(method, target, data=None, **kw) + + +def call_api3(method, target, data, **kw): + req = mkrequest(method, target, data, **kw) + trying = True + currentTimeout = options.min_timeout + while trying: + try: + resp = urllib.request.urlopen(req, timeout=currentTimeout) + payload = resp.read() + trying = False + except urllib.error.URLError as exception: + if currentTimeout < options.max_timeout: + currentTimeout *= TIMEOUTMULTIPLE + else: + sys.exit( + f"Exception raised after maximum timeout {options.max_timeout} seconds reached. " + + f"Exception reason: {exception.reason}.\n Request: {req.full_url}" + ) + + return json.loads(payload) if payload else None + + +def get_osg_co_groups(): + return call_api("co_groups.json", coid=options.osg_co_id) + + +# primary api calls + + +def get_co_group_identifiers(gid): + return call_api("identifiers.json", cogroupid=gid) + + +def get_co_group_members(gid): + return call_api("co_group_members.json", cogroupid=gid) + + +def get_co_person_identifiers(pid): + return call_api("identifiers.json", copersonid=pid) + + +def get_datalist(data, listname): + return data[listname] if data else [] + + +def identifier_index(id_list, id_type): + found_list = list((id["Type"] == id_type for id in id_list)) + if any(found_list): + return found_list.index(True) + return -1 + + +def identifier_matches(id_list, id_type, regex_string): + pattern = re.compile(regex_string) + index = identifier_index(id_list, id_type) + return (index != -1) & (pattern.match(id_list[index]["Identifier"]) is not None) + + +def add_identifier_to_group(gid, type, identifier_name): + new_identifier_info = { + "Version": "1.0", + "Type": type, + "Identifier": identifier_name, + "Login": False, + "Person": {"Type": "Group", "Id": str(gid)}, + "Status": "Active", + } + data = { + "RequestType": "Identifiers", + "Version": "1.0", + "Identifiers": [new_identifier_info], + } + return call_api3(POST, "identifiers.json", data) + + +def parse_options(args): + try: + ops, args = getopt.getopt(args, "u:c:d:f:e:o:t:T:h") + except getopt.GetoptError: + usage() + + if args: + usage("Extra arguments: %s" % repr(args)) + + passfd = None + passfile = None + + for op, arg in ops: + if op == "-h": + usage() + if op == "-u": + options.user = arg + if op == "-c": + options.osg_co_id = int(arg) + if op == "-d": + passfd = int(arg) + if op == "-f": + passfile = arg + if op == "-e": + options.endpoint = arg + if op == "-o": + options.outfile = arg + if op == "-t": + options.min_timeout = float(arg) + if op == "-T": + options.max_timeout = float(arg) + + user, passwd = getpw(options.user, passfd, passfile) + options.authstr = mkauthstr(user, passwd) + + +def main(args): + parse_options(args) + + # get groups with 'OSPool project name' matching "Yes-*" that don't have a 'OSG GID' + + co_groups = get_osg_co_groups()["CoGroups"] + ospool_pattern_str = "Yes-" + highest_osggid = 0 + + for group in co_groups: + gid = group["Id"] + identifier_data = get_co_group_identifiers(gid) + if identifier_data: + identifier_list = identifier_data["Identifiers"] + + project_id_index = identifier_index( + id_list=identifier_list, id_type="ospoolproject" + ) + if project_id_index != -1: + project_id = identifier_list[project_id_index]["Identifier"] + is_project = ( + re.compile(ospool_pattern_str + "*").match(project_id) is not None + ) + else: + is_project = False + + osggid_index = identifier_index(identifier_list, "osggid") + if osggid_index != -1: + highest_osggid = max( + highest_osggid, int(identifier_list[osggid_index]["Identifier"]) + ) + elif is_project is True: + # for each, set a 'OSG GID' starting from 200000 and a 'OSG Group Name' that is the group name + + osggid_to_assign = ( + 200000 if highest_osggid + 1 < 200000 else highest_osggid + 1 + ) + highest_osggid = osggid_to_assign + add_identifier_to_group( + gid, type="osggid", identifier_name=osggid_to_assign + ) + + project_name = project_id.removeprefix(ospool_pattern_str).lower() + add_identifier_to_group(gid, type="osggroup", identifier_name=project_name) + print( + f"project {project_id}: added osggid {osggid_to_assign} and osg project name {project_name}" + ) + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except urllib.error.HTTPError as e: + print(e, file=sys.stderr) + sys.exit(1) From 6f85f2a612334da84e551f9ca794afd619cdba18 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 16 Nov 2023 16:09:44 -0600 Subject: [PATCH 2/5] whitespace change Mis-config of the Black code formatter caused some unhelpful attempts at saving line length. --- group_identifier_assigner.py | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/group_identifier_assigner.py b/group_identifier_assigner.py index 02f6677..1470fc2 100644 --- a/group_identifier_assigner.py +++ b/group_identifier_assigner.py @@ -226,41 +226,29 @@ def main(args): for group in co_groups: gid = group["Id"] identifier_data = get_co_group_identifiers(gid) + if identifier_data: identifier_list = identifier_data["Identifiers"] - project_id_index = identifier_index( - id_list=identifier_list, id_type="ospoolproject" - ) + project_id_index = identifier_index(id_list=identifier_list, id_type="ospoolproject") if project_id_index != -1: project_id = identifier_list[project_id_index]["Identifier"] - is_project = ( - re.compile(ospool_pattern_str + "*").match(project_id) is not None - ) + is_project = (re.compile(ospool_pattern_str + "*").match(project_id) is not None) else: is_project = False osggid_index = identifier_index(identifier_list, "osggid") if osggid_index != -1: - highest_osggid = max( - highest_osggid, int(identifier_list[osggid_index]["Identifier"]) - ) + highest_osggid = max(highest_osggid, int(identifier_list[osggid_index]["Identifier"])) elif is_project is True: # for each, set a 'OSG GID' starting from 200000 and a 'OSG Group Name' that is the group name - - osggid_to_assign = ( - 200000 if highest_osggid + 1 < 200000 else highest_osggid + 1 - ) + osggid_to_assign = (200000 if highest_osggid + 1 < 200000 else highest_osggid + 1) highest_osggid = osggid_to_assign - add_identifier_to_group( - gid, type="osggid", identifier_name=osggid_to_assign - ) + add_identifier_to_group(gid, type="osggid", identifier_name=osggid_to_assign) project_name = project_id.removeprefix(ospool_pattern_str).lower() add_identifier_to_group(gid, type="osggroup", identifier_name=project_name) - print( - f"project {project_id}: added osggid {osggid_to_assign} and osg project name {project_name}" - ) + print(f"project {project_id}: added osggid {osggid_to_assign} and osg project name {project_name}") if __name__ == "__main__": From 47ca49b2b648789a0bb2396fd6a3884b83fbf618 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Fri, 17 Nov 2023 10:03:23 -0600 Subject: [PATCH 3/5] Move ospool pattern and initial project GID For quicker possible future changes and creating command line options. Also slight clean up. --- group_identifier_assigner.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/group_identifier_assigner.py b/group_identifier_assigner.py index 1470fc2..271fa6e 100644 --- a/group_identifier_assigner.py +++ b/group_identifier_assigner.py @@ -20,6 +20,9 @@ POST = "POST" DELETE = "DELETE" +OSPOOL_PATTERN_STR = "Yes-" +PROJECT_GIDS_START = 200000 + _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] @@ -59,6 +62,7 @@ class Options: authstr = None min_timeout = MINTIMEOUT max_timeout = MAXTIMEOUT + project_gid_startval = PROJECT_GIDS_START options = Options() @@ -220,7 +224,6 @@ def main(args): # get groups with 'OSPool project name' matching "Yes-*" that don't have a 'OSG GID' co_groups = get_osg_co_groups()["CoGroups"] - ospool_pattern_str = "Yes-" highest_osggid = 0 for group in co_groups: @@ -230,10 +233,10 @@ def main(args): if identifier_data: identifier_list = identifier_data["Identifiers"] - project_id_index = identifier_index(id_list=identifier_list, id_type="ospoolproject") + project_id_index = identifier_index(identifier_list, "ospoolproject") if project_id_index != -1: project_id = identifier_list[project_id_index]["Identifier"] - is_project = (re.compile(ospool_pattern_str + "*").match(project_id) is not None) + is_project = (re.compile(OSPOOL_PATTERN_STR + "*").match(project_id) is not None) else: is_project = False @@ -242,11 +245,11 @@ def main(args): highest_osggid = max(highest_osggid, int(identifier_list[osggid_index]["Identifier"])) elif is_project is True: # for each, set a 'OSG GID' starting from 200000 and a 'OSG Group Name' that is the group name - osggid_to_assign = (200000 if highest_osggid + 1 < 200000 else highest_osggid + 1) + osggid_to_assign = max(highest_osggid + 1, options.project_gid_startval) highest_osggid = osggid_to_assign add_identifier_to_group(gid, type="osggid", identifier_name=osggid_to_assign) - project_name = project_id.removeprefix(ospool_pattern_str).lower() + project_name = project_id.removeprefix(OSPOOL_PATTERN_STR).lower() add_identifier_to_group(gid, type="osggroup", identifier_name=project_name) print(f"project {project_id}: added osggid {osggid_to_assign} and osg project name {project_name}") From 575a016d902393d252c002fd71958ebaa3be4cdc Mon Sep 17 00:00:00 2001 From: William Swanson Date: Wed, 22 Nov 2023 10:02:39 -0600 Subject: [PATCH 4/5] Updates from Jason review Also "replace"-ed repoveprefix, since I think this script might be running on a machine with only python 3.6.8 (removeprefix needs 3.9) --- group_identifier_assigner.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/group_identifier_assigner.py b/group_identifier_assigner.py index 271fa6e..c71a097 100644 --- a/group_identifier_assigner.py +++ b/group_identifier_assigner.py @@ -20,7 +20,7 @@ POST = "POST" DELETE = "DELETE" -OSPOOL_PATTERN_STR = "Yes-" +OSPOOL_PROJECT_PREFIX_STR = "Yes-" PROJECT_GIDS_START = 200000 _usage = f"""\ @@ -153,10 +153,11 @@ def get_datalist(data, listname): def identifier_index(id_list, id_type): - found_list = list((id["Type"] == id_type for id in id_list)) - if any(found_list): - return found_list.index(True) - return -1 + id_type_list = [id["Type"] for id in id_list] + try: + return id_type_list.index(id_type) + except ValueError: + return -1 def identifier_matches(id_list, id_type, regex_string): @@ -225,18 +226,19 @@ def main(args): co_groups = get_osg_co_groups()["CoGroups"] highest_osggid = 0 + projects_to_assign_identifiers = [] for group in co_groups: gid = group["Id"] identifier_data = get_co_group_identifiers(gid) - + if identifier_data: identifier_list = identifier_data["Identifiers"] project_id_index = identifier_index(identifier_list, "ospoolproject") if project_id_index != -1: - project_id = identifier_list[project_id_index]["Identifier"] - is_project = (re.compile(OSPOOL_PATTERN_STR + "*").match(project_id) is not None) + project_id = str(identifier_list[project_id_index]["Identifier"]) + is_project = re.compile(OSPOOL_PROJECT_PREFIX_STR + "*").match(project_id) is not None else: is_project = False @@ -244,14 +246,16 @@ def main(args): if osggid_index != -1: highest_osggid = max(highest_osggid, int(identifier_list[osggid_index]["Identifier"])) elif is_project is True: - # for each, set a 'OSG GID' starting from 200000 and a 'OSG Group Name' that is the group name - osggid_to_assign = max(highest_osggid + 1, options.project_gid_startval) - highest_osggid = osggid_to_assign - add_identifier_to_group(gid, type="osggid", identifier_name=osggid_to_assign) - - project_name = project_id.removeprefix(OSPOOL_PATTERN_STR).lower() - add_identifier_to_group(gid, type="osggroup", identifier_name=project_name) - print(f"project {project_id}: added osggid {osggid_to_assign} and osg project name {project_name}") + project_name = project_id.replace(OSPOOL_PROJECT_PREFIX_STR, "", 1).lower() + projects_to_assign_identifiers.append(tuple([gid, project_name])) + + for gid, project_name in projects_to_assign_identifiers: + # for each, set a 'OSG GID' starting from 200000 and a 'OSG Group Name' that is the group name + osggid_to_assign = max(highest_osggid + 1, options.project_gid_startval) + highest_osggid = osggid_to_assign + add_identifier_to_group(gid, type="osggid", identifier_name=osggid_to_assign) + add_identifier_to_group(gid, type="osggroup", identifier_name=project_name) + print(f"project {project_name}: added osggid {osggid_to_assign} and osg project name {project_name}") if __name__ == "__main__": From cf48240624b5a475b9eb6dfd53d772f78533e493 Mon Sep 17 00:00:00 2001 From: William Swanson <144060728+williamnswanson@users.noreply.github.com> Date: Wed, 22 Nov 2023 13:20:02 -0600 Subject: [PATCH 5/5] Update group_identifier_assigner.py Co-authored-by: Jason Patton --- group_identifier_assigner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_identifier_assigner.py b/group_identifier_assigner.py index c71a097..950e045 100644 --- a/group_identifier_assigner.py +++ b/group_identifier_assigner.py @@ -247,7 +247,7 @@ def main(args): highest_osggid = max(highest_osggid, int(identifier_list[osggid_index]["Identifier"])) elif is_project is True: project_name = project_id.replace(OSPOOL_PROJECT_PREFIX_STR, "", 1).lower() - projects_to_assign_identifiers.append(tuple([gid, project_name])) + projects_to_assign_identifiers.append((gid, project_name,)) for gid, project_name in projects_to_assign_identifiers: # for each, set a 'OSG GID' starting from 200000 and a 'OSG Group Name' that is the group name