Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bank_table: add max-preempt-after column #556

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

bank_table: add max-preempt-after column #556

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d2c9c19
bank_table: add "max_preempt_after" column
cmoussa1 Dec 19, 2024
da3edd4
add-bank: add --max-preempt-after optional arg
cmoussa1 Dec 19, 2024
d78c69d
edit-bank: add --max-preempt-after optional arg
cmoussa1 Dec 19, 2024
22424a7
t: edit expected output to account for new column
cmoussa1 Dec 19, 2024
c358048
t: add tests for max-preempt-after optional arg
cmoussa1 Dec 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/bindings/python/fluxacct/accounting/__init__.py.in
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
DB_DIR = "@X_LOCALSTATEDIR@/lib/flux/"
DB_PATH = "@X_LOCALSTATEDIR@/lib/flux/FluxAccounting.db"
DB_SCHEMA_VERSION = 24
DB_SCHEMA_VERSION = 25

# flux-accounting DB table column names
ASSOCIATION_TABLE = [
Expand All @@ -21,7 +21,7 @@ ASSOCIATION_TABLE = [
"projects",
"default_project",
]
BANK_TABLE = ["bank_id", "bank", "active", "parent_bank", "shares", "job_usage"]
BANK_TABLE = ["bank_id", "bank", "active", "parent_bank", "shares", "job_usage", "max_preempt_after"]
QUEUE_TABLE = [
"queue",
"min_nodes_per_job",
Expand Down
41 changes: 29 additions & 12 deletions src/bindings/python/fluxacct/accounting/bank_subcommands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
from fluxacct.accounting import formatter as fmt
from fluxacct.accounting import sql_util as sql

from flux.util import parse_fsd

###############################################################
# #
# Helper Functions #
Expand Down Expand Up @@ -218,7 +220,7 @@ def print_parsable_hierarchy(cur, bank, hierarchy_str, indent=""):
###############################################################


def add_bank(conn, bank, shares, parent_bank=""):
def add_bank(conn, bank, shares, parent_bank="", max_preempt_after=None):
cur = conn.cursor()

if parent_bank == "":
Expand Down Expand Up @@ -248,19 +250,24 @@ def add_bank(conn, bank, shares, parent_bank=""):
reactivate_bank(conn, cur, bank, parent_bank)
return 0

if max_preempt_after:
# parse Flux Standard Duration (see RFC 23) and add it to the INSERT statement
duration = parse_fsd(max_preempt_after)
insert_stmt = (
"INSERT INTO bank_table "
"(bank, parent_bank, shares, max_preempt_after) "
"VALUES (?, ?, ?, ?)"
)
stmt_parameters = (bank, parent_bank, shares, duration)
else:
insert_stmt = (
"INSERT INTO bank_table (bank, parent_bank, shares) VALUES (?, ?, ?)"
)
stmt_parameters = (bank, parent_bank, shares)

# insert the bank values into the database
try:
conn.execute(
"""
INSERT INTO bank_table (
bank,
parent_bank,
shares
)
VALUES (?, ?, ?)
""",
(bank, parent_bank, shares),
)
conn.execute(insert_stmt, stmt_parameters)
# commit changes
conn.commit()

Expand Down Expand Up @@ -384,12 +391,14 @@ def edit_bank(
bank=None,
shares=None,
parent_bank=None,
max_preempt_after=None,
):
cur = conn.cursor()
params = locals()
editable_fields = [
"shares",
"parent_bank",
"max_preempt_after",
]
for field in editable_fields:
if params[field] is not None:
Expand All @@ -405,6 +414,14 @@ def edit_bank(
if field == "shares":
if int(shares) <= 0:
raise ValueError("new shares amount must be >= 0")
if field == "max_preempt_after":
if max_preempt_after == str(-1):
duration = None
else:
# convert FSD to seconds
duration = parse_fsd(max_preempt_after)
params[field] = duration
print(f"params[field]: {params[field]}")

update_stmt = "UPDATE bank_table SET " + field

Expand Down
13 changes: 7 additions & 6 deletions src/bindings/python/fluxacct/accounting/create_db.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,13 @@ def create_db(
conn.execute(
"""
CREATE TABLE IF NOT EXISTS bank_table (
bank_id integer PRIMARY KEY AUTOINCREMENT,
bank text NOT NULL,
active int(11) DEFAULT 1 NOT NULL,
parent_bank text DEFAULT '',
shares int NOT NULL,
job_usage real DEFAULT 0.0 NOT NULL
bank_id integer PRIMARY KEY AUTOINCREMENT,
bank text NOT NULL,
active int(11) DEFAULT 1 NOT NULL,
parent_bank text DEFAULT '',
shares int NOT NULL,
job_usage real DEFAULT 0.0 NOT NULL,
max_preempt_after real
);"""
)
logging.info("Created bank_table successfully")
Expand Down
2 changes: 2 additions & 0 deletions src/cmd/flux-account-service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ def add_bank(self, handle, watcher, msg, arg):
msg.payload["bank"],
msg.payload["shares"],
msg.payload["parent_bank"],
msg.payload["max_preempt_after"],
)

payload = {"add_bank": val}
Expand Down Expand Up @@ -312,6 +313,7 @@ def edit_bank(self, handle, watcher, msg, arg):
msg.payload["bank"],
msg.payload["shares"],
msg.payload["parent_bank"],
msg.payload["max_preempt_after"],
)

payload = {"edit_bank": val}
Expand Down
18 changes: 18 additions & 0 deletions src/cmd/flux-account.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,15 @@ def add_add_bank_arg(subparsers):
subparser_add_bank.add_argument(
"shares", help="number of shares to allocate to bank", metavar="SHARES"
)
subparser_add_bank.add_argument(
"--max-preempt-after",
help=(
"max number of time until a job running under this bank can become "
"preemptible; duration must be in Flux Standard Duration"
),
type=str,
metavar="MAX_PREEMPT_AFTER",
)


def add_view_bank_arg(subparsers):
Expand Down Expand Up @@ -356,6 +365,15 @@ def add_edit_bank_arg(subparsers):
help="parent bank",
metavar="PARENT BANK",
)
subparser_edit_bank.add_argument(
"--max-preempt-after",
help=(
"max number of time until a job running under this bank can become "
"preemptible; duration must be in Flux Standard Duration"
),
type=str,
metavar="MAX_PREEMPT_AFTER",
)


def add_list_banks_arg(subparsers):
Expand Down
1 change: 1 addition & 0 deletions t/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ TESTSCRIPTS = \
t1041-view-jobs-by-project.t \
t1042-issue508.t \
t1043-view-jobs-by-bank.t \
t1044-issue553.t \
t5000-valgrind.t \
python/t1000-example.py \
python/t1001_db.py \
Expand Down
4 changes: 2 additions & 2 deletions t/expected/flux_account/A_bank.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
2 A 1 root 1 0.0
bank_id bank active parent_bank shares job_usage max_preempt_after
2 A 1 root 1 0.0 None

Users Under Bank A:

Expand Down
4 changes: 2 additions & 2 deletions t/expected/flux_account/D_bank.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
5 D 1 root 1 0.0
bank_id bank active parent_bank shares job_usage max_preempt_after
5 D 1 root 1 0.0 None

Bank Username RawShares RawUsage Fairshare
D 1 0.0
Expand Down
4 changes: 2 additions & 2 deletions t/expected/flux_account/E_bank.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
6 E 1 D 1 0.0
bank_id bank active parent_bank shares job_usage max_preempt_after
6 E 1 D 1 0.0 None

Bank Username RawShares RawUsage Fairshare
E 1 0.0
Expand Down
4 changes: 2 additions & 2 deletions t/expected/flux_account/F_bank_tree.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
7 F 1 D 1 0.0
bank_id bank active parent_bank shares job_usage max_preempt_after
7 F 1 D 1 0.0 None

Bank Username RawShares RawUsage Fairshare
F 1 0.0
Expand Down
4 changes: 2 additions & 2 deletions t/expected/flux_account/F_bank_users.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
bank_id bank active parent_bank shares job_usage
7 F 1 D 1 0.0
bank_id bank active parent_bank shares job_usage max_preempt_after
7 F 1 D 1 0.0 None

no users under F
4 changes: 2 additions & 2 deletions t/expected/flux_account/deleted_bank.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
bank_id bank active parent_bank shares
4 C 0 root 50
bank_id bank active parent_bank shares job_usage max_preempt_after
4 C 0 root 50 0.0 None

4 changes: 2 additions & 2 deletions t/expected/flux_account/full_hierarchy.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
1 root 1 1 0.0
bank_id bank active parent_bank shares job_usage max_preempt_after
1 root 1 1 0.0 None

Bank Username RawShares RawUsage Fairshare
root 1 0.0
Expand Down
4 changes: 2 additions & 2 deletions t/expected/flux_account/root_bank.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
bank_id bank active parent_bank shares job_usage
1 root 1 1 0.0
bank_id bank active parent_bank shares job_usage max_preempt_after
1 root 1 1 0.0 None

4 changes: 2 additions & 2 deletions t/expected/pop_db/db_hierarchy_base.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
1 root 1 1 0.0
bank_id bank active parent_bank shares job_usage max_preempt_after
1 root 1 1 0.0 None

Bank Username RawShares RawUsage Fairshare
root 1 0.0
Expand Down
4 changes: 2 additions & 2 deletions t/expected/pop_db/db_hierarchy_new_users.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
1 root 1 1 0.0
bank_id bank active parent_bank shares job_usage max_preempt_after
1 root 1 1 0.0 None

Bank Username RawShares RawUsage Fairshare
root 1 0.0
Expand Down
4 changes: 2 additions & 2 deletions t/expected/print_hierarchy/small_no_tie.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
1 root 1 1000 133.0
bank_id bank active parent_bank shares job_usage max_preempt_after
1 root 1 1000 133.0 None

Bank Username RawShares RawUsage Fairshare
root 1000 133.0
Expand Down
4 changes: 2 additions & 2 deletions t/expected/print_hierarchy/small_tie.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
1 root 1 1000 133.0
bank_id bank active parent_bank shares job_usage max_preempt_after
1 root 1 1000 133.0 None

Bank Username RawShares RawUsage Fairshare
root 1000 133.0
Expand Down
4 changes: 2 additions & 2 deletions t/expected/print_hierarchy/small_tie_all.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
1 root 1 1000 1332.0
bank_id bank active parent_bank shares job_usage max_preempt_after
1 root 1 1000 1332.0 None

Bank Username RawShares RawUsage Fairshare
root 1000 1332.0
Expand Down
4 changes: 2 additions & 2 deletions t/expected/update_fshare/post_fshare_update.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
1 root 1 1000 180.0
bank_id bank active parent_bank shares job_usage max_preempt_after
1 root 1 1000 180.0 None

Bank Username RawShares RawUsage Fairshare
root 1000 180.0
Expand Down
4 changes: 2 additions & 2 deletions t/expected/update_fshare/pre_fshare_update.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bank_id bank active parent_bank shares job_usage
1 root 1 1000 133.0
bank_id bank active parent_bank shares job_usage max_preempt_after
1 root 1 1000 133.0 None

Bank Username RawShares RawUsage Fairshare
root 1000 133.0
Expand Down
73 changes: 73 additions & 0 deletions t/t1044-issue553.t
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#!/bin/bash

test_description='test configuring max-preempt-after values for banks'

. `dirname $0`/sharness.sh
DB_PATH=$(pwd)/FluxAccountingTest.db

export TEST_UNDER_FLUX_NO_JOB_EXEC=y
export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="limited=1"
test_under_flux 1 job

flux setattr log-stderr-level 1

test_expect_success 'create flux-accounting DB' '
flux account -p ${DB_PATH} create-db
'

test_expect_success 'start flux-accounting service' '
flux account-service -p ${DB_PATH} -t
'

test_expect_success 'add root bank to the DB' '
flux account add-bank root 1
'

test_expect_success 'add a sub bank with max-preempt in seconds' '
flux account add-bank --parent-bank=root --max-preempt-after=60s A 1 &&
flux account view-bank A > A.test &&
grep "60.0" A.test
'

test_expect_success 'add a sub bank with max-preempt in hours' '
flux account add-bank --parent-bank=root --max-preempt-after=1h B 1 &&
flux account view-bank B > B.test &&
grep "3600.0" B.test
'

test_expect_success 'add a sub bank with max-preempt in days' '
flux account add-bank --parent-bank=root --max-preempt-after=1d C 1 &&
flux account view-bank C > C.test &&
grep "86400.0" C.test
'

test_expect_success 'add a sub bank with no max-preempt specified' '
flux account add-bank --parent-bank=root D 1 &&
flux account view-bank D > D.test &&
grep "None" D.test
'

test_expect_success 'trying to add a sub bank with a bad FSD will raise an error' '
test_must_fail flux account add-bank \
--parent-bank=root \
--max-preempt-after=foo E 1 > error.out 2>&1 &&
grep "error in add_bank: invalid Flux standard duration" error.out
'

test_expect_success 'edit max-preempt-after for a bank' '
flux account edit-bank A --max-preempt-after=200s &&
flux account view-bank A > A_edited.test &&
grep "200.0" A_edited.test
'

test_expect_success 'clear max-preempt-after for a bank' '
flux account edit-bank A --max-preempt-after=-1 &&
flux account view-bank A > A_cleared.test &&
grep "None" A_cleared.test
'

test_expect_success 'shut down flux-accounting service' '
flux python -c "import flux; flux.Flux().rpc(\"accounting.shutdown_service\").get()"
'

test_done
Loading