updating app versions (in container) (#63)

* updating app versions (in container)

we need to use an updated pydantic-settings and fastapi.
This likely will work best in a container, or will
need additional tweaks to install the correct version
after. It does not seem to resolve perfectly in one
requirements.txt, unfortunately.

Signed-off-by: vsoch <[email protected]>

.devcontainer/Dockerfile

@@ -1,35 +1,27 @@
-FROM ghcr.io/rse-ops/pokemon:app-latest
+FROM fluxrm/flux-sched:jammy
 
 LABEL maintainer="Vanessasaurus <@vsoch>"
 
 # Pip not provided in this version
 USER root
-RUN apt-get update && apt-get install -y python3-venv systemctl
+RUN apt-get update && apt-get install -y python3-venv systemctl python3-pip
 COPY ./requirements.txt /requirements.txt
 COPY ./.github/dev-requirements.txt /dev-requirements.txt
 COPY ./docs/requirements.txt /docs-requirements.txt
 
 EXPOSE 5000
-ENV PYTHONPATH=/usr/lib/flux/python3.8
+ENV PYTHONPATH=/usr/lib/flux/python3.10
 
 # For easier Python development.
 RUN python3 -m pip install IPython && \
     python3 -m pip install -r /requirements.txt && \
     python3 -m pip install -r /dev-requirements.txt && \
     python3 -m pip install -r /docs-requirements.txt
 
-RUN python3 -m venv /env && \
-    . /env/bin/activate && \
-    pip install -r /requirements.txt && \
-    pip install -r /dev-requirements.txt && \
-    pip install -r /docs-requirements.txt && \
-    # Only for development - don't add this to a production container
-    sudo useradd -m -p $(openssl passwd '12345') "flux"
-
 RUN mkdir -p /run/flux /var/lib/flux mkdir /etc/flux/system/cron.d /mnt/curve && \
     flux keygen /mnt/curve/curve.cert && \
     # These need to be owned by flux
-    chown -R flux /run/flux /var/lib/flux /mnt/curve && \
+    chown -R fluxuser /run/flux /var/lib/flux /mnt/curve && \
     # flux-imp needs setuid permission
     chmod u+s /usr/libexec/flux/flux-imp
 

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 {
-    "name": "Flux Core Python 3.6",
+    "name": "Flux RESTFul Python 3.10",
     "dockerFile": "Dockerfile",
     "context": "../",
 
@@ -10,12 +10,12 @@
 
           // Ensure that Python autocomplete works out of the box
           "python.autoComplete.extraPaths": [
-            "/usr/lib/flux/python3.8",
-            "/usr/lib/python3.8/site-packages",
+            "/usr/lib/flux/python3.10",
+            "/usr/lib/python3.10/site-packages",
           ],
           "python.analysis.extraPaths": [
-            "/usr/lib/flux/python3.8",
-            "/usr/lib/python3.8/site-packages",
+            "/usr/lib/flux/python3.10",
+            "/usr/lib/python3.10/site-packages",
           ]
         },
         // Note to Flux Developers! We can add extensions here that you like
@@ -24,6 +24,7 @@
 		],
       },
     },
-    // Needed for git security feature, and flux config
-    "postStartCommand": "git config --global --add safe.directory /workspaces/flux-python-api && flux R encode --hosts=$(hostname) > /etc/flux/system/R && sed -i 's@HOSTNAME@'$(hostname)'@' /etc/flux/system/conf.d/broker.toml && sudo service munge start"
+    // Needed for git security feature, and flux config (not added / needed yet)
+    //"postStartCommand": "git config --global --add safe.directory /workspaces/flux-restful-api && flux R encode --hosts=$(hostname) > /etc/flux/system/R && sed -i 's@HOSTNAME@'$(hostname)'@' /etc/flux/system/conf.d/broker.toml && sudo service munge start"
+    "postStartCommand": "git config --global --add safe.directory /workspaces/flux-restful-api && flux R encode --hosts=$(hostname) > /etc/flux/system/R && sudo service munge start"
   }

.github/dev-requirements.txt

@@ -1,5 +1,5 @@
 pre-commit
-black
+black==23.3.0
 isort
 flake8
 pytest

.github/workflows/build-deploy.yaml

@@ -16,7 +16,7 @@ on:
 jobs:
   build:
     env:
-      container: ghcr.io/flux-framework/flux-restful-api
+      container: ghcr.io/flux-framework/flux-restful-api:test
     permissions:
       packages: write
 

.github/workflows/main.yaml

@@ -11,7 +11,7 @@ jobs:
   formatting:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Setup black linter
       run: conda create --quiet --name black pyflakes

.github/workflows/python-tests.yaml

@@ -22,7 +22,7 @@ jobs:
     needs: [prepare-container]
     services:
       api:
-        image: ghcr.io/flux-framework/flux-restful-api:latest
+        image: ghcr.io/flux-framework/flux-restful-api:test
         ports:
           - 5000:5000
         env:
@@ -44,7 +44,7 @@ jobs:
         env:
           INSTALL_BRANCH: ${{ needs.prepare-container.outputs.branch }}
           INSTALL_REPO: ${{ github.repository }}
-        image: ghcr.io/flux-framework/flux-restful-api:latest
+        image: ghcr.io/flux-framework/flux-restful-api:test
         ports:
           - 5000:5000
     steps:

.github/workflows/tests.yaml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: [prepare-container]
     container:
-      image: ghcr.io/flux-framework/flux-restful-api:latest
+      image: ghcr.io/flux-framework/flux-restful-api:test
       ports:
        - 5000
       env:
@@ -30,8 +30,10 @@ jobs:
     - uses: actions/checkout@v3
     - name: Install Dependencies (in case changes)
       run: pip install -r requirements.txt
+
     - name: Run tests
       run: |
+        pip freeze
         # Tests for the API with auth disabled
         flux start pytest -xs tests/test_api.py
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM fluxrm/flux-sched:focal
+FROM fluxrm/flux-sched:jammy
 
 # docker build -t ghcr.io/flux-framework/flux-restful-api .
 
@@ -14,7 +14,7 @@ ENV HOST=${host:-0.0.0.0}
 ENV WORKERS=${workers:-1}
 
 USER root
-RUN apt-get update
+RUN apt-get update && apt-get install -y python3-pip
 COPY ./requirements.txt /requirements.txt
 
 EXPOSE ${port}

VERSION

@@ -1 +1 @@
-0.1.13
+0.2.0

app/core/config.py

@@ -4,8 +4,9 @@
 import secrets
 import shlex
 import string
+from typing import Optional
 
-from pydantic import BaseSettings
+from pydantic_settings import BaseSettings
 
 logger = logging.getLogger(__name__)
 
@@ -85,8 +86,8 @@ class Settings(BaseSettings):
 
     # If you change this, also change in alembic.ini
     db_file: str = "sqlite:///./flux-restful.db"
-    flux_user: str = os.environ.get("FLUX_USER")
-    flux_token: str = os.environ.get("FLUX_TOKEN")
+    flux_user: str = os.environ.get("FLUX_USER") or "fluxuser"
+    flux_token: Optional[str] = os.environ.get("FLUX_TOKEN")
     secret_key: str = os.environ.get("FLUX_SECRET_KEY") or generate_secret_key()
 
     # Expires in 10 hours

app/library/auth.py

@@ -51,7 +51,11 @@ def check_auth(
         db, user_name=credentials.username, password=credentials.password
     )
     if not user:
-        raise HTTPException(status_code=401, detail="Incorrect email or password", headers={"WWW-Authenticate": "Basic"})
+        raise HTTPException(
+            status_code=401,
+            detail="Incorrect email or password",
+            headers={"WWW-Authenticate": "Basic"},
+        )
     elif not crud_user.is_active(user):
         raise HTTPException(status_code=400, detail="Inactive user")
     return credentials.username

app/library/flux.py

@@ -72,6 +72,29 @@ def submit_job(handle, fluxjob, user):
     return job
 
 
+def clean_submit_args(kwargs):
+    """
+    Clean up submit arguments
+    """
+    # Clean up Nones
+    cleaned = {}
+    for k, v in kwargs.items():
+        if k == "option_flags" and v is not None:
+            option_flags = {}
+            flags = v.split(",")
+            for flag in flags:
+                if "=" not in flag:
+                    print('Warning: invalid flag {flag} missing "="')
+                    continue
+                option, value = flag.split("=", 1)
+                option_flags[option] = value
+            v = option_flags
+
+        if v is not None:
+            cleaned[k] = v
+    return cleaned
+
+
 def validate_submit_kwargs(kwargs, envars=None, runtime=None):
     """
     Shared function to validate submit, from API or web UI.

app/routers/api.py

@@ -132,16 +132,16 @@ async def jobs_listing(request: Request, user=user_auth):
 
 
 @router.get("/jobs")
-async def list_jobs(request: Request, user=user_auth):
+async def list_jobs(
+    details: bool = False, limit=None, listing: bool = False, user=user_auth
+):
     """
     List flux jobs associated with the handle.
     """
-    try:
-        payload = await request.json()
-    except Exception:
-        payload = {}
+    payload = {"details": details, "limit": limit, "listing": listing}
 
     # Does the requester want details - in dict or listing form?
+    print(payload)
     if helpers.has_boolean_arg(payload, "details"):
         # Job limit (only relevant for details)
         limit = helpers.get_int_arg(payload, "limit")
@@ -182,7 +182,20 @@ async def cancel_job(jobid, user=user_auth):
 
 
 @router.post("/jobs/submit")
-async def submit_job(request: Request, user=user_auth):
+async def submit_job(
+    command=None,
+    num_tasks: int = None,
+    cores_per_task: int = None,
+    gpus_per_task: int = None,
+    num_nodes: int = None,
+    exclusive: bool = False,
+    option_flags: str = None,
+    envars: dict = None,
+    workdir: str = None,
+    runtime: int = None,
+    is_launcher: bool = False,
+    user=user_auth,
+):
     """
     Submit a job to our running cluster.
 
@@ -192,40 +205,28 @@ async def submit_job(request: Request, user=user_auth):
     """
     from app.main import app
 
-    print(f"User for submit is {user}")
-
     # This can bork if no payload is provided
-    try:
-        payload = await request.json()
-    except Exception:
+    if not command:
         return JSONResponse(
             content={"Message": "A 'command' is minimally required."}, status_code=400
         )
 
-    kwargs = {}
-
-    # Required arguments
-    for required in ["command"]:
-        kwargs[required] = payload.get(required)
-
-    # Optional arguments
-    as_int = ["num_tasks", "cores_per_task", "gpus_per_task", "num_nodes"]
-    as_bool = ["exclusive"]
-    as_is = ["option_flags"]
-
-    for optional in as_int + as_bool + as_is:
-        if optional in payload and payload[optional]:
-            if optional in as_bool:
-                kwargs[optional] = bool(payload[optional])
-            elif optional in as_int:
-                kwargs[optional] = int(payload[optional])
-            else:
-                kwargs[optional] = payload[optional]
+    kwargs = {
+        "command": command,
+        "num_tasks": num_tasks,
+        "cores_per_task": cores_per_task,
+        "gpus_per_task": gpus_per_task,
+        "num_nodes": num_nodes,
+        "exclusive": exclusive,
+        "option_flags": option_flags,
+    }
 
     # One off args not provided to JobspecV1
-    envars = payload.get("envars", {})
-    workdir = payload.get("workdir")
-    runtime = payload.get("runtime", 0) or 0
+    envars = envars or {}
+    runtime = runtime or 0
+
+    # Clean up submit arguments
+    kwargs = flux_cli.clean_submit_args(kwargs)
 
     # Validate the payload, return meaningful message if something is off
     invalid_messages = flux_cli.validate_submit_kwargs(
@@ -238,7 +239,6 @@ async def submit_job(request: Request, user=user_auth):
         )
 
     # Are we using a launcher instead?
-    is_launcher = payload.get("is_launcher", False)
     if is_launcher:
         message = launcher.launch(kwargs, workdir=workdir, envars=envars, user=user)
         result = jsonable_encoder({"Message": message, "id": "MANY"})

clients/python/CHANGELOG.md

@@ -14,6 +14,8 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/flux-framework/flux-restful-api/tree/main) (0.0.x)
+ - Update to use newer versions of fastapi, etc (0.1.15)
+   - option_flags is a flat string list of values
  - Expose host to environment and bug fix for logs (0.1.14)
  - Ensure we update flux environment for user (0.1.13)
  - Add better multi-user mode - running jobs on behalf of user (0.1.12)