Apps: version pinning with immutable snapshots, detail pages, service runtime, and job lifecycle hardening

docs/config.yaml.template

@@ -137,6 +137,15 @@ session_cookie_secure: true
 #   extra_paths:                # Paths appended to $PATH in every job script
 #     - /opt/nextflow/bin       # and used when verifying tool requirements
 #     - /opt/pixi/bin
+#   worker_env_passthrough:     # Extra env var names (exact) or prefixes
+#     - MYSITE_LICENSE_SERVER   # (ending in '_') to pass to per-user workers,
+#     - MYSITE_                 # on top of the built-in allowlist. The worker
+#                               # env is an allowlist so no server secret leaks
+#                               # to the user; add only non-secret vars your
+#                               # scheduler or tools need. FGC_* is never passed.
+#   unknown_timeout_hours: 24    # Give up polling a job stuck in UNKNOWN after
+#                               # this many hours and mark it FAILED (the
+#                               # scheduler can no longer report it). 0 disables.
 
 # Cluster configuration
 # Mirrors py-cluster-api ClusterConfig - all fields are optional

fileglancer/alembic/versions/a7e2f9d31c04_add_status_updated_at_to_jobs.py

@@ -0,0 +1,24 @@
+"""add status_updated_at column to jobs
+
+Revision ID: a7e2f9d31c04
+Revises: f4a1d8c62e97
+Create Date: 2026-07-04 00:00:00.000000
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a7e2f9d31c04'
+down_revision = 'f4a1d8c62e97'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column('jobs', sa.Column('status_updated_at', sa.DateTime(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column('jobs', 'status_updated_at')

fileglancer/alembic/versions/f4a1d8c62e97_add_commit_pinning_to_apps_and_jobs.py

@@ -0,0 +1,56 @@
+"""add commit pinning to user_apps and jobs
+
+Revision ID: f4a1d8c62e97
+Revises: c1f9a4e7b2d8
+Create Date: 2026-07-02 00:00:00.000000
+
+user_apps.commit_sha pins each app to the exact commit it was added or last
+updated at; jobs run from an immutable per-SHA snapshot of that commit rather
+than the mutable branch clone. code_commit_sha is the equivalent pin for a
+manifest's separate code repo (repo_url), when one is declared.
+jobs.commit_sha records the commit whose code the job actually executed, and
+jobs.code_repo_url the repo that commit belongs to when it isn't the app repo.
+
+All columns are nullable: legacy rows have no pin and are backfilled the next
+time the app is launched or updated.
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f4a1d8c62e97'
+down_revision = 'c1f9a4e7b2d8'
+branch_labels = None
+depends_on = None
+
+
+def _missing(table: str, column: str) -> bool:
+    inspector = sa.inspect(op.get_bind())
+    return column not in {c["name"] for c in inspector.get_columns(table)}
+
+
+def upgrade() -> None:
+    # Guarded adds: this revision briefly existed without jobs.code_repo_url,
+    # so a dev database migrated during that window is stamped at this head
+    # while missing the column. Re-running after `alembic stamp c1f9a4e7b2d8`
+    # converges any such state without erroring on the columns that do exist.
+    if _missing('user_apps', 'commit_sha'):
+        op.add_column('user_apps', sa.Column('commit_sha', sa.String(), nullable=True))
+    if _missing('user_apps', 'code_commit_sha'):
+        op.add_column('user_apps', sa.Column('code_commit_sha', sa.String(), nullable=True))
+    if _missing('jobs', 'commit_sha'):
+        op.add_column('jobs', sa.Column('commit_sha', sa.String(), nullable=True))
+    if _missing('jobs', 'code_repo_url'):
+        op.add_column('jobs', sa.Column('code_repo_url', sa.String(), nullable=True))
+
+
+def downgrade() -> None:
+    if not _missing('jobs', 'code_repo_url'):
+        op.drop_column('jobs', 'code_repo_url')
+    if not _missing('jobs', 'commit_sha'):
+        op.drop_column('jobs', 'commit_sha')
+    if not _missing('user_apps', 'code_commit_sha'):
+        op.drop_column('user_apps', 'code_commit_sha')
+    if not _missing('user_apps', 'commit_sha'):
+        op.drop_column('user_apps', 'commit_sha')

fileglancer/apps/__init__.py

@@ -5,17 +5,23 @@
     _ensure_repo_cache,
     clone_url_for_stored_app,
     discover_app_manifests,
+    ensure_repo_snapshot,
     fetch_app_manifest,
+    gc_repo_snapshots,
     get_app_branch,
+    get_remote_head,
+    get_remote_heads,
     canonical_app_url,
     get_or_load_manifest,
     refresh_cached_manifest,
     set_worker_exec,
+    validate_commit_sha,
 )
 from fileglancer.apps.command import (  # noqa: F401
     _TOOL_REGISTRY,
     build_command,
     build_requirements_check,
+    collect_creatable_dirs,
     collect_path_parameters,
     expand_user_path,
     merge_requirements,
@@ -24,7 +30,10 @@
 )
 from fileglancer.apps.jobs import (  # noqa: F401
     _build_container_script,
+    _build_service_url_publisher,
+    _container_bind_paths,
     _container_sif_name,
+    _SERVICE_PORT_HELPER,
     cancel_job,
     start_job_monitor,
     stop_job_monitor,
@@ -34,4 +43,5 @@
     get_job_file_content,
     get_job_file_paths,
     get_service_url,
+    get_service_phase,
 )

fileglancer/apps/command.py

@@ -268,19 +268,26 @@ def validate_path_for_shell(path_value: str) -> str | None:
     return None
 
 
-def validate_path_in_filestore(path_value: str, fsps: list, check_access: bool = True) -> str | None:
+def validate_path_in_filestore(path_value: str, fsps: list, check_access: bool = True,
+                               expected_type: str | None = None) -> str | None:
     """Validate a path within an allowed file share.
 
     Always performs syntax checks and confirms the path resolves within an
     allowed file share; both checks are euid-independent. When check_access is
     True, additionally verifies the path exists and is readable.
 
-    The exists/readable check reflects the *calling process's* identity, so it
-    is only meaningful when this runs as the target user (i.e. in the setuid
-    worker). Callers running on the root server must pass check_access=False —
-    otherwise validation reflects root's access, not the user's (false-pass on
-    local FS where root bypasses perms, false-reject on root-squash NFS) — and
-    defer the access check to the worker (see submit_job).
+    When expected_type is 'file' or 'directory' and the path exists, its type
+    must match — a folder pasted into a file parameter (or vice versa) is
+    rejected here rather than failing at job runtime. A missing path is not
+    type-checked (an exists=false output has no type yet).
+
+    The exists/readable/type checks reflect the *calling process's* identity,
+    so they are only meaningful when this runs as the target user (i.e. in the
+    setuid worker). Callers running on the root server must pass
+    check_access=False — otherwise validation reflects root's access, not the
+    user's (false-pass on local FS where root bypasses perms, false-reject on
+    root-squash NFS) — and defer the access check to the worker (see
+    submit_job).
 
     Returns an error message string if invalid, or None if valid.
     """
@@ -304,14 +311,23 @@ def validate_path_in_filestore(path_value: str, fsps: list, check_access: bool =
     if result is None:
         return "Path is not within an allowed file share"
 
-    if not check_access:
-        return None
-
     fsp, subpath = result
 
-    from fileglancer.filestore import Filestore
-    filestore = Filestore(fsp)
-    return filestore.validate_path(subpath)
+    if check_access:
+        from fileglancer.filestore import Filestore
+        filestore = Filestore(fsp)
+        error = filestore.validate_path(subpath)
+        if error:
+            return error
+
+    if expected_type in ("file", "directory") and os.path.exists(expanded):
+        is_dir = os.path.isdir(expanded)
+        if expected_type == "file" and is_dir:
+            return "Path is a folder, but a file is required"
+        if expected_type == "directory" and not is_dir:
+            return "Path is a file, but a folder is required"
+
+    return None
 
 
 # --- Command Building ---
@@ -375,7 +391,8 @@ def _validate_parameter_value(param: AppParameter, value, session=None, username
         str_val = expand_user_path(str_val, username)
         if session is not None:
             fsps = db.get_file_share_paths(session)
-            error = validate_path_in_filestore(str_val, fsps, check_access=check_access)
+            error = validate_path_in_filestore(str_val, fsps, check_access=check_access,
+                                               expected_type=param.type)
         else:
             error = validate_path_for_shell(str_val)
         if error:
@@ -399,6 +416,15 @@ def _flatten_param_items(items) -> list:
     return result
 
 
+def _is_nextflow_run_command(command: str) -> bool:
+    """Return true when the base command appears to be ``nextflow run``."""
+    try:
+        argv = shlex.split(command)
+    except ValueError:
+        argv = command.strip().split()
+    return len(argv) >= 2 and argv[0] == "nextflow" and argv[1] == "run"
+
+
 def build_command(entry_point: AppEntryPoint, parameters: dict,
                   env_parameters: dict = None, session=None, username=None,
                   check_access: bool = True) -> str:
@@ -456,18 +482,38 @@ def build_command(entry_point: AppEntryPoint, parameters: dict,
 
     # Start with the base command
     parts = [entry_point.command]
+    nextflow_run = _is_nextflow_run_command(entry_point.command)
 
     # Pass 1: Flagged args in declaration order
     for p, value in effective:
         if p.flag is None:
             continue
         validated = _validate_parameter_value(p, value, session=session, username=username,
                                               check_access=check_access)
+        # Nextflow pipeline params (`--foo`) are parsed differently from normal
+        # CLI switches: a separate leading-dash value becomes another param, and
+        # omitting a false boolean leaves any pipeline default in effect. Apply
+        # the Nextflow-safe forms at command-build time too, so cached
+        # auto-generated manifests from older Fileglancer versions are fixed
+        # without requiring users to update/re-add the app.
+        is_nextflow_pipeline_param = nextflow_run and p.flag.startswith("--")
+        value_separator = "equals" if is_nextflow_pipeline_param else p.value_separator
+        boolean_style = "value" if is_nextflow_pipeline_param else p.boolean_style
         if p.type == "boolean":
-            if value is True:
+            if boolean_style == "value":
+                bool_value = "true" if value is True else "false"
+                if value_separator == "equals":
+                    parts.append(f"{p.flag}={bool_value}")
+                else:
+                    parts.append(f"{p.flag} {bool_value}")
+            elif value is True:
                 parts.append(p.flag)
         else:
-            parts.append(f"{p.flag} {shlex.quote(validated)}")
+            quoted = shlex.quote(validated)
+            if value_separator == "equals":
+                parts.append(f"{p.flag}={quoted}")
+            else:
+                parts.append(f"{p.flag} {quoted}")
 
     # Pass 2: Positional args in declaration order
     for p, value in effective:
@@ -488,12 +534,13 @@ def build_command(entry_point: AppEntryPoint, parameters: dict,
 
 
 def collect_path_parameters(entry_point: AppEntryPoint, parameters: dict,
-                            env_parameters: dict = None) -> list[tuple[str, str, str]]:
+                            env_parameters: dict = None) -> list[tuple[AppParameter, str]]:
     """Collect effective file/directory parameter values needing path validation.
 
     Mirrors build_command's effective-value computation (user-provided merged
     with defaults, across both the env and pipeline namespaces) but returns only
-    file/directory parameters as (param_key, param_name, raw_value) tuples.
+    file/directory parameters as (param, raw_value) tuples, so callers can key
+    validation off the parameter's type and exists flag.
 
     Raw (un-expanded) values are returned so that authoritative validation can
     run in the setuid worker, where '~' and access checks resolve as the target
@@ -504,7 +551,7 @@ def collect_path_parameters(entry_point: AppEntryPoint, parameters: dict,
     param_flat = _flatten_param_items(entry_point.parameters)
     groups = ((env_flat, env_parameters), (param_flat, parameters))
 
-    result: list[tuple[str, str, str]] = []
+    result: list[tuple[AppParameter, str]] = []
     for flat, values in groups:
         for param in flat:
             if param.type not in ("file", "directory"):
@@ -515,5 +562,37 @@ def collect_path_parameters(entry_point: AppEntryPoint, parameters: dict,
                 value = param.default
             else:
                 continue
-            result.append((param.key, param.name, str(value)))
+            result.append((param, str(value)))
+    return result
+
+
+def collect_creatable_dirs(entry_point: AppEntryPoint, parameters: dict,
+                           env_parameters: dict = None) -> list[tuple[str, str]]:
+    """Collect effective directory values for params with exists=false.
+
+    Mirrors collect_path_parameters' effective-value logic (user value else
+    default, across both the env and pipeline namespaces) but returns only
+    directory params whose path need not exist yet (exists=false), as
+    (param_name, raw_value) tuples. Raw (un-expanded) values are returned so
+    the setuid worker resolves '~' as the target user. See submit_job.
+    """
+    env_parameters = env_parameters or {}
+    env_flat = _flatten_param_items(entry_point.env_parameters)
+    param_flat = _flatten_param_items(entry_point.parameters)
+    groups = ((env_flat, env_parameters), (param_flat, parameters))
+
+    result: list[tuple[str, str]] = []
+    for flat, values in groups:
+        for param in flat:
+            if param.type != "directory" or param.exists:
+                continue
+            if param.key in values:
+                value = values[param.key]
+            elif param.default is not None:
+                value = param.default
+            else:
+                continue
+            if value == "":
+                continue
+            result.append((param.name, str(value)))
     return result