{
  "schema_version": 2,
  "app": "immich-ml-1job",
  "display_name": "Immich (ML job concurrency 1)",
  "upstream_repo": "https://github.com/immich-app/immich",
  "docs_url": "https://docs.immich.app/install/docker-compose",
  "status": "ok",
  "summary": {
    "primary_service": "immich-server",
    "app_version": "v3.2.4",
    "app_version_pinned": true,
    "idle_total_mem_mib": 1351.89,
    "idle_total_cpu_pct": 1.39,
    "peak_total_mem_mib": 3363.2,
    "peak_is_under_load": true,
    "startup_seconds": 9.0,
    "startup_peak_total_mem_mib": 2215.38,
    "settled": true,
    "settle_seconds_used": 153.2
  },
  "run": {
    "started_at": "2026-10-02T10:00:02Z",
    "date": "2026-10-02",
    "harness_version": "2.0.0",
    "docker_host": null,
    "remote": false
  },
  "host": {
    "cpu_model": "Intel(R) Xeon(R) Processor @ 2.10GHz",
    "cpu_model_note": null,
    "cores": 4,
    "mem_total_mib": 16095,
    "kernel": "6.18.44-fc-v51",
    "os": "Ubuntu 24.04.4 LTS",
    "arch": "x86_64",
    "oci_arch": "amd64",
    "docker_server_version": "29.6.2",
    "docker_client_version": "29.6.2",
    "compose_client_version": "5.3.1",
    "storage_driver": "overlayfs",
    "containerd_image_store": true,
    "source": "docker info + throwaway container reading /proc/cpuinfo on the daemon host",
    "compose_note": "Compose runs on the client machine; Docker server fields are from the daemon host"
  },
  "helper_images": {
    "curlimages/curl:latest": {
      "pulled_as": "curlimages/curl:latest",
      "digest": "curlimages/curl@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777"
    },
    "python:3.12-slim": {
      "pulled_as": "python:3.12-slim",
      "digest": "python@sha256:dddfd7e07f9d15aeeca61529320492139d21cac7f0070c00609243e51e4e0016"
    }
  },
  "pull_seconds": 6.6,
  "startup": {
    "seconds": 9.0,
    "ok": true,
    "checks": [
      {
        "type": "http",
        "url": "http://immich-server:2283/api/server/ping",
        "expect_status": [
          200
        ],
        "ok": true,
        "ready_after_s": 9.0,
        "detail": "OK 200"
      }
    ],
    "up_command_seconds": 0.7,
    "definition": "wall clock from starting `docker compose up -d` until every health check passed; checks poll every 0.5 s from a throwaway container on the compose network, so resolution is about +/- 1 s"
  },
  "services": {
    "database": {
      "image_ref": "ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0@sha256:bcf63357191b76a916ae5eb93464d65c07511da41e3bf7a8416db519b40b1c23",
      "image_repo": "ghcr.io/immich-app/postgres",
      "image_tag": "14-vectorchord0.4.3-pgvectors0.2.0",
      "digest": "sha256:bcf63357191b76a916ae5eb93464d65c07511da41e3bf7a8416db519b40b1c23",
      "digest_note": null,
      "version": "14-vectorchord0.4.3-pgvector0.8.1-pgvectors0.2.0",
      "version_source": "label:org.opencontainers.image.version",
      "version_pinned": true,
      "version_note": null,
      "resolved_tag": "14-vectorchord0.4.3-pgvectors0.2.0",
      "resolved_tag_note": null,
      "image_size": {
        "unpacked_mb": 989.0,
        "unpacked_source": "docker image ls Size: unpacked image on the daemon host, rounded by docker to 3 significant figures",
        "compressed_mb": 205.3,
        "compressed_source": "sum of compressed layer sizes in the registry manifest for linux/amd64",
        "compressed_null_reason": null,
        "unit_note": "MB = 10^6 bytes. unpacked = disk used on the host; compressed = what is downloaded"
      },
      "compose_image": "ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0",
      "compose_comment": "sha256:bcf63357191b76a916ae5eb93464d65c07511da41e3bf7a8416db519b40b1c23  (version 14-vectorchord0.4.3-pgvector0.8.1-pgvectors0.2.0, from label:org.opencontainers.image.version)",
      "restart_count": 0,
      "oom_killed": false
    },
    "immich-machine-learning": {
      "image_ref": "ghcr.io/immich-app/immich-machine-learning:v3",
      "image_repo": "ghcr.io/immich-app/immich-machine-learning",
      "image_tag": "v3",
      "digest": "sha256:e16c2f166a8174901959fdf85e2e4c7bd1ebc4b37e0b6655de97c41408a260c4",
      "digest_note": null,
      "version": "v3.2.4",
      "version_source": "label:org.opencontainers.image.version",
      "version_pinned": true,
      "version_note": null,
      "resolved_tag": "v3.2.4",
      "resolved_tag_note": null,
      "image_size": {
        "unpacked_mb": 1390.0,
        "unpacked_source": "docker image ls Size: unpacked image on the daemon host, rounded by docker to 3 significant figures",
        "compressed_mb": 342.0,
        "compressed_source": "sum of compressed layer sizes in the registry manifest for linux/amd64",
        "compressed_null_reason": null,
        "unit_note": "MB = 10^6 bytes. unpacked = disk used on the host; compressed = what is downloaded"
      },
      "compose_image": "ghcr.io/immich-app/immich-machine-learning:v3.2.4",
      "compose_comment": "sha256:e16c2f166a8174901959fdf85e2e4c7bd1ebc4b37e0b6655de97c41408a260c4  (version v3.2.4, from label:org.opencontainers.image.version)",
      "restart_count": 0,
      "oom_killed": false
    },
    "immich-server": {
      "image_ref": "ghcr.io/immich-app/immich-server:v3",
      "image_repo": "ghcr.io/immich-app/immich-server",
      "image_tag": "v3",
      "digest": "sha256:d317916b28090c33eb36b308464ea391f8b7df1d850fcfea227a39ec879718c2",
      "digest_note": null,
      "version": "v3.2.4",
      "version_source": "label:org.opencontainers.image.version",
      "version_pinned": true,
      "version_note": null,
      "resolved_tag": "v3.2.4",
      "resolved_tag_note": null,
      "image_size": {
        "unpacked_mb": 2480.0,
        "unpacked_source": "docker image ls Size: unpacked image on the daemon host, rounded by docker to 3 significant figures",
        "compressed_mb": 616.7,
        "compressed_source": "sum of compressed layer sizes in the registry manifest for linux/amd64",
        "compressed_null_reason": null,
        "unit_note": "MB = 10^6 bytes. unpacked = disk used on the host; compressed = what is downloaded"
      },
      "compose_image": "ghcr.io/immich-app/immich-server:v3.2.4",
      "compose_comment": "sha256:d317916b28090c33eb36b308464ea391f8b7df1d850fcfea227a39ec879718c2  (version v3.2.4, from label:org.opencontainers.image.version)",
      "restart_count": 0,
      "oom_killed": false
    },
    "redis": {
      "image_ref": "docker.io/valkey/valkey:9@sha256:70739f85ad2ee01a726a965584a0f94895f01b0c60b3cc8b0aeef11eaa6888cf",
      "image_repo": "docker.io/valkey/valkey",
      "image_tag": "9",
      "digest": "sha256:70739f85ad2ee01a726a965584a0f94895f01b0c60b3cc8b0aeef11eaa6888cf",
      "digest_note": null,
      "version": "9.1.1",
      "version_source": "exec",
      "version_pinned": true,
      "version_note": null,
      "resolved_tag": null,
      "resolved_tag_note": "no candidate tag among ['9.1.1', 'v9.1.1'] could be verified against the registry",
      "image_size": {
        "unpacked_mb": 167.0,
        "unpacked_source": "docker image ls Size: unpacked image on the daemon host, rounded by docker to 3 significant figures",
        "compressed_mb": 44.3,
        "compressed_source": "sum of compressed layer sizes in the registry manifest for linux/amd64",
        "compressed_null_reason": null,
        "unit_note": "MB = 10^6 bytes. unpacked = disk used on the host; compressed = what is downloaded"
      },
      "compose_image": "docker.io/valkey/valkey:9@sha256:70739f85ad2ee01a726a965584a0f94895f01b0c60b3cc8b0aeef11eaa6888cf",
      "compose_comment": "version 9.1.1 (from exec); no registry tag for it could be verified, so the digest is pinned instead",
      "restart_count": 0,
      "oom_killed": false
    }
  },
  "setup": [
    {
      "exit_code": 0,
      "duration_s": 1.1,
      "reported": {
        "admin_created": true
      },
      "output_tail": [
        "admin created, isAdmin = True"
      ],
      "description": "Create the first (admin) user through the API."
    },
    {
      "exit_code": 0,
      "duration_s": 0.9,
      "reported": {
        "smart_search_face_detection_concurrency": 1
      },
      "output_tail": [
        "concurrency before: {'smartSearch': 2, 'faceDetection': 2}",
        "concurrency after: {'smartSearch': 1, 'faceDetection': 1}"
      ],
      "description": "Set the Smart Search and Face Detection job concurrency to 1 (Administration > Settings > Job Settings)."
    }
  ],
  "idle": {
    "per_service": {
      "database": {
        "samples": 16,
        "mem_mean_mib": 418.99,
        "mem_max_mib": 419.0,
        "cpu_mean_pct": 0.08,
        "cpu_max_pct": 1.28
      },
      "immich-machine-learning": {
        "samples": 16,
        "mem_mean_mib": 161.9,
        "mem_max_mib": 161.9,
        "cpu_mean_pct": 0.85,
        "cpu_max_pct": 10.71
      },
      "immich-server": {
        "samples": 16,
        "mem_mean_mib": 766.21,
        "mem_max_mib": 766.3,
        "cpu_mean_pct": 0.16,
        "cpu_max_pct": 0.7
      },
      "redis": {
        "samples": 16,
        "mem_mean_mib": 4.79,
        "mem_max_mib": 4.88,
        "cpu_mean_pct": 0.29,
        "cpu_max_pct": 0.48
      }
    },
    "total": {
      "samples": 16,
      "dropped_for_total": 0,
      "mem_mean_mib": 1351.89,
      "mem_max_mib": 1351.97,
      "cpu_mean_pct": 1.39,
      "cpu_max_pct": 10.97
    },
    "window_seconds": 30.2,
    "series": [
      {
        "t_offset_s": 0.0,
        "total_mem_mib": 1351.79,
        "total_cpu_pct": 0.5
      },
      {
        "t_offset_s": 2.0,
        "total_mem_mib": 1351.79,
        "total_cpu_pct": 1.3
      },
      {
        "t_offset_s": 4.0,
        "total_mem_mib": 1351.75,
        "total_cpu_pct": 1.73
      },
      {
        "t_offset_s": 6.0,
        "total_mem_mib": 1351.75,
        "total_cpu_pct": 10.97
      },
      {
        "t_offset_s": 8.1,
        "total_mem_mib": 1351.74,
        "total_cpu_pct": 0.47
      },
      {
        "t_offset_s": 10.1,
        "total_mem_mib": 1351.73,
        "total_cpu_pct": 0.5
      },
      {
        "t_offset_s": 12.1,
        "total_mem_mib": 1351.98,
        "total_cpu_pct": 1.24
      },
      {
        "t_offset_s": 14.1,
        "total_mem_mib": 1351.98,
        "total_cpu_pct": 0.46
      },
      {
        "t_offset_s": 16.1,
        "total_mem_mib": 1351.97,
        "total_cpu_pct": 0.46
      },
      {
        "t_offset_s": 18.1,
        "total_mem_mib": 1351.97,
        "total_cpu_pct": 0.51
      },
      {
        "t_offset_s": 20.2,
        "total_mem_mib": 1351.97,
        "total_cpu_pct": 0.52
      },
      {
        "t_offset_s": 22.2,
        "total_mem_mib": 1351.97,
        "total_cpu_pct": 1.33
      },
      {
        "t_offset_s": 24.2,
        "total_mem_mib": 1351.97,
        "total_cpu_pct": 0.49
      },
      {
        "t_offset_s": 26.2,
        "total_mem_mib": 1351.97,
        "total_cpu_pct": 0.53
      },
      {
        "t_offset_s": 28.2,
        "total_mem_mib": 1351.97,
        "total_cpu_pct": 0.6
      },
      {
        "t_offset_s": 30.2,
        "total_mem_mib": 1351.97,
        "total_cpu_pct": 0.56
      }
    ]
  },
  "settle": {
    "settled": true,
    "mode": "detected",
    "settle_seconds_used": 153.2,
    "settle_started_after_healthy_s": 16.3,
    "params": {
      "window_seconds": 60,
      "threshold_pct": 2.0,
      "threshold_min_mib": 10.0,
      "min_seconds": 120,
      "max_seconds": 600
    },
    "last_window": {
      "n": 30,
      "mean_mib": 1350.8,
      "min_mib": 1348.59,
      "max_mib": 1351.79,
      "spread_mib": 3.2,
      "threshold_mib": 27.02
    },
    "rechecks": 0,
    "rule": "settled = over the last window_seconds the stack total varied (max - min) by less than max(threshold_pct of the window mean, threshold_min_mib); not before min_seconds, at the latest max_seconds, both counted from the end of the setup steps; the idle window must also be flat and agree with it. If max_seconds passes without settling, settled=false and the idle window is measured anyway."
  },
  "startup_peak": {
    "per_service_mem_mib": {
      "database": 547.3,
      "immich-machine-learning": 177.6,
      "immich-server": 1572.86,
      "redis": 5.37
    },
    "total_mem_mib": 2215.38,
    "samples": 84,
    "note": "highest value across docker stats samples (about one every 2-3 s) from healthy until the stack settled; shorter spikes between samples are not seen"
  },
  "timeline": {
    "step_s": 10,
    "t0": "healthy",
    "offsets_s": [
      10,
      20,
      30,
      40,
      50,
      60,
      70,
      80,
      90,
      100,
      110,
      120,
      130,
      140,
      150,
      160,
      170,
      180,
      190,
      200
    ],
    "total_mem_mib": [
      2084.5,
      2109.0,
      2126.2,
      2126.6,
      2138.4,
      2157.0,
      2157.1,
      2156.9,
      2159.7,
      2156.5,
      1951.9,
      1349.4,
      1350.4,
      1350.6,
      1351.0,
      1351.6,
      1351.7,
      1351.8,
      1352.0,
      1352.0
    ],
    "per_service_mem_mib": {
      "database": [
        426.9,
        410.8,
        417.0,
        417.0,
        417.0,
        417.9,
        418.5,
        418.5,
        418.5,
        418.5,
        418.5,
        419.2,
        419.0,
        419.0,
        418.9,
        418.9,
        418.9,
        419.0,
        419.0,
        419.0
      ],
      "immich-machine-learning": [
        161.6,
        161.6,
        161.6,
        161.6,
        161.6,
        161.8,
        161.8,
        161.8,
        164.8,
        161.5,
        161.5,
        161.6,
        161.7,
        161.7,
        161.8,
        161.9,
        161.9,
        161.9,
        161.9,
        161.9
      ],
      "immich-server": [
        1490.7,
        1531.7,
        1543.0,
        1543.2,
        1555.0,
        1572.9,
        1572.2,
        1571.8,
        1571.8,
        1571.8,
        1367.2,
        764.0,
        765.1,
        765.3,
        765.6,
        766.1,
        766.2,
        766.2,
        766.2,
        766.2
      ],
      "redis": [
        5.3,
        4.9,
        4.6,
        4.8,
        4.7,
        4.5,
        4.6,
        4.8,
        4.6,
        4.7,
        4.7,
        4.6,
        4.6,
        4.6,
        4.7,
        4.7,
        4.7,
        4.7,
        4.9,
        4.9
      ]
    }
  },
  "load": {
    "per_service": {
      "database": {
        "samples": 20,
        "mem_mean_mib": 423.88,
        "mem_max_mib": 424.2,
        "cpu_mean_pct": 0.98,
        "cpu_max_pct": 5.91
      },
      "immich-machine-learning": {
        "samples": 20,
        "mem_mean_mib": 916.28,
        "mem_max_mib": 1812.48,
        "cpu_mean_pct": 105.12,
        "cpu_max_pct": 371.7
      },
      "immich-server": {
        "samples": 20,
        "mem_mean_mib": 1115.72,
        "mem_max_mib": 1123.33,
        "cpu_mean_pct": 27.24,
        "cpu_max_pct": 358.73
      },
      "redis": {
        "samples": 20,
        "mem_mean_mib": 5.47,
        "mem_max_mib": 5.66,
        "cpu_mean_pct": 0.64,
        "cpu_max_pct": 3.83
      }
    },
    "total": {
      "samples": 20,
      "dropped_for_total": 0,
      "mem_mean_mib": 2461.35,
      "mem_max_mib": 3363.2,
      "cpu_mean_pct": 133.99,
      "cpu_max_pct": 377.87
    },
    "description": "Upload 20 generated 4000x3000 JPEGs (about 1 MB each, synthetic gradients and shapes, no people) through the API as the admin user, then wait until every Immich background queue (thumbnails, metadata, CLIP smart search, face detection, ...) is empty. Smart Search and Face Detection run one job at a time (concurrency 1, default 2). The machine-learning container really runs; the images contain no faces, so face recognition has nothing to find. Peak and mean are taken from the first upload until the queues are idle.",
    "duration_s": 37.7,
    "window": "from LAB_MARK load_start (or step start) until 2 s after the step ended",
    "reported_by_script": {
      "images_uploaded": 20,
      "image_size_px": "4000x3000",
      "images_total_mb": 23.1,
      "upload_seconds": 1.1,
      "seconds_until_all_queues_idle": 36.2,
      "failed_jobs_by_queue": {},
      "smart_search_hits": 20,
      "ml_enabled": true
    },
    "output_tail": [
      "generated 20 JPEGs 4000x3000, 23.1 MB total in 25.1s",
      "uploaded 20 assets in 1.1s"
    ],
    "series": [
      {
        "t_offset_s": 0.0,
        "total_mem_mib": 1636.05,
        "total_cpu_pct": 375.95
      },
      {
        "t_offset_s": 2.0,
        "total_mem_mib": 1748.97,
        "total_cpu_pct": 131.82
      },
      {
        "t_offset_s": 4.0,
        "total_mem_mib": 1953.01,
        "total_cpu_pct": 82.34
      },
      {
        "t_offset_s": 6.1,
        "total_mem_mib": 1933.29,
        "total_cpu_pct": 153.73
      },
      {
        "t_offset_s": 8.1,
        "total_mem_mib": 2006.55,
        "total_cpu_pct": 166.06
      },
      {
        "t_offset_s": 10.1,
        "total_mem_mib": 2325.1,
        "total_cpu_pct": 140.44
      },
      {
        "t_offset_s": 12.1,
        "total_mem_mib": 2319.22,
        "total_cpu_pct": 85.94
      },
      {
        "t_offset_s": 14.1,
        "total_mem_mib": 2530.42,
        "total_cpu_pct": 73.27
      },
      {
        "t_offset_s": 16.2,
        "total_mem_mib": 3037.78,
        "total_cpu_pct": 147.09
      },
      {
        "t_offset_s": 18.2,
        "total_mem_mib": 3363.2,
        "total_cpu_pct": 101.93
      },
      {
        "t_offset_s": 20.2,
        "total_mem_mib": 2765.18,
        "total_cpu_pct": 131.67
      },
      {
        "t_offset_s": 22.2,
        "total_mem_mib": 2572.91,
        "total_cpu_pct": 372.1
      },
      {
        "t_offset_s": 24.3,
        "total_mem_mib": 2618.95,
        "total_cpu_pct": 377.87
      },
      {
        "t_offset_s": 26.3,
        "total_mem_mib": 2622.0,
        "total_cpu_pct": 210.02
      },
      {
        "t_offset_s": 28.3,
        "total_mem_mib": 2544.6,
        "total_cpu_pct": 54.79
      },
      {
        "t_offset_s": 30.3,
        "total_mem_mib": 2544.6,
        "total_cpu_pct": 0.97
      },
      {
        "t_offset_s": 32.3,
        "total_mem_mib": 2544.58,
        "total_cpu_pct": 4.95
      },
      {
        "t_offset_s": 34.3,
        "total_mem_mib": 2638.0,
        "total_cpu_pct": 20.83
      },
      {
        "t_offset_s": 36.4,
        "total_mem_mib": 2773.05,
        "total_cpu_pct": 47.28
      },
      {
        "t_offset_s": 38.4,
        "total_mem_mib": 2749.47,
        "total_cpu_pct": 0.71
      }
    ],
    "peak_note": "peak = highest value seen across docker stats samples (about one every 2-3 s); shorter spikes between samples are not seen"
  },
  "finished_at": "2026-10-02T10:04:53Z",
  "elapsed_seconds": 291.2,
  "teardown": {
    "done": true,
    "volumes_removed": true,
    "images_removed": false,
    "error": null
  },
  "load_defined": true,
  "notes": [
    "Same stack, .env and upload load as apps/immich; the only change is job concurrency 1 for Smart Search and Face Detection, set through the admin settings API (set_concurrency.py). Docs: https://docs.immich.app/FAQ#can-i-lower-cpu-and-ram-usage (read 2026-10-01).",
    "The .env adds IMMICH_HOST=0.0.0.0: the lab host has no IPv6, and the ML container binds [::] by default and crash-loops without it.",
    "Hardware acceleration (GPU/iGPU) is not tested: the lab host has none.",
    "Run behind a TLS-intercepting proxy: an extra CA bundle was mounted read-only into every container and the SSL_CERT_FILE-style variables pointed at it (lab-only override, NOT in the published compose). It changes outbound TLS trust, nothing else."
  ],
  "warnings": [],
  "files": {
    "compose": "immich-ml-1job.compose.yaml",
    "env": "immich-ml-1job.env"
  },
  "definitions": {
    "memory": "cgroup working set as reported by `docker stats` (usage minus inactive file cache), MiB = 2^20 bytes",
    "cpu": "percent of ONE core as reported by `docker stats` (4 busy cores = 400)",
    "total": "stats over the sum of all containers at the same instant",
    "sample_interval": "docker stats --no-stream takes about 1-2 s per call, so samples are 2-3 s apart"
  }
}