feat: maintenance mode (#23431)

* feat: add a `maintenance.enabled` config flag

* feat: implement graceful restart
feat: restart when maintenance config is toggled

* feat: boot a stripped down maintenance api if enabled

* feat: cli command to toggle maintenance mode

* chore: fallback IMMICH_SERVER_URL environment variable in process

* chore: add additional routes to maintenance controller

* fix: don't wait for nest application to close to finish request response

* chore: add a failsafe on restart to prevent other exit codes from preventing restart

* feat: redirect into/from maintenance page

* refactor: use system metadata for maintenance status

* refactor: wait on WebSocket connection to refresh

* feat: broadcast websocket event on server restart
refactor: listen to WS instead of polling

* refactor: bubble up maintenance information instead of hijacking in fetch function
feat: show modal when server is restarting

* chore: increase timeout for ungraceful restart

* refactor: deduplicate code between api/maintenance workers

* fix: skip config check if database is not initialised

* fix: add `maintenanceMode` field to system config test

* refactor: move maintenance resolution code to static method in service

* chore: clean up linter issues

* chore: generate dart openapi

* refactor: use try{} block for maintenance mode check

* fix: logic error in server redirect

* chore: include `maintenanceMode` key in e2e test

* chore: add i18n entries for maintenance screens

* chore: remove negated condition from hook

* fix: should set default value not override in service

* fix: minor error in page

* feat: initial draft of maintenance module, repo., worker controller, worker service

* refactor: move broadcast code into notification service

* chore: connect websocket on client if in maintenance

* chore: set maintenance module app name

* refactor: rename repository to include worker
chore: configure websocket adapter

* feat: reimplement maintenance mode exit with new module

* refactor: add a constant enum for ExitCode

* refactor: remove redundant route for maintenance

* refactor: only spin up kysely on boot (rather than a Nest app)

* refactor(web): move redirect logic into +layout file where modal is setup

* feat: add Maintenance permission

* refactor: merge common code between api/maintenance

* fix: propagate changes from the CLI to servers

* feat: maintenance authentication guard

* refactor: unify maintenance code into repository
feat: add a step to generate maintenance mode token

* feat: jwt auth for maintenance

* refactor: switch from nest jwt to just jsonwebtokens

* feat: log into maintenance mode from CLI command

* refactor: use `secret` instead of `token` in jwt terminology
chore: log maintenance mode login URL on boot
chore: don't make CLI actions reload if already in target state

* docs: initial draft for maintenance mode page

* refactor: always validate the maintenance auth on the server

* feat: add a link to maintenance mode documentation

* feat: redirect users back to the last page they were on when exiting maintenance

* refactor: provide closeFn in both maintenance repos.

* refactor: ensure the user is also redirected by the server

* chore: swap jsonwebtoken for jose

* refactor: introduce AppRestartEvent w/o secret passing

* refactor: use navigation goto

* refactor: use `continue` instead of `next`

* chore: lint fixes for server

* chore: lint fixes for web

* test: add mock for maintenance repository

* test: add base service dependency to maintenance

* chore: remove @types/jsonwebtoken

* refactor: close database connection after startup check

* refactor: use `request#auth` key

* refactor: use service instead of repository
chore: read token from cookie if possible
chore: rename client event to AppRestartV1

* refactor: more concise redirect logic on web

* refactor: move redirect check into utils
refactor: update translation strings to be more sensible

* refactor: always validate login (i.e. check cookie)

* refactor: lint, open-api, remove old dto

* refactor: encode at point of usage

* refactor: remove business logic from repositories

* chore: fix server/web lints

* refactor: remove repository mock

* chore: fix formatting

* test: write service mocks for maintenance mode

* test: write cli service tests

* fix: catch errors when closing app

* fix: always report no maintenance when usual API is available

* test: api e2e maintenance spec

* chore: add response builder

* chore: add helper to set maint. auth cookie

* feat: add SSR to maintenance API

* test(e2e): write web spec for maintenance

* chore: clean up lint issues

* chore: format files

* feat: perform 302 redirect at server level during maintenance

* fix: keep trying to stop immich until it succeeds (CLI issue)

* chore: lint/format

* refactor: annotate references to other services in worker service

* chore: lint

* refactor: remove unnecessary await

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* refactor: move static methods into util

* refactor: assert secret exists in maintenance worker

* refactor: remove assertion which isn't necessary anymore

* refactor: remove assertion

* refactor: remove outer try {} catch block from loadMaintenanceAuth

* refactor: undo earlier change to vite.config.ts

* chore: update tests due to refactors

* revert: vite.config.ts

* test: expect string jwt

* chore: move blanket exceptions into controllers

* test: update tests according with last change

* refactor: use respondWithCookie
refactor: merge start/end into one route
refactor: rename MaintenanceRepository to AppRepository
chore: use new ApiTag/Endpoint
refactor: apply other requested changes

* chore: regenerate openapi

* chore: lint/format

* chore: remove secureOnly for maint. cookie

* refactor: move maintenance worker code into src/maintenance\nfix: various test fixes

* refactor: use `action` property for setting maint. mode

* refactor: remove Websocket#restartApp in favour of individual methods

* chore: incomplete commit

* chore: remove stray log

* fix: call exitApp from maintenance worker on exit

* fix: add app repository mock

* fix: ensure maintenance cookies are secure

* fix: run playwright tests over secure context (localhost)

* test: update other references to 127.0.0.1

* refactor: use serverSideEmitWithAck

* chore: correct the logic in tryTerminate

* test: juggle cookies ourselves

* chore: fix lint error for e2e spec

* chore: format e2e test

* fix: set cookie secure/non-secure depending on context

* chore: format files

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

e2e/src/api/specs/maintenance.e2e-spec.ts

@@ -0,0 +1,172 @@
+import { LoginResponseDto } from '@immich/sdk';
+import { createUserDto } from 'src/fixtures';
+import { errorDto } from 'src/responses';
+import { app, utils } from 'src/utils';
+import request from 'supertest';
+import { beforeAll, describe, expect, it } from 'vitest';
+
+describe('/admin/maintenance', () => {
+  let cookie: string | undefined;
+  let admin: LoginResponseDto;
+  let nonAdmin: LoginResponseDto;
+
+  beforeAll(async () => {
+    await utils.resetDatabase();
+    admin = await utils.adminSetup();
+    nonAdmin = await utils.userSetup(admin.accessToken, createUserDto.user1);
+  });
+
+  // => outside of maintenance mode
+
+  describe('GET ~/server/config', async () => {
+    it('should indicate we are out of maintenance mode', async () => {
+      const { status, body } = await request(app).get('/server/config');
+      expect(status).toBe(200);
+      expect(body.maintenanceMode).toBeFalsy();
+    });
+  });
+
+  describe('POST /login', async () => {
+    it('should not work out of maintenance mode', async () => {
+      const { status, body } = await request(app).post('/admin/maintenance/login').send({ token: 'token' });
+      expect(status).toBe(400);
+      expect(body).toEqual(errorDto.badRequest('Not in maintenance mode'));
+    });
+  });
+
+  // => enter maintenance mode
+
+  describe.sequential('POST /', () => {
+    it('should require authentication', async () => {
+      const { status, body } = await request(app).post('/admin/maintenance').send({
+        action: 'end',
+      });
+      expect(status).toBe(401);
+      expect(body).toEqual(errorDto.unauthorized);
+    });
+
+    it('should only work for admins', async () => {
+      const { status, body } = await request(app)
+        .post('/admin/maintenance')
+        .set('Authorization', `Bearer ${nonAdmin.accessToken}`)
+        .send({ action: 'end' });
+      expect(status).toBe(403);
+      expect(body).toEqual(errorDto.forbidden);
+    });
+
+    it('should be a no-op if try to exit maintenance mode', async () => {
+      const { status } = await request(app)
+        .post('/admin/maintenance')
+        .set('Authorization', `Bearer ${admin.accessToken}`)
+        .send({ action: 'end' });
+      expect(status).toBe(201);
+    });
+
+    it('should enter maintenance mode', async () => {
+      const { status, headers } = await request(app)
+        .post('/admin/maintenance')
+        .set('Authorization', `Bearer ${admin.accessToken}`)
+        .send({
+          action: 'start',
+        });
+      expect(status).toBe(201);
+
+      cookie = headers['set-cookie'][0].split(';')[0];
+      expect(cookie).toEqual(
+        expect.stringMatching(/^immich_maintenance_token=[A-Za-z0-9-_]*\.[A-Za-z0-9-_]*\.[A-Za-z0-9-_]*$/),
+      );
+
+      await expect
+        .poll(
+          async () => {
+            const { body } = await request(app).get('/server/config');
+            return body.maintenanceMode;
+          },
+          {
+            interval: 5e2,
+            timeout: 1e4,
+          },
+        )
+        .toBeTruthy();
+    });
+  });
+
+  // => in maintenance mode
+
+  describe.sequential('in maintenance mode', () => {
+    describe('GET ~/server/config', async () => {
+      it('should indicate we are in maintenance mode', async () => {
+        const { status, body } = await request(app).get('/server/config');
+        expect(status).toBe(200);
+        expect(body.maintenanceMode).toBeTruthy();
+      });
+    });
+
+    describe('POST /login', async () => {
+      it('should fail without cookie or token in body', async () => {
+        const { status, body } = await request(app).post('/admin/maintenance/login').send({});
+        expect(status).toBe(401);
+        expect(body).toEqual(errorDto.unauthorizedWithMessage('Missing JWT Token'));
+      });
+
+      it('should succeed with cookie', async () => {
+        const { status, body } = await request(app).post('/admin/maintenance/login').set('cookie', cookie!).send({});
+        expect(status).toBe(201);
+        expect(body).toEqual(
+          expect.objectContaining({
+            username: 'Immich Admin',
+          }),
+        );
+      });
+
+      it('should succeed with token', async () => {
+        const { status, body } = await request(app)
+          .post('/admin/maintenance/login')
+          .send({
+            token: cookie!.split('=')[1].trim(),
+          });
+        expect(status).toBe(201);
+        expect(body).toEqual(
+          expect.objectContaining({
+            username: 'Immich Admin',
+          }),
+        );
+      });
+    });
+
+    describe('POST /', async () => {
+      it('should be a no-op if try to enter maintenance mode', async () => {
+        const { status } = await request(app)
+          .post('/admin/maintenance')
+          .set('cookie', cookie!)
+          .send({ action: 'start' });
+        expect(status).toBe(201);
+      });
+    });
+  });
+
+  // => exit maintenance mode
+
+  describe.sequential('POST /', () => {
+    it('should exit maintenance mode', async () => {
+      const { status } = await request(app).post('/admin/maintenance').set('cookie', cookie!).send({
+        action: 'end',
+      });
+
+      expect(status).toBe(201);
+
+      await expect
+        .poll(
+          async () => {
+            const { body } = await request(app).get('/server/config');
+            return body.maintenanceMode;
+          },
+          {
+            interval: 5e2,
+            timeout: 1e4,
+          },
+        )
+        .toBeFalsy();
+    });
+  });
+});

e2e/src/api/specs/server.e2e-spec.ts

@@ -136,6 +136,7 @@ describe('/server', () => {
         externalDomain: '',
         publicUsers: true,
         isOnboarded: false,
+        maintenanceMode: false,
         mapDarkStyleUrl: 'https://tiles.immich.cloud/v1/style/dark.json',
         mapLightStyleUrl: 'https://tiles.immich.cloud/v1/style/light.json',
       });

e2e/src/responses.ts

@@ -7,6 +7,12 @@ export const errorDto = {
     message: 'Authentication required',
     correlationId: expect.any(String),
   },
+  unauthorizedWithMessage: (message: string) => ({
+    error: 'Unauthorized',
+    statusCode: 401,
+    message,
+    correlationId: expect.any(String),
+  }),
   forbidden: {
     error: 'Forbidden',
     statusCode: 403,

e2e/src/utils.ts

@@ -6,6 +6,7 @@ import {
   CheckExistingAssetsDto,
   CreateAlbumDto,
   CreateLibraryDto,
+  MaintenanceAction,
   MetadataSearchDto,
   Permission,
   PersonCreateDto,
@@ -36,6 +37,7 @@ import {
   scanLibrary,
   searchAssets,
   setBaseUrl,
+  setMaintenanceMode,
   signUpAdmin,
   tagAssets,
   updateAdminOnboarding,
@@ -514,6 +516,42 @@ export const utils = {
       },
     ]),
 
+  setMaintenanceAuthCookie: async (context: BrowserContext, token: string, domain = '127.0.0.1') =>
+    await context.addCookies([
+      {
+        name: 'immich_maintenance_token',
+        value: token,
+        domain,
+        path: '/',
+        expires: 2_058_028_213,
+        httpOnly: true,
+        secure: false,
+        sameSite: 'Lax',
+      },
+    ]),
+
+  enterMaintenance: async (accessToken: string) => {
+    let setCookie: string[] | undefined;
+
+    await setMaintenanceMode(
+      {
+        setMaintenanceModeDto: {
+          action: MaintenanceAction.Start,
+        },
+      },
+      {
+        headers: asBearerAuth(accessToken),
+        fetch: (...args: Parameters<typeof fetch>) =>
+          fetch(...args).then((response) => {
+            setCookie = response.headers.getSetCookie();
+            return response;
+          }),
+      },
+    );
+
+    return setCookie;
+  },
+
   resetTempFolder: () => {
     rmSync(`${testAssetDir}/temp`, { recursive: true, force: true });
     mkdirSync(`${testAssetDir}/temp`, { recursive: true });

e2e/src/web/specs/maintenance.e2e-spec.ts

@@ -0,0 +1,52 @@
+import { LoginResponseDto } from '@immich/sdk';
+import { expect, test } from '@playwright/test';
+import { utils } from 'src/utils';
+
+test.describe.configure({ mode: 'serial' });
+
+test.describe('Maintenance', () => {
+  let admin: LoginResponseDto;
+
+  test.beforeAll(async () => {
+    utils.initSdk();
+    await utils.resetDatabase();
+    admin = await utils.adminSetup();
+  });
+
+  test('enter and exit maintenance mode', async ({ context, page }) => {
+    await utils.setAuthCookies(context, admin.accessToken);
+
+    await page.goto('/admin/system-settings?isOpen=maintenance');
+    await page.getByRole('button', { name: 'Start maintenance mode' }).click();
+
+    await page.waitForURL(`/maintenance?${new URLSearchParams({ continue: '/admin/system-settings' })}`);
+    await expect(page.getByText('Temporarily Unavailable')).toBeVisible();
+    await page.getByRole('button', { name: 'End maintenance mode' }).click();
+    await page.waitForURL('/admin/system-settings');
+  });
+
+  test('maintenance shows no options to users until they authenticate', async ({ page }) => {
+    const setCookie = await utils.enterMaintenance(admin.accessToken);
+    const cookie = setCookie
+      ?.map((cookie) => cookie.split(';')[0].split('='))
+      ?.find(([name]) => name === 'immich_maintenance_token');
+
+    expect(cookie).toBeTruthy();
+
+    await expect(async () => {
+      await page.goto('/');
+      await page.waitForURL('/maintenance?**', {
+        timeout: 1e3,
+      });
+    }).toPass({ timeout: 1e4 });
+
+    await expect(page.getByText('Temporarily Unavailable')).toBeVisible();
+    await expect(page.getByRole('button', { name: 'End maintenance mode' })).toHaveCount(0);
+
+    await page.goto(`/maintenance?${new URLSearchParams({ token: cookie![1] })}`);
+    await expect(page.getByText('Temporarily Unavailable')).toBeVisible();
+    await expect(page.getByRole('button', { name: 'End maintenance mode' })).toBeVisible();
+    await page.getByRole('button', { name: 'End maintenance mode' }).click();
+    await page.waitForURL('/auth/login');
+  });
+});