feat(http): Implement soft and hard TTL for all cache providers

lib/modules/platform/github/index.spec.ts

@@ -53,9 +53,7 @@ describe('modules/platform/github/index', () => {
       token: '123test',
     });
 
-    const repoCache = repository.getCache();
-    delete repoCache.platform;
-    delete process.env.RENOVATE_X_GITHUB_HOST_RULES;
+    repository.resetCache();
   });
 
   describe('initPlatform()', () => {

lib/util/http/cache/abstract-http-cache-provider.ts

@@ -1,3 +1,4 @@
+import { DateTime } from 'luxon';
 import { logger } from '../../../logger';
 import { HttpCacheStats } from '../../stats';
 import type { GotOptions, HttpResponse } from '../types';
@@ -6,6 +7,9 @@
 import type { HttpCacheProvider } from './types';
 
 export abstract class AbstractHttpCacheProvider implements HttpCacheProvider {
+  protected abstract softTtlMinutes: number;
+  protected abstract hardTtlMinutes: number;
+
   protected abstract load(url: string): Promise<unknown>;
   protected abstract persist(url: string, data: HttpCache): Promise<void>;
 
@@ -19,10 +23,14 @@
     return httpCache as HttpCache;
   }
 
-  async setCacheHeaders<T extends Pick<GotOptions, 'headers'>>(
+  async setCacheHeaders<T extends Pick<GotOptions, 'headers' | 'method'>>(
     url: string,
     opts: T,
   ): Promise<void> {
+    if (opts.method?.toLowerCase() !== 'get') {
+      return;
+    }
+
     const httpCache = await this.get(url);
     if (!httpCache) {
       return;
@@ -39,11 +47,29 @@
     }
   }
 
-  bypassServer<T>(
-    _url: string,
-    _ignoreSoftTtl: boolean,
+  async bypassServer<T>(
+    url: string,
+    ignoreSoftTtl: boolean,
   ): Promise<HttpResponse<T> | null> {
-    return Promise.resolve(null);
+    const cached = await this.get(url);
+    if (!cached) {
+      return null;
+    }
+
+    if (ignoreSoftTtl) {
+      return cached.httpResponse as HttpResponse<T>;
+    }
+
+    const cachedAt = DateTime.fromISO(cached.timestamp);
+    const deadline = cachedAt.plus({ minutes: this.softTtlMinutes });
+    const now = DateTime.now();
+    if (now >= deadline) {
+      HttpCacheStats.incLocalMisses(url);
+      return null;
+    }
+
+    HttpCacheStats.incLocalHits(url);
+    return cached.httpResponse as HttpResponse<T>;
   }
 
   async wrapServerResponse<T>(
@@ -65,6 +91,7 @@
         httpResponse,
         timestamp,
       });
+      /* istanbul ignore else */
       if (newHttpCache) {
         logger.debug(
           `http cache: saving ${url} (etag=${etag}, lastModified=${lastModified})`,

lib/util/http/cache/package-http-cache-provider.ts

@@ -1,9 +1,6 @@
-import { DateTime } from 'luxon';
 import { get, set } from '../../cache/package'; // Import the package cache functions
 import { resolveTtlValues } from '../../cache/package/ttl';
 import type { PackageCacheNamespace } from '../../cache/package/types';
-import { HttpCacheStats } from '../../stats';
-import type { HttpResponse } from '../types';
 import { AbstractHttpCacheProvider } from './abstract-http-cache-provider';
 import type { HttpCache } from './schema';
 
@@ -15,12 +12,14 @@ export interface PackageHttpCacheProviderOptions {
 export class PackageHttpCacheProvider extends AbstractHttpCacheProvider {
   private namespace: PackageCacheNamespace;
 
-  private softTtlMinutes: number;
-  private hardTtlMinutes: number;
+  protected override softTtlMinutes: number;
+  protected override hardTtlMinutes: number;
 
   constructor({ namespace, ttlMinutes = 15 }: PackageHttpCacheProviderOptions) {
     super();
+
     this.namespace = namespace;
+
     const { softTtlMinutes, hardTtlMinutes } = resolveTtlValues(
       this.namespace,
       ttlMinutes,
@@ -36,29 +35,4 @@ export class PackageHttpCacheProvider extends AbstractHttpCacheProvider {
   async persist(url: string, data: HttpCache): Promise<void> {
     await set(this.namespace, url, data, this.hardTtlMinutes);
   }
-
-  override async bypassServer<T>(
-    url: string,
-    ignoreSoftTtl = false,
-  ): Promise<HttpResponse<T> | null> {
-    const cached = await this.get(url);
-    if (!cached) {
-      return null;
-    }
-
-    if (ignoreSoftTtl) {
-      return cached.httpResponse as HttpResponse<T>;
-    }
-
-    const cachedAt = DateTime.fromISO(cached.timestamp);
-    const deadline = cachedAt.plus({ minutes: this.softTtlMinutes });
-    const now = DateTime.now();
-    if (now >= deadline) {
-      HttpCacheStats.incLocalMisses(url);
-      return null;
-    }
-
-    HttpCacheStats.incLocalHits(url);
-    return cached.httpResponse as HttpResponse<T>;
-  }
 }

lib/util/http/cache/repository-http-cache-provider.spec.ts

@@ -1,6 +1,5 @@
 import { Http } from '..';
 import * as httpMock from '../../../../test/http-mock';
-import { logger } from '../../../../test/util';
 import { resetCache } from '../../cache/repository';
 import { repoCacheProvider } from './repository-http-cache-provider';
 
@@ -13,7 +12,7 @@
     cacheProvider: repoCacheProvider,
   });
 
   it('reuses data with etag', async () => {
     const scope = httpMock.scope('https://example.com');
 
     scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' });
@@ -26,14 +25,14 @@
 
     scope.get('/foo/bar').reply(304);
     const res2 = await http.getJsonUnchecked('https://example.com/foo/bar');
     expect(res2).toMatchObject({
       statusCode: 200,
       body: { msg: 'Hello, world!' },
       authorization: false,
     });
   });
 
   it('reuses data with last-modified', async () => {
     const scope = httpMock.scope('https://example.com');
 
     scope
@@ -52,26 +51,13 @@
 
     scope.get('/foo/bar').reply(304);
     const res2 = await http.getJsonUnchecked('https://example.com/foo/bar');
     expect(res2).toMatchObject({
       statusCode: 200,
       body: { msg: 'Hello, world!' },
       authorization: false,
     });
   });
 
-  it('reports if cache could not be persisted', async () => {
-    httpMock
-      .scope('https://example.com')
-      .get('/foo/bar')
-      .reply(200, { msg: 'Hello, world!' });
-
-    await http.getJsonUnchecked('https://example.com/foo/bar');
-
-    expect(logger.logger.debug).toHaveBeenCalledWith(
-      'http cache: failed to persist cache for https://example.com/foo/bar',
-    );
-  });
-
   it('handles abrupt cache reset', async () => {
     const scope = httpMock.scope('https://example.com');
 
@@ -105,7 +91,7 @@
     });
   });
 
   it('supports authorization', async () => {
     const scope = httpMock.scope('https://example.com');
 
     scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' });
@@ -122,7 +108,7 @@
     const res2 = await http.getJsonUnchecked('https://example.com/foo/bar', {
       headers: { authorization: 'Bearer 123' },
     });
     expect(res2).toMatchObject({
       statusCode: 200,
       body: { msg: 'Hello, world!' },
       authorization: true,

lib/util/http/cache/repository-http-cache-provider.ts

@@ -3,6 +3,9 @@ import { AbstractHttpCacheProvider } from './abstract-http-cache-provider';
 import type { HttpCache } from './schema';
 
 export class RepositoryHttpCacheProvider extends AbstractHttpCacheProvider {
+  protected override softTtlMinutes = 15;
+  protected override hardTtlMinutes = 365 * 24 * 60;
+
   override load(url: string): Promise<unknown> {
     const cache = getCache();
     cache.httpCache ??= {};

lib/util/http/cache/schema.ts

@@ -7,10 +7,6 @@ export const HttpCacheSchema = z
     httpResponse: z.unknown(),
     timestamp: z.string(),
   })
-  .refine(
-    ({ etag, lastModified }) => etag ?? lastModified,
-    'Cache object should have `etag` or `lastModified` fields',
-  )
   .nullable()
   .catch(null);
 export type HttpCache = z.infer<typeof HttpCacheSchema>;