bundle-crc-cache (!274)

This PR is required by SPT-AKI/Modules!104 in order for it to function correctly.

## Overview

- Adds the package `buffer-crc32`, it can generate CRC32 hashes from buffers or strings
- Splits `HashCacheService` into 2 classes `ModHashCacheService` does exactly the same `HashCacheService` used to do, and added a new `BundleHashCacheService`
- `BundleLoader` now generates a CRC32 hash of every bundle file from every loaded mod
- Reworked `BundleInfo` to better represent the data expected by the client when requesting `/singleplayer/bundles`
- Removes all checks on `BundleLoader` that verified if the request was made to a localhost address, this is now addressed by the client.

## Testing

The code has been tested by @Senko-san and me.

Co-authored-by: chomp <chomp@noreply.dev.sp-tarkov.com>
Reviewed-on: https://dev.sp-tarkov.com/SPT-AKI/Server/pulls/274
Co-authored-by: TheSparta <thesparta@noreply.dev.sp-tarkov.com>
Co-committed-by: TheSparta <thesparta@noreply.dev.sp-tarkov.com>

project/package.json

@@ -32,6 +32,7 @@
     },
     "dependencies": {
         "atomically": "~1.7",
+        "buffer-crc32": "^1.0.0",
         "date-fns": "~2.30",
         "date-fns-tz": "~2.0",
         "i18n": "~0.15",

project/src/callbacks/BundleCallbacks.ts

@@ -25,8 +25,7 @@ export class BundleCallbacks
      */
     public getBundles(url: string, info: any, sessionID: string): string
     {
-        const local = this.httpConfig.ip === "127.0.0.1" || this.httpConfig.ip === "localhost";
+        return this.httpResponse.noBody(this.bundleLoader.getBundles());
-        return this.httpResponse.noBody(this.bundleLoader.getBundles(local));
     }
 
     public getBundle(url: string, info: any, sessionID: string): string

project/src/di/Container.ts

@@ -199,7 +199,6 @@ import { BotWeaponModLimitService } from "@spt-aki/services/BotWeaponModLimitSer
 import { CustomLocationWaveService } from "@spt-aki/services/CustomLocationWaveService";
 import { FenceService } from "@spt-aki/services/FenceService";
 import { GiftService } from "@spt-aki/services/GiftService";
-import { HashCacheService } from "@spt-aki/services/HashCacheService";
 import { InsuranceService } from "@spt-aki/services/InsuranceService";
 import { ItemBaseClassService } from "@spt-aki/services/ItemBaseClassService";
 import { ItemFilterService } from "@spt-aki/services/ItemFilterService";
@@ -228,6 +227,8 @@ import { SeasonalEventService } from "@spt-aki/services/SeasonalEventService";
 import { TraderAssortService } from "@spt-aki/services/TraderAssortService";
 import { TraderPurchasePersisterService } from "@spt-aki/services/TraderPurchasePersisterService";
 import { TraderServicesService } from "@spt-aki/services/TraderServicesService";
+import { BundleHashCacheService } from "@spt-aki/services/cache/BundleHashCacheService";
+import { ModHashCacheService } from "@spt-aki/services/cache/ModHashCacheService";
 import { CustomItemService } from "@spt-aki/services/mod/CustomItemService";
 import { DynamicRouterModService } from "@spt-aki/services/mod/dynamicRouter/DynamicRouterModService";
 import { HttpListenerModService } from "@spt-aki/services/mod/httpListener/HttpListenerModService";
@@ -690,7 +691,10 @@ export class Container
             lifecycle: Lifecycle.Singleton,
         });
         depContainer.register<ModCompilerService>("ModCompilerService", ModCompilerService);
-        depContainer.register<HashCacheService>("HashCacheService", HashCacheService, {
+        depContainer.register<BundleHashCacheService>("BundleHashCacheService", BundleHashCacheService, {
+            lifecycle: Lifecycle.Singleton,
+        });
+        depContainer.register<ModHashCacheService>("ModHashCacheService", ModHashCacheService, {
             lifecycle: Lifecycle.Singleton,
         });
         depContainer.register<LocaleService>("LocaleService", LocaleService, { lifecycle: Lifecycle.Singleton });

project/src/loaders/BundleLoader.ts

@@ -1,25 +1,24 @@
+import path from "node:path";
 import { inject, injectable } from "tsyringe";
 
-import path from "path";
 import { HttpServerHelper } from "@spt-aki/helpers/HttpServerHelper";
+import { BundleHashCacheService } from "@spt-aki/services/cache/BundleHashCacheService";
 import { JsonUtil } from "@spt-aki/utils/JsonUtil";
 import { VFS } from "@spt-aki/utils/VFS";
 
 export class BundleInfo
 {
-    modPath: string;
+    modpath: string;
-    key: string;
+    filename: string;
-    path: string;
+    crc: number;
-    filepath: string;
+    dependencies: string[];
-    dependencyKeys: string[];
 
-    constructor(modpath: string, bundle: any, bundlePath: string, bundleFilepath: string)
+    constructor(modpath: string, bundle: BundleManifestEntry, bundleHash: number)
     {
-        this.modPath = modpath;
+        this.modpath = modpath;
-        this.key = bundle.key;
+        this.filename = bundle.key;
-        this.path = bundlePath;
+        this.crc = bundleHash;
-        this.filepath = bundleFilepath;
+        this.dependencies = bundle.dependencyKeys || [];
-        this.dependencyKeys = bundle.dependencyKeys || [];
     }
 }
 
@@ -32,47 +31,48 @@ export class BundleLoader
         @inject("HttpServerHelper") protected httpServerHelper: HttpServerHelper,
         @inject("VFS") protected vfs: VFS,
         @inject("JsonUtil") protected jsonUtil: JsonUtil,
+        @inject("BundleHashCacheService") protected bundleHashCacheService: BundleHashCacheService,
     )
     {}
 
     /**
      * Handle singleplayer/bundles
      */
-    public getBundles(local: boolean): BundleInfo[]
+    public getBundles(): BundleInfo[]
     {
         const result: BundleInfo[] = [];
 
         for (const bundle in this.bundles)
         {
-            result.push(this.getBundle(bundle, local));
+            result.push(this.getBundle(bundle));
         }
 
         return result;
     }
 
-    public getBundle(key: string, local: boolean): BundleInfo
+    public getBundle(key: string): BundleInfo
     {
-        const bundle = this.jsonUtil.clone(this.bundles[key]);
+        return this.jsonUtil.clone(this.bundles[key]);
-
-        if (local)
-        {
-            bundle.path = path.join(process.cwd(), bundle.filepath);
-        }
-
-        delete bundle.filepath;
-        return bundle;
     }
 
     public addBundles(modpath: string): void
     {
-        const manifest =
+        const bundleManifestArr =
             this.jsonUtil.deserialize<BundleManifest>(this.vfs.readFile(`${modpath}bundles.json`)).manifest;
 
-        for (const bundle of manifest)
+        for (const bundleManifest of bundleManifestArr)
         {
-            const bundlePath = `${this.httpServerHelper.getBackendUrl()}/files/bundle/${bundle.key}`;
+            const absoluteModPath = path.join(process.cwd(), modpath).slice(0, -1).replace(/\\/g, "/");
-            const bundleFilepath = bundle.path || `${modpath}bundles/${bundle.key}`.replace(/\\/g, "/");
+            const bundleLocalPath = `${modpath}bundles/${bundleManifest.key}`.replace(/\\/g, "/");
-            this.addBundle(bundle.key, new BundleInfo(modpath, bundle, bundlePath, bundleFilepath));
+
+            if (!this.bundleHashCacheService.calculateAndMatchHash(bundleLocalPath))
+            {
+                this.bundleHashCacheService.calculateAndStoreHash(bundleLocalPath);
+            }
+
+            const bundleHash = this.bundleHashCacheService.getStoredValue(bundleLocalPath);
+
+            this.addBundle(bundleManifest.key, new BundleInfo(absoluteModPath, bundleManifest, bundleHash));
         }
     }
 
@@ -84,11 +84,11 @@ export class BundleLoader
 
 export interface BundleManifest
 {
-    manifest: Array<BundleManifestEntry>;
+    manifest: BundleManifestEntry[];
 }
 
 export interface BundleManifestEntry
 {
     key: string;
-    path: string;
+    dependencyKeys: string[];
 }

project/src/routers/serializers/BundleSerializer.ts

@@ -23,10 +23,9 @@ export class BundleSerializer extends Serializer
         this.logger.info(`[BUNDLE]: ${req.url}`);
 
         const key = req.url.split("/bundle/")[1];
-        const bundle = this.bundleLoader.getBundle(key, true);
+        const bundle = this.bundleLoader.getBundle(key);
 
-        // send bundle
+        this.httpFileUtil.sendFile(resp, `${bundle.modpath}/bundles/${bundle.filename}`);
-        this.httpFileUtil.sendFile(resp, bundle.path);
     }
 
     public override canHandle(route: string): boolean

project/src/servers/HttpServer.ts

@@ -1,4 +1,4 @@
-import http, { IncomingMessage, ServerResponse } from "node:http";
+import http, { IncomingMessage, ServerResponse, Server } from "node:http";
 import { inject, injectAll, injectable } from "tsyringe";
 
 import { ApplicationContext } from "@spt-aki/context/ApplicationContext";
@@ -38,7 +38,9 @@ export class HttpServer
     public load(): void
     {
         /* create server */
-        const httpServer: http.Server = http.createServer((req, res) =>
+        const httpServer: Server = http.createServer();
+
+        httpServer.on("request", (req, res) =>
         {
             this.handleRequest(req, res);
         });
@@ -104,7 +106,7 @@ export class HttpServer
         }
     }
 
-    protected getCookies(req: http.IncomingMessage): Record<string, string>
+    protected getCookies(req: IncomingMessage): Record<string, string>
     {
         const found: Record<string, string> = {};
         const cookies = req.headers.cookie;

project/src/servers/http/AkiHttpListener.ts

@@ -48,7 +48,7 @@ export class AkiHttpListener implements IHttpListener
                 // kinda big), on a slow connection. We need to re-assemble the entire http payload
                 // before processing it.
 
-                const requestLength = parseInt(req.headers["content-length"]);
+                const requestLength = Number.parseInt(req.headers["content-length"]);
                 const buffer = Buffer.alloc(requestLength);
                 let written = 0;
 

project/src/services/HashCacheService.ts

@@ -1,80 +0,0 @@
-import { inject, injectable } from "tsyringe";
-
-import { ILogger } from "@spt-aki/models/spt/utils/ILogger";
-import { HashUtil } from "@spt-aki/utils/HashUtil";
-import { JsonUtil } from "@spt-aki/utils/JsonUtil";
-import { VFS } from "@spt-aki/utils/VFS";
-
-@injectable()
-export class HashCacheService
-{
-    protected jsonHashes = null;
-    protected modHashes = null;
-    protected readonly modCachePath = "./user/cache/modCache.json";
-
-    constructor(
-        @inject("VFS") protected vfs: VFS,
-        @inject("HashUtil") protected hashUtil: HashUtil,
-        @inject("JsonUtil") protected jsonUtil: JsonUtil,
-        @inject("WinstonLogger") protected logger: ILogger,
-    )
-    {
-        if (!this.vfs.exists(this.modCachePath))
-        {
-            this.vfs.writeFile(this.modCachePath, "{}");
-        }
-
-        // get mod hash file
-        if (!this.modHashes)
-        { // empty
-            this.modHashes = this.jsonUtil.deserialize(this.vfs.readFile(`${this.modCachePath}`), this.modCachePath);
-        }
-    }
-
-    /**
-     * Return a stored hash by key
-     * @param modName Name of mod to get hash for
-     * @returns Mod hash
-     */
-    public getStoredModHash(modName: string): string
-    {
-        return this.modHashes[modName];
-    }
-
-    /**
-     * Does the generated hash match the stored hash
-     * @param modName name of mod
-     * @param modContent
-     * @returns True if they match
-     */
-    public modContentMatchesStoredHash(modName: string, modContent: string): boolean
-    {
-        const storedModHash = this.getStoredModHash(modName);
-        const generatedHash = this.hashUtil.generateSha1ForData(modContent);
-
-        return storedModHash === generatedHash;
-    }
-
-    public hashMatchesStoredHash(modName: string, modHash: string): boolean
-    {
-        const storedModHash = this.getStoredModHash(modName);
-
-        return storedModHash === modHash;
-    }
-
-    public storeModContent(modName: string, modContent: string): void
-    {
-        const generatedHash = this.hashUtil.generateSha1ForData(modContent);
-
-        this.storeModHash(modName, generatedHash);
-    }
-
-    public storeModHash(modName: string, modHash: string): void
-    {
-        this.modHashes[modName] = modHash;
-
-        this.vfs.writeFile(this.modCachePath, this.jsonUtil.serialize(this.modHashes));
-
-        this.logger.debug(`Mod ${modName} hash stored in ${this.modCachePath}`);
-    }
-}

project/src/services/ModCompilerService.ts

@@ -5,7 +5,7 @@ import { inject, injectable } from "tsyringe";
 import ts from "typescript";
 
 import type { ILogger } from "@spt-aki/models/spt/utils/ILogger";
-import { HashCacheService } from "@spt-aki/services/HashCacheService";
+import { ModHashCacheService } from "@spt-aki/services/cache/ModHashCacheService";
 import { VFS } from "@spt-aki/utils/VFS";
 
 @injectable()
@@ -15,7 +15,7 @@ export class ModCompilerService
 
     constructor(
         @inject("WinstonLogger") protected logger: ILogger,
-        @inject("HashCacheService") protected hashCacheService: HashCacheService,
+        @inject("ModHashCacheService") protected modHashCacheService: ModHashCacheService,
         @inject("VFS") protected vfs: VFS,
     )
     {
@@ -47,7 +47,7 @@ export class ModCompilerService
             }
         }
 
-        const hashMatches = this.hashCacheService.modContentMatchesStoredHash(modName, tsFileContents);
+        const hashMatches = this.modHashCacheService.calculateAndCompareHash(modName, tsFileContents);
 
         if (fileExists && hashMatches)
         {
@@ -58,7 +58,7 @@ export class ModCompilerService
         if (!hashMatches)
         {
             // Store / update hash in json file
-            this.hashCacheService.storeModContent(modName, tsFileContents);
+            this.modHashCacheService.calculateAndStoreHash(modName, tsFileContents);
         }
 
         return this.compile(modTypeScriptFiles, {

project/src/services/cache/BundleHashCacheService.ts

@@ -0,0 +1,64 @@
+import { inject, injectable } from "tsyringe";
+
+import { ILogger } from "@spt-aki/models/spt/utils/ILogger";
+import { HashUtil } from "@spt-aki/utils/HashUtil";
+import { JsonUtil } from "@spt-aki/utils/JsonUtil";
+import { VFS } from "@spt-aki/utils/VFS";
+
+@injectable()
+export class BundleHashCacheService
+{
+    protected bundleHashes: Record<string, number>;
+    protected readonly bundleHashCachePath = "./user/cache/bundleHashCache.json";
+
+    constructor(
+        @inject("VFS") protected vfs: VFS,
+        @inject("HashUtil") protected hashUtil: HashUtil,
+        @inject("JsonUtil") protected jsonUtil: JsonUtil,
+        @inject("WinstonLogger") protected logger: ILogger,
+    )
+    {
+        if (!this.vfs.exists(this.bundleHashCachePath))
+        {
+            this.vfs.writeFile(this.bundleHashCachePath, "{}");
+        }
+
+        this.bundleHashes = this.jsonUtil.deserialize(
+            this.vfs.readFile(this.bundleHashCachePath),
+            this.bundleHashCachePath,
+        );
+    }
+
+    public getStoredValue(key: string): number
+    {
+        return this.bundleHashes[key];
+    }
+
+    public storeValue(key: string, value: number): void
+    {
+        this.bundleHashes[key] = value;
+
+        this.vfs.writeFile(this.bundleHashCachePath, this.jsonUtil.serialize(this.bundleHashes));
+
+        this.logger.debug(`Bundle ${key} hash stored in ${this.bundleHashCachePath}`);
+    }
+
+    public matchWithStoredHash(bundlePath: string, hash: number): boolean
+    {
+        return this.getStoredValue(bundlePath) === hash;
+    }
+
+    public calculateAndMatchHash(bundlePath: string): boolean
+    {
+        const generatedHash = this.hashUtil.generateCRC32ForFile(bundlePath);
+
+        return this.matchWithStoredHash(bundlePath, generatedHash);
+    }
+
+    public calculateAndStoreHash(bundlePath: string): void
+    {
+        const generatedHash = this.hashUtil.generateCRC32ForFile(bundlePath);
+
+        this.storeValue(bundlePath, generatedHash);
+    }
+}

project/src/services/cache/ModHashCacheService.ts

@@ -0,0 +1,61 @@
+import { inject, injectable } from "tsyringe";
+
+import { ILogger } from "@spt-aki/models/spt/utils/ILogger";
+import { HashUtil } from "@spt-aki/utils/HashUtil";
+import { JsonUtil } from "@spt-aki/utils/JsonUtil";
+import { VFS } from "@spt-aki/utils/VFS";
+
+@injectable()
+export class ModHashCacheService
+{
+    protected modHashes: Record<string, string>;
+    protected readonly modCachePath = "./user/cache/modCache.json";
+
+    constructor(
+        @inject("VFS") protected vfs: VFS,
+        @inject("HashUtil") protected hashUtil: HashUtil,
+        @inject("JsonUtil") protected jsonUtil: JsonUtil,
+        @inject("WinstonLogger") protected logger: ILogger,
+    )
+    {
+        if (!this.vfs.exists(this.modCachePath))
+        {
+            this.vfs.writeFile(this.modCachePath, "{}");
+        }
+
+        this.modHashes = this.jsonUtil.deserialize(this.vfs.readFile(this.modCachePath), this.modCachePath);
+    }
+
+    public getStoredValue(key: string): string
+    {
+        return this.modHashes[key];
+    }
+
+    public storeValue(key: string, value: string): void
+    {
+        this.modHashes[key] = value;
+
+        this.vfs.writeFile(this.modCachePath, this.jsonUtil.serialize(this.modHashes));
+
+        this.logger.debug(`Mod ${key} hash stored in ${this.modCachePath}`);
+    }
+
+    public matchWithStoredHash(modName: string, hash: string): boolean
+    {
+        return this.getStoredValue(modName) === hash;
+    }
+
+    public calculateAndCompareHash(modName: string, modContent: string): boolean
+    {
+        const generatedHash = this.hashUtil.generateSha1ForData(modContent);
+
+        return this.matchWithStoredHash(modName, generatedHash);
+    }
+
+    public calculateAndStoreHash(modName: string, modContent: string): void
+    {
+        const generatedHash = this.hashUtil.generateSha1ForData(modContent);
+
+        this.storeValue(modName, generatedHash);
+    }
+}

project/src/utils/HashUtil.ts

@@ -1,4 +1,6 @@
 import crypto from "node:crypto";
+import fs from "node:fs";
+import crc32 from "buffer-crc32";
 import { inject, injectable } from "tsyringe";
 
 import { TimeUtil } from "@spt-aki/utils/TimeUtil";
@@ -32,6 +34,11 @@ export class HashUtil
         return this.generateHashForData("sha1", data);
     }
 
+    public generateCRC32ForFile(filePath: fs.PathLike): number
+    {
+        return crc32.unsigned(fs.readFileSync(filePath));
+    }
+
     /**
      * Create a hash for the data parameter
      * @param algorithm algorithm to use to hash

project/src/utils/HttpFileUtil.ts

@@ -11,12 +11,12 @@ export class HttpFileUtil
     {
     }
 
-    public sendFile(resp: ServerResponse, file: any): void
+    public sendFile(resp: ServerResponse, filePath: string): void
     {
-        const pathSlic = file.split("/");
+        const pathSlic = filePath.split("/");
         const type = this.httpServerHelper.getMimeText(pathSlic[pathSlic.length - 1].split(".").at(-1))
             || this.httpServerHelper.getMimeText("txt");
-        const fileStream = fs.createReadStream(file);
+        const fileStream = fs.createReadStream(filePath);
 
         fileStream.on("open", () =>
         {