chore: bump lmdb to 2.3.10 (#35585)

gatsbyjs · May 6, 2022 · c410214 · c410214
1 parent 4e415cc
commit c410214
Show file tree

Hide file tree

Showing 12 changed files with 112 additions and 28 deletions.
diff --git a/packages/gatsby-core-utils/package.json b/packages/gatsby-core-utils/package.json
@@ -50,7 +50,7 @@
     "fs-extra": "^10.1.0",
     "got": "^11.8.3",
     "import-from": "^4.0.0",
-    "lmdb": "^2.2.6",
+    "lmdb": "2.3.10",
     "lock": "^1.1.0",
     "node-object-hash": "^2.3.10",
     "proper-lockfile": "^4.1.2",

diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json
@@ -112,7 +112,7 @@
     "joi": "^17.4.2",
     "json-loader": "^0.5.7",
     "latest-version": "5.1.0",
-    "lmdb": "~2.2.3",
+    "lmdb": "2.3.10",
     "lodash": "^4.17.21",
     "md5-file": "^5.0.0",
     "meant": "^1.0.3",

diff --git a/packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts b/packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts
@@ -47,28 +47,40 @@ let fullDbPath
 let rootDb
 let databases
 
+/* eslint-disable @typescript-eslint/no-namespace */
+declare global {
+  namespace NodeJS {
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    interface Global {
+      __GATSBY_OPEN_LMDBS?: Map<string, ILmdbDatabases>
+      __GATSBY_OPEN_ROOT_LMDBS?: Map<string, RootDatabase>
+    }
+  }
+}
+
 function getRootDb(): RootDatabase {
   if (!rootDb) {
     if (!fullDbPath) {
       throw new Error(`LMDB path is not set!`)
     }
+
+    if (!globalThis.__GATSBY_OPEN_ROOT_LMDBS) {
+      globalThis.__GATSBY_OPEN_ROOT_LMDBS = new Map()
+    }
+    rootDb = globalThis.__GATSBY_OPEN_ROOT_LMDBS.get(fullDbPath)
+    if (rootDb) {
+      return rootDb
+    }
+
     rootDb = open({
       name: `root`,
       path: fullDbPath,
       compression: true,
     })
-  }
-  return rootDb
-}
 
-/* eslint-disable @typescript-eslint/no-namespace */
-declare global {
-  namespace NodeJS {
-    // eslint-disable-next-line @typescript-eslint/naming-convention
-    interface Global {
-      __GATSBY_OPEN_LMDBS?: Map<string, ILmdbDatabases>
-    }
+    globalThis.__GATSBY_OPEN_ROOT_LMDBS.set(fullDbPath, rootDb)
   }
+  return rootDb
 }
 
 function getDatabases(): ILmdbDatabases {

diff --git a/packages/gatsby/src/schema/graphql-engine/lmdb-bundling-patch.ts b/packages/gatsby/src/schema/graphql-engine/lmdb-bundling-patch.ts
@@ -31,7 +31,19 @@ export default function (this: any, source: string): string {
       path.dirname(this.resourcePath).replace(`/dist`, ``)
 
     const lmdbRequire = createRequire(this.resourcePath)
-    const nodeGypBuild = lmdbRequire(`node-gyp-build`)
+    let nodeGypBuild
+    try {
+      nodeGypBuild = lmdbRequire(`node-gyp-build-optional-packages`)
+    } catch (e) {
+      // [email protected] way of loading binaries failed, we will try to fallback to
+      // old way before failing completely
+    }
+
+    if (!nodeGypBuild) {
+      // if [email protected] didn't import expected node-gyp-build fork (node-gyp-build-optional-packages)
+      // let's try falling back to upstream package - if that doesn't work, we will fail compilation
+      nodeGypBuild = lmdbRequire(`node-gyp-build`)
+    }
 
     lmdbBinaryLocation = slash(
       path.relative(
@@ -43,6 +55,10 @@ export default function (this: any, source: string): string {
     return source
   }
   return source
+    .replace(
+      `require$1('node-gyp-build-optional-packages')(dirName)`,
+      `require(${JSON.stringify(lmdbBinaryLocation)})`
+    )
     .replace(
       `require$1('node-gyp-build')(dirName)`,
       `require(${JSON.stringify(lmdbBinaryLocation)})`

diff --git a/packages/gatsby/src/utils/worker/__tests__/config.ts b/packages/gatsby/src/utils/worker/__tests__/config.ts
@@ -17,9 +17,9 @@ beforeEach(() => {
   store.dispatch({ type: `DELETE_CACHE` })
 })
 
-afterEach(() => {
+afterEach(async () => {
   if (worker) {
-    worker.end()
+    await Promise.all(worker.end())
     worker = undefined
   }
 })

diff --git a/packages/gatsby/src/utils/worker/__tests__/datastore.ts b/packages/gatsby/src/utils/worker/__tests__/datastore.ts
@@ -7,6 +7,8 @@ import { store } from "../../../redux"
 import { actions } from "../../../redux/actions"
 import { getDataStore } from "../../../datastore"
 
+jest.setTimeout(15000)
+
 jest.mock(`gatsby-telemetry`, () => {
   return {
     decorateEvent: jest.fn(),
@@ -26,9 +28,9 @@ beforeEach(() => {
   store.dispatch({ type: `DELETE_CACHE` })
 })
 
-afterEach(() => {
+afterEach(async () => {
   if (worker) {
-    worker.end()
+    await Promise.all(worker.end())
     worker = undefined
   }
 })

diff --git a/packages/gatsby/src/utils/worker/__tests__/jobs.ts b/packages/gatsby/src/utils/worker/__tests__/jobs.ts
@@ -71,9 +71,9 @@ describe(`worker (jobs)`, () => {
     workerStateAfter = await Promise.all(worker.all.getReduxJobs())
   })
 
-  afterAll(() => {
+  afterAll(async () => {
     if (worker) {
-      worker.end()
+      await Promise.all(worker.end())
       worker = undefined
     }
   })

diff --git a/packages/gatsby/src/utils/worker/__tests__/queries.ts b/packages/gatsby/src/utils/worker/__tests__/queries.ts
@@ -186,9 +186,9 @@ describeWhenLMDB(`worker (queries)`, () => {
     await Promise.all(worker.all.buildSchema())
   })
 
-  afterAll(() => {
+  afterAll(async () => {
     if (worker) {
-      worker.end()
+      await Promise.all(worker.end())
       worker = undefined
     }
     for (const watcher of mockWatchersToClose) {

diff --git a/packages/gatsby/src/utils/worker/__tests__/reporter.ts b/packages/gatsby/src/utils/worker/__tests__/reporter.ts
@@ -11,12 +11,12 @@ const spies: Record<keyof typeof ActionCreators, jest.SpyInstance> = (
   return acc
 }, {} as any)
 
-afterEach(() => {
+afterEach(async () => {
   for (const spy of Object.values(spies)) {
     spy.mockClear()
   }
   if (worker) {
-    worker.end()
+    await Promise.all(worker.end())
     worker = undefined
   }
 })

diff --git a/packages/gatsby/src/utils/worker/__tests__/schema.ts b/packages/gatsby/src/utils/worker/__tests__/schema.ts
@@ -75,9 +75,9 @@ describeWhenLMDB(`worker (schema)`, () => {
     stateFromWorker = await worker.single.getState()
   })
 
-  afterAll(() => {
+  afterAll(async () => {
     if (worker) {
-      worker.end()
+      await Promise.all(worker.end())
       worker = undefined
     }
     for (const watcher of mockWatchersToClose) {

diff --git a/packages/gatsby/src/utils/worker/__tests__/share-state.ts b/packages/gatsby/src/utils/worker/__tests__/share-state.ts
@@ -29,9 +29,9 @@ describe(`worker (share-state)`, () => {
     store.dispatch({ type: `DELETE_CACHE` })
   })
 
-  afterEach(() => {
+  afterEach(async () => {
     if (worker) {
-      worker.end()
+      await Promise.all(worker.end())
       worker = undefined
     }
   })

diff --git a/yarn.lock b/yarn.lock
@@ -15719,7 +15719,56 @@ livereload-js@^2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/livereload-js/-/livereload-js-2.3.0.tgz#c3ab22e8aaf5bf3505d80d098cbad67726548c9a"
 
-lmdb@^2.0.2, lmdb@^2.1.7, lmdb@^2.2.6, lmdb@~2.2.3:
+[email protected]:
+  version "2.3.10"
+  resolved "https://registry.yarnpkg.com/lmdb-darwin-arm64/-/lmdb-darwin-arm64-2.3.10.tgz#4e20f75770eeedc60af3d4630975fd105a89ffe8"
+  integrity sha512-LVXbH2MYu7/ZuQ8+P9rv+SwNyBKltxo7vHAGJS94HWyfwnCbKEYER9PImBvNBwzvgtaYk6x0RMX3oor6e6KdDQ==
+
+[email protected]:
+  version "2.3.10"
+  resolved "https://registry.yarnpkg.com/lmdb-darwin-x64/-/lmdb-darwin-x64-2.3.10.tgz#e53637a6735488eaa15feb7c0e9da142015b9476"
+  integrity sha512-gAc/1b/FZOb9yVOT+o0huA+hdW82oxLo5r22dFTLoRUFG1JMzxdTjmnW6ONVOHdqC9a5bt3vBCEY3jmXNqV26A==
+
+[email protected]:
+  version "2.3.10"
+  resolved "https://registry.yarnpkg.com/lmdb-linux-arm64/-/lmdb-linux-arm64-2.3.10.tgz#ac7db8bdfe0e9dbf2be1cc3362d6f2b79e2a9722"
+  integrity sha512-Ihr8mdICTK3jA4GXHxrXGK2oekn0mY6zuDSXQDNtyRSH19j3D2Y04A7SEI9S0EP/t5sjKSudYgZbiHDxRCsI5A==
+
+[email protected]:
+  version "2.3.10"
+  resolved "https://registry.yarnpkg.com/lmdb-linux-arm/-/lmdb-linux-arm-2.3.10.tgz#74235418bbe7bf41e8ea5c9d52365c4ff5ca4b49"
+  integrity sha512-Rb8+4JjsThuEcJ7GLLwFkCFnoiwv/3hAAbELWITz70buQFF+dCZvCWWgEgmDTxwn5r+wIkdUjmFv4dqqiKQFmQ==
+
+[email protected]:
+  version "2.3.10"
+  resolved "https://registry.yarnpkg.com/lmdb-linux-x64/-/lmdb-linux-x64-2.3.10.tgz#d790b95061d03c5c99a57b3ad5126f7723c60a2f"
+  integrity sha512-E3l3pDiCA9uvnLf+t3qkmBGRO01dp1EHD0x0g0iRnfpAhV7wYbayJGfG93BUt22Tj3fnq4HDo4dQ6ZWaDI1nuw==
+
+[email protected]:
+  version "2.3.10"
+  resolved "https://registry.yarnpkg.com/lmdb-win32-x64/-/lmdb-win32-x64-2.3.10.tgz#bff73d12d94084343c569b16069d8d38626eb2d6"
+  integrity sha512-gspWk34tDANhjn+brdqZstJMptGiwj4qFNVg0Zey9ds+BUlif+Lgf5szrfOVzZ8gVRkk1Lgbz7i78+V7YK7SCA==
+
+[email protected]:
+  version "2.3.10"
+  resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-2.3.10.tgz#640fc60815846babcbe088d7f8ed0a51da857f6a"
+  integrity sha512-GtH+nStn9V59CfYeQ5ddx6YTfuFCmu86UJojIjJAweG+/Fm0PDknuk3ovgYDtY/foMeMdZa8/P7oSljW/d5UPw==
+  dependencies:
+    msgpackr "^1.5.4"
+    nan "^2.14.2"
+    node-addon-api "^4.3.0"
+    node-gyp-build-optional-packages "^4.3.2"
+    ordered-binary "^1.2.4"
+    weak-lru-cache "^1.2.2"
+  optionalDependencies:
+    lmdb-darwin-arm64 "2.3.10"
+    lmdb-darwin-x64 "2.3.10"
+    lmdb-linux-arm "2.3.10"
+    lmdb-linux-arm64 "2.3.10"
+    lmdb-linux-x64 "2.3.10"
+    lmdb-win32-x64 "2.3.10"
+
+lmdb@^2.0.2, lmdb@^2.1.7:
   version "2.2.6"
   resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-2.2.6.tgz#a52ef533812b8abcbe0033fc9d74d215e7dfc0a0"
   integrity sha512-UmQV0oZZcV3EN6rjcAjIiuWcc3MYZGWQ0GUYz46Ron5fuTa/dUow7WSQa6leFkvZIKVUdECBWVw96tckfEzUFQ==
@@ -17693,6 +17742,11 @@ [email protected], node-fetch@^2.5.0, node-fetch@^2.6.0, node-fetch@^2.6.1, node-
   dependencies:
     whatwg-url "^5.0.0"
 
+node-gyp-build-optional-packages@^4.3.2:
+  version "4.3.2"
+  resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-4.3.2.tgz#82de9bdf9b1ad042457533afb2f67469dc2264bb"
+  integrity sha512-P5Ep3ISdmwcCkZIaBaQamQtWAG0facC89phWZgi5Z3hBU//J6S48OIvyZWSPPf6yQMklLZiqoosWAZUj7N+esA==
+
 node-gyp-build@^4.2.3, node-gyp-build@^4.3.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.3.0.tgz#9f256b03e5826150be39c764bf51e993946d71a3"