Added readStream and big json parsing to avoid memory leaks

index.js

@@ -1,5 +1,6 @@
 const express = require('express')
 const cors = require('cors')
+const json = require('big-json')
 
 const app = express()
 app.use(cors())
@@ -30,10 +31,12 @@ app.use('/rep', async (req, res) => {
 
     codeDepartement = codeDepartement.padStart(3, "0");
 
-    const f = fs.readFileSync(path.join(__dirname, `public/cirs/${codeDepartement}.json`), 'utf-8')
+    const pathToFeatures = path.join(__dirname, `public/cirs/${codeDepartement}.json`)
-    const data = JSON.parse(f)
+    const readStream = fs.createReadStream(pathToFeatures)
+    const parseStream = json.createParseStream()
 
-    gl = new GeoJsonGeometriesLookup(data)
+    parseStream.on('data', (pojo) => {
+        gl = new GeoJsonGeometriesLookup(pojo)
     
         const pos = {
             type: "Point",
@@ -54,6 +57,9 @@ app.use('/rep', async (req, res) => {
         res.end(repFile)
     })
 
+    readStream.pipe(parseStream);
+})
+
 app.listen(PORT, () => console.log(`Server listening on port: ${PORT}`));
 
 /**

package-lock.json

@@ -6,6 +6,7 @@
         "": {
             "license": "ISC",
             "dependencies": {
+                "big-json": "^3.2.0",
                 "cors": "^2.8.5",
                 "express": "^4.18.2",
                 "geojson-geometries-lookup": "^0.5.0"
@@ -109,6 +110,27 @@
             "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
             "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
         },
+        "node_modules/assert-plus": {
+            "version": "1.0.0",
+            "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+            "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==",
+            "engines": {
+                "node": ">=0.8"
+            }
+        },
+        "node_modules/big-json": {
+            "version": "3.2.0",
+            "resolved": "https://registry.npmjs.org/big-json/-/big-json-3.2.0.tgz",
+            "integrity": "sha512-2Etdurszm1CAFxqpH08lItXyf5CI1OBKRn7imCeI8Lh+a2UvdN2WpuSduxB/3ccao6v93SxiS5fIlE/v1QLoPg==",
+            "dependencies": {
+                "assert-plus": "^1.0.0",
+                "into-stream": "^5.1.0",
+                "json-stream-stringify": "^2.0.1",
+                "JSONStream": "^1.3.1",
+                "once": "^1.4.0",
+                "through2": "^3.0.1"
+            }
+        },
         "node_modules/body-parser": {
             "version": "1.20.1",
             "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
@@ -184,6 +206,11 @@
             "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
             "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
         },
+        "node_modules/core-util-is": {
+            "version": "1.0.3",
+            "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
+            "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
+        },
         "node_modules/cors": {
             "version": "2.8.5",
             "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
@@ -321,6 +348,15 @@
                 "node": ">= 0.6"
             }
         },
+        "node_modules/from2": {
+            "version": "2.3.0",
+            "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz",
+            "integrity": "sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==",
+            "dependencies": {
+                "inherits": "^2.0.1",
+                "readable-stream": "^2.0.0"
+            }
+        },
         "node_modules/function-bind": {
             "version": "1.1.1",
             "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
@@ -414,6 +450,18 @@
             "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
             "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
         },
+        "node_modules/into-stream": {
+            "version": "5.1.1",
+            "resolved": "https://registry.npmjs.org/into-stream/-/into-stream-5.1.1.tgz",
+            "integrity": "sha512-krrAJ7McQxGGmvaYbB7Q1mcA+cRwg9Ij2RfWIeVesNBgVDZmzY/Fa4IpZUT3bmdRzMzdf/mzltCG2Dq99IZGBA==",
+            "dependencies": {
+                "from2": "^2.3.0",
+                "p-is-promise": "^3.0.0"
+            },
+            "engines": {
+                "node": ">=8"
+            }
+        },
         "node_modules/ipaddr.js": {
             "version": "1.9.1",
             "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -422,6 +470,39 @@
                 "node": ">= 0.10"
             }
         },
+        "node_modules/isarray": {
+            "version": "1.0.0",
+            "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+            "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
+        },
+        "node_modules/json-stream-stringify": {
+            "version": "2.0.4",
+            "resolved": "https://registry.npmjs.org/json-stream-stringify/-/json-stream-stringify-2.0.4.tgz",
+            "integrity": "sha512-gIPoa6K5w6j/RnQ3fOtmvICKNJGViI83A7dnTIL+0QJ/1GKuNvCPFvbFWxt0agruF4iGgDFJvge4Gua4ZoiggQ=="
+        },
+        "node_modules/jsonparse": {
+            "version": "1.3.1",
+            "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz",
+            "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==",
+            "engines": [
+                "node >= 0.2.0"
+            ]
+        },
+        "node_modules/JSONStream": {
+            "version": "1.3.5",
+            "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz",
+            "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==",
+            "dependencies": {
+                "jsonparse": "^1.2.0",
+                "through": ">=2.2.7 <3"
+            },
+            "bin": {
+                "JSONStream": "bin.js"
+            },
+            "engines": {
+                "node": "*"
+            }
+        },
         "node_modules/media-typer": {
             "version": "0.3.0",
             "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@@ -513,6 +594,22 @@
                 "node": ">= 0.8"
             }
         },
+        "node_modules/once": {
+            "version": "1.4.0",
+            "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+            "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+            "dependencies": {
+                "wrappy": "1"
+            }
+        },
+        "node_modules/p-is-promise": {
+            "version": "3.0.0",
+            "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-3.0.0.tgz",
+            "integrity": "sha512-Wo8VsW4IRQSKVXsJCn7TomUaVtyfjVDn3nUP7kE967BQk0CwFpdbZs0X0uk5sW9mkBa9eNM7hCMaG93WUAwxYQ==",
+            "engines": {
+                "node": ">=8"
+            }
+        },
         "node_modules/parseurl": {
             "version": "1.3.3",
             "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -526,6 +623,11 @@
             "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
             "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
         },
+        "node_modules/process-nextick-args": {
+            "version": "2.0.1",
+            "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+            "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+        },
         "node_modules/proxy-addr": {
             "version": "2.0.7",
             "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -587,6 +689,25 @@
                 "quickselect": "^2.0.0"
             }
         },
+        "node_modules/readable-stream": {
+            "version": "2.3.8",
+            "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+            "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+            "dependencies": {
+                "core-util-is": "~1.0.0",
+                "inherits": "~2.0.3",
+                "isarray": "~1.0.0",
+                "process-nextick-args": "~2.0.0",
+                "safe-buffer": "~5.1.1",
+                "string_decoder": "~1.1.1",
+                "util-deprecate": "~1.0.1"
+            }
+        },
+        "node_modules/readable-stream/node_modules/safe-buffer": {
+            "version": "5.1.2",
+            "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+            "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+        },
         "node_modules/safe-buffer": {
             "version": "5.2.1",
             "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -679,6 +800,33 @@
                 "node": ">= 0.8"
             }
         },
+        "node_modules/string_decoder": {
+            "version": "1.1.1",
+            "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+            "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+            "dependencies": {
+                "safe-buffer": "~5.1.0"
+            }
+        },
+        "node_modules/string_decoder/node_modules/safe-buffer": {
+            "version": "5.1.2",
+            "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+            "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+        },
+        "node_modules/through": {
+            "version": "2.3.8",
+            "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
+            "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg=="
+        },
+        "node_modules/through2": {
+            "version": "3.0.2",
+            "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz",
+            "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==",
+            "dependencies": {
+                "inherits": "^2.0.4",
+                "readable-stream": "2 || 3"
+            }
+        },
         "node_modules/toidentifier": {
             "version": "1.0.1",
             "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
@@ -707,6 +855,11 @@
                 "node": ">= 0.8"
             }
         },
+        "node_modules/util-deprecate": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+            "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
+        },
         "node_modules/utils-merge": {
             "version": "1.0.1",
             "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
@@ -722,6 +875,11 @@
             "engines": {
                 "node": ">= 0.8"
             }
+        },
+        "node_modules/wrappy": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+            "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
         }
     }
 }

package.json

@@ -3,6 +3,7 @@
     "author": "",
     "license": "ISC",
     "dependencies": {
+        "big-json": "^3.2.0",
         "cors": "^2.8.5",
         "express": "^4.18.2",
         "geojson-geometries-lookup": "^0.5.0"