nhj

more

unified-ai-platform/node_modules/file-entry-cache/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Roy Riojas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

unified-ai-platform/node_modules/file-entry-cache/cache.js

@@ -0,0 +1,291 @@
+var path = require('path');
+var crypto = require('crypto');
+
+module.exports = {
+  createFromFile: function (filePath, useChecksum) {
+    var fname = path.basename(filePath);
+    var dir = path.dirname(filePath);
+    return this.create(fname, dir, useChecksum);
+  },
+
+  create: function (cacheId, _path, useChecksum) {
+    var fs = require('fs');
+    var flatCache = require('flat-cache');
+    var cache = flatCache.load(cacheId, _path);
+    var normalizedEntries = {};
+
+    var removeNotFoundFiles = function removeNotFoundFiles() {
+      const cachedEntries = cache.keys();
+      // remove not found entries
+      cachedEntries.forEach(function remover(fPath) {
+        try {
+          fs.statSync(fPath);
+        } catch (err) {
+          if (err.code === 'ENOENT') {
+            cache.removeKey(fPath);
+          }
+        }
+      });
+    };
+
+    removeNotFoundFiles();
+
+    return {
+      /**
+       * the flat cache storage used to persist the metadata of the `files
+       * @type {Object}
+       */
+      cache: cache,
+
+      /**
+       * Given a buffer, calculate md5 hash of its content.
+       * @method getHash
+       * @param  {Buffer} buffer   buffer to calculate hash on
+       * @return {String}          content hash digest
+       */
+      getHash: function (buffer) {
+        return crypto.createHash('md5').update(buffer).digest('hex');
+      },
+
+      /**
+       * Return whether or not a file has changed since last time reconcile was called.
+       * @method hasFileChanged
+       * @param  {String}  file  the filepath to check
+       * @return {Boolean}       wheter or not the file has changed
+       */
+      hasFileChanged: function (file) {
+        return this.getFileDescriptor(file).changed;
+      },
+
+      /**
+       * given an array of file paths it return and object with three arrays:
+       *  - changedFiles: Files that changed since previous run
+       *  - notChangedFiles: Files that haven't change
+       *  - notFoundFiles: Files that were not found, probably deleted
+       *
+       * @param  {Array} files the files to analyze and compare to the previous seen files
+       * @return {[type]}       [description]
+       */
+      analyzeFiles: function (files) {
+        var me = this;
+        files = files || [];
+
+        var res = {
+          changedFiles: [],
+          notFoundFiles: [],
+          notChangedFiles: [],
+        };
+
+        me.normalizeEntries(files).forEach(function (entry) {
+          if (entry.changed) {
+            res.changedFiles.push(entry.key);
+            return;
+          }
+          if (entry.notFound) {
+            res.notFoundFiles.push(entry.key);
+            return;
+          }
+          res.notChangedFiles.push(entry.key);
+        });
+        return res;
+      },
+
+      getFileDescriptor: function (file) {
+        var fstat;
+
+        try {
+          fstat = fs.statSync(file);
+        } catch (ex) {
+          this.removeEntry(file);
+          return { key: file, notFound: true, err: ex };
+        }
+
+        if (useChecksum) {
+          return this._getFileDescriptorUsingChecksum(file);
+        }
+
+        return this._getFileDescriptorUsingMtimeAndSize(file, fstat);
+      },
+
+      _getFileDescriptorUsingMtimeAndSize: function (file, fstat) {
+        var meta = cache.getKey(file);
+        var cacheExists = !!meta;
+
+        var cSize = fstat.size;
+        var cTime = fstat.mtime.getTime();
+
+        var isDifferentDate;
+        var isDifferentSize;
+
+        if (!meta) {
+          meta = { size: cSize, mtime: cTime };
+        } else {
+          isDifferentDate = cTime !== meta.mtime;
+          isDifferentSize = cSize !== meta.size;
+        }
+
+        var nEntry = (normalizedEntries[file] = {
+          key: file,
+          changed: !cacheExists || isDifferentDate || isDifferentSize,
+          meta: meta,
+        });
+
+        return nEntry;
+      },
+
+      _getFileDescriptorUsingChecksum: function (file) {
+        var meta = cache.getKey(file);
+        var cacheExists = !!meta;
+
+        var contentBuffer;
+        try {
+          contentBuffer = fs.readFileSync(file);
+        } catch (ex) {
+          contentBuffer = '';
+        }
+
+        var isDifferent = true;
+        var hash = this.getHash(contentBuffer);
+
+        if (!meta) {
+          meta = { hash: hash };
+        } else {
+          isDifferent = hash !== meta.hash;
+        }
+
+        var nEntry = (normalizedEntries[file] = {
+          key: file,
+          changed: !cacheExists || isDifferent,
+          meta: meta,
+        });
+
+        return nEntry;
+      },
+
+      /**
+       * Return the list o the files that changed compared
+       * against the ones stored in the cache
+       *
+       * @method getUpdated
+       * @param files {Array} the array of files to compare against the ones in the cache
+       * @returns {Array}
+       */
+      getUpdatedFiles: function (files) {
+        var me = this;
+        files = files || [];
+
+        return me
+          .normalizeEntries(files)
+          .filter(function (entry) {
+            return entry.changed;
+          })
+          .map(function (entry) {
+            return entry.key;
+          });
+      },
+
+      /**
+       * return the list of files
+       * @method normalizeEntries
+       * @param files
+       * @returns {*}
+       */
+      normalizeEntries: function (files) {
+        files = files || [];
+
+        var me = this;
+        var nEntries = files.map(function (file) {
+          return me.getFileDescriptor(file);
+        });
+
+        //normalizeEntries = nEntries;
+        return nEntries;
+      },
+
+      /**
+       * Remove an entry from the file-entry-cache. Useful to force the file to still be considered
+       * modified the next time the process is run
+       *
+       * @method removeEntry
+       * @param entryName
+       */
+      removeEntry: function (entryName) {
+        delete normalizedEntries[entryName];
+        cache.removeKey(entryName);
+      },
+
+      /**
+       * Delete the cache file from the disk
+       * @method deleteCacheFile
+       */
+      deleteCacheFile: function () {
+        cache.removeCacheFile();
+      },
+
+      /**
+       * remove the cache from the file and clear the memory cache
+       */
+      destroy: function () {
+        normalizedEntries = {};
+        cache.destroy();
+      },
+
+      _getMetaForFileUsingCheckSum: function (cacheEntry) {
+        var contentBuffer = fs.readFileSync(cacheEntry.key);
+        var hash = this.getHash(contentBuffer);
+        var meta = Object.assign(cacheEntry.meta, { hash: hash });
+        delete meta.size;
+        delete meta.mtime;
+        return meta;
+      },
+
+      _getMetaForFileUsingMtimeAndSize: function (cacheEntry) {
+        var stat = fs.statSync(cacheEntry.key);
+        var meta = Object.assign(cacheEntry.meta, {
+          size: stat.size,
+          mtime: stat.mtime.getTime(),
+        });
+        delete meta.hash;
+        return meta;
+      },
+
+      /**
+       * Sync the files and persist them to the cache
+       * @method reconcile
+       */
+      reconcile: function (noPrune) {
+        removeNotFoundFiles();
+
+        noPrune = typeof noPrune === 'undefined' ? true : noPrune;
+
+        var entries = normalizedEntries;
+        var keys = Object.keys(entries);
+
+        if (keys.length === 0) {
+          return;
+        }
+
+        var me = this;
+
+        keys.forEach(function (entryName) {
+          var cacheEntry = entries[entryName];
+
+          try {
+            var meta = useChecksum
+              ? me._getMetaForFileUsingCheckSum(cacheEntry)
+              : me._getMetaForFileUsingMtimeAndSize(cacheEntry);
+            cache.setKey(entryName, meta);
+          } catch (err) {
+            // if the file does not exists we don't save it
+            // other errors are just thrown
+            if (err.code !== 'ENOENT') {
+              throw err;
+            }
+          }
+        });
+
+        cache.save(noPrune);
+      },
+    };
+  },
+};

unified-ai-platform/node_modules/file-entry-cache/changelog.md

@@ -0,0 +1,29 @@
+
+# file-entry-cache - Changelog
+## v6.0.1
+- **Other changes**
+  - Delete previous mtime when checksum is used and vice versa - [abcf0f9]( https://github.com/royriojas/file-entry-cache/commit/abcf0f9 ), [Milos Djermanovic](https://github.com/Milos Djermanovic), 19/02/2021 18:19:43
+
+    
+  - Adds travis jobs on ppc64le - [92e4d4a]( https://github.com/royriojas/file-entry-cache/commit/92e4d4a ), [dineshks1](https://github.com/dineshks1), 25/11/2020 04:52:11
+
+    
+## v6.0.0
+- **Refactoring**
+  - Align file-entry-cache with latest eslint - [4c6f1fb]( https://github.com/royriojas/file-entry-cache/commit/4c6f1fb ), [Roy Riojas](https://github.com/Roy Riojas), 08/11/2020 02:43:09
+
+    
+  - Upgrade deps - [8ab3257]( https://github.com/royriojas/file-entry-cache/commit/8ab3257 ), [Roy Riojas](https://github.com/Roy Riojas), 08/11/2020 02:41:53
+
+    
+  - updated packages - [3dd4231]( https://github.com/royriojas/file-entry-cache/commit/3dd4231 ), [Roy Riojas](https://github.com/Roy Riojas), 08/11/2020 02:29:37
+
+    
+  - Upgrade flat-cache to version 3 - [d7c60ef]( https://github.com/royriojas/file-entry-cache/commit/d7c60ef ), [Roy Riojas](https://github.com/Roy Riojas), 08/11/2020 01:18:04
+
+    
+## v5.0.1
+- **Bug Fixes**
+  - Fix missing checksum comparison from reconcile since now we use mtime and size by default. - [e858aa9]( https://github.com/royriojas/file-entry-cache/commit/e858aa9 ), [Roy Riojas](https://github.com/Roy Riojas), 04/02/2019 09:30:22
+
+    Old mode using checkSum can still be used by passing the `useCh

unified-ai-platform/node_modules/file-entry-cache/package.json

@@ -0,0 +1,80 @@
+{
+  "name": "file-entry-cache",
+  "version": "6.0.1",
+  "description": "Super simple cache for file metadata, useful for process that work o a given series of files and that only need to repeat the job on the changed ones since the previous run of the process",
+  "repository": "royriojas/file-entry-cache",
+  "license": "MIT",
+  "author": {
+    "name": "Roy Riojas",
+    "url": "http://royriojas.com"
+  },
+  "main": "cache.js",
+  "files": [
+    "cache.js"
+  ],
+  "engines": {
+    "node": "^10.12.0 || >=12.0.0"
+  },
+  "scripts": {
+    "eslint": "eslint --cache --cache-location=node_modules/.cache/ 'cache.js' 'test/**/*.js' 'perf.js'",
+    "autofix": "npm run eslint -- --fix",
+    "install-hooks": "prepush install && changelogx install-hook && precommit install",
+    "changelog": "changelogx -f markdown -o ./changelog.md",
+    "do-changelog": "npm run changelog && git add ./changelog.md && git commit -m 'DOC: Generate changelog' --no-verify",
+    "pre-v": "npm run test",
+    "post-v": "npm run do-changelog && git push --no-verify && git push --tags --no-verify",
+    "bump-major": "npm run pre-v && npm version major -m 'BLD: Release v%s' && npm run post-v",
+    "bump-minor": "npm run pre-v && npm version minor -m 'BLD: Release v%s' && npm run post-v",
+    "bump-patch": "npm run pre-v && npm version patch -m 'BLD: Release v%s' && npm run post-v",
+    "test": "npm run eslint --silent && mocha -R spec test/specs",
+    "perf": "node perf.js",
+    "cover": "istanbul cover test/runner.js html text-summary",
+    "watch": "watch-run -i -p 'test/specs/**/*.js' istanbul cover test/runner.js html text-summary"
+  },
+  "prepush": [
+    "npm run eslint --silent"
+  ],
+  "precommit": [
+    "npm run eslint --silent"
+  ],
+  "keywords": [
+    "file cache",
+    "task cache files",
+    "file cache",
+    "key par",
+    "key value",
+    "cache"
+  ],
+  "changelogx": {
+    "ignoreRegExp": [
+      "BLD: Release",
+      "DOC: Generate Changelog",
+      "Generated Changelog"
+    ],
+    "issueIDRegExp": "#(\\d+)",
+    "commitURL": "https://github.com/royriojas/file-entry-cache/commit/{0}",
+    "authorURL": "https://github.com/{0}",
+    "issueIDURL": "https://github.com/royriojas/file-entry-cache/issues/{0}",
+    "projectName": "file-entry-cache"
+  },
+  "devDependencies": {
+    "chai": "^4.2.0",
+    "changelogx": "^5.0.6",
+    "del": "^6.0.0",
+    "eslint": "^7.13.0",
+    "eslint-config-prettier": "^6.15.0",
+    "eslint-plugin-mocha": "^8.0.0",
+    "eslint-plugin-prettier": "^3.1.4",
+    "glob-expand": "^0.2.1",
+    "istanbul": "^0.4.5",
+    "mocha": "^8.2.1",
+    "precommit": "^1.2.2",
+    "prepush": "^3.1.11",
+    "prettier": "^2.1.2",
+    "watch-run": "^1.2.5",
+    "write": "^2.0.0"
+  },
+  "dependencies": {
+    "flat-cache": "^3.0.4"
+  }
+}