diff --git a/fixtures/cat.png b/fixtures/cat.png new file mode 100644 index 0000000..d4be8f2 Binary files /dev/null and b/fixtures/cat.png differ diff --git a/fixtures/neocities.png b/fixtures/neocities.png new file mode 100644 index 0000000..73e6030 Binary files /dev/null and b/fixtures/neocities.png differ diff --git a/lib/folder-diff.js b/lib/folder-diff.js new file mode 100644 index 0000000..de3f7e8 --- /dev/null +++ b/lib/folder-diff.js @@ -0,0 +1,159 @@ +const crypto = require('crypto') +const util = require('util') +const fs = require('fs') +const ppump = util.promisify(require('pump')) + +/** + * neocitiesLocalDiff returns an array of files to delete and update and some useful stats. + */ +async function neocitiesLocalDiff (neocitiesFiles, localListing, opts = {}) { + opts = { + ...opts + } + + const localIndex = {} + const ncIndex = {} + + const neoCitiesFiltered = neocitiesFiles.filter(f => !f.is_directory) + neoCitiesFiltered.forEach(f => { ncIndex[f.path] = f }) // index + const ncFiles = new Set(neoCitiesFiltered.map(f => f.path)) // shape + + const localListingFiltered = localListing.filter(f => !f.stat.isDirectory()) // files only + localListingFiltered.forEach(f => { localIndex[f.relname] = f }) // index + // TODO: convert windows to unix paths + const localFiles = new Set(localListingFiltered.map(f => f.relname)) // shape + + const filesToAdd = difference(localFiles, ncFiles) + const filesToDelete = difference(ncFiles, localFiles) + + const maybeUpdate = intersection(localFiles, ncFiles) + const skipped = new Set() + + for (const p of maybeUpdate) { + const local = localIndex[p] + const remote = ncIndex[p] + + if (local.stat.size !== remote.size) { filesToAdd.add(p); continue } + + const localSha1 = await sha1FromPath(local.filepath) + if (localSha1 !== remote.sha1_hash) { filesToAdd.add(p); continue } + + skipped.add(p) + } + + return { + filesToUpload: Array.from(filesToAdd).map(p => ({ + name: localIndex[p].relname, + path: localIndex[p].filepath + })), + filesToDelete: Array.from(filesToDelete).map(p => ncIndex[p].path), + filesSkipped: Array.from(skipped).map(p => localIndex[p]) + } +} + +module.exports = { + neocitiesLocalDiff +} + +/** + * sha1FromPath returns a sha1 hex from a path + * @param {String} p string of the path of the file to hash + * @return {Promise} the hex representation of the sha1 + */ +async function sha1FromPath (p) { + const rs = fs.createReadStream(p) + const hash = crypto.createHash('sha1') + + await ppump(rs, hash) + + return hash.digest('hex') +} + +// From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set#Implementing_basic_set_operations + +/** + * difference betwen setA and setB + * @param {Set} setA LHS set + * @param {Set} setB RHS set + * @return {Set} The difference Set + */ +function difference (setA, setB) { + const _difference = new Set(setA) + for (const elem of setB) { + _difference.delete(elem) + } + return _difference +} + +function intersection (setA, setB) { + const _intersection = new Set() + for (const elem of setB) { + if (setA.has(elem)) { + _intersection.add(elem) + } + } + return _intersection +} + +// [ +// { +// path: 'img', +// is_directory: true, +// updated_at: 'Thu, 21 Nov 2019 04:06:17 -0000' +// }, +// { +// path: 'index.html', +// is_directory: false, +// size: 1094, +// updated_at: 'Mon, 11 Nov 2019 22:23:16 -0000', +// sha1_hash: '7f15617e87d83218223662340f4052d9bb9d096d' +// }, +// { +// path: 'neocities.png', +// is_directory: false, +// size: 13232, +// updated_at: 'Mon, 11 Nov 2019 22:23:16 -0000', +// sha1_hash: 'fd2ee41b1922a39a716cacb88c323d613b0955e4' +// }, +// { +// path: 'not_found.html', +// is_directory: false, +// size: 347, +// updated_at: 'Mon, 11 Nov 2019 22:23:16 -0000', +// sha1_hash: 'd7f004e9d3b2eaaa8827f741356f1122dc9eb030' +// }, +// { +// path: 'style.css', +// is_directory: false, +// size: 298, +// updated_at: 'Mon, 11 Nov 2019 22:23:16 -0000', +// sha1_hash: 'e516457acdb0d00710ab62cc257109ef67209ce8' +// } +// ] + +// [{ +// root: '/Users/bret/repos/async-folder-walker/fixtures', +// filepath: '/Users/bret/repos/async-folder-walker/fixtures/sub-folder/sub-sub-folder', +// stat: Stats { +// dev: 16777220, +// mode: 16877, +// nlink: 3, +// uid: 501, +// gid: 20, +// rdev: 0, +// blksize: 4096, +// ino: 30244023, +// size: 96, +// blocks: 0, +// atimeMs: 1574381262779.8396, +// mtimeMs: 1574380914743.5474, +// ctimeMs: 1574380914743.5474, +// birthtimeMs: 1574380905232.5996, +// atime: 2019-11-22T00:07:42.780Z, +// mtime: 2019-11-22T00:01:54.744Z, +// ctime: 2019-11-22T00:01:54.744Z, +// birthtime: 2019-11-22T00:01:45.233Z +// }, +// relname: 'sub-folder/sub-sub-folder', +// basename: 'sub-sub-folder' +// }] diff --git a/lib/folder-diff.test.js b/lib/folder-diff.test.js new file mode 100644 index 0000000..a727943 --- /dev/null +++ b/lib/folder-diff.test.js @@ -0,0 +1,65 @@ +const tap = require('tap') +const afw = require('async-folder-walker') +const path = require('path') +const { neocitiesLocalDiff } = require('./folder-diff') + +const remoteFiles = [ + { + path: 'img', + is_directory: true, + updated_at: 'Thu, 21 Nov 2019 04:06:17 -0000' + }, + { + path: 'index.html', + is_directory: false, + size: 1094, + updated_at: 'Mon, 11 Nov 2019 22:23:16 -0000', + sha1_hash: '7f15617e87d83218223662340f4052d9bb9d096d' + }, + { + path: 'neocities.png', + is_directory: false, + size: 13232, + updated_at: 'Mon, 11 Nov 2019 22:23:16 -0000', + sha1_hash: 'fd2ee41b1922a39a716cacb88c323d613b0955e4' + }, + { + path: 'not_found.html', + is_directory: false, + size: 347, + updated_at: 'Mon, 11 Nov 2019 22:23:16 -0000', + sha1_hash: 'd7f004e9d3b2eaaa8827f741356f1122dc9eb030' + }, + { + path: 'style.css', + is_directory: false, + size: 298, + updated_at: 'Mon, 11 Nov 2019 22:23:16 -0000', + sha1_hash: 'e516457acdb0d00710ab62cc257109ef67209ce8' + } +] + +tap.test('test differ', async t => { + const localFiles = await afw.allFiles(path.join(__dirname, '../fixtures'), { + shaper: f => f + }) + + const { filesToUpload, filesToDelete, filesSkipped } = await neocitiesLocalDiff(remoteFiles, localFiles) + + t.true(['tootzzz.png', 'toot.gif', 'cat.png'].every(path => { + const found = filesToUpload.find(ftu => ftu.name === path) + t.ok(found.path && found.name, 'each file to upload has a name and path') + return found + }), 'every file to upload is included') + + t.deepEqual(filesToDelete, [ + 'index.html', + 'not_found.html', + 'style.css' + ], 'filesToDelete returned correctly') + + t.true(['neocities.png'].every(path => { + const found = filesSkipped.find(fs => fs.relname === path) + return found + }), 'every file skipped is included') +}) diff --git a/package.json b/package.json index 261f0b6..e7ca46e 100644 --- a/package.json +++ b/package.json @@ -34,9 +34,11 @@ "dependencies": { "@actions/core": "^1.2.0", "@actions/github": "^1.1.0", + "async-folder-walker": "^2.0.0", "fetch-errors": "^1.0.1", "form-data": "^3.0.0", "nanoassert": "^2.0.0", + "pump": "^3.0.0", "qs": "^6.9.1" }, "standard": { diff --git a/test.js b/test.js index 279e01d..bd4030b 100644 --- a/test.js +++ b/test.js @@ -33,10 +33,10 @@ tap.test('can get info about site', async t => { const client = new NeocitiesAPIClient(token) const info = await client.info() - console.log(info) + // console.log(info) t.equal(info.result, 'success', 'info requesst successfull') const list = await client.list() - console.log(list) + // console.log(list) t.equal(list.result, 'success', 'list result successfull') }) @@ -70,13 +70,13 @@ tap.test('can upload and delete files', async t => { } ]) - console.log(uploadResults) + // console.log(uploadResults) t.equal(uploadResults.result, 'success', 'list result successfull') const deleteResults = await client.delete([ 'toot.gif', 'img/tootzzz.png' ]) - console.log(deleteResults) + // console.log(deleteResults) t.equal(deleteResults.result, 'success', 'list result successfull') })