knight/deploy-to-neocities
1
0
Fork 0
mirror of https://github.com/bcomnes/deploy-to-neocities.git synced 2026-01-21 00:46:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update logging

Browse Source
This commit is contained in:
Bret Comnes
2020-02-12 16:52:32 -07:00
parent e49d0b8d3a
commit 383cb4ca53
36 changed files with 824 additions and 597 deletions
Download Patch File Download Diff File Expand all files Collapse all files

312
node_modules/async-neocities/index.js generated vendored
View File

@@ -1,18 +1,41 @@
const { handleResponse } = require('fetch-errors')
const { createReadStream } = require('fs')
const afw = require('async-folder-walker')
const FormData = require('form-data')
const assert = require('nanoassert')
const fetch = require('node-fetch')
const { URL } = require('url')
const qs = require('qs')
const os = require('os')
const { createReadStream } = require('fs')
const FormData = require('form-data')
const { handleResponse } = require('fetch-errors')
const afw = require('async-folder-walker')
const pkg = require('./package.json')
const { neocitiesLocalDiff } = require('./lib/folder-diff')
const pkg = require('./package.json')
const SimpleTimer = require('./lib/timer')
const { getStreamLength, meterStream } = require('./lib/stream-meter')
const defaultURL = 'https://neocities.org'
// Progress API constants
const START = 'start'
const PROGRESS = 'progress' // progress updates
const STOP = 'stop'
// Progress stages
const INSPECTING = 'inspecting'
const DIFFING = 'diffing'
const APPLYING = 'applying'
/**
* NeocitiesAPIClient class representing a neocities api client.
*/
class NeocitiesAPIClient {
/**
* getKey returns an apiKey from a sitename and password.
* @param {String} sitename username/sitename to log into.
* @param {String} password password to log in with.
* @param {Object} [opts] Options object.
* @param {Object} [opts.url=https://neocities.org] Base URL to request to.
* @return {Promise<String>} An api key for the sitename..
*/
static getKey (sitename, password, opts) {
assert(sitename, 'must pass sitename as first arg')
assert(typeof sitename === 'string', 'user arg must be a string')
@@ -20,11 +43,11 @@ class NeocitiesAPIClient {
assert(typeof password, 'password arg must be a string')
opts = Object.assign({
baseURL: defaultURL
url: defaultURL
}, opts)
const baseURL = opts.baseURL
delete opts.baseURL
const baseURL = opts.url
delete opts.url
const url = new URL('/api/key', baseURL)
url.username = sitename
@@ -32,6 +55,13 @@ class NeocitiesAPIClient {
return fetch(url, opts)
}
/**
* Create an async-neocities api client.
* @param {string} apiKey An apiKey to make requests with.
* @param {Object} [opts] Options object.
* @param {Object} [opts.url=https://neocities.org] Base URL to make requests to.
* @return {Object} An api client instance.
*/
constructor (apiKey, opts) {
assert(apiKey, 'must pass apiKey as first argument')
assert(typeof apiKey === 'string', 'apiKey must be a string')
@@ -39,7 +69,6 @@ class NeocitiesAPIClient {
url: defaultURL
})
this.opts = opts
this.url = opts.url
this.apiKey = apiKey
}
@@ -48,12 +77,18 @@ class NeocitiesAPIClient {
return {
Authorization: `Bearer ${this.apiKey}`,
Accept: 'application/json',
'User-Agent': `deploy-to-neocities/${pkg.version} (${os.type()})`
'User-Agent': `async-neocities/${pkg.version} (${os.type()})`
}
}
/**
* Generic get request to neocities
* Generic GET request to neocities.
* @param {String} endpoint An endpoint path to GET request.
* @param {Object} [quieries] An object that gets added to the request in the form of a query string.
* @param {Object} [opts] Options object.
* @param {String} [opts.method=GET] The http method to use.
* @param {Object} [opts.headers] Headers to include in the request.
* @return {Object} The parsed JSON from the request response.
*/
get (endpoint, quieries, opts) {
assert(endpoint, 'must pass endpoint as first argument')
@@ -70,19 +105,46 @@ class NeocitiesAPIClient {
}
/**
* Generic post request to neocities
* Low level POST request to neocities with FormData.
* @param {String} endpoint The endpoint to make the request to.
* @param {Array.<{name: String, value: String}>} formEntries Array of form entries.
* @param {Object} [opts] Options object.
* @param {String} [opts.method=POST] HTTP Method.
* @param {Object} [opts.headers] Additional headers to send.
* @return {Object} The parsed JSON response object.
*/
post (endpoint, formEntries, opts) {
async post (endpoint, formEntries, opts) {
assert(endpoint, 'must pass endpoint as first argument')
const form = new FormData()
assert(formEntries, 'must pass formEntries as second argument')
function createForm () {
const form = new FormData()
for (const { name, value } of formEntries) {
form.append(name, value)
}
return form
}
opts = Object.assign({
method: 'POST',
body: form
statsCb: () => {}
}, opts)
const statsCb = opts.statsCb
delete opts.statsCb
for (const { name, value } of formEntries) {
form.append(name, value)
const stats = {
totalBytes: await getStreamLength(createForm()),
bytesWritten: 0
}
statsCb(stats)
const form = createForm()
opts.body = meterStream(form, bytesRead => {
stats.bytesWritten = bytesRead
statsCb(stats)
})
opts.headers = Object.assign(
{},
@@ -97,7 +159,11 @@ class NeocitiesAPIClient {
/**
* Upload files to neocities
*/
upload (files) {
upload (files, opts = {}) {
opts = {
statsCb: () => {},
...opts
}
const formEntries = files.map(({ name, path }) => {
const streamCtor = (next) => next(createReadStream(path))
streamCtor.path = path
@@ -107,22 +173,26 @@ class NeocitiesAPIClient {
}
})
return this.post('upload', formEntries).then(handleResponse)
return this.post('upload', formEntries, { statsCb: opts.statsCb }).then(handleResponse)
}
/**
* delete files from your website
*/
delete (filenames) {
delete (filenames, opts = {}) {
assert(filenames, 'filenames is a required first argument')
assert(Array.isArray(filenames), 'filenames argument must be an array of file paths in your website')
opts = {
statsCb: () => {},
...opts
}
const formEntries = filenames.map(file => ({
name: 'filenames[]',
value: file
}))
return this.post('delete', formEntries).then(handleResponse)
return this.post('delete', formEntries, { statsCb: opts.statsCb }).then(handleResponse)
}
list (queries) {
@@ -141,34 +211,210 @@ class NeocitiesAPIClient {
}
/**
* Deploy a folder to neocities, skipping already uploaded files and optionally cleaning orphaned files.
* @param {String} folder The path of the folder to deploy.
* Deploy a directory to neocities, skipping already uploaded files and optionally cleaning orphaned files.
* @param {String} directory The path of the directory to deploy.
* @param {Object} opts Options object.
* @param {Boolean} opts.cleanup Boolean to delete orphaned files nor not. Defaults to false.
* @param {Boolean} opts.statsCb Get access to stat info before uploading is complete.
* @return {Promise} Promise containing stats about the deploy
*/
async deploy (folder, opts) {
async deploy (directory, opts) {
opts = {
cleanup: false, // delete remote orphaned files
statsCb: () => {},
...opts
}
const [localFiles, remoteFiles] = await Promise.all([
afw.allFiles(folder, { shaper: f => f }),
this.list()
])
const statsCb = opts.statsCb
const startDeployTime = Date.now()
const totalTime = new SimpleTimer(startDeployTime)
// Inspection stage stats initializer
const inspectionStats = {
stage: INSPECTING,
status: START,
timer: new SimpleTimer(startDeployTime),
totalTime,
tasks: {
localScan: {
numberOfFiles: 0,
totalSize: 0,
timer: new SimpleTimer(startDeployTime)
},
remoteScan: {
numberOfFiles: 0,
totalSize: 0,
timer: new SimpleTimer(startDeployTime)
}
}
}
const sendInspectionUpdate = (status) => {
if (status) inspectionStats.status = status
statsCb(inspectionStats)
}
sendInspectionUpdate(START)
// Remote scan timers
const remoteScanJob = this.list()
remoteScanJob.then(({ files }) => { // Comes in the form of a response object
const { tasks: { remoteScan } } = inspectionStats
remoteScan.numberOfFiles = files.length
remoteScan.totalSize = files.reduce((accum, cur) => {
return accum + cur.size || 0
}, 0)
remoteScan.timer.stop()
sendInspectionUpdate(PROGRESS)
})
// Local scan timers and progress accumulator
const localScanJob = progressAccum(
afw.asyncFolderWalker(directory, { shaper: f => f })
)
async function progressAccum (iterator) {
const localFiles = []
const { tasks: { localScan } } = inspectionStats
for await (const file of iterator) {
localFiles.push(file)
localScan.numberOfFiles += 1
localScan.totalSize += file.stat.blksize
sendInspectionUpdate(PROGRESS)
}
return localFiles
}
localScanJob.then(files => {
const { tasks: { localScan } } = inspectionStats
localScan.timer.stop()
sendInspectionUpdate(PROGRESS)
})
// Inspection stage finalizer
const [localFiles, remoteFiles] = await Promise.all([
localScanJob,
this.list().then(res => res.files)
])
inspectionStats.timer.stop()
sendInspectionUpdate(STOP)
// DIFFING STAGE
const diffingStats = {
stage: DIFFING,
status: START,
timer: new SimpleTimer(Date.now()),
totalTime,
tasks: {
diffing: {
uploadCount: 0,
deleteCount: 0,
skipCount: 0
}
}
}
statsCb(diffingStats)
const { tasks: { diffing } } = diffingStats
const { filesToUpload, filesToDelete, filesSkipped } = await neocitiesLocalDiff(remoteFiles, localFiles)
diffingStats.timer.stop()
diffingStats.status = STOP
diffing.uploadCount = filesToUpload.length
diffing.deleteCount = filesToDelete.length
diffing.skipCount = filesSkipped.length
statsCb(diffingStats)
const applyingStartTime = Date.now()
const applyingStats = {
stage: APPLYING,
status: START,
timer: new SimpleTimer(applyingStartTime),
totalTime,
tasks: {
uploadFiles: {
timer: new SimpleTimer(applyingStartTime),
bytesWritten: 0,
totalBytes: 0,
get percent () {
return this.totalBytes === 0 ? 0 : this.bytesWritten / this.totalBytes
},
get speed () {
return this.bytesWritten / this.timer.elapsed
}
},
deleteFiles: {
timer: new SimpleTimer(applyingStartTime),
bytesWritten: 0,
totalBytes: 0,
get percent () {
return this.totalBytes === 0 ? 0 : this.bytesWritten / this.totalBytes
},
get speed () {
return this.bytesWritten / this.timer.elapsed
}
},
skippedFiles: {
count: filesSkipped.length,
size: filesSkipped.reduce((accum, file) => accum + file.stat.blksize, 0)
}
}
}
const sendApplyingUpdate = (status) => {
if (status) applyingStats.status = status
statsCb(applyingStats)
}
sendApplyingUpdate(START)
const { filesToUpload, filesToDelete, filesSkipped } = await neocitiesLocalDiff(remoteFiles.files, localFiles)
opts.statsCb({ filesToUpload, filesToDelete, filesSkipped })
const work = []
if (filesToUpload.length > 0) work.push(this.upload(filesToUpload))
if (opts.cleanup && filesToDelete.length > 0) { work.push(this.delete(filesToDelete)) }
const { tasks: { uploadFiles, deleteFiles } } = applyingStats
if (filesToUpload.length > 0) {
const uploadJob = this.upload(filesToUpload, {
statsCb: ({ bytesWritten, totalBytes }) => {
uploadFiles.bytesWritten = bytesWritten
uploadFiles.totalBytes = totalBytes
sendApplyingUpdate(PROGRESS)
}
})
work.push(uploadJob)
uploadJob.then(res => {
uploadFiles.timer.stop()
sendApplyingUpdate(PROGRESS)
})
} else {
uploadFiles.timer.stop()
}
if (opts.cleanup && filesToDelete.length > 0) {
const deleteJob = this.delete(filesToDelete, {
statsCb: ({ bytesWritten, totalBytes }) => {
deleteFiles.bytesWritten = bytesWritten
deleteFiles.totalBytes = totalBytes
sendApplyingUpdate(PROGRESS)
}
})
work.push(deleteJob)
deleteJob.then(res => {
deleteFiles.timer.stop()
sendApplyingUpdate(PROGRESS)
})
} else {
deleteFiles.timer.stop()
}
await Promise.all(work)
applyingStats.timer.stop()
sendApplyingUpdate(STOP)
return { filesToUpload, filesToDelete, filesSkipped }
totalTime.stop()
const statsSummary = {
time: totalTime,
inspectionStats,
diffingStats,
applyingStats
}
return statsSummary
}
}
module.exports = NeocitiesAPIClient