diff --git a/.jshintrc b/.jshintrc index 6343d28..606ba7b 100644 --- a/.jshintrc +++ b/.jshintrc @@ -24,7 +24,7 @@ "undef": true, "unused": true, "maxparams": 4, - "maxstatements": 14, + "maxstatements": 15, "maxcomplexity": 6, "maxdepth": 3, "maxlen": 80, diff --git a/README.md b/README.md index 5f033c3..1dee378 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Make sure you have the following prerequisites installed: wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.33.0/install.sh | bash ``` -Close your shell and open an new one. Now that you can call the `nvm` program, +Close your shell and open an new one. Now that you can call the `nvm` program, install Node.js (which comes with NPM): ``` @@ -93,7 +93,7 @@ npm install --global windows-build-tools --- -Once build dependencies have been installed for your platform, install the +Once build dependencies have been installed for your platform, install the package globally using Node Package Manager: ``` @@ -102,7 +102,7 @@ npm install --global storjshare-daemon ## Usage (CLI) -Once installed, you will have access to the `storjshare` program, so start by +Once installed, you will have access to the `storjshare` program, so start by asking it for some help. ``` @@ -152,16 +152,16 @@ storjshare help create --tunnelportmin specify min gateway port --tunnelportmax specify max gateway port --manualforwarding do not use nat traversal strategies - --logfile specify the logfile path + --logdir specify the log directory --noedit do not open generated config in editor -o, --outfile write config to path ``` ## Usage (Programmatic) -The Storj Share daemon uses a local [dnode](https://github.com/substack/dnode) -server to handle RPC message from the CLI and other applications. Assuming the -daemon is running, your program can communicate with it using this interface. +The Storj Share daemon uses a local [dnode](https://github.com/substack/dnode) +server to handle RPC message from the CLI and other applications. Assuming the +daemon is running, your program can communicate with it using this interface. The example that follows is using Node.js, but dnode is implemented in many [other languages](https://github.com/substack/dnode#dnode-in-other-languages). @@ -181,7 +181,7 @@ daemon.on('remote', (rpc) => { }); ``` -You can also easily start the daemon from your program by creating a dnode +You can also easily start the daemon from your program by creating a dnode server and passing it an instance of the `RPC` class exposed from this package. ```js @@ -194,10 +194,10 @@ dnode(api.methods).listen(45015, '127.0.0.1'); ## Configuring the Daemon -The Storj Share daemon loads configuration from anywhere the -[rc](https://www.npmjs.com/package/rc) package can read it. The first time you -run the daemon, it will create a directory in `$HOME/.config/storjshare`, so -the simplest way to change the daemon's behavior is to create a file at +The Storj Share daemon loads configuration from anywhere the +[rc](https://www.npmjs.com/package/rc) package can read it. The first time you +run the daemon, it will create a directory in `$HOME/.config/storjshare`, so +the simplest way to change the daemon's behavior is to create a file at `$HOME/.config/storjshare/config` containing the following: ```json @@ -209,15 +209,15 @@ the simplest way to change the daemon's behavior is to create a file at } ``` -Modify these parameters to your liking, see `example/daemon.config.json` for +Modify these parameters to your liking, see `example/daemon.config.json` for detailed explanation of these properties. ## Debugging the Daemon -The daemon logs activity to the configured log file, which by default is -`$HOME/.config/storjshare/logs/daemon.log`. However if you find yourself -needing to frequently restart the daemon and check the logs during -development, you can run the daemon as a foreground process for a tighter +The daemon logs activity to the configured log file, which by default is +`$HOME/.config/storjshare/logs/daemon.log`. However if you find yourself +needing to frequently restart the daemon and check the logs during +development, you can run the daemon as a foreground process for a tighter feedback loop. ``` @@ -233,13 +233,13 @@ choose to migrate from the old storjshare-gui to the CLI version of storjshare-daemon, please follow the instructions below. #### storjshare-cli -Storj Share provides a simple method for creating new shares, but if you were -previously using the `storjshare-cli` package superceded by this one, you'll -want to migrate your configuration to the new format. To do this, first you'll +Storj Share provides a simple method for creating new shares, but if you were +previously using the `storjshare-cli` package superceded by this one, you'll +want to migrate your configuration to the new format. To do this, first you'll need to dump your private key **before** installing this package. -> If you accidentally overwrote your old `storjshare-cli` installation with -> this package, don't worry - just reinstall the old package to dump the key, +> If you accidentally overwrote your old `storjshare-cli` installation with +> this package, don't worry - just reinstall the old package to dump the key, > then reinstall this package. ### Step 0: Dump Your Private Key @@ -270,7 +270,7 @@ information and go on with Step 1 and 2. ``` #### storjshare-cli -You can print your cleartext private key from storjshare-cli, using the +You can print your cleartext private key from storjshare-cli, using the `dump-key` command: ``` @@ -285,15 +285,15 @@ storjshare dump-key ``` If you are using a custom data directory, be sure to add the `--datadir ` -option to be sure you get the correct key. Also be sure to note your defined +option to be sure you get the correct key. Also be sure to note your defined payout address and data directory. ### Step 1: Install Storj Share and Create Config -Now that you have your private key, you can generate a new configuration file. -To do this, first install the `storjshare-daemon` package globally and use the -`create` command. You'll need to remove the `storjshare-cli` package first, so -make sure you perform the previous step for all shared drives before +Now that you have your private key, you can generate a new configuration file. +To do this, first install the `storjshare-daemon` package globally and use the +`create` command. You'll need to remove the `storjshare-cli` package first, so +make sure you perform the previous step for all shared drives before proceeding forward. ``` @@ -301,21 +301,21 @@ npm remove -g storjshare-cli npm install -g storjshare-daemon ``` -Now that you have Storj Share installed, use the `create` command to generate +Now that you have Storj Share installed, use the `create` command to generate your configuration. ``` storjshare create --key 4154e8... --sjcx 1K1rPg... --storage -o ``` -This will generate your configuration file given the parameters you passed in, -write the file to the path following the `-o` option, and open it in your text -editor. Here, you can make other changes to the configuration following the +This will generate your configuration file given the parameters you passed in, +write the file to the path following the `-o` option, and open it in your text +editor. Here, you can make other changes to the configuration following the detailed comments in the generated file. ### Step 2: Use The New Configuration -Now that you have successfully migrated your configuration file, you can use +Now that you have successfully migrated your configuration file, you can use it to start the share. ``` @@ -343,4 +343,3 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see http://www.gnu.org/licenses/. - diff --git a/bin/storjshare-create.js b/bin/storjshare-create.js index a3f15d9..b1c2932 100755 --- a/bin/storjshare-create.js +++ b/bin/storjshare-create.js @@ -39,7 +39,7 @@ storjshare_create .option('--tunnelportmin ', 'specify min gateway port') .option('--tunnelportmax ', 'specify max gateway port') .option('--manualforwarding', 'do not use nat traversal strategies') - .option('--logfile ', 'specify the logfile path') + .option('--logdir ', 'specify the log directory') .option('--noedit', 'do not open generated config in editor') .option('-o, --outfile ', 'write config to path') .parse(process.argv); @@ -62,14 +62,6 @@ if (!storjshare_create.storage) { mkdirp.sync(storjshare_create.storage); } -if (!storjshare_create.logfile) { - storjshare_create.logfile = path.join( - homedir(), - '.config/storjshare/logs', - storj.KeyPair(storjshare_create.key).getNodeID() + '.log' - ); -} - if (!storjshare_create.outfile) { storjshare_create.outfile = path.join( homedir(), @@ -78,6 +70,13 @@ if (!storjshare_create.outfile) { ); } +if (!storjshare_create.logdir) { + storjshare_create.logdir = path.join( + homedir(), + '.config/storjshare/logs' + ); +} + let exampleConfigPath = path.join(__dirname, '../example/farmer.config.json'); let exampleConfigString = fs.readFileSync(exampleConfigPath).toString(); @@ -116,7 +115,7 @@ replaceDefaultConfigValue('networkPrivateKey', storjshare_create.key); replaceDefaultConfigValue('storagePath', path.normalize(storjshare_create.storage)); replaceDefaultConfigValue('loggerOutputFile', - path.normalize(storjshare_create.logfile)); + path.normalize(storjshare_create.logdir)); const optionalReplacements = [ { option: storjshare_create.size, name: 'storageAllocation' }, diff --git a/bin/storjshare-daemon.js b/bin/storjshare-daemon.js index 89ccfc6..cc28151 100755 --- a/bin/storjshare-daemon.js +++ b/bin/storjshare-daemon.js @@ -22,7 +22,7 @@ const api = new RPC({ function startDaemonRpcServer() { dnode(api.methods) - .on('error', (err) => api.logger.warn(err.message)) + .on('error', (err) => api.jsonlogger.warn(err.message)) .listen(config.daemonRpcPort, config.daemonRpcAddress); } @@ -35,12 +35,12 @@ utils.checkDaemonRpcStatus(config.daemonRpcPort, (isRunning) => { } else { if (storjshare_daemon.foreground) { console.info('\n * starting daemon in foreground\n'); - api.logger.pipe(process.stdout); + api.jsonlogger.pipe(process.stdout); startDaemonRpcServer(); } else { console.info('\n * starting daemon in background'); daemonize(); - api.logger.pipe(logFile); + api.jsonlogger.pipe(logFile); startDaemonRpcServer(); } } diff --git a/bin/storjshare-logs.js b/bin/storjshare-logs.js index 907b602..fc77547 100755 --- a/bin/storjshare-logs.js +++ b/bin/storjshare-logs.js @@ -8,6 +8,7 @@ const {Tail} = require('tail'); const colors = require('colors/safe'); const storjshare_logs = require('commander'); const fs = require('fs'); +const FsLogger = require('fslogger'); storjshare_logs .description('tails the logs for the given share id') @@ -78,28 +79,44 @@ utils.connectToDaemon(config.daemonRpcPort, function(rpc, sock) { return sock.end(); } - let logFilePath = null; + let logFileDir = null; for (let i = 0; i < shares.length; i++) { if (shares[i].id === storjshare_logs.nodeid) { - logFilePath = shares[i].config.loggerOutputFile; + logFileDir = shares[i].config.loggerOutputFile; break; } } - if (!utils.existsSync(logFilePath)) { - console.error(`\n no logs to show for ${storjshare_logs.nodeid}`); - return sock.end(); - } + const fslogger = new FsLogger(logFileDir, storjshare_logs.nodeid); - let logTail = new Tail(logFilePath); - let numLines = storjshare_logs.lines - ? parseInt(storjshare_logs.lines) - : 20; + let currentFile = null; + let logTail = null; + + setInterval(function() { + if (currentFile !== fslogger._todaysFile()) { + if (logTail instanceof Tail) { + logTail.unwatch(); + } + + currentFile = fslogger._todaysFile(); + if (!utils.existsSync(fslogger._todaysFile())) { + console.error(`\n no logs to show for ${storjshare_logs.nodeid}`); + return sock.end(); + } + + logTail = new Tail(fslogger._todaysFile()); + let numLines = storjshare_logs.lines + ? parseInt(storjshare_logs.lines) + : 20; + + getLastLines(fslogger._todaysFile(), numLines, (lines) => { + lines.forEach((line) => prettyLog(line)); + logTail.on('line', (line) => prettyLog(line)); + }); + } + + }, 1000); - getLastLines(logFilePath, numLines, (lines) => { - lines.forEach((line) => prettyLog(line)); - logTail.on('line', (line) => prettyLog(line)); - }); }); }); diff --git a/example/farmer.config.json b/example/farmer.config.json index 628b38a..17a6bf8 100644 --- a/example/farmer.config.json +++ b/example/farmer.config.json @@ -16,11 +16,11 @@ // Known preferred seeds in form of a storj URI // Example: "storj://[ip.or.hostname]:[port]/[nodeid]" "seedList": [], - // Interface to bind RPC server, use 0.0.0.0 for all interfaces or if you + // Interface to bind RPC server, use 0.0.0.0 for all interfaces or if you // have a public address, use that, else leave 127.0.0.1 and Storj Share // will try to determine your address "rpcAddress": "127.0.0.1", - // Port to bind for RPC server, make sure this is forwarded if behind a + // Port to bind for RPC server, make sure this is forwarded if behind a // NAT or firewall - otherwise Storj Share will try to punch out "rpcPort": 4000, // Enables NAT traversal strategies, first UPnP, then reverse HTTP tunnel @@ -31,7 +31,7 @@ "maxTunnels": 3, // Maximum number of concurrent connections to allow "maxConnections": 150, - // If providing tunnels, the starting and ending port range to open for + // If providing tunnels, the starting and ending port range to open for // them "tunnelGatewayRange": { "min": 4001, @@ -42,7 +42,7 @@ "times": 3, "interval": 5000 }, - // Temporarily stop sending OFFER messages if more than this number of shard + // Temporarily stop sending OFFER messages if more than this number of shard // transfers are active "offerBackoffLimit": 4, // ECDSA private key for your network identity, your Node ID is derived from @@ -52,14 +52,14 @@ // 4 - DEBUG | 3 - INFO | 2 - WARN | 1 - ERROR | 0 - SILENT "loggerVerbosity": 3, // Path to write the log file to disk, leave empty to default to: - // $HOME/.config/storjshare/logs/[nodeid].log + // $HOME/.config/storjshare/logs/[nodeid]_date.log "loggerOutputFile": "", // Directory path to store contracts and shards "storagePath": "", // Amount of space to lease to the network, as human readable string // Valid units are B, KB, MB, GB, TB "storageAllocation": "2GB", - // Periodically report your used and free capacity to Storj Labs to improve + // Periodically report your used and free capacity to Storj Labs to improve // the network - no personally identifiable information is sent "enableTelemetryReporting": true } diff --git a/lib/api.js b/lib/api.js index e844dd8..1d9064e 100644 --- a/lib/api.js +++ b/lib/api.js @@ -5,11 +5,13 @@ const storj = require('storj-lib'); const fs = require('fs'); const {statSync, readFileSync} = require('fs'); const stripJsonComments = require('strip-json-comments'); +const FsLogger = require('fslogger'); const JsonLogger = require('kad-logger-json'); const {fork} = require('child_process'); const utils = require('./utils'); const path = require('path'); const { cpus } = require('os'); +const {homedir} = require('os'); /** Class representing a local RPC API's handlers */ class RPC { @@ -20,7 +22,7 @@ class RPC { * @param {Number} options.logVerbosity */ constructor(options={}) { - this.logger = new JsonLogger(options.logVerbosity); + this.jsonlogger = new JsonLogger(options.logVerbosity); this.shares = new Map(); } @@ -30,7 +32,7 @@ class RPC { * @param {String} level */ _log(msg, level='info') { - this.logger[level](msg); + this.jsonlogger[level](msg); } /** @@ -69,7 +71,6 @@ class RPC { } catch (err) { throw new Error(err.message.toLowerCase()); } - return config; } @@ -131,13 +132,18 @@ class RPC { ); share.readyState = RPC.SHARE_STARTED; - let logFile = fs.createWriteStream(share.config.loggerOutputFile, { - flags: 'a' + const fslogger = new FsLogger((!share.config.loggerOutputFile) ? + path.join(homedir(),'.config/storjshare/logs') : + share.config.loggerOutputFile, nodeId); + fslogger.setLogLevel(config.logVerbosity); + + share.process.stderr.on('data', function(data) { + fslogger.write(data); }); - // NB: Pipe the stdio to the configured log file - share.process.stdout.pipe(logFile); - share.process.stderr.pipe(logFile); + share.process.stdout.on('data', function(data) { + fslogger.write(data); + }); // NB: Listen for state changes to update the share's record share.process.on('error', (err) => { diff --git a/lib/config/farmer.js b/lib/config/farmer.js index 9b03209..1c5e712 100644 --- a/lib/config/farmer.js +++ b/lib/config/farmer.js @@ -1,7 +1,6 @@ 'use strict'; const mkdirp = require('mkdirp'); -const storj = require('storj-lib'); const path = require('path'); const {homedir} = require('os'); const datadir = path.join(homedir(), '.config', 'storjshare'); @@ -43,10 +42,8 @@ const config = require('rc')('storjfarmer', { enableTelemetryReporting: true }); -let nodeId = storj.KeyPair(config.networkPrivateKey).getNodeID(); - if (!config.loggerOutputFile) { - config.loggerOutputFile = path.join(datadir, 'logs', nodeId, '.log'); + config.loggerOutputFile = path.join(homedir(),'.config/storjshare/logs'); } module.exports = config; diff --git a/package.json b/package.json index 942b1e8..40ed55a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "storjshare-daemon", - "version": "2.5.4", + "version": "3.0.0", "description": "daemon + process manager for sharing space on the storj network", "main": "index.js", "bin": { @@ -66,6 +66,7 @@ "du": "^0.1.0", "editor": "^1.0.0", "fd-diskspace": "git+https://github.com/littleskunk/fd-diskspace#c339b792a3fa7b6bb3af6f004b9873f270ce729b", + "fslogger": "^2.0.1", "kad-logger-json": "^0.1.2", "mkdirp": "^0.5.1", "pretty-ms": "^2.1.0", diff --git a/test/api.unit.js b/test/api.unit.js index 20888e4..a6c16f5 100644 --- a/test/api.unit.js +++ b/test/api.unit.js @@ -13,7 +13,7 @@ describe('class:RPC', function() { it('should call the log method', function() { let rpc = new RPC({ loggerVerbosity: 0 }); - let info = sinon.stub(rpc.logger, 'info'); + let info = sinon.stub(rpc.jsonlogger, 'info'); rpc._log('test'); expect(info.called).to.equal(true); }); @@ -158,6 +158,9 @@ describe('class:RPC', function() { let _proc = new EventEmitter(); _proc.stdout = new Readable({ read: () => null }); _proc.stderr = new Readable({ read: () => null }); + var MockFsLogger = function(){}; + MockFsLogger.prototype.setLogLevel = sinon.stub(); + MockFsLogger.prototype.write = sinon.stub(); let _RPC = proxyquire('../lib/api', { fs: { createWriteStream: sinon.stub().returns(new Writable({ @@ -172,11 +175,12 @@ describe('class:RPC', function() { }, child_process: { fork: sinon.stub().returns(_proc) - } + }, + fslogger: MockFsLogger }); let rpc = new _RPC({ loggerVerbosity: 0 }); let _ipc = sinon.stub(rpc, '_processShareIpc'); - rpc.start('path/to/config', function() { + rpc.start('/tmp/', function() { let id = rpc.shares.keys().next().value; let share = rpc.shares.get(id); share.meta.uptimeMs = 6000; @@ -196,6 +200,49 @@ describe('class:RPC', function() { }); }); + it('should call fslogger.write on data', function(done) { + let _proc = new EventEmitter(); + _proc.stdout = new Readable({ read: () => null }); + _proc.stderr = new Readable({ read: () => null }); + var MockFsLogger = function(){}; + MockFsLogger.prototype.setLogLevel = sinon.stub(); + MockFsLogger.prototype.write = sinon.stub(); + let _RPC = proxyquire('../lib/api', { + fs: { + createWriteStream: sinon.stub().returns(new Writable({ + write: (d, e, cb) => cb() + })), + statSync: sinon.stub(), + readFileSync: sinon.stub().returns(Buffer.from('{}')) + }, + './utils': { + validate: sinon.stub(), + validateAllocation: sinon.stub().callsArg(1) + }, + child_process: { + fork: sinon.stub().returns(_proc) + }, + fslogger: MockFsLogger + }); + let rpc = new _RPC({ loggerVerbosity: 0 }); + rpc.start('/tmp/', function() { + let id = rpc.shares.keys().next().value; + let share = rpc.shares.get(id); + share.meta.uptimeMs = 6000; + _proc.stdout.emit('data', {}); + setImmediate(() => { + expect(MockFsLogger.prototype.write.called).to.equal(true); + MockFsLogger.prototype.write.called = false; + _proc.stderr.emit('data', {}); + setImmediate(() => { + expect(MockFsLogger.prototype.write.called).to.equal(true); + _proc.emit('error', new Error()); + done(); + }); + }); + }); + }); + }); describe('#stop', function() {