Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Commit

Permalink
fix(lnd): ensure graceful shutdown
Browse files Browse the repository at this point in the history 
Lnd doesn't currently do a graceful shutdown when a `SIGTERM` is issued.
To ensure a graceful shutdown, call `stopDaemon` on the gRPC interface
prior to terminating lnd. If a graceful shutdown is unsuccessful then
force kill the daemon.
  • Loading branch information
@mrfelton
mrfelton committed Aug 27, 2018
1 parent ae304e2 commit a16fdec
Show file tree
Hide file tree
Showing 4 changed files with 159 additions and 70 deletions.
69 changes: 60 additions & 9 deletions app/lib/lnd/lightning.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import grpc from 'grpc'
import { loadSync } from '@grpc/proto-loader'
import { BrowserWindow } from 'electron'
import StateMachine from 'javascript-state-machine'
import LndConfig from './config'
import { getDeadline, validateHost, createSslCreds, createMacaroonCreds } from './util'
import methods from './methods'
Expand All @@ -25,24 +26,43 @@ type LightningSubscriptionsType = {
class Lightning {
mainWindow: BrowserWindow
lnd: any
lndConfig: LndConfig
subscriptions: LightningSubscriptionsType
_fsm: StateMachine

constructor() {
// Transitions provided by the state machine.
connect: any
disconnect: any
terminate: any
is: any
can: any
state: string

constructor(lndConfig: LndConfig) {
this.mainWindow = null
this.lnd = null
this.lndConfig = lndConfig
this.subscriptions = {
channelGraph: null,
invoices: null,
transactions: null
}

// Initialize the state machine.
this._fsm()
}

// ------------------------------------
// FSM Callbacks
// ------------------------------------

/**
* Connect to the gRPC interface and verify it is functional.
* @return {Promise<rpc.lnrpc.Lightning>}
*/
async connect(lndConfig: LndConfig) {
const { rpcProtoPath, host, cert, macaroon } = lndConfig
async onBeforeConnect() {
mainLog.info('Connecting to Lightning gRPC service')
const { rpcProtoPath, host, cert, macaroon } = this.lndConfig

// Verify that the host is valid before creating a gRPC client that is connected to it.
return await validateHost(host).then(async () => {
Expand All @@ -69,16 +89,15 @@ class Lightning {
const credentials = grpc.credentials.combineChannelCredentials(sslCreds, macaroonCreds)

// Create a new gRPC client instance.
const lnd = new rpc.lnrpc.Lightning(host, credentials)
this.lnd = new rpc.lnrpc.Lightning(host, credentials)

// Call the getInfo method to ensure that we can make successful calls to the gRPC interface.
// Wait for the gRPC connection to be established.
return new Promise((resolve, reject) => {
lnd.getInfo({}, { deadline: getDeadline(2) }, err => {
grpc.waitForClientReady(this.lnd, getDeadline(2), err => {
if (err) {
return reject(err)
}
this.lnd = lnd
return resolve(lnd)
return resolve()
})
})
})
Expand All @@ -87,13 +106,34 @@ class Lightning {
/**
* Discomnnect the gRPC service.
*/
disconnect() {
onBeforeDisconnect() {
mainLog.info('Disconnecting from Lightning gRPC service')
this.unsubscribe()
if (this.lnd) {
this.lnd.close()
}
}

/**
* Gracefully shutdown the gRPC service.
*/
async onBeforeTerminate() {
mainLog.info('Shutting down Lightning daemon')
this.unsubscribe()
return new Promise((resolve, reject) => {
this.lnd.stopDaemon({}, (err, data) => {
if (err) {
return reject(err)
}
resolve(data)
})
})
}

// ------------------------------------
// Helpers
// ------------------------------------

/**
* Hook up lnd restful methods.
*/
Expand All @@ -105,6 +145,7 @@ class Lightning {
* Subscribe to all bi-directional streams.
*/
subscribe(mainWindow: BrowserWindow) {
mainLog.info('Subscribing to Lightning gRPC streams')
this.mainWindow = mainWindow

this.subscriptions.channelGraph = subscribeToChannelGraph.call(this)
Expand All @@ -116,6 +157,7 @@ class Lightning {
* Unsubscribe from all bi-directional streams.
*/
unsubscribe() {
mainLog.info('Unsubscribing from Lightning gRPC streams')
this.mainWindow = null
Object.keys(this.subscriptions).forEach(subscription => {
if (this.subscriptions[subscription]) {
Expand All @@ -126,4 +168,13 @@ class Lightning {
}
}

StateMachine.factory(Lightning, {
init: 'ready',
transitions: [
{ name: 'connect', from: 'ready', to: 'connected' },
{ name: 'disconnect', from: 'connected', to: 'ready' },
{ name: 'terminate', from: 'connected', to: 'ready' }
]
})

export default Lightning
8 changes: 6 additions & 2 deletions app/lib/lnd/neutrino.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,10 @@ class Neutrino extends EventEmitter {

if (this.is(CHAIN_SYNC_PENDING) || this.is(CHAIN_SYNC_IN_PROGRESS)) {
// If we cant get a connectionn to the backend.
if (line.includes('Waiting for chain backend to finish sync')) {
if (
line.includes('Waiting for chain backend to finish sync') ||
line.includes('Waiting for block headers to sync, then will start cfheaders sync')
) {
this.setState(CHAIN_SYNC_WAITING)
}
// If we are still waiting for the back end to finish synncing.
Expand Down Expand Up @@ -223,8 +226,9 @@ class Neutrino extends EventEmitter {
/**
* Stop the Lnd process.
*/
stop() {
kill() {
if (this.process) {
mainLog.info('Killing Neutrino process...')
this.process.kill()
this.process = null
}
Expand Down
148 changes: 91 additions & 57 deletions app/lib/zap/controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,9 @@ const grpcSslCipherSuites = connectionType =>
*/
class ZapController {
mainWindow: BrowserWindow
neutrino: any
lightning: any
neutrino: Neutrino
lightning: Lightning
splashScreenTime: number
lightningGrpcConnected: boolean
lndConfig: LndConfig
_fsm: StateMachine

Expand All @@ -73,18 +72,9 @@ class ZapController {
// Variable to hold the main window instance.
this.mainWindow = mainWindow

// Keep a reference any neutrino process started by us.
this.neutrino = undefined

// Keep a reference to the lightning gRPC instance.
this.lightning = undefined

// Time for the splash screen to remain visible.
this.splashScreenTime = 500

// Boolean indicating wether the lightning grpc is connected ot not.
this.lightningGrpcConnected = false

// Initialize the state machine.
this._fsm()

Expand Down Expand Up @@ -127,7 +117,7 @@ class ZapController {
// FSM Callbacks
// ------------------------------------

onOnboarding() {
async onOnboarding(lifecycle: any) {
mainLog.debug('[FSM] onOnboarding...')

// Remove any existing IPC listeners so that we can start fresh.
Expand All @@ -136,12 +126,14 @@ class ZapController {
// Register IPC listeners so that we can react to instructions coming from the app.
this._registerIpcListeners()

// Ensure wallet is disconnected.
this.disconnectLightningWallet()
// Disconnect any pre-existing lightning wallet connection.
if (lifecycle.from === 'connected' && this.lightning && this.lightning.can('disconnect')) {
this.lightning.disconnect()
}

// If Neutrino is running, kill it.
if (this.neutrino) {
this.neutrino.stop()
// If we are comming from a running state, stop the Neutrino process.
else if (lifecycle.from === 'running') {
await this.shutdownNeutrino()
}

// Give the grpc connections a chance to be properly closed out.
Expand Down Expand Up @@ -209,15 +201,16 @@ class ZapController {
})
}

onTerminated() {
async onTerminated(lifecycle: any) {
mainLog.debug('[FSM] onTerminated...')
// Unsubscribe the gRPC streams before thhe window closes. This ensures that we can properly reestablish a fresh
// connection when a new window is opened.
this.disconnectLightningWallet()

// If Neutrino is running, kill it.
if (this.neutrino) {
this.neutrino.stop()
// Disconnect from any existing lightning wallet connection.
if (lifecycle.from === 'connected' && this.lightning && this.lightning.can('disconnect')) {
this.lightning.disconnect()
}
// If we are comming from a running state, stop the Neutrino process.
else if (lifecycle.from === 'running') {
await this.shutdownNeutrino()
}
}

Expand Down Expand Up @@ -251,7 +244,7 @@ class ZapController {
* Start the wallet unlocker.
*/
startWalletUnlocker() {
mainLog.info('Starting wallet unlocker...')
mainLog.info('Establishing connection to Wallet Unlocker gRPC interface...')
try {
const walletUnlockerMethods = initWalletUnlocker(this.lndConfig)

Expand All @@ -275,42 +268,23 @@ class ZapController {
* Create and subscribe to the Lightning service.
*/
async startLightningWallet() {
if (this.lightningGrpcConnected) {
return
}
mainLog.info('Starting lightning wallet...')
this.lightning = new Lightning()
mainLog.info('Establishing connection to Lightning gRPC interface...')
this.lightning = new Lightning(this.lndConfig)

// Connect to the Lightning interface.
await this.lightning.connect(this.lndConfig)

// Subscribe the main window to receive streams.
this.lightning.subscribe(this.mainWindow)

// Listen for all gRPC restful methods and pass to gRPC.
ipcMain.on('lnd', (event, { msg, data }) => this.lightning.lndMethods(event, msg, data))
try {
await this.lightning.connect()

// Let the renderer know that we are connected.
this.sendMessage('lightningGrpcActive')
this.lightning.subscribe(this.mainWindow)

// Update our internal state.
this.lightningGrpcConnected = true
}
// Listen for all gRPC restful methods and pass to gRPC.
ipcMain.on('lnd', (event, { msg, data }) => this.lightning.lndMethods(event, msg, data))

/**
* Unsubscribe from the Lightning service.
*/
disconnectLightningWallet() {
if (!this.lightningGrpcConnected) {
return
// Let the renderer know that we are connected.
this.sendMessage('lightningGrpcActive')
} catch (err) {
mainLog.warn('Unable to connect to Lighitnng gRPC interface: %o', err)
}
mainLog.info('Disconnecting lightning Wallet...')

// Disconnect streams.
this.lightning.disconnect()

// Update our internal state.
this.lightningGrpcConnected = false
}

/**
Expand All @@ -337,7 +311,7 @@ class ZapController {
if (this.is('running') || this.is('connected')) {
dialog.showMessageBox({
type: 'error',
message: `Lnd has unexpectadly quit: ${lastError}`
message: `Lnd has unexpectedly quit: ${lastError}`
})
this.terminate()
}
Expand Down Expand Up @@ -383,6 +357,66 @@ class ZapController {
this.neutrino.start()
}

/**
* Gracefully shutdown LND.
*/
async shutdownNeutrino() {
// We only want to shut down LND if we are running it locally.
if (this.lndConfig.type !== 'local') {
return Promise.resolve()
}

// Attempt a graceful shutdown if we can.
if (this.lightning && this.lightning.can('terminate')) {
mainLog.info('Shutting down Neutrino...')

return new Promise(resolve => {
// HACK: Sometimes there are errors during the shutdown process that prevent the daeming from shutting down at
// all. If we haven't received notification of the process closing within 10 seconds, kill it.
// See https://github.com/lightningnetwork/lnd/pull/1781
// See https://github.com/lightningnetwork/lnd/pull/1783
const shutdownTimeout = setTimeout(() => {
this.neutrino.removeListener('close', closeHandler)
if (this.neutrino) {
mainLog.warn('Graceful shutdown failed to complete within 30 seconds.')
this.neutrino.kill()
resolve()
}
}, 1000 * 10)

// HACK: The Lightning.stopDaemon` call returns before lnd has actually fully completed the shutdown process
// so we add a listener on the close event so that we can wrap things up once the process has been fully closed
// out.
const closeHandler = function() {
mainLog.info('Neutrino shutdown complete.')
clearTimeout(shutdownTimeout)
resolve()
}
this.neutrino.once('close', closeHandler)

this.lightning
.terminate()
.then(() => mainLog.info('Neutrino Daemon shutdown complete'))
.catch(err => {
mainLog.error('Unable to gracefully shutdown LND: %o', err)
// Kill the process ourselves here to ensure that we don't leave hanging processes.
if (this.neutrino) {
this.neutrino.kill()
resolve()
}
})
})
}

// The Lightning service is only active after the wallet has been unlocked and a gRPC connection has been
// established. In this case, kill the Neutrino process to ensure that we don't leave hanging process.
// FIXME: This currencly doesn't do a graceful shutdown as LND does not properly handle SIGTERM.
// See https://github.com/lightningnetwork/lnd/issues/1028
else if (this.neutrino) {
this.neutrino.kill()
}
}

finishOnboarding(options: onboardingOptions) {
mainLog.info('Finishing onboarding')
// Save the lnd config options that we got from the renderer.
Expand Down
Loading

0 comments on commit a16fdec

Please sign in to comment.