Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add automated health checks #733

Merged
merged 7 commits into from
May 5, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,24 @@ Add your Check UUID to the relevant network in your `networks.local.json` config
}
```

If you wish for your health checks to be created automatically, you can provide an [API key](https://healthchecks.io/docs/api/) to manage it for you.
The default behavior is for the check to be named after the network, but this can be overridden with the `name` property.

**Note that the `uuid` is not necessary when using an `apiKey` to create checks automatically.**

```JSON
{
"cheqd": {
"healthCheck": {
"apiKey": "12_CanM1Q3T72uGH4kc32G14BdA4Emc4y",
"name": "cheqd every 12 hours", // optional, defaults to the network name
"timeout": 43200, // optional, expected seconds between each run
"gracePeriod": 86400, // optional, grace period seconds before it notifies
}
}
}
```

### Submitting your operator

#### Setup your REStake operator
Expand Down
60 changes: 47 additions & 13 deletions src/autostake/Health.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,51 +4,85 @@ import { timeStamp } from '../utils/Helpers.mjs'

class Health {
constructor(config, opts) {
const { address, uuid } = config || {}
const { dryRun } = opts || {}
const { address, uuid, name, apiKey, timeout, gracePeriod } = config || {}
const { dryRun, networkName } = opts || {}
this.address = address || 'https://hc-ping.com'
this.name = name || networkName
this.gracePeriod = gracePeriod || 86400 // default 24 hours
this.timeout = timeout || 86400 // default 24 hours
this.uuid = uuid
this.apiKey = apiKey
this.dryRun = dryRun
this.logs = []
this.getOrCreateHealthCheck()

if (address) {
// This is necessary as the default provider - hc-ping.com - has a built in ping mechanism
// whereas providing self-hosted addresses do NOT.
// https://healthchecks.selfhosted.com/ping/{uuid} rather than https://hc-ping.com/{uuid}
this.address = this.address + "/ping"
}
}

started(...args){
started(...args) {
timeStamp(...args)
if(this.uuid) timeStamp('Starting health', [this.address, this.uuid].join('/'))
if (this.uuid) timeStamp('Starting health', [this.address, this.uuid].join('/'))
return this.ping('start', [args.join(' ')])
}

success(...args){
success(...args) {
timeStamp(...args)
return this.ping(undefined, [...this.logs, args.join(' ')])
}

failed(...args){
failed(...args) {
timeStamp(...args)
return this.ping('fail', [...this.logs, args.join(' ')])
}

log(...args){
log(...args) {
timeStamp(...args)
this.logs = [...this.logs, args.join(' ')]
}

addLogs(logs){
addLogs(logs) {
this.logs = this.logs.concat(logs)
}

async sendLog(){
async getOrCreateHealthCheck(...args) {
if (!this.apiKey) return;

let config = {
headers: {
"X-Api-Key": this.apiKey,
}
}

let data = {
"name": this.name, "channels": "*", "timeout": this.timeout, "grace": this.gracePeriod, "unique": ["name"]
}

try {
await axios.post([this.address, 'api/v2/checks/'].join('/'), data, config).then((res) => {
this.uuid = res.data.ping_url.split('/')[4]
});
} catch (error) {
timeStamp("Health Check creation failed: " + error)
}
}

async sendLog() {
await this.ping('log', this.logs)
this.logs = []
}

async ping(action, logs){
if(!this.uuid) return
if(this.dryRun) return timeStamp('DRYRUN: Skipping health check ping')
async ping(action, logs) {
if (!this.uuid) return
if (this.dryRun) return timeStamp('DRYRUN: Skipping health check ping')

return axios.request({
method: 'POST',
url: _.compact([this.address, this.uuid, action]).join('/'),
url: _.compact([this.address, this.uuid, action]).join('/'),
data: logs.join("\n")
}).catch(error => {
timeStamp('Health ping failed', error.message)
Expand Down
2 changes: 1 addition & 1 deletion src/autostake/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export default function Autostake(mnemonic, opts) {
if (networkNames && networkNames.length && !networkNames.includes(data.name)) return
if (data.enabled === false) return

const health = new Health(data.healthCheck, { dryRun: opts.dryRun })
const health = new Health(data.healthCheck, { dryRun: opts.dryRun, networkName: data.name })
health.started('⚛')
const results = await runWithRetry(data, health)
const { success, skipped } = results[results.length - 1] || {}
Expand Down