From 8d5a2d666fc0cf997dad9f26af705583f611b710 Mon Sep 17 00:00:00 2001 From: Franck Date: Mon, 22 Apr 2019 20:53:27 -0700 Subject: [PATCH] Add country column to identity table (#2083) * Add country column to identity table * tweak comments * Fix unit tests * linter * prettier * Make call to handleEvent synchronous --- .travis.yml | 1 + .../src/listener/handler_identity.js | 44 ++++++++++-- infra/discovery/src/listener/listener.js | 2 +- infra/discovery/test/listener-handler.test.js | 11 ++- .../scripts/oneoff/backfillIdentityCountry.js | 71 +++++++++++++++++++ .../migrations/20190420064537-add-country.js | 19 +++++ infra/identity/src/models/identity.js | 3 +- 7 files changed, 141 insertions(+), 10 deletions(-) create mode 100644 infra/growth/src/scripts/oneoff/backfillIdentityCountry.js create mode 100644 infra/identity/migrations/20190420064537-add-country.js diff --git a/.travis.yml b/.travis.yml index fa685daaec2f..f32237ebe8d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -78,6 +78,7 @@ matrix: - ATTESTATION_ACCOUNT=0x99C03fBb0C995ff1160133A8bd210D0E77bCD101 before_script: - psql -c 'create database travis_ci_test;' -U postgres + - lerna run migrate --scope @origin/bridge - lerna run migrate --scope @origin/discovery - lerna run migrate --scope @origin/growth - lerna run migrate --scope @origin/identity diff --git a/infra/discovery/src/listener/handler_identity.js b/infra/discovery/src/listener/handler_identity.js index e61a412e1faa..45059699d37a 100644 --- a/infra/discovery/src/listener/handler_identity.js +++ b/infra/discovery/src/listener/handler_identity.js @@ -14,6 +14,7 @@ const { AttestationServiceToEventType, GrowthEvent } = require('@origin/growth/src/resources/event') +const { ip2geo } = require('@origin/growth/src/util/ip2geo') class IdentityEventHandler { constructor(config, graphqlClient) { @@ -47,7 +48,8 @@ class IdentityEventHandler { } } - /* Get details about an account from @origin/graphql + /** + * Get details about an account from @origin/graphql * @param {String} account: eth address of account * @returns {Object} result of GraphQL query * @private @@ -94,6 +96,30 @@ class IdentityEventHandler { return attestation.value } + /** + * Returns the country of the identity based on IP from the most recent attestation. + * @param {string} ethAddress + * @returns {Promise || null} 2 letters country code or null if lookup failed. + * @private + */ + async _countryLookup(ethAddress) { + // Load the most recent attestation. + const attestation = await db.Attestation.findOne({ + where: { ethAddress: ethAddress.toLowerCase() }, + order: [['createdAt', 'DESC']] + }) + if (!attestation) { + return null + } + + // Do the IP to geo lookup. + const geo = await ip2geo(attestation.remoteIpAddress) + if (!geo) { + return null + } + return geo.countryCode + } + /** * Decorates an identity object with attestation data. * @param {{}} identity - result of identityQuery @@ -102,6 +128,8 @@ class IdentityEventHandler { */ async _decorateIdentity(identity) { const decoratedIdentity = Object.assign({}, identity) + + // Load attestation data. await Promise.all( decoratedIdentity.attestations.map(async attestationJson => { const attestation = JSON.parse(attestationJson) @@ -142,6 +170,10 @@ class IdentityEventHandler { } }) ) + + // Add country of origin information based on IP. + decoratedIdentity.country = await this._countryLookup(identity.id) + return decoratedIdentity } @@ -153,7 +185,7 @@ class IdentityEventHandler { * @private */ async _indexIdentity(identity, blockInfo) { - // Decorate the user object with extra attestation related info. + // Decorate the identity object with extra attestation related info. const decoratedIdentity = await this._decorateIdentity(identity) logger.info(`Indexing identity ${decoratedIdentity.id} in DB`) @@ -162,7 +194,7 @@ class IdentityEventHandler { throw new Error(`Invalid eth address: ${decoratedIdentity.id}`) } - // Construct an decoratedIdentity object based on the user's profile + // Construct a decoratedIdentity object based on the user's profile // and data loaded from the attestation table. const identityRow = { ethAddress: decoratedIdentity.id.toLowerCase(), @@ -173,7 +205,9 @@ class IdentityEventHandler { airbnb: decoratedIdentity.airbnb, twitter: decoratedIdentity.twitter, facebookVerified: decoratedIdentity.facebookVerified || false, - data: { blockInfo } + googleVerified: decoratedIdentity.googleVerified || false, + data: { blockInfo }, + country: decoratedIdentity.country } logger.debug('Identity=', identityRow) @@ -212,7 +246,7 @@ class IdentityEventHandler { /** * Records AttestationPublished events in the growth_event table. - * @param {Object} user - Origin js user model object. + * @param {Object} identity * @param {{blockNumber: number, logIndex: number}} blockInfo * @param {Date} Event date. * @returns {Promise} diff --git a/infra/discovery/src/listener/listener.js b/infra/discovery/src/listener/listener.js index 4b8c44b67781..35205cfaf63f 100644 --- a/infra/discovery/src/listener/listener.js +++ b/infra/discovery/src/listener/listener.js @@ -189,7 +189,7 @@ async function main() { // In case all retries fails, it indicates something is wrong at the system // level and a process restart may fix it. await withRetrys(async () => { - handleEvent(event, context) + return handleEvent(event, context) }) } } diff --git a/infra/discovery/test/listener-handler.test.js b/infra/discovery/test/listener-handler.test.js index 1dedf3e5b92e..050be3808d10 100644 --- a/infra/discovery/test/listener-handler.test.js +++ b/infra/discovery/test/listener-handler.test.js @@ -36,7 +36,7 @@ const mockIdentity = { data: { attestation: { verificationMethod: { - email: true + phone: true }, phone: '+00 00000000' } @@ -46,7 +46,7 @@ const mockIdentity = { data: { attestation: { verificationMethod: { - phone: true + email: true }, email: 'test@originprotocol.com' } @@ -228,6 +228,10 @@ describe('Listener Handlers', () => { } } + handler._countryLookup = () => { + return 'FR' + } + const result = await handler.process({ timestamp: 1 }, this.identityEvent) // Check output. @@ -244,7 +248,8 @@ describe('Listener Handlers', () => { firstName: 'Origin', lastName: 'Protocol', email: 'test@originprotocol.com', - phone: '+00 00000000' + phone: '+00 00000000', + country: 'FR' } }) expect(identityRow.length).to.equal(1) diff --git a/infra/growth/src/scripts/oneoff/backfillIdentityCountry.js b/infra/growth/src/scripts/oneoff/backfillIdentityCountry.js new file mode 100644 index 000000000000..d5822f68b753 --- /dev/null +++ b/infra/growth/src/scripts/oneoff/backfillIdentityCountry.js @@ -0,0 +1,71 @@ +// One-off script to backfill the country column of the identity table. +// +// Note: logically this script belongs more to the identity package +// rather than growth package. But identity does not have all the dependencies +// required (e.g. bridge, growth)... Since this is a one-off script that will +// get deleted after it gets run in production, we made the decision +// to put it in the growth package. + +const _identityModels = require('@origin/identity/src/models') +const _bridgeModels = require('@origin/bridge/src/models') +const db = { ..._identityModels, ..._bridgeModels } +const { ip2geo } = require('../../util/ip2geo') +const parseArgv = require('../../util/args') + +const Logger = require('logplease') +Logger.setLogLevel(process.env.LOG_LEVEL || 'INFO') +const logger = Logger.create('backfill', { showTimestamp: false }) + +async function main(dryRun) { + // Load all identity rows. + // It's a small amount of rows so ok to load them all up in memory. + const identities = await db.Identity.findAll() + logger.info(`Loaded ${identities.length} rows from identity table.`) + + for (const identity of identities) { + if (!identity.country) { + // Get the IP from the most recent attestation + const attestation = await db.Attestation.findOne({ + where: { ethAddress: identity.ethAddress }, + order: [['createdAt', 'DESC']] + }) + if (!attestation) { + logger.info(`No attestation data for identity ${identity.ethAddress}`) + continue + } + const ip = attestation.remoteIpAddress + + // Get the country by doing a geo lookup. + const geo = await ip2geo(ip) + if (!geo) { + logger.info( + `IP lookup failed for identity ${identity.ethAddress} ip=${ip}` + ) + continue + } + const country = geo.countryCode + + if (dryRun) { + logger.info( + `Would update identity row with ethAddress ${ + identity.ethAddress + } country: ${country}` + ) + } else { + identity.update({ country }) + logger.info( + `Updated identity row with ethAddress ${ + identity.ethAddress + } country: ${country}` + ) + } + } + } +} + +const args = parseArgv() +const dryRun = args['--dryRun'] === 'false' ? false : true + +logger.info('Starting backfill...') +logger.info('DryRun mode=', dryRun) +main(dryRun).then(() => logger.info('Done')) diff --git a/infra/identity/migrations/20190420064537-add-country.js b/infra/identity/migrations/20190420064537-add-country.js new file mode 100644 index 000000000000..f46784d75840 --- /dev/null +++ b/infra/identity/migrations/20190420064537-add-country.js @@ -0,0 +1,19 @@ +'use strict' + +const tableName='identity' + +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.addColumn( + tableName, + 'country', + Sequelize.CHAR(2) + ) + }, + down: (queryInterface) => { + return queryInterface.removeColumn( + tableName, + 'country' + ) + } +} diff --git a/infra/identity/src/models/identity.js b/infra/identity/src/models/identity.js index 5cb41f1c9576..f38d73631c5a 100644 --- a/infra/identity/src/models/identity.js +++ b/infra/identity/src/models/identity.js @@ -13,7 +13,8 @@ module.exports = (sequelize, DataTypes) => { twitter: DataTypes.STRING, facebookVerified: DataTypes.BOOLEAN, googleVerified: DataTypes.BOOLEAN, - data: DataTypes.JSONB + data: DataTypes.JSONB, + country: DataTypes.CHAR(2) }, { tableName: 'identity'