Add country column to identity table (#2083)

* Add country column to identity table

* tweak comments

* Fix unit tests

* linter

* prettier

* Make call to handleEvent synchronous

.travis.yml

@@ -78,6 +78,7 @@ matrix:
         - ATTESTATION_ACCOUNT=0x99C03fBb0C995ff1160133A8bd210D0E77bCD101
       before_script:
         - psql -c 'create database travis_ci_test;' -U postgres
+        - lerna run migrate --scope @origin/bridge
         - lerna run migrate --scope @origin/discovery
         - lerna run migrate --scope @origin/growth
         - lerna run migrate --scope @origin/identity

infra/discovery/src/listener/handler_identity.js

@@ -14,6 +14,7 @@ const {
   AttestationServiceToEventType,
   GrowthEvent
 } = require('@origin/growth/src/resources/event')
+const { ip2geo } = require('@origin/growth/src/util/ip2geo')
 
 class IdentityEventHandler {
   constructor(config, graphqlClient) {
@@ -47,7 +48,8 @@ class IdentityEventHandler {
     }
   }
 
-  /* Get details about an account from @origin/graphql
+  /**
+   * Get details about an account from @origin/graphql
    * @param {String} account: eth address of account
    * @returns {Object} result of GraphQL query
    * @private
@@ -94,6 +96,30 @@ class IdentityEventHandler {
     return attestation.value
   }
 
+  /**
+   * Returns the country of the identity based on IP from the most recent attestation.
+   * @param {string} ethAddress
+   * @returns {Promise<string> || null} 2 letters country code or null if lookup failed.
+   * @private
+   */
+  async _countryLookup(ethAddress) {
+    // Load the most recent attestation.
+    const attestation = await db.Attestation.findOne({
+      where: { ethAddress: ethAddress.toLowerCase() },
+      order: [['createdAt', 'DESC']]
+    })
+    if (!attestation) {
+      return null
+    }
+
+    // Do the IP to geo lookup.
+    const geo = await ip2geo(attestation.remoteIpAddress)
+    if (!geo) {
+      return null
+    }
+    return geo.countryCode
+  }
+
   /**
    * Decorates an identity object with attestation data.
    * @param {{}} identity - result of identityQuery
@@ -102,6 +128,8 @@ class IdentityEventHandler {
    */
   async _decorateIdentity(identity) {
     const decoratedIdentity = Object.assign({}, identity)
+
+    // Load attestation data.
     await Promise.all(
       decoratedIdentity.attestations.map(async attestationJson => {
         const attestation = JSON.parse(attestationJson)
@@ -142,6 +170,10 @@ class IdentityEventHandler {
         }
       })
     )
+
+    // Add country of origin information based on IP.
+    decoratedIdentity.country = await this._countryLookup(identity.id)
+
     return decoratedIdentity
   }
 
@@ -153,7 +185,7 @@ class IdentityEventHandler {
    * @private
    */
   async _indexIdentity(identity, blockInfo) {
-    // Decorate the user object with extra attestation related info.
+    // Decorate the identity object with extra attestation related info.
     const decoratedIdentity = await this._decorateIdentity(identity)
 
     logger.info(`Indexing identity ${decoratedIdentity.id} in DB`)
@@ -162,7 +194,7 @@ class IdentityEventHandler {
       throw new Error(`Invalid eth address: ${decoratedIdentity.id}`)
     }
 
-    // Construct an decoratedIdentity object based on the user's profile
+    // Construct a decoratedIdentity object based on the user's profile
     // and data loaded from the attestation table.
     const identityRow = {
       ethAddress: decoratedIdentity.id.toLowerCase(),
@@ -173,7 +205,9 @@ class IdentityEventHandler {
       airbnb: decoratedIdentity.airbnb,
       twitter: decoratedIdentity.twitter,
       facebookVerified: decoratedIdentity.facebookVerified || false,
-      data: { blockInfo }
+      googleVerified: decoratedIdentity.googleVerified || false,
+      data: { blockInfo },
+      country: decoratedIdentity.country
     }
 
     logger.debug('Identity=', identityRow)
@@ -212,7 +246,7 @@ class IdentityEventHandler {
 
   /**
    * Records AttestationPublished events in the growth_event table.
-   * @param {Object} user - Origin js user model object.
+   * @param {Object} identity
    * @param {{blockNumber: number, logIndex: number}} blockInfo
    * @param {Date} Event date.
    * @returns {Promise<void>}

infra/discovery/src/listener/listener.js

@@ -189,7 +189,7 @@ async function main() {
         // In case all retries fails, it indicates something is wrong at the system
         // level and a process restart may fix it.
         await withRetrys(async () => {
-          handleEvent(event, context)
+          return handleEvent(event, context)
         })
       }
     }

infra/discovery/test/listener-handler.test.js

@@ -36,7 +36,7 @@ const mockIdentity = {
       data: {
         attestation: {
           verificationMethod: {
-            email: true
+            phone: true
           },
           phone: '+00 00000000'
         }
@@ -46,7 +46,7 @@ const mockIdentity = {
       data: {
         attestation: {
           verificationMethod: {
-            phone: true
+            email: true
           },
           email: 'test@originprotocol.com'
         }
@@ -228,6 +228,10 @@ describe('Listener Handlers', () => {
       }
     }
 
+    handler._countryLookup = () => {
+      return 'FR'
+    }
+
     const result = await handler.process({ timestamp: 1 }, this.identityEvent)
 
     // Check output.
@@ -244,7 +248,8 @@ describe('Listener Handlers', () => {
         firstName: 'Origin',
         lastName: 'Protocol',
         email: 'test@originprotocol.com',
-        phone: '+00 00000000'
+        phone: '+00 00000000',
+        country: 'FR'
       }
     })
     expect(identityRow.length).to.equal(1)

infra/growth/src/scripts/oneoff/backfillIdentityCountry.js

@@ -0,0 +1,71 @@
+// One-off script to backfill the country column of the identity table.
+//
+// Note: logically this script belongs more to the identity package
+// rather than growth package. But identity does not have all the dependencies
+// required (e.g. bridge, growth)... Since this is a one-off script that will
+// get deleted after it gets run in production, we made the decision
+// to put it in the growth package.
+
+const _identityModels = require('@origin/identity/src/models')
+const _bridgeModels = require('@origin/bridge/src/models')
+const db = { ..._identityModels, ..._bridgeModels }
+const { ip2geo } = require('../../util/ip2geo')
+const parseArgv = require('../../util/args')
+
+const Logger = require('logplease')
+Logger.setLogLevel(process.env.LOG_LEVEL || 'INFO')
+const logger = Logger.create('backfill', { showTimestamp: false })
+
+async function main(dryRun) {
+  // Load all identity rows.
+  // It's a small amount of rows so ok to load them all up in memory.
+  const identities = await db.Identity.findAll()
+  logger.info(`Loaded ${identities.length} rows from identity table.`)
+
+  for (const identity of identities) {
+    if (!identity.country) {
+      // Get the IP from the most recent attestation
+      const attestation = await db.Attestation.findOne({
+        where: { ethAddress: identity.ethAddress },
+        order: [['createdAt', 'DESC']]
+      })
+      if (!attestation) {
+        logger.info(`No attestation data for identity ${identity.ethAddress}`)
+        continue
+      }
+      const ip = attestation.remoteIpAddress
+
+      // Get the country by doing a geo lookup.
+      const geo = await ip2geo(ip)
+      if (!geo) {
+        logger.info(
+          `IP lookup failed for identity ${identity.ethAddress} ip=${ip}`
+        )
+        continue
+      }
+      const country = geo.countryCode
+
+      if (dryRun) {
+        logger.info(
+          `Would update identity row with ethAddress ${
+            identity.ethAddress
+          } country: ${country}`
+        )
+      } else {
+        identity.update({ country })
+        logger.info(
+          `Updated identity row with ethAddress ${
+            identity.ethAddress
+          } country: ${country}`
+        )
+      }
+    }
+  }
+}
+
+const args = parseArgv()
+const dryRun = args['--dryRun'] === 'false' ? false : true
+
+logger.info('Starting backfill...')
+logger.info('DryRun mode=', dryRun)
+main(dryRun).then(() => logger.info('Done'))

infra/identity/migrations/20190420064537-add-country.js

@@ -0,0 +1,19 @@
+'use strict'
+
+const tableName='identity'
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn(
+      tableName,
+      'country',
+      Sequelize.CHAR(2)
+    )
+  },
+  down: (queryInterface) => {
+    return queryInterface.removeColumn(
+      tableName,
+      'country'
+    )
+  }
+}

infra/identity/src/models/identity.js

@@ -13,7 +13,8 @@ module.exports = (sequelize, DataTypes) => {
       twitter: DataTypes.STRING,
       facebookVerified: DataTypes.BOOLEAN,
       googleVerified: DataTypes.BOOLEAN,
-      data: DataTypes.JSONB
+      data: DataTypes.JSONB,
+      country: DataTypes.CHAR(2)
     },
     {
       tableName: 'identity'