feat: update with data from ITU database

I finally have gotten access to the database!
bifravst · Dec 20, 2024 · f5b1f5f · f5b1f5f
1 parent 2368086
commit f5b1f5f
Show file tree

Hide file tree

Showing 5 changed files with 390 additions and 115 deletions.
diff --git a/README.md b/README.md
@@ -10,20 +10,19 @@ List of issuer identification numbers for the international telecommunication
 charge card
 ([ITU-T E.118](https://www.itu.int/rec/dologin_pub.asp?lang=e&id=T-REC-E.118-200605-I!!PDF-E&type=items)).
 
-> **Note**  
+> [!NOTE]  
+> Last update: 2024-12-20
+
+> [!IMPORTANT]  
 > Up-to-date with
-> [Operational Bulletin No. 1295 (1.VII.2024)](https://www.itu.int/pub/T-SP-OB.1295-2024)
+> [the ITU-T E.118 Issuer Identifier Numbers database](https://www.itu.int/net/itu-t/inrdb/secured/e118iin.aspx)
 > and also includes
 > [E.164 shared country code entries](http://www.itu.int/net/itu-t/inrdb/e164_intlsharedcc.aspx?cc=881,882,883)
 > (which has some overlapping entries).
 
 Data source as
 [Google Spreadsheet](https://docs.google.com/spreadsheets/d/1ErJzksU5bF2YA8tQQ9QJleEZHsdvDRDk0Rvi0nf3fh4/edit?usp=sharing).
 
-> _Note:_ There is actually
-> [a database](https://www.itu.int/net/itu-t/inrdb/secured/e118iin.aspx) for
-> this information, but the access is restricted to ITU-T Sector Members. 🤷
-
 ## Motivation
 
 Since E.118's issuer identification number is of variable length (it can be 4–7
@@ -93,18 +92,16 @@ The maximum length of the visible card number (primary account number) shall be
 
 Sources:
 
-- http://www.itu.int/pub/T-SP-E.118
-- https://www.itu.int/pub/T-SP-OB
+- https://www.itu.int/net/itu-t/inrdb/secured/e118iin.aspx
 - http://www.itu.int/net/itu-t/inrdb/e164_intlsharedcc.aspx?cc=881,882,883
 
 Process:
 
-1. Download the latest Word Documents from http://www.itu.int/pub/T-SP-E.118,
-   and copy and past the table into a Google Spreadsheet
-2. Download the operational bulletins from https://www.itu.int/pub/T-SP-OB and
-   incorporate the changes into the spreadsheet
-3. Export list of shared country codes (E.164) from
+1. Export
+   [the database](https://www.itu.int/net/itu-t/inrdb/secured/e118iin.aspx) as
+   Excel spreadsheet and copy and past the table into a Google Spreadsheet
+2. Export list of shared country codes (E.164) from
    http://www.itu.int/net/itu-t/inrdb/e164_intlsharedcc.aspx?cc=881,882,883 and
    filter out `CRS` records (inactive), add to the spreadsheet
-4. Export that to CSV and store it as `list.csv`
-5. Convert to JSON using `npm run convert`
+3. Export that to CSV and store it as `list.csv`
+4. Convert to JSON using `npm run convert`
diff --git a/src/convert.ts b/src/convert.ts
@@ -2,92 +2,124 @@ import csv from 'csv-parser'
 import * as fs from 'fs'
 import * as path from 'path'
 import prettier from 'prettier'
+import { darkIssuersList } from './darkIssuers.js'
 import type { IssuerList } from './types.js'
 
 type ParsedCSVEntry = {
-	CountryGeographicalarea: string[]
-	CompanyNameAddress: string[]
-	IssuerIdentifierNumber: string[]
-	Contact: string[]
+	CountryGeographicalarea: string
+	CompanyNameAddress: string
+	IssuerIdentifierNumber: string
+	Contact: string
 }
 
 const results: ParsedCSVEntry[] = []
 
 const target = path.join(process.cwd(), 'src', 'list.ts')
 
-fs.createReadStream('list.csv')
-	.pipe(
-		csv({
-			mapHeaders: ({ header }) =>
-				header.replace(/\n/g, ' ').replace(/[ /]/g, '').trim(),
-			mapValues: ({ value }) => value.split('\n').map((v: string) => v.trim()),
+const notBlank = (s: string) => (s.trim().length > 0 ? s.trim() : undefined)
+const removeBlanks = (o: Record<string, string | undefined>) => {
+	const result: Record<string, string> = {}
+	for (const [k, v] of Object.entries(o)) {
+		if (v !== undefined) {
+			result[k] = v
+		}
+	}
+	return result
+}
+
+await new Promise<void>((resolve) =>
+	fs
+		.createReadStream('list.csv')
+		.pipe(csv())
+		.on('data', (data) => results.push(data))
+		.on('end', () => resolve()),
+)
+type Issuers = Record<
+	string,
+	{
+		company: string
+		country?: string
+		contact: string[]
+	}
+>
+const issuerData: Issuers = {}
+
+let currentICCID: string | undefined
+for (const [country, company, iccid, contact] of results.map((r) =>
+	Object.values(r),
+)) {
+	if (iccid.startsWith('89')) {
+		currentICCID = notBlank(iccid)
+	}
+	if (currentICCID === undefined) continue
+	const current = issuerData[currentICCID] ?? {}
+	issuerData[currentICCID] = {
+		...current,
+		...removeBlanks({
+			country: notBlank(country),
+			company: notBlank(current.company ?? company),
 		}),
-	)
-	.on('data', (data) => results.push(data))
-	.on('end', async () => {
-		const list: IssuerList = results.reduce(
-			(
-				list,
-				{
-					IssuerIdentifierNumber,
-					CountryGeographicalarea,
-					CompanyNameAddress,
-					Contact,
-				},
-			) => {
-				const [, countryCode, issuerIdentifierNumber] =
-					IssuerIdentifierNumber[0].split(' ')
-				const iin = parseInt(IssuerIdentifierNumber[0].replace(/ /g, ''), 10)
-				const key = `${countryCode}${issuerIdentifierNumber}`
-				const emailRegEx = /e-mail ?: ?(.+)/i
-				const companyURLs = Contact.reduce(
-					(urls, s) => {
-						const m = emailRegEx.exec(s)
-						if (!m) return urls
-						return m[1]
-							.replace(/ /g, '')
-							.split(';')
-							.map((email) => email.replace(/^.+@/, 'http://').toLowerCase())
-							.filter((url, k, urls) => urls.indexOf(url) === k)
-					},
-					undefined as undefined | string[],
-				)
-				const cc = parseInt(countryCode, 10)
-				const result = {
-					...list,
-					[key]: [
-						iin,
-						issuerIdentifierNumber,
-						cc,
-						CountryGeographicalarea[0],
-						CompanyNameAddress[0],
-						companyURLs ?? [],
-					],
-				}
-				if (cc === 1) {
-					// USA: Some vendors prefix the 1 with a 0 in the ICCID
-					result[`0${key}`] = result[key]
-				}
-				return result as IssuerList
+		contact: [...(current.contact ?? []), notBlank(contact)].filter(
+			(s) => s !== undefined,
+		),
+	}
+}
+
+const list: IssuerList = Object.entries(issuerData).reduce<IssuerList>(
+	(list, [iccid, { company, country, contact }]) => {
+		const [, countryCode, issuerIdentifierNumber] = iccid.split(' ')
+		const iin = parseInt(iccid.replace(/ /g, ''), 10)
+		const key = `${countryCode}${issuerIdentifierNumber}`
+		const emailRegEx = /e-mail ?: ?(.+)/i
+		const companyURLs = contact.reduce(
+			(urls, s) => {
+				const m = emailRegEx.exec(s)
+				if (!m) return urls
+				return m[1]
+					.replace(/ /g, '')
+					.split(';')
+					.map((email) => email.replace(/^.+@/, 'http://').toLowerCase())
+					.filter((url, k, urls) => urls.indexOf(url) === k)
 			},
-			{} as IssuerList,
-		)
-		fs.writeFileSync(
-			target,
-			await prettier.format(
-				[
-					`/* Auto-generated file. Do not change! */`,
-					`import type { IssuerList } from './types.js';`,
-					`export const iinRegEx = /^89(${Object.keys(list).join('|')})/;`,
-					`export const e118IINList: IssuerList = ${JSON.stringify(
-						list,
-						null,
-						2,
-					)} as const;`,
-				].join('\n\n'),
-				{ parser: 'typescript' },
-			),
-			'utf-8',
+			undefined as undefined | string[],
 		)
-		console.log(`${target} written.`)
-	})
+		const cc = parseInt(countryCode, 10)
+		const result = {
+			...list,
+			[key]: [
+				iin,
+				issuerIdentifierNumber,
+				cc,
+				[881, 882, 883].includes(cc) ? 'Global' : country,
+				company,
+				companyURLs ?? [],
+			],
+		}
+		if (cc === 1) {
+			// USA: Some vendors prefix the 1 with a 0 in the ICCID
+			result[`0${key}`] = result[key]
+		}
+		return result as IssuerList
+	},
+	darkIssuersList,
+)
+
+fs.writeFileSync(
+	target,
+	await prettier.format(
+		[
+			`/* Auto-generated file. Do not change! */`,
+			`/* Generated: ${new Date().toISOString()} */`,
+			`import type { IssuerList } from './types.js';`,
+			`export const iinRegEx = /^89(${Object.keys(list).join('|')})/;`,
+			`export const e118IINList: IssuerList = ${JSON.stringify(
+				list,
+				null,
+				2,
+			)} as const;`,
+		].join('\n\n'),
+		{ parser: 'typescript' },
+	),
+	'utf-8',
+)
+console.log(`${target} written.`)
diff --git a/src/darkIssuers.ts b/src/darkIssuers.ts
@@ -0,0 +1,91 @@
+import type { IssuerList } from './types.js'
+
+/**
+ * This is a list of issuers, that we know operate, but they are not (yet) listed in the ITU-T E.118 database.
+ */
+const darkIssuers: Array<{
+	prefix: number
+	countryCode: number
+	networkCode: string
+	country: string
+	company: string
+	url?: URL
+}> = [
+	{
+		prefix: 893108,
+		networkCode: '08',
+		countryCode: 31,
+		country: 'Netherlands',
+		company: 'KPN Telecom B.V., Card Services',
+	},
+	{
+		prefix: 894573,
+		networkCode: '73',
+		countryCode: 45,
+		country: 'Denmark',
+		company: 'Onomondo ApS',
+	},
+	{
+		prefix: 894604,
+		networkCode: '04',
+		countryCode: 46,
+		country: 'Sweden',
+		company: 'eRate Sverige AB',
+		url: new URL('http://telavox.se'),
+	},
+	{
+		prefix: 8985220,
+		networkCode: '20',
+		countryCode: 852,
+		country: 'Hong Kong, China',
+		company: 'Internet Initiative Japan Inc.',
+		url: new URL('http://iij.ad.jp'),
+	},
+	{
+		prefix: 8988280,
+		networkCode: '80',
+		countryCode: 882,
+		country: 'Germany',
+		company: '1NCE GmbH',
+		url: new URL('http://1nce.com'),
+	},
+	{
+		prefix: 89883440,
+		networkCode: '440',
+		countryCode: 883,
+		country: 'Global',
+		company: 'Truphone Limited',
+	},
+	{
+		prefix: 898835100,
+		networkCode: '5100',
+		countryCode: 883,
+		country: 'Global',
+		company: 'Voxbone SA',
+	},
+	{
+		prefix: 898835110,
+		networkCode: '5110',
+		countryCode: 883,
+		country: 'Global',
+		company: 'Bandwidth.com Inc',
+	},
+]
+
+type Mutable<T> = {
+	-readonly [P in keyof T]: T[P]
+}
+
+export const darkIssuersList: Mutable<IssuerList> = darkIssuers.reduce<
+	Mutable<IssuerList>
+>((list, issuer) => {
+	list[`${issuer.countryCode}${issuer.networkCode}`] = [
+		issuer.prefix,
+		issuer.networkCode,
+		issuer.countryCode,
+		issuer.country,
+		issuer.company,
+		issuer.url ? [issuer.url.toString()] : [],
+	]
+	return list
+}, {})
diff --git a/src/identifyIssuer.spec.ts b/src/identifyIssuer.spec.ts
@@ -1,6 +1,6 @@
-import { identifyIssuer } from './identifyIssuer.js'
-import { describe, it } from 'node:test'
 import assert from 'node:assert/strict'
+import { describe, it } from 'node:test'
+import { identifyIssuer } from './identifyIssuer.js'
 
 void describe('identifyIssuer', () => {
 	for (const [iccid, issuer] of [
@@ -32,7 +32,7 @@ void describe('identifyIssuer', () => {
 				issuerIdentifierNumber: '80',
 				countryName: 'Germany',
 				companyName: '1NCE GmbH',
-				companyURLs: ['http://1nce.com'],
+				companyURLs: ['http://1nce.com/'],
 			},
 		],
 		[
@@ -43,7 +43,7 @@ void describe('identifyIssuer', () => {
 				issuerIdentifierNumber: '20',
 				countryName: 'Hong Kong, China',
 				companyName: 'Internet Initiative Japan Inc.',
-				companyURLs: ['http://iij.ad.jp'],
+				companyURLs: ['http://iij.ad.jp/'],
 			},
 		],
 		[