From b0277f4b8f239082ec3065ab31c93aa19c225582 Mon Sep 17 00:00:00 2001 From: Sacha Greif Date: Tue, 19 Oct 2021 10:20:23 +0900 Subject: [PATCH] Load entities from different repo/dir through github API and filesystem --- src/compute/experience.ts | 2 +- src/compute/generic.ts | 2 +- src/compute/matrices.ts | 2 +- src/entities.ts | 140 +++++++++++++++++++++++++++++++ src/helpers.ts | 35 +------- src/i18n.ts | 22 +---- src/resolvers/features.ts | 6 +- src/resolvers/features_others.ts | 4 +- src/resolvers/query.ts | 2 +- src/resolvers/surveys.ts | 44 +++++----- src/rpcs.ts | 2 +- src/server.ts | 2 + stateofjs-api.code-workspace | 4 +- 13 files changed, 184 insertions(+), 83 deletions(-) create mode 100644 src/entities.ts diff --git a/src/compute/experience.ts b/src/compute/experience.ts index bf6abc9d..ddb62e1a 100644 --- a/src/compute/experience.ts +++ b/src/compute/experience.ts @@ -2,7 +2,7 @@ import _ from 'lodash' import { Db } from 'mongodb' import config from '../config' import { ratioToPercentage, appendCompletionToYearlyResults } from './common' -import { getEntity } from '../helpers' +import { getEntity } from '../entities' import { Completion, SurveyConfig } from '../types' import { Filters, generateFiltersQuery } from '../filters' import { computeCompletionByYear } from './generic' diff --git a/src/compute/generic.ts b/src/compute/generic.ts index eb755939..acd8adf4 100644 --- a/src/compute/generic.ts +++ b/src/compute/generic.ts @@ -5,7 +5,7 @@ import config from '../config' import { Completion, SurveyConfig } from '../types' import { Filters, generateFiltersQuery } from '../filters' import { ratioToPercentage } from './common' -import { getEntity } from '../helpers' +import { getEntity } from '../entities' import { getParticipationByYearMap } from './demographics' import { useCache } from '../caching' diff --git a/src/compute/matrices.ts b/src/compute/matrices.ts index eb875600..da971568 100644 --- a/src/compute/matrices.ts +++ b/src/compute/matrices.ts @@ -3,7 +3,7 @@ import _ from 'lodash' import { Db } from 'mongodb' import config from '../config' import { ratioToPercentage } from './common' -import { getEntity } from '../helpers' +import { getEntity } from '../entities' import { SurveyConfig } from '../types' import { ToolExperienceFilterId, toolExperienceConfigById } from './tools' diff --git a/src/entities.ts b/src/entities.ts new file mode 100644 index 00000000..acd6b2a5 --- /dev/null +++ b/src/entities.ts @@ -0,0 +1,140 @@ +import { Entity } from './types' +import { Octokit } from '@octokit/core' +import fetch from 'node-fetch' +import yaml from 'js-yaml' +import { readdir, readFile } from 'fs/promises' +import last from 'lodash/last' +import { logToFile } from './debug' + +let entities: Entity[] = [] + +const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN }) + +// load locales if not yet loaded +export const loadOrGetEntities = async () => { + if (entities.length === 0) { + entities = await loadEntities() + } + return entities +} + +export const loadFromGitHub = async () => { + const entities: Entity[] = [] + console.log(`-> loading entities repo`) + + const options = { + owner: 'StateOfJS', + repo: 'entities', + path: '' + } + + const contents = await octokit.request('GET /repos/{owner}/{repo}/contents/{path}', options) + const files = contents.data as any[] + + // loop over repo contents and fetch raw yaml files + for (const file of files) { + const extension: string = last(file?.name.split('.')) || '' + if (['yml', 'yaml'].includes(extension)) { + const response = await fetch(file.download_url) + const contents = await response.text() + try { + const yamlContents: any = yaml.load(contents) + const category = file.name.replace('./', '').replace('.yml', '') + yamlContents.forEach((entity: Entity) => { + const tags = entity.tags ? [...entity.tags, category] : [category] + entities.push({ + ...entity, + category, + tags + }) + }) + } catch (error) { + console.log(`// Error loading file ${file.name}`) + console.log(error) + } + } + } + return entities +} + +// when developing locally, load from local files +export const loadLocally = async () => { + console.log(`-> loading entities locally`) + + const entities: Entity[] = [] + + const devDir = __dirname.split('/').slice(1, -2).join('/') + const path = `/${devDir}/stateof-entities/` + const files = await readdir(path) + const yamlFiles = files.filter((f: String) => f.includes('.yml')) + + // loop over dir contents and fetch raw yaml files + for (const fileName of yamlFiles) { + const filePath = path + '/' + fileName + const contents = await readFile(filePath, 'utf8') + const yamlContents: any = yaml.load(contents) + const category = fileName.replace('./', '').replace('.yml', '') + yamlContents.forEach((entity: Entity) => { + const tags = entity.tags ? [...entity.tags, category] : [category] + entities.push({ + ...entity, + category, + tags + }) + }) + } + + return entities +} + +// load locales contents through GitHub API or locally +export const loadEntities = async () => { + console.log('// loading entities') + + const entities: Entity[] = + process.env.LOAD_LOCALES === 'local' ? await loadLocally() : await loadFromGitHub() + console.log('// done loading entities') + + return entities +} + +export const initEntities = async () => { + console.log('// initializing locales…') + const entities = await loadOrGetEntities() + logToFile('entities.json', entities, { mode: 'overwrite' }) +} + +export const getEntities = async ({ type, tag, tags }: { type?: string; tag?: string, tags?: string[] }) => { + let entities = await loadOrGetEntities() + if (type) { + entities = entities.filter(e => e.type === type) + } + if (tag) { + entities = entities.filter(e => e.tags && e.tags.includes(tag)) + } + if (tags) { + entities = entities.filter(e => tags.every(t => e.tags && e.tags.includes(t))) + } + return entities +} + +// Look up entities by id, name, or aliases (case-insensitive) +export const getEntity = async ({ id }: { id: string }) => { + const entities = await loadOrGetEntities() + + if (!id || typeof id !== 'string') { + return + } + + const lowerCaseId = id.toLowerCase() + const entity = entities.find(e => { + return ( + (e.id && e.id.toLowerCase() === lowerCaseId) || + (e.id && e.id.toLowerCase().replace(/\-/g, '_') === lowerCaseId) || + (e.name && e.name.toLowerCase() === lowerCaseId) || + (e.aliases && e.aliases.find((a: string) => a.toLowerCase() === lowerCaseId)) + ) + }) + + return entity || {} +} \ No newline at end of file diff --git a/src/helpers.ts b/src/helpers.ts index d192f5f8..c7a6f02a 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -1,45 +1,14 @@ import { EnumTypeDefinitionNode } from 'graphql' import typeDefs from './type_defs/schema.graphql' -import allEntities from './data/entities/index' +// import allEntities from './data/entities/index' import { RequestContext, ResolverDynamicConfig, SurveyConfig } from './types' import { computeTermAggregationAllYearsWithCache, computeTermAggregationSingleYearWithCache } from './compute' import { Filters } from './filters' +import {loadOrGetEntities} from './entities' -export const getEntities = ({ type, tag, tags }: { type?: string; tag?: string, tags?: string[] }) => { - let entities = allEntities - if (type) { - entities = entities.filter(e => e.type === type) - } - if (tag) { - entities = entities.filter(e => e.tags && e.tags.includes(tag)) - } - if (tags) { - entities = entities.filter(e => tags.every(t => e.tags && e.tags.includes(t))) - } - return entities -} - -// Look up entities by id, name, or aliases (case-insensitive) -export const getEntity = ({ id }: { id: string }) => { - if (!id || typeof id !== 'string') { - return - } - - const lowerCaseId = id.toLowerCase() - const entity = allEntities.find(e => { - return ( - (e.id && e.id.toLowerCase() === lowerCaseId) || - (e.id && e.id.toLowerCase().replace(/\-/g, '_') === lowerCaseId) || - (e.name && e.name.toLowerCase() === lowerCaseId) || - (e.aliases && e.aliases.find((a: string) => a.toLowerCase() === lowerCaseId)) - ) - }) - - return entity || {} -} /** * Return either e.g. other_tools.browsers.choices or other_tools.browsers.others_normalized diff --git a/src/i18n.ts b/src/i18n.ts index 41d4a7c5..4fe67724 100644 --- a/src/i18n.ts +++ b/src/i18n.ts @@ -1,6 +1,5 @@ import { EnumTypeDefinitionNode } from 'graphql' import { Entity, StringFile, Locale, TranslationStringObject } from './types' -import entities from './data/entities/index' import typeDefs from './type_defs/schema.graphql' import { Octokit } from '@octokit/core' import fetch from 'node-fetch' @@ -10,6 +9,7 @@ import marked from 'marked' import { logToFile } from './debug' import { readdir, readFile } from 'fs/promises' import last from 'lodash/last' +import { loadOrGetEntities } from './entities' let locales: Locale[] = [] @@ -68,6 +68,7 @@ export const loadFromGitHub = async (localesWithRepos: any) => { } return locales } + // when developing locally, load from local files export const loadLocally = async (localesWithRepos: any) => { let i = 0 @@ -119,25 +120,6 @@ export const loadLocales = async () => { return locales } -// Look up entities by id, name, or aliases (case-insensitive) -export const getEntity = ({ id }: { id: string }) => { - if (!id || typeof id !== 'string') { - return - } - - const lowerCaseId = id.toLowerCase() - const entity = entities.find(e => { - return ( - (e.id && e.id.toLowerCase() === lowerCaseId) || - (e.id && e.id.toLowerCase().replace(/\-/g, '_') === lowerCaseId) || - (e.name && e.name.toLowerCase() === lowerCaseId) || - (e.aliases && e.aliases.find((a: string) => a.toLowerCase() === lowerCaseId)) - ) - }) - - return entity || {} -} - /** * Return either e.g. other_tools.browsers.choices or other_tools.browsers.others_normalized */ diff --git a/src/resolvers/features.ts b/src/resolvers/features.ts index 3092c0a7..50a8912a 100644 --- a/src/resolvers/features.ts +++ b/src/resolvers/features.ts @@ -1,11 +1,11 @@ import { Db } from 'mongodb' import { useCache } from '../caching' import { fetchMdnResource } from '../external_apis' -import features from '../data/entities/features.yml' import { RequestContext, SurveyConfig } from '../types' import { computeTermAggregationByYear } from '../compute' import { Filters } from '../filters' import { Entity } from '../types' +import { getEntities } from '../entities' const computeFeatureExperience = async ( db: Db, @@ -32,12 +32,14 @@ export default { } }, Feature: { - name: ({ id }: { id: string }) => { + name: async ({ id }: { id: string }) => { + const features = await getEntities({ tag: 'feature' }) const feature = features.find((f: Entity) => f.id === id) return feature && feature.name }, mdn: async ({ id }: { id: string }) => { + const features = await getEntities({ tag: 'feature' }) const feature = features.find((f: Entity) => f.id === id) if (!feature || !feature.mdn) { return diff --git a/src/resolvers/features_others.ts b/src/resolvers/features_others.ts index 5969f7fd..8ebacdd3 100644 --- a/src/resolvers/features_others.ts +++ b/src/resolvers/features_others.ts @@ -1,11 +1,11 @@ import { Db } from 'mongodb' -import features from '../data/entities/features.yml' import { useCache } from '../caching' import { computeTermAggregationByYear } from '../compute' import { getOtherKey } from '../helpers' import { RequestContext, SurveyConfig } from '../types' import { Filters } from '../filters' import { Entity } from '../types' +import { getEntities } from '../entities' interface OtherFeaturesConfig { survey: SurveyConfig @@ -19,6 +19,8 @@ const computeOtherFeatures = async ( id: string, filters?: Filters ) => { + const features = await getEntities({ tag: 'feature'}) + const otherFeaturesByYear = await useCache(computeTermAggregationByYear, db, [ survey, `features_others.${getOtherKey(id)}`, diff --git a/src/resolvers/query.ts b/src/resolvers/query.ts index f8b2087a..257f1562 100644 --- a/src/resolvers/query.ts +++ b/src/resolvers/query.ts @@ -1,5 +1,5 @@ import { SurveyType } from '../types' -import { getEntities, getEntity } from '../helpers' +import { getEntities, getEntity } from '../entities' import { getLocales, getLocaleObject, getTranslation } from '../i18n' import { SurveyConfig } from '../types' diff --git a/src/resolvers/surveys.ts b/src/resolvers/surveys.ts index bb025c82..6c96c557 100644 --- a/src/resolvers/surveys.ts +++ b/src/resolvers/surveys.ts @@ -1,7 +1,12 @@ -import { getEntity, getGraphQLEnumValues, getDemographicsResolvers } from '../helpers' +import { getGraphQLEnumValues, getDemographicsResolvers } from '../helpers' +import { getEntity } from '../entities' import { RequestContext, SurveyConfig } from '../types' import { Filters } from '../filters' -import {computeToolExperienceGraph, computeToolsCardinalityByUser, ToolExperienceId} from '../compute' +import { + computeToolExperienceGraph, + computeToolsCardinalityByUser, + ToolExperienceId +} from '../compute' import { useCache } from '../caching' const toolIds = getGraphQLEnumValues('ToolID') @@ -77,10 +82,7 @@ export default { id, filters }), - happiness: ( - survey: SurveyConfig, - { id, filters }: { id: string; filters?: Filters } - ) => ({ + happiness: (survey: SurveyConfig, { id, filters }: { id: string; filters?: Filters }) => ({ survey, id, filters @@ -141,20 +143,20 @@ export default { { db }: RequestContext ) => useCache(computeToolExperienceGraph, db, [survey, id, filters]) })), - tools_cardinality_by_user: (survey: SurveyConfig, { - year, - // tool IDs - ids, - experienceId, - }: { - year: number - ids: string[] - experienceId: ToolExperienceId - }, context: RequestContext) => useCache( - computeToolsCardinalityByUser, - context.db, - [survey, year, ids, experienceId] - ), + tools_cardinality_by_user: ( + survey: SurveyConfig, + { + year, + // tool IDs + ids, + experienceId + }: { + year: number + ids: string[] + experienceId: ToolExperienceId + }, + context: RequestContext + ) => useCache(computeToolsCardinalityByUser, context.db, [survey, year, ids, experienceId]), tools_others: ( survey: SurveyConfig, { id, filters }: { id: string; filters?: Filters } @@ -171,6 +173,6 @@ export default { ids, filters }), - totals: (survey: SurveyConfig) => survey, + totals: (survey: SurveyConfig) => survey } } diff --git a/src/rpcs.ts b/src/rpcs.ts index 93679dc6..10c72858 100644 --- a/src/rpcs.ts +++ b/src/rpcs.ts @@ -1,7 +1,7 @@ import { TwitterStat } from './types/twitter' import { MongoClient } from 'mongodb' import config from './config' -import { getEntities } from './helpers' +import { getEntities } from './entities' import { getTwitterUser, getTwitterFollowings } from './external_apis/twitter' function sleep(ms: number) { diff --git a/src/server.ts b/src/server.ts index 4a906380..866ef5d2 100644 --- a/src/server.ts +++ b/src/server.ts @@ -8,6 +8,7 @@ import { RequestContext } from './types' import resolvers from './resolvers' import express from 'express' import { initLocales } from './i18n' +import { initEntities } from './entities' import { analyzeTwitterFollowings } from './rpcs' import { clearCache } from './caching' @@ -106,6 +107,7 @@ const start = async () => { const port = process.env.PORT || 4000 await initLocales() + await initEntities() app.listen({ port: port }, () => console.log(`🚀 Server ready at http://localhost:${port}${server.graphqlPath}`) diff --git a/stateofjs-api.code-workspace b/stateofjs-api.code-workspace index 443f5a5b..86194378 100644 --- a/stateofjs-api.code-workspace +++ b/stateofjs-api.code-workspace @@ -3,5 +3,7 @@ { "path": "." } - ] + ], + "settings": { + } } \ No newline at end of file