feat: unify meta data

Signed-off-by: frank-zsy <syzhao1988@126.com>
X-lab2017 · Aug 3, 2023 · b7f2369 · b7f2369
1 parent be70207
commit b7f2369
Showing 1 changed file with 18 additions and 5 deletions.
diff --git a/src/cron/tasks/monthly_export.ts b/src/cron/tasks/monthly_export.ts
@@ -1,3 +1,4 @@
+// @ts-nocheck
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
 import { join } from 'path';
 import { Task } from '..';
@@ -110,6 +111,7 @@ const task: Task = {
           writeFileSync(join(exportBasePath, name, 'meta.json'), JSON.stringify({
             updatedAt: new Date().getTime(),
             type: option.type ?? undefined,
+            id: parseInt(row.id),
           }));
           if (!Array.isArray(fields)) fields = [fields];
           const aggContent: any = {};
@@ -194,6 +196,7 @@ const task: Task = {
       for (let i = 0; i < repoPartitions.length; i++) {
         const { min, max } = repoPartitions[i];
         option.whereClause = `repo_id BETWEEN ${min} AND ${max} AND repo_id IN (SELECT id FROM ${exportRepoTableName})`;
+        option.type = 'repo';
         // [X-lab index] repo activity
         await processMetric(getRepoActivity, { ...option, options: { developerDetail: true } }, [getField('activity'), getField('details', { targetKey: 'activity_details', ...arrayFieldOption, parser: arr => arr.length <= 100 ? arr : arr.filter(i => i[1] >= 2) })]);
         // [X-lab index] repo openrank
@@ -292,6 +295,7 @@ const task: Task = {
         const [owner, repos, ids, orgId] = row;
         updateMetaData(join(exportBasePath, owner, 'meta.json'), {
           updatedAt: date.getTime(),
+          id: orgId === '0' ? undefined : parseInt(orgId),
           type: orgId === '0' ? 'user' : 'org',
           repos: repos.map((name, index) => {
             return {
@@ -312,18 +316,27 @@ const task: Task = {
       for (const l of labelData) {
         const update = (name: any) => {
           if (!labelMap.has(name)) labelMap.set(name, []);
-          labelMap.get(name)!.push({
-            id: l.identifier,
-            name: l.name,
-            type: l.type,
-          });
+          // avoid duplicate label
+          if (labelMap.get(name)!.indexOf(i => i.id === l.id) < 0) {
+            labelMap.get(name)!.push({
+              id: l.identifier,
+              name: l.name,
+              type: l.type,
+            });
+          }
         };
         if (l.githubOrgs.length > 0) {
           const sql = `SELECT argMax(org_login, created_at) FROM gh_events WHERE org_id IN (${l.githubOrgs.join(',')}) GROUP BY org_id`;
           await queryStream(sql, row => {
             const [orgLogin] = row;
             update(orgLogin);
           });
+          // add the label to repos under the org too.
+          const repoInOrgSql = `SELECT repo_name FROM gh_export_repo WHERE org_id IN (${l.githubOrgs.join(',')})`;
+          await queryStream(repoInOrgSql, row => {
+            const [repoName] = row;
+            update(repoName);
+          });
         }
         if (l.githubRepos.length > 0) {
           const sql = `SELECT argMax(repo_name, created_at) FROM gh_events WHERE repo_id IN (${l.githubRepos.join(',')}) GROUP BY repo_id`;