Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(dev): upgrade env /2 #595

Merged
merged 7 commits into from
Jun 4, 2021
Merged

fix(dev): upgrade env /2 #595

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2114d7e
fix(dev): upgrade env /2
bodinsamuel Jun 4, 2021
8999a2a
tmp
bodinsamuel Jun 4, 2021
1de01ed
Merge branch 'master' into fix/upgrade-dev-env2
Jun 4, 2021
d01fff4
Update src/utils/request.js
Jun 4, 2021
9d9cbe2
lint
bodinsamuel Jun 4, 2021
7903f92
Merge branch 'master' into fix/upgrade-dev-env2
Jun 4, 2021
bc56565
review
bodinsamuel Jun 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,21 @@
},
"license": "MIT",
"dependencies": {
"@sentry/node": "5.30.0",
"algoliasearch": "3.35.1",
"@sentry/node": "6.5.1",
"algoliasearch": "4.9.1",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

aren't there breaking changes in the algolia usage?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

well you are better suited to answer to that, but the basic client creation and save objects seems to work fine.
Note I tested locally with bootstrap with 100 Packages.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it was typescript we could find more of the errors, but for example:

async function putDefaultSettings(index, config) {
  await index.setSettings(config.indexSettings);

  await index.batchSynonyms(config.indexSynonyms, {
    replaceExistingSynonyms: true,
  });
  const { taskID } = await index.batchRules(config.indexRules, {
    replaceExistingRules: true,
  });

  await index.waitTask(taskID);
}

must become:

  await index.setSettings(config.indexSettings);

  await index.saveSynonyms(config.indexSynonyms, {
    replaceExistingSynonyms: true,
  });
  const { taskID } = await index.saveRules(config.indexRules, {
    replaceExistingRules: true,
  });

  await index.waitTask(taskID);
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah good catch, this one is executed only if the original index does not exists. I missed it 👌🏻
in the next PRs I'll try to create integration test, a lot of things are untested right now, too easy to make mistake (and I do a lot of them ahah)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Screen Shot 2021-06-04 at 17 21 09

"async": "3.2.0",
"babel-cli": "6.26.0",
"babel-preset-env": "1.7.0",
"babel-preset-stage-2": "6.24.1",
"bunyan": "1.8.15",
"bunyan-debug-stream": "2.0.0",
"dotenv": "8.2.0",
"dotenv": "10.0.0",
"dtrace-provider": "0.8.8",
"escape-html": "1.0.3",
"got": "9.6.0",
"got": "11.8.2",
"gravatar-url": "3.1.0",
"hosted-git-info": "2.7.1",
"hot-shots": "6.8.7",
"hot-shots": "8.3.2",
"lodash": "4.17.21",
"ms": "2.1.3",
"nano": "8.2.3",
Expand All @@ -49,11 +49,11 @@
"truncate-utf8-bytes": "1.0.2"
},
"devDependencies": {
"@babel/core": "7.13.10",
"@babel/preset-env": "7.13.10",
"@babel/core": "7.14.3",
"@babel/preset-env": "7.14.4",
"babel-eslint": "10.1.0",
"babel-jest": "24.9.0",
"doctoc": "1.4.0",
"babel-jest": "27.0.2",
"doctoc": "2.0.0",
"eslint": "7.27.0",
"eslint-config-algolia": "18.0.0",
"eslint-config-prettier": "8.3.0",
Expand All @@ -70,7 +70,7 @@
"pre-commit": "1.2.2",
"prettier": "2.3.0",
"renovate-config-algolia": "2.1.3",
"validator": "11.1.0"
"validator": "13.6.0"
},
"engines": {
"node": "14.16.1",
Expand Down
4 changes: 2 additions & 2 deletions src/changelog.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import got from 'got';
import race from 'promise-rat-race';

import config from './config.js';
import datadog from './datadog.js';
import { request } from './utils/request.js';

export const baseUrlMap = new Map([
[
Expand All @@ -29,7 +29,7 @@ export const baseUrlMap = new Map([
]);

async function handledGot(file) {
const result = await got(file, { method: 'HEAD' });
const result = await request(file, { method: 'HEAD' });

if (
// bitbucket returns 200 for private repos
Expand Down
2 changes: 1 addition & 1 deletion src/config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ const defaultConfig = {
indexName: 'npm-search',
bootstrapIndexName: 'npm-search-bootstrap',
replicateConcurrency: 10,
bootstrapConcurrency: 100,
bootstrapConcurrency: 25,
timeToRedoBootstrap: ms('2 weeks'),
seq: null,
indexSettings: {
Expand Down
11 changes: 5 additions & 6 deletions src/jsDelivr/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import got from 'got';

import config from '../config.js';
import datadog from '../datadog.js';
import log from '../log.js';
import { request } from '../utils/request.js';

const hits = new Map();

Expand All @@ -14,8 +13,8 @@ async function loadHits() {
log.info('📦 Loading hits from jsDelivr');

try {
const { body: hitsJSON } = await got(config.jsDelivrHitsEndpoint, {
json: true,
const { body: hitsJSON } = await request(config.jsDelivrHitsEndpoint, {
responseType: 'json',
});
hits.clear();
hitsJSON.forEach((pkg) => {
Expand Down Expand Up @@ -81,10 +80,10 @@ async function getFilesList(pkg) {

let files = [];
try {
const response = await got(
const response = await request(
`${config.jsDelivrPackageEndpoint}/${pkg.name}@${pkg.version}/flat`,
{
json: true,
responseType: 'json',
}
);
files = response.body.files;
Expand Down
17 changes: 8 additions & 9 deletions src/npm/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import got from 'got';
import chunk from 'lodash/chunk.js';
import nano from 'nano';
import numeral from 'numeral';

import config from '../config.js';
import datadog from '../datadog.js';
import log from '../log.js';
import { request } from '../utils/request.js';

const registry = nano({
url: config.npmRegistryEndpoint,
Expand Down Expand Up @@ -97,8 +97,8 @@ async function getInfo() {

const {
body: { doc_count: nbDocs, update_seq: seq },
} = await got(config.npmRegistryEndpoint, {
json: true,
} = await request(config.npmRegistryEndpoint, {
responseType: 'json',
});

datadog.timing('npm.info', Date.now() - start);
Expand All @@ -119,8 +119,7 @@ async function validatePackageExists(pkgName) {

let exists;
try {
const response = await got(`${config.npmRootEndpoint}/${pkgName}`, {
json: true,
const response = await request(`${config.npmRootEndpoint}/${pkgName}`, {
method: 'HEAD',
});
exists = response.statusCode === 200;
Expand Down Expand Up @@ -153,8 +152,8 @@ function getDependents(pkgs) {
async function getTotalDownloads() {
const {
body: { downloads: totalNpmDownloadsPerDay },
} = await got(`${config.npmDownloadsEndpoint}/range/last-month`, {
json: true,
} = await request(`${config.npmDownloadsEndpoint}/range/last-month`, {
responseType: 'json',
});

return totalNpmDownloadsPerDay.reduce(
Expand All @@ -170,10 +169,10 @@ async function getTotalDownloads() {
*/
async function getDownload(pkgNames) {
try {
const response = await got(
const response = await request(
`${config.npmDownloadsEndpoint}/point/last-month/${pkgNames}`,
{
json: true,
responseType: 'json',
}
);
if (response.body.downloads) {
Expand Down
2 changes: 1 addition & 1 deletion src/saveDocs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ export default async function saveDocs({ docs, index }) {
datadog.timing('saveDocs.addMetaData', Date.now() - start2);

start2 = Date.now();
index.saveObjects(pkgs);
await index.saveObjects(pkgs);
datadog.timing('saveDocs.saveObjects', Date.now() - start2);

datadog.timing('saveDocs', Date.now() - start);
Expand Down
5 changes: 2 additions & 3 deletions src/unpkg.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
import got from 'got';

import config from './config.js';
import { request } from './utils/request.js';

// make a head request to a route like:
// https://unpkg.com/lodash@4.17.11/_LazyWrapper.js
// to validate the existence of a particular file
export async function fileExistsInUnpkg(pkg, version, path) {
const uri = `${config.unpkgRoot}/${pkg}@${version}/${path}`;
try {
const response = await got(uri, {
const response = await request(uri, {
method: 'HEAD',
});
return response.statusCode === 200;
Expand Down
25 changes: 25 additions & 0 deletions src/utils/request.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import http from 'http';
import https from 'https';

import got from 'got';

const options = {
keepAlive: true,
timeout: 60000,
bodinsamuel marked this conversation as resolved.
Show resolved Hide resolved
maxFreeSockets: 2000,
scheduling: 'fifo',
};

// The agents will pool TCP connections
export const httpAgent = new http.Agent(options);
export const httpsAgent = new https.Agent(options);

export async function request(url, opts) {
return await got(url, {
...opts,
agent: {
http: httpAgent,
https: httpsAgent,
},
});
}
Loading