Skip to content

Commit

Permalink
Fixed behavior for multiple queues.
Browse files Browse the repository at this point in the history
  • Loading branch information
otaviomacedo committed May 12, 2023
1 parent 974194f commit b2056a2
Show file tree
Hide file tree
Showing 6 changed files with 54 additions and 31 deletions.
12 changes: 8 additions & 4 deletions .projen/deps.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 1 addition & 3 deletions .projenrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,12 @@ const project = new typescript.TypeScriptProject({
defaultReleaseBranch: 'main',
name: 'aws-batch-simulator',
deps: [
'@aws-cdk/aws-batch-alpha',
'constructs',
],
description: 'A client-side tool to simulate the behavior of an AWS Batch application',
devDeps: ['aws-cdk-lib', 'constructs', 'ts-node'],
packageName: 'aws-batch-simulator',
gitignore: ['.idea'],
peerDeps: ['aws-cdk-lib'],
peerDeps: ['aws-cdk-lib', '@aws-cdk/aws-batch-alpha', 'constructs'],
releaseToNpm: true,
repository: 'https://github.com/otaviomacedo/aws-batch-simulator.git',
keywords: ['aws', 'batch', 'simulation', 'queueing', 'cdk', 'stochastic', 'markov'],
Expand Down
5 changes: 2 additions & 3 deletions package.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 8 additions & 0 deletions src/scheduling.ts
Original file line number Diff line number Diff line change
Expand Up @@ -884,10 +884,16 @@ export interface QueueMetrics {
readonly size: number;
}

export interface QueueHistory {
id: string;
metrics: QueueMetrics[];
}

export interface JobQueueProps {
readonly computeEnvironments: IComputeEnvironment[];
readonly schedulingPolicy: ISchedulingPolicy;
readonly eventLoop: EventLoop;
readonly queueId: string;
}

export class JobQueue implements IComputeEnvironmentListener {
Expand All @@ -898,12 +904,14 @@ export class JobQueue implements IComputeEnvironmentListener {

public readonly executionMetrics: ExecutionMetrics[] = [];
public readonly queueMetrics: QueueMetrics[] = [];
public readonly queueId: string;

constructor(props: JobQueueProps) {
this.computeEnvironments = props.computeEnvironments;
props.computeEnvironments.forEach(env => env.addListener(this));
this.schedulingPolicy = props.schedulingPolicy;
this.eventLoop = props.eventLoop;
this.queueId = props.queueId;
}

push(job: Job): void {
Expand Down
54 changes: 34 additions & 20 deletions src/simulation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ import {
IComputeEnvironment,
ISchedulingPolicy,
JobQueue,
QueueMetrics,
QueueHistory,
} from './scheduling';

const NUMBER_OF_JOBS: number = 20000;
const DEFAULT_NUMBER_OF_JOBS: number = 10000;

export interface StochasticModel {
readonly interArrivalTimeDistribution: IDistribution;
Expand Down Expand Up @@ -120,8 +120,8 @@ export class JobGenerator {

private generateRandomJobs(): Job[] {
let time = 0;
const jobs: Job[] = new Array<Job>(NUMBER_OF_JOBS);
for (let i = 0; i < NUMBER_OF_JOBS; i++) {
const jobs: Job[] = new Array<Job>(DEFAULT_NUMBER_OF_JOBS);
for (let i = 0; i < DEFAULT_NUMBER_OF_JOBS; i++) {
const c = Math.floor(Math.random() * this.models.length);
const jobDefinition: IJobDefinition = this.models[c].jobDefinition;
const rng: RandomNumberGenerator = this.rngs[c];
Expand Down Expand Up @@ -154,8 +154,10 @@ export class JobGenerator {
* Generates a random number from a geometric distribution
*/
function geometric(successProbability: number): number {
if (successProbability === 1) return 1;
return Math.ceil(Math.log(Math.random()) / Math.log(1 - successProbability));
}

function validateConfigs(configs: StochasticModel[]) {
// TODO test this
if (configs.some(c => c.weightFactorProbabilities != null
Expand Down Expand Up @@ -200,7 +202,7 @@ interface SimulationResult {
readonly timesByJobDefinition: Map<string, Histogram>;
readonly meanTimeByShareIdentifier: Map<string, number>;
readonly meanTimeByJobDefinition: Map<string, number>;
readonly queueSize: QueueMetrics[];
readonly queueHistories: QueueHistory[];
}

export interface IBatchSimulator<M> {
Expand Down Expand Up @@ -291,7 +293,10 @@ export class Simulator {
eventLoop.start();

const metrics: ExecutionMetrics[] = backlogs.flatMap(b => b.queue.executionMetrics);
const queueSizeMetric: QueueMetrics[] = backlogs.flatMap(b => b.queue.queueMetrics);
const queueHistories: QueueHistory[] = backlogs.map(b => ({
id: b.queue.queueId,
metrics: b.queue.queueMetrics,
}));
const map: Map<Job, number> = new Map(metrics.map(m => [m.job, m.time]));

const timesByShareIdentifier = this.aggregateByShareIdentifier(map);
Expand All @@ -313,7 +318,7 @@ export class Simulator {
timesByJobDefinition,
meanTimeByShareIdentifier,
meanTimeByJobDefinition,
queueSize: queueSizeMetric,
queueHistories,
});
}

Expand Down Expand Up @@ -364,6 +369,7 @@ class ConstructConverter {
schedulingPolicy: this.convertSchedulingPolicy(queue.schedulingPolicy),
computeEnvironments: queue.computeEnvironments.map(e => this.convertComputeEnvironment(e.computeEnvironment)),
eventLoop: this.eventLoop,
queueId: queue.node.path,
});
}

Expand Down Expand Up @@ -433,6 +439,14 @@ export class SimulationReport {
return result;
};

const generateQueueDivs = () => {
const result: string[] = [];
this.simulationResult.queueHistories.forEach(history => {
result.push(`<div id="q-${history.id}"></div>`);
});
return result;
};

const generateCalls = () => {
const calls: string[] = [];
const header: [any, any][] = [['Time', '']];
Expand All @@ -442,20 +456,20 @@ export class SimulationReport {
this.simulationResult.timesByJobDefinition.forEach((histogram, jobDefinition) => {
calls.push(`drawChart('jd-${jobDefinition}', '${jobDefinition} (μ = ${this.simulationResult.meanTimeByJobDefinition.get(jobDefinition)})', ${JSON.stringify(header.concat(histogram.entries()))});`);
});
calls.push(...generateQueueHistories());
return calls.join('\n');
};

const generateQueueSize = () => {
const header: [any, any][] = [['Time', 'Number of jobs']];
const data: [number, number][] = this.simulationResult.queueSize
.map(m => [m.time, m.size]);
const sortedData: [number, number][] = data
.sort((a, b) => a[0] - b[0])
.filter((_, i) => i % 10 === 0);

return `drawChart('queue-size', 'Queue size', ${JSON.stringify(header.concat(sortedData))}, 'Line');`;
const generateQueueHistories = () => {
return this.simulationResult.queueHistories.map(history => {
const header: [any, any][] = [['Time', 'Number of jobs']];
const data: [number, number][] = history.metrics.map(p => ([p.time, p.size]));
const sortedData: [number, number][] = data
.sort((a, b) => a[0] - b[0])
.filter((_, i) => i % 10 === 0);
return `drawChart('q-${history.id}', '${history.id}', ${JSON.stringify(header.concat(sortedData))}, 'Line');`;
});
};

return `
<html>
<head>
Expand All @@ -482,7 +496,7 @@ export class SimulationReport {
function drawCharts() {
${generateCalls()}
${generateQueueSize()}
}
</script>
</head>
Expand All @@ -499,9 +513,9 @@ ${generateShareIdDivs().join('')}
${generateJobDefinitionDivs().join('')}
</div>
<h2>Queue size</h2>
<h2>Queue histories</h2>
<div bp="full-width">
<div id="queue-size" />
${generateQueueDivs().join('')}
</div>
</body>
Expand Down
2 changes: 1 addition & 1 deletion yarn.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit b2056a2

Please sign in to comment.