Scheduler - configurable minimum number of jobs per process

ondrejmirtes · ondrejmirtes · commit 7e28fb88186d · 2020-02-16T14:37:50.000+01:00
diff --git a/conf/config.neon b/conf/config.neon
@@ -42,6 +42,7 @@ parameters:
 		jobSize: 20
 		processTimeout: 60.0
 		maximumNumberOfProcesses: 32
+		minimumNumberOfJobsPerProcess: 2
 	polluteScopeWithLoopInitialAssignments: true
 	polluteScopeWithAlwaysIterableForeach: true
 	polluteCatchScopeWithTryAssignments: false
@@ -152,7 +153,8 @@ parametersSchema:
 	parallel: structure([
 		jobSize: int(),
 		processTimeout: float(),
-		maximumNumberOfProcesses: int()
+		maximumNumberOfProcesses: int(),
+		minimumNumberOfJobsPerProcess: int()
 	])
 	polluteScopeWithLoopInitialAssignments: bool()
 	polluteScopeWithAlwaysIterableForeach: bool()
@@ -388,6 +390,7 @@ services:
 		arguments:
 			jobSize: %parallel.jobSize%
 			maximumNumberOfProcesses: %parallel.maximumNumberOfProcesses%
+			minimumNumberOfJobsPerProcess: %parallel.minimumNumberOfJobsPerProcess%
 
 	-
 		class: PHPStan\Parser\CachedParser
diff --git a/src/Parallel/Scheduler.php b/src/Parallel/Scheduler.php
@@ -11,13 +11,18 @@ class Scheduler
 	/** @var int */
 	private $maximumNumberOfProcesses;
 
+	/** @var int */
+	private $minimumNumberOfJobsPerProcess;
+
 	public function __construct(
 		int $jobSize,
-		int $maximumNumberOfProcesses
+		int $maximumNumberOfProcesses,
+		int $minimumNumberOfJobsPerProcess
 	)
 	{
 		$this->jobSize = $jobSize;
 		$this->maximumNumberOfProcesses = $maximumNumberOfProcesses;
+		$this->minimumNumberOfJobsPerProcess = $minimumNumberOfJobsPerProcess;
 	}
 
 	/**
@@ -31,7 +36,10 @@ public function scheduleWork(
 	): Schedule
 	{
 		$jobs = array_chunk($files, $this->jobSize);
-		$numberOfProcesses = min(count($jobs), $cpuCores);
+		$numberOfProcesses = min(
+			(int) floor(count($jobs) / $this->minimumNumberOfJobsPerProcess),
+			$cpuCores
+		);
 
 		return new Schedule(min($numberOfProcesses, $this->maximumNumberOfProcesses), $jobs);
 	}
diff --git a/tests/PHPStan/Parallel/SchedulerTest.php b/tests/PHPStan/Parallel/SchedulerTest.php
@@ -13,6 +13,7 @@ public function dataSchedule(): array
 			[
 				1,
 				16,
+				1,
 				50,
 				115,
 				1,
@@ -21,6 +22,7 @@ public function dataSchedule(): array
 			[
 				16,
 				16,
+				1,
 				30,
 				124,
 				5,
@@ -29,6 +31,7 @@ public function dataSchedule(): array
 			[
 				16,
 				3,
+				1,
 				30,
 				124,
 				3,
@@ -37,18 +40,29 @@ public function dataSchedule(): array
 			[
 				16,
 				16,
+				1,
 				10,
 				298,
 				16,
 				[10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 8],
 			],
+			[
+				16,
+				16,
+				2,
+				30,
+				124,
+				2,
+				[30, 30, 30, 30, 4],
+			],
 		];
 	}
 
 	/**
 	 * @dataProvider dataSchedule
 	 * @param int $cpuCores
 	 * @param int $maximumNumberOfProcesses
+	 * @param int $minimumNumberOfJobsPerProcess
 	 * @param int $jobSize
 	 * @param int $numberOfFiles
 	 * @param int $expectedNumberOfProcesses
@@ -57,14 +71,15 @@ public function dataSchedule(): array
 	public function testSchedule(
 		int $cpuCores,
 		int $maximumNumberOfProcesses,
+		int $minimumNumberOfJobsPerProcess,
 		int $jobSize,
 		int $numberOfFiles,
 		int $expectedNumberOfProcesses,
 		array $expectedJobSizes
 	): void
 	{
 		$files = array_fill(0, $numberOfFiles, 'file.php');
-		$scheduler = new Scheduler($jobSize, $maximumNumberOfProcesses);
+		$scheduler = new Scheduler($jobSize, $maximumNumberOfProcesses, $minimumNumberOfJobsPerProcess);
 		$schedule = $scheduler->scheduleWork($cpuCores, $files);
 
 		$this->assertSame($expectedNumberOfProcesses, $schedule->getNumberOfProcesses());

Original file line number	Diff line number	Diff line change
`@@ -11,13 +11,18 @@ class Scheduler`
`11`	`11`	`/** @var int */`
`12`	`12`	`private $maximumNumberOfProcesses;`
`13`	`13`
	`14`	`+ /** @var int */`
	`15`	`+ private $minimumNumberOfJobsPerProcess;`
	`16`	`+`
`14`	`17`	`public function __construct(`
`15`	`18`	`int $jobSize,`
`16`		`- int $maximumNumberOfProcesses`
	`19`	`+ int $maximumNumberOfProcesses,`
	`20`	`+ int $minimumNumberOfJobsPerProcess`
`17`	`21`	`)`
`18`	`22`	`{`
`19`	`23`	`$this->jobSize = $jobSize;`
`20`	`24`	`$this->maximumNumberOfProcesses = $maximumNumberOfProcesses;`
	`25`	`+ $this->minimumNumberOfJobsPerProcess = $minimumNumberOfJobsPerProcess;`
`21`	`26`	`}`
`22`	`27`
`23`	`28`	`/**`
`@@ -31,7 +36,10 @@ public function scheduleWork(`
`31`	`36`	`): Schedule`
`32`	`37`	`{`
`33`	`38`	`$jobs = array_chunk($files, $this->jobSize);`
`34`		`- $numberOfProcesses = min(count($jobs), $cpuCores);`
	`39`	`+ $numberOfProcesses = min(`
	`40`	`+ (int) floor(count($jobs) / $this->minimumNumberOfJobsPerProcess),`
	`41`	`+ $cpuCores`
	`42`	`+ );`
`35`	`43`
`36`	`44`	`return new Schedule(min($numberOfProcesses, $this->maximumNumberOfProcesses), $jobs);`
`37`	`45`	`}`