diff --git a/test/linters_test.go b/test/linters_test.go
index db2ce7d0..0293a74f 100644
--- a/test/linters_test.go
+++ b/test/linters_test.go
@@ -61,7 +61,7 @@ func testSourcesFromDir(t *testing.T, dir string) {
 		t.Run(filepath.Base(source), func(subTest *testing.T) {
 			subTest.Parallel()
 
-			rel, err := filepath.Rel(dir, sources[0])
+			rel, err := filepath.Rel(dir, source)
 			require.NoError(t, err)
 
 			testOneSource(subTest, log, binPath, rel)
diff --git a/test/testshared/runner.go b/test/testshared/runner.go
index 1c3e3fba..0c54af7d 100644
--- a/test/testshared/runner.go
+++ b/test/testshared/runner.go
@@ -104,7 +104,17 @@ func (b *RunnerBuilder) WithRunContext(rc *RunContext) *RunnerBuilder {
 		return b
 	}
 
-	return b.WithConfigFile(rc.ConfigPath).WithArgs(rc.Args...)
+	dir, err := os.Getwd()
+	require.NoError(b.tb, err)
+
+	configPath := filepath.FromSlash(rc.ConfigPath)
+
+	base := filepath.Base(dir)
+	if strings.HasPrefix(configPath, base) {
+		configPath = strings.TrimPrefix(configPath, base+string(filepath.Separator))
+	}
+
+	return b.WithConfigFile(configPath).WithArgs(rc.Args...)
 }
 
 func (b *RunnerBuilder) WithDirectives(sourcePath string) *RunnerBuilder {