Skip to content

Commit 71b6afb

Browse files
frhuelszfrhuelsz
authored
infra: Improve pr pipeline speed (#522)
# 🔍 Description Parallelize a couple things where possible
1 parent 9f694f7 commit 71b6afb

3 files changed

Lines changed: 161 additions & 84 deletions

File tree

‎.pipelines/templates/stages/validate_makefile/dev-build.yml‎

Lines changed: 49 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,50 @@ stages:
1414
dependsOn: ${{ parameters.dependsOnStage }}
1515

1616
jobs:
17+
- job: Make_Pipelines
18+
displayName: Validate Pipelines
19+
timeoutInMinutes: 30
20+
pool:
21+
type: linux
22+
name: trident-ubuntu-1es-pool-eastus2
23+
hostArchitecture: amd64
24+
variables:
25+
ob_outputDirectory: $(Build.SourcesDirectory)/build
26+
steps:
27+
- template: ../common_tasks/avoid-pypi-usage.yml
28+
29+
- script: |
30+
set -eux
31+
32+
make check-pipelines BRANCH=$(Build.SourceBranch) NO_PARALLEL=true
33+
displayName: "Validating pipeline templates work"
34+
workingDirectory: $(Build.SourcesDirectory)
35+
env:
36+
AZURE_DEVOPS_EXT_PAT: $(System.AccessToken)
37+
OVERRIDE_RUST_FEED: false
38+
39+
- job: Check_Licenses
40+
displayName: Check Licenses
41+
timeoutInMinutes: 30
42+
pool:
43+
type: linux
44+
name: trident-ubuntu-1es-pool-eastus2
45+
hostArchitecture: amd64
46+
variables:
47+
ob_outputDirectory: $(Build.SourcesDirectory)/build
48+
steps:
49+
- template: ../common_tasks/avoid-pypi-usage.yml
50+
- template: ../common_tasks/rustup.yml
51+
- template: ../common_tasks/cargo-auth.yml
52+
53+
- script: |
54+
# Use a lower optimization level to speed up cargo-deny installation.
55+
CARGO_PROFILE_RELEASE_OPT_LEVEL=0 cargo install --locked cargo-deny@0.16.2
56+
displayName: Install cargo-deny
57+
58+
- script: cargo deny --all-features --workspace check licenses
59+
displayName: Check licenses
60+
1761
- job: Make
1862
displayName: make
1963
timeoutInMinutes: 30
@@ -56,6 +100,11 @@ stages:
56100

57101
- bash: |
58102
set -eux
103+
104+
# Use a lower optimization level to speed up the build, since
105+
# we're only validating that the Makefile targets work.
106+
export CARGO_PROFILE_RELEASE_OPT_LEVEL=0
107+
59108
make validate-configs
60109
make bin/trident-rpms.tar.gz
61110
make docker-build
@@ -71,19 +120,3 @@ stages:
71120
workingDirectory: $(Build.SourcesDirectory)
72121
env:
73122
OVERRIDE_RUST_FEED: false
74-
75-
- script: |
76-
set -eux
77-
78-
make check-pipelines BRANCH=$(Build.SourceBranch)
79-
displayName: "Validating pipeline templates work"
80-
workingDirectory: $(Build.SourcesDirectory)
81-
env:
82-
AZURE_DEVOPS_EXT_PAT: $(System.AccessToken)
83-
OVERRIDE_RUST_FEED: false
84-
85-
- script: cargo install --locked cargo-deny@0.16.2
86-
displayName: Install cargo-deny
87-
88-
- script: cargo deny --all-features --workspace check licenses
89-
displayName: Check licenses

‎Makefile‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,23 @@ check-pipelines:
3434
ifdef BRANCH
3535
$(eval BRANCH_FLAG := -b $(BRANCH))
3636
endif
37-
./scripts/test-pipeline prism-cicd -q $(BRANCH_FLAG)
38-
./scripts/test-pipeline azl-cicd -q $(BRANCH_FLAG)
39-
./scripts/test-pipeline pr -q $(BRANCH_FLAG)
40-
./scripts/test-pipeline pr-e2e -q $(BRANCH_FLAG)
41-
./scripts/test-pipeline pr-e2e-azure -q $(BRANCH_FLAG)
42-
./scripts/test-pipeline ci -q $(BRANCH_FLAG)
43-
./scripts/test-pipeline pre -q $(BRANCH_FLAG)
44-
./scripts/test-pipeline rel -q $(BRANCH_FLAG)
45-
./scripts/test-pipeline testing -q $(BRANCH_FLAG)
46-
./scripts/test-pipeline tester -q $(BRANCH_FLAG)
47-
./scripts/test-pipeline scale-official -q $(BRANCH_FLAG)
48-
./scripts/test-pipeline full-validation -q $(BRANCH_FLAG)
37+
ifndef NO_PARALLEL
38+
$(eval PARALLEL_FLAG := --parallel)
39+
endif
40+
# Note: the az-cli version in pipelines does not like --parallel, so run sequentially.
41+
./scripts/test-pipeline $(PARALLEL_FLAG) -q $(BRANCH_FLAG) \
42+
prism-cicd \
43+
azl-cicd \
44+
pr \
45+
pr-e2e \
46+
pr-e2e-azure \
47+
ci \
48+
pre \
49+
rel \
50+
testing \
51+
tester \
52+
scale-official \
53+
full-validation
4954

5055
.PHONY: check-sh
5156
check-sh:

‎scripts/test-pipeline‎

Lines changed: 95 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ parser.add_argument(
5656
type=str,
5757
help="The pipeline to preview",
5858
choices=pipeline_metadata_map.keys(),
59+
nargs="+",
5960
)
6061

6162
parser.add_argument(
@@ -70,14 +71,75 @@ parser.add_argument(
7071
"-q", "--quiet", action="store_true", help="Suppress YAML output", default=False
7172
)
7273

74+
parser.add_argument(
75+
"--parallel",
76+
action="store_true",
77+
help="Preview pipelines in parallel",
78+
)
79+
7380
args = parser.parse_args()
7481

82+
if args.parallel and not args.quiet:
83+
parser.error("--parallel requires --quiet (-q)")
84+
85+
86+
def check_pipeline(pipeline_name: str, selected_branch: str, quiet: bool = False):
87+
pipeline_metadata = pipeline_metadata_map[pipeline_name]
88+
89+
if pipeline_metadata.id == -1:
90+
raise Exception(f"Pipeline '{pipeline_name}' is not defined in the script")
91+
92+
print(
93+
f"Checking pipeline '{pipeline_name}' with ID '{pipeline_metadata.id}' on branch '{selected_branch}'",
94+
file=sys.stderr,
95+
)
96+
97+
payload = {
98+
"previewRun": True,
99+
"resources": {"repositories": {"self": {"refName": selected_branch}}},
100+
}
101+
102+
if pipeline_metadata.parameters:
103+
payload["templateParameters"] = pipeline_metadata.parameters
104+
105+
with tempfile.NamedTemporaryFile() as payload_file:
106+
payload_file.write(json.dumps(payload).encode("utf-8"))
107+
payload_file.flush()
108+
109+
cmd = [
110+
"az",
111+
"devops",
112+
"invoke",
113+
"--org",
114+
"https://dev.azure.com/mariner-org",
115+
"--api-version",
116+
"7.0",
117+
"--area",
118+
"pipelines",
119+
"--resource",
120+
"runs",
121+
"--route-parameters",
122+
f"project={pipeline_metadata.project}",
123+
f"pipelineId={pipeline_metadata.id}",
124+
"--http-method",
125+
"POST",
126+
"--in-file",
127+
payload_file.name,
128+
]
129+
output = subprocess.run(
130+
cmd,
131+
capture_output=True,
132+
)
133+
134+
if output.returncode != 0:
135+
raise Exception(f"Pipeline preview failed: {output.stderr.decode('utf-8')}")
136+
137+
print(f"Pipeline '{pipeline_name}' previewed successfully", file=sys.stderr)
75138

76-
pipeline_metadata = pipeline_metadata_map[args.pipeline]
139+
if not quiet:
140+
out_json = json.loads(output.stdout.decode("utf-8"))
141+
print(out_json["finalYaml"])
77142

78-
if pipeline_metadata.id == -1:
79-
print("Pipeline does not exist yet", file=sys.stderr)
80-
exit(2)
81143

82144
if args.branch:
83145
selected_branch = args.branch
@@ -93,55 +155,32 @@ else:
93155
.strip()
94156
)
95157

96-
print(
97-
f"Checking pipeline '{args.pipeline}' with ID '{pipeline_metadata.id}' on branch '{selected_branch}'",
98-
file=sys.stderr,
99-
)
100-
101-
payload = {
102-
"previewRun": True,
103-
"resources": {"repositories": {"self": {"refName": selected_branch}}},
104-
}
105-
106-
if pipeline_metadata.parameters:
107-
payload["templateParameters"] = pipeline_metadata.parameters
108-
109-
with tempfile.NamedTemporaryFile() as payload_file:
110-
payload_file.write(json.dumps(payload).encode("utf-8"))
111-
payload_file.flush()
112-
113-
cmd = [
114-
"az",
115-
"devops",
116-
"invoke",
117-
"--org",
118-
"https://dev.azure.com/mariner-org",
119-
"--api-version",
120-
"7.0",
121-
"--area",
122-
"pipelines",
123-
"--resource",
124-
"runs",
125-
"--route-parameters",
126-
f"project={pipeline_metadata.project}",
127-
f"pipelineId={pipeline_metadata.id}",
128-
"--http-method",
129-
"POST",
130-
"--in-file",
131-
payload_file.name,
132-
]
133-
output = subprocess.run(
134-
cmd,
135-
capture_output=True,
136-
)
137-
138-
if output.returncode != 0:
139-
print("Failed to preview pipeline:", file=sys.stderr)
140-
print(output.stderr.decode("utf-8"), file=sys.stderr)
141-
exit(1)
142-
143-
print("Pipeline previewed successfully", file=sys.stderr)
144-
145-
if not args.quiet:
146-
out_json = json.loads(output.stdout.decode("utf-8"))
147-
print(out_json["finalYaml"])
158+
if args.parallel:
159+
import concurrent.futures
160+
161+
with concurrent.futures.ThreadPoolExecutor() as executor:
162+
futures = {
163+
executor.submit(check_pipeline, name, selected_branch): name
164+
for name in args.pipeline
165+
}
166+
failures = []
167+
for future in concurrent.futures.as_completed(futures):
168+
name = futures[future]
169+
try:
170+
future.result()
171+
except Exception as e:
172+
print(f"Error previewing pipeline '{name}': {e}", file=sys.stderr)
173+
failures.append(name)
174+
if failures:
175+
print(
176+
f"Failed pipelines: {', '.join(failures)}",
177+
file=sys.stderr,
178+
)
179+
exit(1)
180+
else:
181+
for pipeline_name in args.pipeline:
182+
try:
183+
check_pipeline(pipeline_name, selected_branch, args.quiet)
184+
except Exception as e:
185+
print(f"Error previewing pipeline '{pipeline_name}': {e}", file=sys.stderr)
186+
exit(1)

0 commit comments

Comments
 (0)