-
Notifications
You must be signed in to change notification settings - Fork 519
(WIP) OCPBUGS-112465 #6431
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
(WIP) OCPBUGS-112465 #6431
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -252,37 +252,45 @@ func (b *buildReconciler) AddJob(ctx context.Context, job *batchv1.Job) error { | |
|
|
||
| // Executes whenever a build Job is updated | ||
| func (b *buildReconciler) UpdateJob(ctx context.Context, oldJob, curJob *batchv1.Job) error { | ||
| klog.Errorf("in UpdateJob") | ||
| return b.timeObjectOperation(curJob, updatingVerb, func() error { | ||
| mosb, err := b.getMachineOSBuildForJob(curJob) | ||
| if err == nil && mosb != nil { | ||
| if curJob.Status.Succeeded > 0 && (oldJob.Status.Succeeded == 0) { | ||
| klog.Errorf("in curJob.Status.Succeeded > 0 && (oldJob.Status.Succeeded == 0)") | ||
| b.eventRecorder.RecordJobCompleted(mosb, curJob) | ||
| } | ||
|
|
||
| if curJob.Status.Failed > 0 && (oldJob.Status.Failed == 0) { | ||
| klog.Errorf("in curJob.Status.Failed > 0 && (oldJob.Status.Failed == 0)") | ||
| b.eventRecorder.RecordJobFailed(mosb, curJob) | ||
| } | ||
|
|
||
| if curJob.Status.Active > 0 && (oldJob.Status.Active == 0) { | ||
| klog.Errorf("in curJob.Status.Active > 0 && (oldJob.Status.Active == 0)") | ||
| b.eventRecorder.RecordJobStarted(mosb, curJob) | ||
| b.eventRecorder.RecordBuildBuilding(mosb) | ||
| } | ||
|
|
||
| mosc, err := utils.GetMachineOSConfigForMachineOSBuild(mosb, b.utilListers()) | ||
| if err == nil { | ||
| klog.Errorf("in err == nil") | ||
| poolName := mosc.Spec.MachineConfigPool.Name | ||
|
|
||
| if curJob.Status.Succeeded > 0 && (oldJob.Status.Succeeded == 0) { | ||
| klog.Errorf("in curJob.Status.Succeeded > 0 && (oldJob.Status.Succeeded == 0)") | ||
| RecordBuildJobState(poolName, StateSucceeded) | ||
| RecordImagePushCompleted(poolName) | ||
| } | ||
|
|
||
| if curJob.Status.Failed > 0 && (oldJob.Status.Failed == 0) { | ||
| klog.Errorf("in curJob.Status.Failed > 0 && (oldJob.Status.Failed == 0)") | ||
| RecordBuildJobState(poolName, StateFailed) | ||
| RecordImagePushFailed(poolName) | ||
| } | ||
|
|
||
| if curJob.Status.Failed > oldJob.Status.Failed && curJob.Status.Failed <= constants.JobMaxRetries { | ||
| klog.Errorf("in curJob.Status.Failed > oldJob.Status.Failed && curJob.Status.Failed <= constants.JobMaxRetries") | ||
|
Comment on lines
+255
to
+293
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🩺 Stability & Availability | 🟠 Major | ⚡ Quick win Adjust log severity and boolean formatting. 📍 Affects 1 file
🤖 Prompt for AI Agents |
||
| RecordBuildRetry(poolName) | ||
| } | ||
| } | ||
|
|
@@ -614,7 +622,6 @@ func (b *buildReconciler) startBuild(ctx context.Context, mosb *mcfgv1.MachineOS | |
| // Retrieves a deep-copy of the MachineOSConfig from the lister so that the cache is not mutated during the update. | ||
| func (b *buildReconciler) getMachineOSConfigForUpdate(mosc *mcfgv1.MachineOSConfig) (*mcfgv1.MachineOSConfig, error) { | ||
| out, err := b.machineOSConfigLister.Get(mosc.Name) | ||
|
|
||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -635,7 +642,6 @@ func (b *buildReconciler) getMachineOSBuildForJob(job *batchv1.Job) (*mcfgv1.Mac | |
| // Retrieves a deep-copy of the MachineOSBuild from the lister so that the cache is not mutated during the update. | ||
| func (b *buildReconciler) getMachineOSBuildForUpdate(mosb *mcfgv1.MachineOSBuild) (*mcfgv1.MachineOSBuild, error) { | ||
| out, err := b.machineOSBuildLister.Get(mosb.Name) | ||
|
|
||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -730,7 +736,6 @@ func (b *buildReconciler) createNewMachineOSBuildOrReuseExisting(ctx context.Con | |
| MachineOSConfig: mosc, | ||
| MachineConfigPool: mcp, | ||
| }) | ||
|
|
||
| if err != nil { | ||
| return fmt.Errorf("could not instantiate new MachineOSBuild: %w", err) | ||
| } | ||
|
|
@@ -861,7 +866,9 @@ func (b *buildReconciler) getMachineOSBuildStatusForBuilder(ctx context.Context, | |
| // the decision off to setStatusOnMachineOSBuildIfNeeded. | ||
| func (b *buildReconciler) updateMachineOSBuildWithStatusIfNeeded(ctx context.Context, oldBuilder, curBuilder metav1.Object) error { | ||
| oldStatus, _, err := b.getMachineOSBuildStatusForBuilder(ctx, oldBuilder) | ||
| klog.Errorf("oldStatus: %v", oldStatus) | ||
| if err != nil { | ||
| klog.Errorf("got err for getMachineOSBuildStatusForBuilder for old") | ||
| // If we can't find the MachineOSConfig, MachineOSBuild, or any of the | ||
| // ephemeral build objects, it means that it was probably deleted. Instead | ||
| // of trying to reconcile the status, we'll return nil here to avoid | ||
|
|
@@ -870,7 +877,9 @@ func (b *buildReconciler) updateMachineOSBuildWithStatusIfNeeded(ctx context.Con | |
| } | ||
|
|
||
| curStatus, mosb, err := b.getMachineOSBuildStatusForBuilder(ctx, curBuilder) | ||
| klog.Errorf("curStatus: %v", curStatus) | ||
| if err != nil { | ||
| klog.Errorf("got err for getMachineOSBuildStatusForBuilder for cur") | ||
| // If we can't find the MachineOSConfig, MachineOSBuild, or any of the | ||
| // ephemeral build objects, it means that it was probably deleted. Instead | ||
| // of trying to reconcile the status, we'll return nil here to avoid | ||
|
|
@@ -909,9 +918,11 @@ func (b *buildReconciler) updateMachineOSBuildWithStatusIfNeeded(ctx context.Con | |
|
|
||
| // Sets the status on the MachineOSBuild object after comparing the statuses according to very specific state transitions. | ||
| func (b *buildReconciler) setStatusOnMachineOSBuildIfNeeded(ctx context.Context, mosb *mcfgv1.MachineOSBuild, oldStatus, curStatus mcfgv1.MachineOSBuildStatus) error { | ||
| klog.Errorf("in setStatusOnMachineOSBuildIfNeeded") | ||
| // Compare the old status and the current status to determine if an update is | ||
| // needed. This is handled according to very specific state transitions. | ||
| isUpdateNeeded, reason := isMachineOSBuildStatusUpdateNeeded(oldStatus, curStatus) | ||
| klog.Errorf("isUpdateNeeded: %v", isUpdateNeeded) | ||
| if !isUpdateNeeded { | ||
| if reason != "" { | ||
| klog.Infof("MachineOSBuild %q %s; skipping update because of invalid transition", mosb.Name, reason) | ||
|
|
@@ -1212,7 +1223,6 @@ func (b *buildReconciler) syncAll(ctx context.Context) error { | |
|
|
||
| return nil | ||
| }) | ||
|
|
||
| if err != nil { | ||
| return fmt.Errorf("could not sync all: %w", err) | ||
| } | ||
|
|
@@ -1236,7 +1246,6 @@ func (b *buildReconciler) syncMachineOSBuilds(ctx context.Context) error { | |
|
|
||
| return nil | ||
| }) | ||
|
|
||
| if err != nil { | ||
| return fmt.Errorf("could not sync MachineOSBuilds: %w", err) | ||
| } | ||
|
|
@@ -1249,7 +1258,6 @@ func (b *buildReconciler) syncMachineOSBuilds(ctx context.Context) error { | |
| // builder associated with it that one should be created. | ||
| func (b *buildReconciler) syncMachineOSBuild(ctx context.Context, mosb *mcfgv1.MachineOSBuild) error { | ||
| return b.timeObjectOperation(mosb, syncingVerb, func() error { | ||
|
|
||
| // It could be the case that the MCP the mosb in queue was targeting no longer is valid | ||
| mcp, err := b.machineConfigPoolLister.Get(mosb.ObjectMeta.Labels[constants.TargetMachineConfigPoolLabelKey]) | ||
| if err != nil { | ||
|
|
@@ -1392,7 +1400,6 @@ func (b *buildReconciler) syncMachineOSConfigs(ctx context.Context) error { | |
|
|
||
| return nil | ||
| }) | ||
|
|
||
| if err != nil { | ||
| return fmt.Errorf("could not sync MachineOSConfigs: %w", err) | ||
| } | ||
|
|
@@ -1476,7 +1483,6 @@ func (b *buildReconciler) syncMachineConfigPools(ctx context.Context) error { | |
|
|
||
| return nil | ||
| }) | ||
|
|
||
| if err != nil { | ||
| return fmt.Errorf("could not sync MachineConfigPools: %w", err) | ||
| } | ||
|
|
@@ -1612,7 +1618,6 @@ func (b *buildReconciler) reconcilePoolChange(ctx context.Context, mcp *mcfgv1.M | |
| return b.reuseImageForNewMOSB(ctx, mosc, oldMOSB) | ||
| } | ||
| return b.createNewMachineOSBuildOrReuseExisting(ctx, mosc, needsImageRebuild) | ||
|
|
||
| } | ||
|
|
||
| // reuseImageForNewMOSB creates a new MOSB (for the new rendered-MC name) | ||
|
|
@@ -1637,7 +1642,6 @@ func (b *buildReconciler) reuseImageForNewMOSB(ctx context.Context, mosc *mcfgv1 | |
| MachineOSConfig: mosc, | ||
| MachineConfigPool: mcp, | ||
| }) | ||
|
|
||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
@@ -1805,7 +1809,6 @@ func (b *buildReconciler) shouldPreventBuildDueToDegradation(mcp *mcfgv1.Machine | |
| // reconcileImageRebuild calls RequiresRebuild to see if an MC changes the kernel args, ext, or osimageurl. | ||
| // if it does, we build a new image in our new MOSB | ||
| func (b *buildReconciler) reconcileImageRebuild(oldMCP, curMCP *mcfgv1.MachineConfigPool) (bool, error) { | ||
|
|
||
| curr, err := b.machineConfigLister.Get(oldMCP.Spec.Configuration.Name) | ||
| if err != nil { | ||
| return false, err | ||
|
|
@@ -1982,7 +1985,6 @@ func (b *buildReconciler) seedMachineOSConfigWithExistingImage(ctx context.Conte | |
| MachineConfigPool: mcp, | ||
| MachineOSConfig: mosc, | ||
| }) | ||
|
|
||
| if err != nil { | ||
| return fmt.Errorf("could not generate MachineOSBuild template for MachineOSConfig %q: %w", mosc.Name, err) | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,7 @@ import ( | |
| mcfgv1 "github.com/openshift/api/machineconfiguration/v1" | ||
| "github.com/openshift/machine-config-operator/pkg/apihelpers" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/klog/v2" | ||
| ) | ||
|
|
||
| // This is intended to provide a singular way to interrogate MachineConfigPool | ||
|
|
@@ -115,7 +116,9 @@ func (b *MachineOSBuildState) IsInInitialState() bool { | |
|
|
||
| // Determines if an OS image build is in its terminal state where build success, build failure, or build interrupted condition is set. | ||
| func (b *MachineOSBuildState) IsInTerminalState() bool { | ||
| return b.GetTerminalState() != "" | ||
| test := b.GetTerminalState() | ||
| klog.Errorf("test: %v", test) | ||
| return test != "" | ||
| } | ||
|
|
||
| // Determines if an OS image build is in a transient state where it is either prepared, pending, or running. | ||
|
|
@@ -137,6 +140,7 @@ func (b *MachineOSBuildState) GetTransientState() mcfgv1.BuildProgress { | |
| // Gets the current terminal state, if any is set. Otherwise, returns an empty string. | ||
| func (b *MachineOSBuildState) GetTerminalState() mcfgv1.BuildProgress { | ||
| for terminalState := range MachineOSBuildTerminalStates() { | ||
| klog.Errorf("terminalState: %s", terminalState) | ||
| if apihelpers.IsMachineOSBuildConditionTrue(b.Build.Status.Conditions, terminalState) { | ||
| return terminalState | ||
| } | ||
|
|
@@ -165,11 +169,13 @@ func (b *MachineOSBuildState) SetBuildConditions(conditions []metav1.Condition) | |
| for _, condition := range conditions { | ||
| condition := condition | ||
| currentCondition := apihelpers.GetMachineOSBuildCondition(b.Build.Status, mcfgv1.BuildProgress(condition.Type)) | ||
| klog.Errorf("SetBuildConditions currentCondition %v", currentCondition) | ||
| if currentCondition != nil && isConditionEqual(*currentCondition, condition) { | ||
| continue | ||
| } | ||
|
|
||
| mosbCondition := apihelpers.NewMachineOSBuildCondition(condition.Type, condition.Status, condition.Reason, condition.Message) | ||
| klog.Errorf("SetBuildConditions mosbCondition %v", mosbCondition) | ||
|
Comment on lines
+172
to
+178
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔒 Security & Privacy | 🟡 Minor | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
rg -n -C 4 \
'NewMachineOSBuildCondition|metav1\.Condition|SetBuildConditions|klog\.(Errorf|Infof|InfoS|V\()' \
--glob '*.go' .Repository: openshift/machine-config-operator Length of output: 50390 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- target implementation ---'
sed -n '1,230p' pkg/controller/common/mos_state.go
printf '%s\n' '--- condition helper and call sites ---'
rg -n -C 5 --glob '!vendor/**' \
'func NewMachineOSBuildCondition|NewMachineOSBuildCondition\(|SetBuildConditions\(' .
printf '%s\n' '--- focused tests ---'
sed -n '1,260p' pkg/controller/common/mos_state_test.go
printf '%s\n' '--- project logging conventions ---'
rg -n --glob '!vendor/**' --glob '*.go' \
'klog\.(Errorf|ErrorS|Infof|InfoS|V\([0-9]+\)\.(Infof|InfoS))' pkg | head -n 240Repository: openshift/machine-config-operator Length of output: 44490 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- condition helper ---'
cat -n pkg/apihelpers/machineosbuild.go | sed -n '1,220p'
printf '%s\n' '--- condition constructors and messages outside vendor ---'
rg -n -C 6 --glob '!vendor/**' --glob '*.go' \
'MachineOSBuild(Initial|Pending|Running|Succeeded|Failed|Interrupted)Conditions|NewMachineOSBuildCondition|Message:' \
pkg | head -n 500
printf '%s\n' '--- reconciler status source ---'
sed -n '880,970p' pkg/controller/build/reconciler.go
rg -n -C 8 --glob '!vendor/**' --glob '*.go' \
'curStatus\.Conditions|Status\.Conditions.*Message|MachineOSBuild.*Condition' pkg/controller/build pkg | head -n 500Repository: openshift/machine-config-operator Length of output: 50389 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- status propagation into curStatus ---'
rg -n -C 12 --glob '!vendor/**' --glob '*.go' \
'func \(b \*buildReconciler\) getMachineOSBuildStatusForBuilder|func getMachineOSBuildStatusForBuilder|getMachineOSBuildStatusForBuilder\(' \
pkg/controller/build
printf '%s\n' '--- all assignments to MachineOSBuildStatus conditions ---'
rg -n -C 8 --glob '!vendor/**' --glob '*.go' \
'Conditions\s*:\s*|\.Conditions\s*=|SetMachineOSBuildCondition' \
pkg/controller/build pkg/imagebuilder pkg | grep -v '/vendor/' | head -n 500
printf '%s\n' '--- exact changed-file history summary ---'
git diff --stat
git status --shortRepository: openshift/machine-config-operator Length of output: 46723 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- imagebuilder files ---'
fd -t f -e go 'imagebuilder|observer|status' pkg | head -n 120
printf '%s\n' '--- MachineOSBuildStatus implementations ---'
rg -n -C 12 --glob '!vendor/**' --glob '*.go' \
'func .*MachineOSBuildStatus|MachineOSBuildStatus\(ctx|MachineOSBuildStatus\(' pkg | head -n 500
printf '%s\n' '--- pre-built condition flow ---'
sed -n '2025,2085p' pkg/controller/build/reconciler.go
sed -n '1660,1720p' pkg/controller/build/reconciler.goRepository: openshift/machine-config-operator Length of output: 36734 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- status mapping implementation ---'
sed -n '60,145p' pkg/controller/build/imagebuilder/base.go
sed -n '1,180p' pkg/controller/build/imagebuilder/jobimagebuilder.go
printf '%s\n' '--- status condition producers in imagebuilder ---'
rg -n -C 10 --glob '*.go' \
'MapJobStatusToBuildStatus|conditions :=|MachineOSBuild.*Conditions|Message:' \
pkg/controller/build/imagebuilderRepository: openshift/machine-config-operator Length of output: 20310 Use verbosity-gated, redacted diagnostics.
🤖 Prompt for AI AgentsSource: Coding guidelines |
||
| apihelpers.SetMachineOSBuildCondition(&b.Build.Status, *mosbCondition) | ||
| } | ||
| } | ||
|
|
@@ -262,7 +268,6 @@ func HasBuildObjectForCurrentMachineConfig(pool *mcfgv1.MachineConfigPool, mosb | |
| // Determines if we should do a build based upon the state of our | ||
| // MachineConfigPool, the presence of a build pod, etc. | ||
| func BuildDueToPoolChange(oldPool, curPool *mcfgv1.MachineConfigPool, moscNew *mcfgv1.MachineOSConfig, mosbNew *mcfgv1.MachineOSBuild) bool { | ||
|
|
||
| moscState := NewMachineOSConfigState(moscNew) | ||
| mosbState := NewMachineOSBuildState(mosbNew) | ||
|
|
||
|
|
@@ -273,7 +278,6 @@ func BuildDueToPoolChange(oldPool, curPool *mcfgv1.MachineConfigPool, moscNew *m | |
| (IsPoolConfigChange(oldPool, curPool) || !moscState.HasOSImage()) | ||
|
|
||
| return poolStateSuggestsBuild | ||
|
|
||
| } | ||
|
|
||
| // Checks our pool to see if we can do a build. We base this off of a few criteria: | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🩺 Stability & Availability | 🟡 Minor | ⚡ Quick win
🧩 Analysis chain
🏁 Script executed:
Repository: openshift/machine-config-operator
Length of output: 50390
🏁 Script executed:
Repository: openshift/machine-config-operator
Length of output: 50389
Use a non-error log level for normal status evaluation.
This helper runs during normal reconciliation and emits at least two
ERRORrecords on every path. Replace the addedklog.Errorfcalls withklog.Infoforklog.V(4).Infof. Reserve error-level logging for actual failures.🤖 Prompt for AI Agents