diff --git a/pkg/canary/scaled_object_reconciler.go b/pkg/canary/scaled_object_reconciler.go
index 954fd4015..2c0a362bd 100644
--- a/pkg/canary/scaled_object_reconciler.go
+++ b/pkg/canary/scaled_object_reconciler.go
@@ -212,11 +212,13 @@ func setPrimaryScaledObjectQueries(cd *flaggerv1.Canary, triggers []keda.ScaleTr
 }
 
 func makeObjectMetaSo(name string, labels map[string]string, annotations map[string]string, cd *flaggerv1.Canary) metav1.ObjectMeta {
+	filteredAnnotations := filterMetadata(annotations)
+	delete(filteredAnnotations, keda.PausedReplicasAnnotation)
 	return metav1.ObjectMeta{
 		Name:        name,
 		Namespace:   cd.Namespace,
 		Labels:      filterMetadata(labels),
-		Annotations: filterMetadata(annotations),
+		Annotations: filteredAnnotations,
 		OwnerReferences: []metav1.OwnerReference{
 			*metav1.NewControllerRef(cd, schema.GroupVersionKind{
 				Group:   flaggerv1.SchemeGroupVersion.Group,
diff --git a/pkg/canary/scaled_object_reconciler_test.go b/pkg/canary/scaled_object_reconciler_test.go
index 175fd0541..e3babb0d1 100644
--- a/pkg/canary/scaled_object_reconciler_test.go
+++ b/pkg/canary/scaled_object_reconciler_test.go
@@ -23,12 +23,16 @@ func Test_reconcilePrimaryScaledObject(t *testing.T) {
 
 	so, err := mocks.flaggerClient.KedaV1alpha1().ScaledObjects("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
 	require.NoError(t, err)
+	so.Annotations[keda.PausedReplicasAnnotation] = "0"
+	_, err = mocks.flaggerClient.KedaV1alpha1().ScaledObjects("default").Update(context.TODO(), so, metav1.UpdateOptions{})
+	require.NoError(t, err)
 
 	err = soReconciler.reconcilePrimaryScaler(mocks.canary, true)
 	require.NoError(t, err)
 
 	primarySO, err := mocks.flaggerClient.KedaV1alpha1().ScaledObjects("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
 	require.NoError(t, err)
+	assert.NotContains(t, primarySO.Annotations, keda.PausedReplicasAnnotation)
 	// test that the hpa ownership annotation is added to the primarySO
 	assert.Equal(t, primarySO.ObjectMeta.Annotations["scaledobject.keda.sh/transfer-hpa-ownership"], "true")
 	// test that the horizontalpodautoscalerconfig is set to 'podinfo-primary', so that it takes over ownership of the HPA