fix(serialization): references inlining (#3124)

fix(serialization): potential infinite loop when inlining references

* fix(serialization): additional recursive paths

Author: baywet

src/Microsoft.OpenApi/Extensions/OpenAPIWriterExtensions.cs

@@ -15,5 +15,13 @@ internal static OpenApiWriterSettings GetSettings(this IOpenApiWriter openApiWri
             }
             return new();
         }
+
+        internal static void CountInlineReference(this IOpenApiWriter openApiWriter)
+        {
+            if (openApiWriter is OpenApiWriterBase @base)
+            {
+                @base.CountInlineReference();
+            }
+        }
     }
 }

src/Microsoft.OpenApi/Models/OpenApiHeader.cs

@@ -98,7 +98,7 @@ public virtual void SerializeAsV3(IOpenApiWriter writer)
             SerializeInternal(writer, OpenApiSpecVersion.OpenApi3_0, (writer, element) => element.SerializeAsV3(writer));
         }
 
-        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version, 
+        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version,
             Action<IOpenApiWriter, IOpenApiSerializable> callback)
         {
             Utils.CheckArgumentNull(writer);
@@ -175,7 +175,12 @@ public virtual void SerializeAsV2(IOpenApiWriter writer)
             writer.WriteProperty(OpenApiConstants.AllowReserved, AllowReserved, false);
 
             // schema
-            var targetSchema = Schema switch {
+            if (Schema is OpenApiSchemaReference)
+            {
+                writer.CountInlineReference();
+            }
+            var targetSchema = Schema switch
+            {
                 OpenApiSchemaReference schemaReference => schemaReference.RecursiveTarget,
                 OpenApiSchema schema => schema,
                 _ => null,

src/Microsoft.OpenApi/Models/OpenApiParameter.cs

@@ -113,11 +113,11 @@ public virtual void SerializeAsV3(IOpenApiWriter writer)
             SerializeInternal(writer, OpenApiSpecVersion.OpenApi3_0, (writer, element) => element.SerializeAsV3(writer));
         }
 
-        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version, 
+        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version,
             Action<IOpenApiWriter, IOpenApiSerializable> callback)
         {
             Utils.CheckArgumentNull(writer);
-            
+
             // Validate that Cookie style is only used in OpenAPI 3.2 and later
             if (Style == ParameterStyle.Cookie && version < OpenApiSpecVersion.OpenApi3_2)
             {
@@ -228,7 +228,12 @@ internal virtual void WriteRequestBodySchemaForV2(IOpenApiWriter writer, Diction
                 // uniqueItems
                 // enum
                 // multipleOf
-                var targetSchema = Schema switch {
+                if (Schema is OpenApiSchemaReference)
+                {
+                    writer.CountInlineReference();
+                }
+                var targetSchema = Schema switch
+                {
                     OpenApiSchemaReference schemaReference => schemaReference.RecursiveTarget,
                     OpenApiSchema schema => schema,
                     _ => null,
@@ -273,7 +278,7 @@ internal virtual void WriteRequestBodySchemaForV2(IOpenApiWriter writer, Diction
         public virtual void SerializeAsV2(IOpenApiWriter writer)
         {
             Utils.CheckArgumentNull(writer);
-            
+
             // Validate that Cookie style is only used in OpenAPI 3.2 and later
             if (Style == ParameterStyle.Cookie)
             {

src/Microsoft.OpenApi/Models/References/BaseOpenApiReferenceHolder.cs

@@ -207,7 +207,7 @@ protected BaseOpenApiReferenceHolder(string referenceId, OpenApiDocument? hostDo
     /// <inheritdoc/>
     public virtual void SerializeAsV3(IOpenApiWriter writer)
     {
-        if (!writer.GetSettings().ShouldInlineReference(Reference) 
+        if (!writer.GetSettings().ShouldInlineReference(Reference)
             || Reference.Type == ReferenceType.Tag) // tags are held as references need to drop in.
         {
             Reference.SerializeAsV3(writer);
@@ -267,6 +267,7 @@ private protected void SerializeInternal(IOpenApiWriter writer,
         Action<IOpenApiWriter, U> action)
     {
         Utils.CheckArgumentNull(writer);
+        writer.CountInlineReference();
         ApplyToTarget(element => action(writer, element));
     }
 }

src/Microsoft.OpenApi/Models/References/OpenApiMediaTypeReference.cs

@@ -77,11 +77,13 @@ protected override BaseOpenApiReference CopyReference(BaseOpenApiReference sourc
         /// <inheritdoc/>
         public override void SerializeAsV31(IOpenApiWriter writer)
         {
+            writer.CountInlineReference();
             CopyReferenceAsTargetElementWithOverrides(new OpenApiMediaType()).SerializeAsV31(writer);
         }
         /// <inheritdoc/>
         public override void SerializeAsV3(IOpenApiWriter writer)
         {
+            writer.CountInlineReference();
             CopyReferenceAsTargetElementWithOverrides(new OpenApiMediaType()).SerializeAsV3(writer);
         }
     }

src/Microsoft.OpenApi/Models/References/OpenApiRequestBodyReference.cs

@@ -64,6 +64,7 @@ public override void SerializeAsV2(IOpenApiWriter writer)
         {
             if (writer.GetSettings().ShouldInlineReference(Reference))
             {
+                writer.CountInlineReference();
                 return GetFromTarget(target => target.ConvertToBodyParameter(writer));
             }
 
@@ -74,6 +75,7 @@ public override void SerializeAsV2(IOpenApiWriter writer)
         {
             if (writer.GetSettings().ShouldInlineReference(Reference))
             {
+                writer.CountInlineReference();
                 return GetFromTarget(target => target.ConvertToFormDataParameters(writer));
             }
 

src/Microsoft.OpenApi/Models/References/OpenApiSchemaReference.cs

@@ -256,8 +256,14 @@ private void SerializeAsWithoutLoops(IOpenApiWriter writer, Action<IOpenApiWrite
             }
             else
             {
-                SerializeInternal(writer, (w, element) => action(w, element));
-                writer.GetSettings().LoopDetector.PopLoop<IOpenApiSchema>();
+                try
+                {
+                    SerializeInternal(writer, (w, element) => action(w, element));
+                }
+                finally
+                {
+                    writer.GetSettings().LoopDetector.PopLoop<IOpenApiSchema>();
+                }
             }
 
         }

src/Microsoft.OpenApi/Properties/SRResource.Designer.cs

@@ -132,6 +132,9 @@
   <data name="IndentationLevelInvalid" xml:space="preserve">
     <value>Indentation level cannot be lower than 0.</value>
   </data>
+  <data name="MaximumInlineReferenceCountExceeded" xml:space="preserve">
+    <value>The maximum number of inline references ({0}) has been exceeded.</value>
+  </data>
   <data name="InputItemShouldBeType" xml:space="preserve">
     <value>The input item should be in type of '{0}'.</value>
   </data>

src/Microsoft.OpenApi/Properties/SRResource.resx

@@ -1,2 +1,4 @@
 #nullable enable
 Microsoft.OpenApi.OpenApiDocument.GetOperationById(string! operationId) -> Microsoft.OpenApi.OpenApiOperation?
+Microsoft.OpenApi.OpenApiWriterSettings.MaxInlineReferenceCount.get -> int
+Microsoft.OpenApi.OpenApiWriterSettings.MaxInlineReferenceCount.set -> void