fix/circular schema serialization to v2 (#3125)

* fix(serialization): potential infinite loop when inlining references

* fix(serialization): additional recursive paths

Author: baywet

src/Microsoft.OpenApi/Extensions/OpenAPIWriterExtensions.cs

@@ -15,5 +15,13 @@ internal static OpenApiWriterSettings GetSettings(this IOpenApiWriter openApiWri
             }
             return new();
         }
+
+        internal static void CountInlineReference(this IOpenApiWriter openApiWriter)
+        {
+            if (openApiWriter is OpenApiWriterBase @base)
+            {
+                @base.CountInlineReference();
+            }
+        }
     }
 }

src/Microsoft.OpenApi/Models/OpenApiHeader.cs

@@ -90,7 +90,7 @@ public virtual void SerializeAsV3(IOpenApiWriter writer)
             SerializeInternal(writer, OpenApiSpecVersion.OpenApi3_0, (writer, element) => element.SerializeAsV3(writer));
         }
 
-        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version, 
+        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version,
             Action<IOpenApiWriter, IOpenApiSerializable> callback)
         {
             Utils.CheckArgumentNull(writer);
@@ -167,7 +167,12 @@ public virtual void SerializeAsV2(IOpenApiWriter writer)
             writer.WriteProperty(OpenApiConstants.AllowReserved, AllowReserved, false);
 
             // schema
-            var targetSchema = Schema switch {
+            if (Schema is OpenApiSchemaReference)
+            {
+                writer.CountInlineReference();
+            }
+            var targetSchema = Schema switch
+            {
                 OpenApiSchemaReference schemaReference => schemaReference.RecursiveTarget,
                 OpenApiSchema schema => schema,
                 _ => null,

src/Microsoft.OpenApi/Models/OpenApiParameter.cs

@@ -107,7 +107,7 @@ public virtual void SerializeAsV3(IOpenApiWriter writer)
             SerializeInternal(writer, OpenApiSpecVersion.OpenApi3_0, (writer, element) => element.SerializeAsV3(writer));
         }
 
-        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version, 
+        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version,
             Action<IOpenApiWriter, IOpenApiSerializable> callback)
         {
             Utils.CheckArgumentNull(writer);
@@ -202,7 +202,12 @@ internal virtual void WriteRequestBodySchemaForV2(IOpenApiWriter writer, Diction
                 // uniqueItems
                 // enum
                 // multipleOf
-                var targetSchema = Schema switch {
+                if (Schema is OpenApiSchemaReference)
+                {
+                    writer.CountInlineReference();
+                }
+                var targetSchema = Schema switch
+                {
                     OpenApiSchemaReference schemaReference => schemaReference.RecursiveTarget,
                     OpenApiSchema schema => schema,
                     _ => null,

src/Microsoft.OpenApi/Models/References/BaseOpenApiReferenceHolder.cs

@@ -207,7 +207,7 @@ protected BaseOpenApiReferenceHolder(string referenceId, OpenApiDocument? hostDo
     /// <inheritdoc/>
     public virtual void SerializeAsV3(IOpenApiWriter writer)
     {
-        if (!writer.GetSettings().ShouldInlineReference(Reference) 
+        if (!writer.GetSettings().ShouldInlineReference(Reference)
             || Reference.Type == ReferenceType.Tag) // tags are held as references need to drop in.
         {
             Reference.SerializeAsV3(writer);
@@ -254,6 +254,7 @@ private protected void SerializeInternal(IOpenApiWriter writer,
         Action<IOpenApiWriter, U> action)
     {
         Utils.CheckArgumentNull(writer);
+        writer.CountInlineReference();
         ApplyToTarget(element => action(writer, element));
     }
 }

src/Microsoft.OpenApi/Models/References/OpenApiRequestBodyReference.cs

@@ -64,6 +64,7 @@ public override void SerializeAsV2(IOpenApiWriter writer)
         {
             if (writer.GetSettings().ShouldInlineReference(Reference))
             {
+                writer.CountInlineReference();
                 return GetFromTarget(target => target.ConvertToBodyParameter(writer));
             }
 
@@ -74,6 +75,7 @@ public override void SerializeAsV2(IOpenApiWriter writer)
         {
             if (writer.GetSettings().ShouldInlineReference(Reference))
             {
+                writer.CountInlineReference();
                 return GetFromTarget(target => target.ConvertToFormDataParameters(writer));
             }
 

src/Microsoft.OpenApi/Models/References/OpenApiSchemaReference.cs

@@ -250,8 +250,14 @@ private void SerializeAsWithoutLoops(IOpenApiWriter writer, Action<IOpenApiWrite
             }
             else
             {
-                SerializeInternal(writer, (w, element) => action(w, element));
-                writer.GetSettings().LoopDetector.PopLoop<IOpenApiSchema>();
+                try
+                {
+                    SerializeInternal(writer, (w, element) => action(w, element));
+                }
+                finally
+                {
+                    writer.GetSettings().LoopDetector.PopLoop<IOpenApiSchema>();
+                }
             }
 
         }

src/Microsoft.OpenApi/Properties/SRResource.Designer.cs

@@ -132,6 +132,9 @@
   <data name="IndentationLevelInvalid" xml:space="preserve">
     <value>Indentation level cannot be lower than 0.</value>
   </data>
+  <data name="MaximumInlineReferenceCountExceeded" xml:space="preserve">
+    <value>The maximum number of inline references ({0}) has been exceeded.</value>
+  </data>
   <data name="InputItemShouldBeType" xml:space="preserve">
     <value>The input item should be in type of '{0}'.</value>
   </data>

src/Microsoft.OpenApi/Properties/SRResource.resx

@@ -1,2 +1,4 @@
 #nullable enable
 Microsoft.OpenApi.OpenApiDocument.GetOperationById(string! operationId) -> Microsoft.OpenApi.OpenApiOperation?
+Microsoft.OpenApi.OpenApiWriterSettings.MaxInlineReferenceCount.get -> int
+Microsoft.OpenApi.OpenApiWriterSettings.MaxInlineReferenceCount.set -> void

src/Microsoft.OpenApi/PublicAPI.Unshipped.txt

@@ -35,6 +35,7 @@ public abstract class OpenApiWriterBase : IOpenApiWriter
         /// Number which specifies the level of indentation.
         /// </summary>
         private int _indentLevel;
+        private int _inlineReferenceCount;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="OpenApiWriterBase"/> class.
@@ -70,6 +71,20 @@ protected OpenApiWriterBase(TextWriter textWriter, OpenApiWriterSettings? settin
         /// </summary>
         protected TextWriter Writer { get; }
 
+        internal void CountInlineReference()
+        {
+            if (_inlineReferenceCount >= Settings.MaxInlineReferenceCount)
+            {
+                throw new OpenApiWriterException(
+                    string.Format(
+                        CultureInfo.InvariantCulture,
+                        SRResource.MaximumInlineReferenceCountExceeded,
+                        Settings.MaxInlineReferenceCount));
+            }
+
+            _inlineReferenceCount++;
+        }
+
         /// <summary>
         /// Write start object.
         /// </summary>
@@ -319,7 +334,7 @@ public virtual void WriteIndentation()
                 Writer.Write(IndentationString);
             }
         }
-        
+
         /// <summary>
         /// Get current scope.
         /// </summary>