From da480d99e6d384bd1507cd3c46dcaf063b61bb01 Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:56:23 -0400 Subject: [PATCH 1/5] Add DAFFODIL_TDML_TUNABLES to run TDML suites under tunables The DAFFODIL_TDML_TUNABLES environment variable holds a comma-separated list of name=value tunables that apply to every TDML test. A test's own defineConfig tunables override them, so a whole suite can run under any combination of tunables without editing any schema or TDML file. The list is parsed by a small helper that ignores blank entries, trims whitespace, lets a value contain '=', and lets a later duplicate win. An entry with no name or no '=' fails with an error that names the entry, so a typo in the variable is reported instead of silently ignored. A tunable name Daffodil does not know is rejected when the test compiles, as it would be from defineConfig. Unit tests cover each parsing case. DAFFODIL-3065 --- .../org/apache/daffodil/tdml/TDMLRunner.scala | 38 +++++++++- .../daffodil/cliTest/TestCLIEnvTunables.scala | 70 +++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 daffodil-test-integration/src/test/scala/org/apache/daffodil/cliTest/TestCLIEnvTunables.scala diff --git a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala index 74443cd32b..11c044552f 100644 --- a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala +++ b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala @@ -387,6 +387,15 @@ class DFDLTestSuite private[tdml] ( var checkAllTopLevel: Boolean = compileAllTopLevel + /** + * Tunables from the DAFFODIL_TDML_TUNABLES environment variable, a + * comma-separated list of name=value pairs. They apply to every test and + * are overridden by a test's own defineConfig tunables, so a whole suite + * can run under any combination of tunables. + */ + private[tdml] lazy val envTunables: Map[String, String] = + TDMLEnvTunables.parse(sys.env.getOrElse(TDMLEnvTunables.VariableName, "")) + def setCheckAllTopLevel(flag: Boolean): Unit = { checkAllTopLevel = flag } @@ -902,7 +911,8 @@ abstract class TestCase(testCaseXML: NodeSeq, val parent: DFDLTestSuite) { } } - lazy val tunables = cfg.map { _.tunablesMap }.getOrElse(Map.empty) + lazy val tunables = + parent.envTunables ++ cfg.map { _.tunablesMap }.getOrElse(Map.empty) lazy val tunableObj = DaffodilTunables(tunables) @@ -3151,3 +3161,29 @@ object GlobalTDMLCompileResultCache { new TDMLCompileResultCache(Some(expireTimeSeconds)) } } + +private[tdml] object TDMLEnvTunables { + val VariableName = "DAFFODIL_TDML_TUNABLES" + + /** + * Parses a comma-separated list of name=value pairs, as in + * "key=value,key=value". Blank entries are ignored, and a value may itself + * contain '='. An entry without a name or without '=' is an error. + */ + def parse(value: String): Map[String, String] = + value + .split(',') + .map(_.trim) + .filter(_.nonEmpty) + .map { pair => + pair.split("=", 2) match { + case Array(name, tunableValue) if name.trim.nonEmpty => + (name.trim, tunableValue.trim) + case _ => + throw new IllegalArgumentException( + s"Invalid $VariableName entry '$pair'; expected name=value" + ) + } + } + .toMap +} diff --git a/daffodil-test-integration/src/test/scala/org/apache/daffodil/cliTest/TestCLIEnvTunables.scala b/daffodil-test-integration/src/test/scala/org/apache/daffodil/cliTest/TestCLIEnvTunables.scala new file mode 100644 index 0000000000..97a2c5adfa --- /dev/null +++ b/daffodil-test-integration/src/test/scala/org/apache/daffodil/cliTest/TestCLIEnvTunables.scala @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.cliTest + +import org.apache.daffodil.cli.Main.ExitCode +import org.apache.daffodil.cli.cliTest.Util.* + +import org.junit.Test + +class TestCLIEnvTunables { + + // parses a 4 byte hexBinary and sets no tunables of its own + private val simpleTypes = path( + "daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/SimpleTypes.tdml" + ) + + // sets maxHexBinaryLengthInBytes to 10 in its own config and expects an 11 byte hexBinary to fail + private val tunables = path( + "daffodil-test/src/test/resources/org/apache/daffodil/section00/general/tunables.tdml" + ) + + @Test def test_CLI_EnvTunables_smallHexLimitFails(): Unit = { + val envs = Map("DAFFODIL_TDML_TUNABLES" -> "maxHexBinaryLengthInBytes=1") + + runCLI(args"test $simpleTypes hexBinary_01", envs = envs) { cli => + cli.expect("[Fail] hexBinary_01") + }(ExitCode.TestError) + } + + @Test def test_CLI_EnvTunables_largeHexLimitPasses(): Unit = { + val envs = Map("DAFFODIL_TDML_TUNABLES" -> "maxHexBinaryLengthInBytes=100") + + runCLI(args"test $simpleTypes hexBinary_01", envs = envs) { cli => + cli.expect("[Pass] hexBinary_01") + }(ExitCode.Success) + } + + @Test def test_CLI_EnvTunables_multipleTunables(): Unit = { + val envs = Map( + "DAFFODIL_TDML_TUNABLES" -> "maxSkipLengthInBytes=4, maxHexBinaryLengthInBytes=1" + ) + + runCLI(args"test $simpleTypes hexBinary_01", envs = envs) { cli => + cli.expect("[Fail] hexBinary_01") + }(ExitCode.TestError) + } + + @Test def test_CLI_EnvTunables_testConfigOverridesEnv(): Unit = { + val envs = Map("DAFFODIL_TDML_TUNABLES" -> "maxHexBinaryLengthInBytes=100000") + + runCLI(args"test $tunables maxHexBinaryError", envs = envs) { cli => + cli.expect("[Pass] maxHexBinaryError") + }(ExitCode.Success) + } +} From 751fe37a4b0954ea673cec266172eeb12168d5fe Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:50:28 -0400 Subject: [PATCH 2/5] Inline withRetryIfBlocking and the typed equality operators withRetryIfBlocking evaluates its body in place, since a blocked evaluation's RetryableException already propagates to the suspension machinery that retries it later. Making it inline means the by-name body no longer allocates a closure on every call, and the separate helper that wrapped it is removed. The typed equality operators (=#=, !=#=, =:=, !=:=, _eq_, _ne_) are inline so each call site compares using its operands' static types and expands in place. As ordinary methods they compared through the erased type and went through boxed equality on every call. The implicit TypeEquality is now only the compile-time proof that the two types are related. Behavior is unchanged. DAFFODIL-3065 --- .../core/dsom/RuntimePropertyMixins.scala | 2 +- .../primitives/ElementCombinator.scala | 2 +- .../apache/daffodil/lib/TypedEquality.scala | 53 ++++++----- .../annotation/props/ByHandMixins.scala | 2 +- .../apache/daffodil/lib/xml/QNameBase.scala | 2 +- .../runtime1/dpath/DFDLFunctions2.scala | 46 +++++----- .../daffodil/runtime1/dpath/DState.scala | 89 +------------------ .../daffodil/runtime1/dpath/UpDownMoves.scala | 17 ++-- .../runtime1/infoset/InfosetImpl.scala | 3 +- .../processors/PackedBinaryTraits.scala | 2 +- .../runtime1/processors/dfa/Registers.scala | 2 +- .../runtime1/DelimitedUnparsers.scala | 2 +- .../io/TestInputSourceDataInputStream.scala | 2 +- 13 files changed, 67 insertions(+), 157 deletions(-) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala index 4145d8fed5..59413a1394 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala @@ -17,7 +17,7 @@ package org.apache.daffodil.core.dsom -import org.apache.daffodil.lib.equality.TypeEqual +import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.Found import org.apache.daffodil.lib.schema.annotation.props.TextStandardExponentRepMixin diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala index 0dcc7ad5cd..fb5ee24af9 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala @@ -22,7 +22,7 @@ import org.apache.daffodil.core.grammar.EmptyGram import org.apache.daffodil.core.grammar.Gram import org.apache.daffodil.core.grammar.NamedGram import org.apache.daffodil.core.grammar.Terminal -import org.apache.daffodil.lib.equality.TypeEqual +import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind import org.apache.daffodil.lib.schema.annotation.props.gen.Representation diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/lib/TypedEquality.scala b/daffodil-core/src/main/scala/org/apache/daffodil/lib/TypedEquality.scala index bea3883631..c23ca9b942 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/lib/TypedEquality.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/lib/TypedEquality.scala @@ -43,9 +43,12 @@ package object equality { // Convertible types - strongly typed equality - implicit class ViewEqual[T](val left: T) extends AnyVal { - @inline def =#=(right: T) = left == right - @inline def !=#=(right: T) = left != right + // The operators are inline extension methods with inline receivers so each + // call site compares with its operands' static types, and neither a wrapper + // object nor a boxed operand is created. + extension [T](inline left: T) { + inline def =#=(inline right: T): Boolean = left == right + inline def !=#=(inline right: T): Boolean = left != right } // implicit class ViewEqual[L](val left: L) extends AnyVal { // def =#=[R](right: R)(implicit equality: ViewEquality[L, R]): Boolean = @@ -87,41 +90,43 @@ package object equality { // Type wise - allows bi-directional subtypes, not just subtype on right. - implicit class TypeEqual[L <: AnyRef](val left: L) extends AnyVal { - @inline def =:=[R <: AnyRef](right: R)(implicit equality: TypeEquality[L, R]): Boolean = - equality.areEqual(left, right) - @inline def !=:=[R <: AnyRef](right: R)(implicit equality: TypeEquality[L, R]): Boolean = - !equality.areEqual(left, right) - @inline def _eq_[R <: AnyRef](right: R)(implicit equality: TypeEquality[L, R]): Boolean = - equality.areEq(left, right) - @inline def _ne_[R <: AnyRef](right: R)(implicit equality: TypeEquality[L, R]): Boolean = - !equality.areEq(left, right) + // The given TypeEquality is only the compile-time proof that L and R are + // in a subtype relationship; it is an inline parameter so it is never evaluated. + extension [L <: AnyRef](inline left: L) { + inline def =:=[R <: AnyRef](inline right: R)(using + inline equality: TypeEquality[L, R] + ): Boolean = + left == right + inline def !=:=[R <: AnyRef](inline right: R)(using + inline equality: TypeEquality[L, R] + ): Boolean = + left != right + inline def _eq_[R <: AnyRef](inline right: R)(using + inline equality: TypeEquality[L, R] + ): Boolean = + left eq right + inline def _ne_[R <: AnyRef](inline right: R)(using + inline equality: TypeEquality[L, R] + ): Boolean = + left ne right } @implicitNotFound("Typed equality requires ${L} and ${R} to be in a subtype relationship!") - private[equality] sealed trait TypeEquality[L <: AnyRef, R <: AnyRef] { - def areEqual(left: L, right: R): Boolean - def areEq(left: L, right: R): Boolean - } + private[equality] sealed trait TypeEquality[L <: AnyRef, R <: AnyRef] private[equality] object TypeEquality extends LowPriorityTypeEqualityImplicits { - @inline implicit def rightSubtypeOfLeftEquality[L <: AnyRef, R <: L]: TypeEquality[L, R] = + inline given rightSubtypeOfLeftEquality[L <: AnyRef, R <: L]: TypeEquality[L, R] = AnyTypeEquality.asInstanceOf[TypeEquality[L, R]] } private[equality] trait LowPriorityTypeEqualityImplicits { - @inline implicit def leftSubtypeOfRightEquality[R <: AnyRef, L <: R]: TypeEquality[L, R] = + inline given leftSubtypeOfRightEquality[R <: AnyRef, L <: R]: TypeEquality[L, R] = AnyTypeEquality.asInstanceOf[TypeEquality[L, R]] } // must be public or scala compiler complains that it can't embed the // static reference. // - object AnyTypeEquality extends TypeEquality[AnyRef, AnyRef] { - @inline override def areEqual(left: AnyRef, right: AnyRef): Boolean = - left == right - @inline override def areEq(left: AnyRef, right: AnyRef): Boolean = - left eq right - } + object AnyTypeEquality extends TypeEquality[AnyRef, AnyRef] // exact type equality // diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/lib/schema/annotation/props/ByHandMixins.scala b/daffodil-core/src/main/scala/org/apache/daffodil/lib/schema/annotation/props/ByHandMixins.scala index cc1229a581..92052dcdf1 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/lib/schema/annotation/props/ByHandMixins.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/lib/schema/annotation/props/ByHandMixins.scala @@ -31,7 +31,7 @@ import org.apache.daffodil.lib.cookers.TextStandardInfinityRepCooker import org.apache.daffodil.lib.cookers.TextStandardNaNRepCooker import org.apache.daffodil.lib.cookers.TextStandardZeroRepCooker import org.apache.daffodil.lib.cookers.TextStringPadCharacterCooker -import org.apache.daffodil.lib.equality.ViewEqual +import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.exceptions.ThrowsSDE import org.apache.daffodil.lib.iapi.DaffodilTunables diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/lib/xml/QNameBase.scala b/daffodil-core/src/main/scala/org/apache/daffodil/lib/xml/QNameBase.scala index 6ed812a6ef..8c44cef2ce 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/lib/xml/QNameBase.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/lib/xml/QNameBase.scala @@ -21,7 +21,7 @@ import java.net.URI import java.net.URISyntaxException import scala.util.Try -import org.apache.daffodil.lib.equality.TypeEqual +import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.iapi.UnqualifiedPathStepPolicy diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/DFDLFunctions2.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/DFDLFunctions2.scala index fa3c0153ff..dfd87721a4 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/DFDLFunctions2.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/DFDLFunctions2.scala @@ -32,32 +32,30 @@ sealed abstract class DFDLLengthFunctionBase(kind: String, recipes: List[Compile protected def lengthState(elt: DIElement): LengthState - protected def getLength(elt: DIElement, units: LengthUnits, dstate: DState): ULong = { + protected def getLength(elt: DIElement, units: LengthUnits): ULong = { val len: ULong = - DState.withRetryIfBlocking(dstate) { - units match { - case LengthUnits.Bits => lengthState(elt).lengthInBits - case LengthUnits.Bytes => lengthState(elt).lengthInBytes - case LengthUnits.Characters => { - // - // TODO: We could warn about taking lengthInCharacters of something - // that isn't all text, but it's not required that it be purely - // text. DFDL lets you mix text and binary and then search it for - // delimiters or run regex patterns to parse it. You have to know what - // you are doing. - // Furthermore, in fixed-width encodings, this length can be computed - // from the length-in-bits by just dividing by a codepoint width - // code point width. - // - val nyi = new IllegalArgumentException( - "dfdl:%sLength's second argument of 'characters' is not yet supported.".format( - kind - ) + units match { + case LengthUnits.Bits => lengthState(elt).lengthInBits + case LengthUnits.Bytes => lengthState(elt).lengthInBytes + case LengthUnits.Characters => { + // + // TODO: We could warn about taking lengthInCharacters of something + // that isn't all text, but it's not required that it be purely + // text. DFDL lets you mix text and binary and then search it for + // delimiters or run regex patterns to parse it. You have to know what + // you are doing. + // Furthermore, in fixed-width encodings, this length can be computed + // from the length-in-bits by just dividing by a codepoint width + // code point width. + // + val nyi = new IllegalArgumentException( + "dfdl:%sLength's second argument of 'characters' is not yet supported.".format( + kind ) - elt.erd.SDE(nyi) - // lengthState(elt).lengthInCharacters - } + ) + elt.erd.SDE(nyi) + // lengthState(elt).lengthInCharacters } } len @@ -89,7 +87,7 @@ sealed abstract class DFDLLengthFunctionBase(kind: String, recipes: List[Compile ) } - val jLen: JLong = getLength(elt, units, dstate).longValue + val jLen: JLong = getLength(elt, units).longValue jLen } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/DState.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/DState.scala index 183eeb21dc..e0590a8fee 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/DState.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/DState.scala @@ -31,7 +31,6 @@ import org.apache.daffodil.runtime1.infoset.DISimple import org.apache.daffodil.runtime1.infoset.DataValue import org.apache.daffodil.runtime1.infoset.FakeDINode import org.apache.daffodil.runtime1.infoset.InfosetNoNextSiblingException -import org.apache.daffodil.runtime1.infoset.RetryableException import org.apache.daffodil.runtime1.processors.ParseOrUnparseState import org.apache.daffodil.runtime1.processors.SchemaSetRuntimeData; object EqualityNoWarn2 { EqualitySuppressUnusedImportWarning() } @@ -142,60 +141,12 @@ case class DState( def mode = _mode - // - // The purpose of this commented out code was to retry expressions, not - // from the begining, but closer to where the expression failed. That is - // for a big complex expression, we would not redo the whole thing, but would - // block the co-routine right where it was unable to proceed e.g., in a call to - // node.dataValue, such that when it was resumed, it would resume right there. - // retry the call to node.dataValue, and then carry on with the rest of the - // expression. - // - // If we don't do this, then we're not taking advantage of that optimization - // Then we don't really need coroutines at all to implement blocking. We just - // keep the saved/cloned state around, and retry whatever needed to be done, from scratch. - // - // The vast bulk of expressions are going to be fairly small, so this extra - // overhead vs. the complexity of coroutines?? I think we should move away - // From coroutines. - // - // However, if expressions were to block on the infoset, by queuing themselves - // adjacent to what they need, then... well you can still just retry the whole - // expression again, this time knowing it won't block in the same place. - // - // private var thisExpressionCoroutine_ : Maybe[Coroutine[AnyRef]] = Nope - // - // def setThisExpressionCoroutine(co: Maybe[Coroutine[AnyRef]]) { - // Assert.usage(mode eq UnparserBlocking) - // thisExpressionCoroutine_ = co - // } - // - // def thisExpressionCoroutine = { - // Assert.usage(mode eq UnparserBlocking) - // thisExpressionCoroutine_ - // } - // - // private var coroutineToResumeIfBlocked_ : Maybe[Coroutine[AnyRef]] = Nope - // - // def setCoroutineToResumeIfBlocked(co: Maybe[Coroutine[AnyRef]]) { - // Assert.usage(mode eq UnparserBlocking) - // coroutineToResumeIfBlocked_ = co - // } - // - // def coroutineToResumeIfBlocked = { - // Assert.usage(mode eq UnparserBlocking) - // coroutineToResumeIfBlocked_ - // } - def resetValue(): Unit = { _currentValue = DataValue.NoValue } def currentValue: DataValuePrimitiveNullable = { - if (_currentValue.isEmpty) - withRetryIfBlocking { - currentSimple.dataValue - } + if (_currentValue.isEmpty) currentSimple.dataValue else _currentValue } @@ -363,44 +314,6 @@ case class DState( def fnExists(): Unit = { // do nothing } - - // @inline // TODO: Performance maybe this won't allocate a closure if this is inline? If not replace with macro - final def withRetryIfBlocking[T](body: => T): T = - DState.withRetryIfBlocking(this)(body) -} - -object DState { - - // private object ToBeIgnored - - // @inline - final def withRetryIfBlocking[T](ds: DState)(body: => T): T = { // TODO: Performance maybe this won't allocate a closure if this is inline? If not replace with macro - ds.mode match { - case _: ParserMode => body - case UnparserNonBlocking => body - case UnparserBlocking => { - var isDone = false - var res: T = null.asInstanceOf[T] - while (!isDone) { - try { - res = body - isDone = true - } catch { - case e: RetryableException => { - // we're to block here, and retry subsequently. - isDone = false - // if (ds.thisExpressionCoroutine.isDefined && ds.coroutineToResumeIfBlocked.isDefined) { - // ds.thisExpressionCoroutine.get.resume(ds.coroutineToResumeIfBlocked.get, ToBeIgnored) - // } else { - throw e - // } - } - } - } - res - } - } - } } class DStateForConstantFolding( diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/UpDownMoves.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/UpDownMoves.scala index d64cff1f68..51a1a682f6 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/UpDownMoves.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/dpath/UpDownMoves.scala @@ -73,7 +73,7 @@ case class DownElement(nqn: NamedQName) extends RecipeOp { val now = dstate.currentComplex // TODO PE ? if doesn't exist should be a processing error. // It will throw and so will be a PE, but may be poor diagnostic. - val c = dstate.withRetryIfBlocking(now.getChild(nqn, dstate.tunable)) + val c = now.getChild(nqn, dstate.tunable) dstate.setCurrentNode(c.asInstanceOf[DIElement]) } @@ -100,14 +100,9 @@ case class DownArrayOccurrence(nqn: NamedQName, indexRecipe: CompiledDPath) // current node to null. And future calls depend on a current node to be set dstate.setCurrentNode(savedCurrentElement) val childArrayElementERD = - dstate.withRetryIfBlocking( - savedCurrentElement.getChildArray(nqn, dstate.tunable).asInstanceOf[DIArray].erd - ) - val arr = dstate.withRetryIfBlocking( - savedCurrentElement.getChildArray(childArrayElementERD, dstate.tunable) - ) - val occurrence = - dstate.withRetryIfBlocking(arr(index)) // will throw on out of bounds + savedCurrentElement.getChildArray(nqn, dstate.tunable).asInstanceOf[DIArray].erd + val arr = savedCurrentElement.getChildArray(childArrayElementERD, dstate.tunable) + val occurrence = arr(index) // will throw on out of bounds dstate.setCurrentNode(occurrence.asInstanceOf[DIElement]) } @@ -124,7 +119,7 @@ case class DownArray(nqn: NamedQName) extends RecipeOp { override def run(dstate: DState): Unit = { val now = dstate.currentComplex - val arr = dstate.withRetryIfBlocking(now.getChildArray(nqn, dstate.tunable)) + val arr = now.getChildArray(nqn, dstate.tunable) Assert.invariant(arr ne null) dstate.setCurrentNode(arr.asInstanceOf[DIArray]) } @@ -139,7 +134,7 @@ case class DownArrayExists(nqn: NamedQName) extends RecipeOp { override def run(dstate: DState): Unit = { val now = dstate.currentComplex - val arr = dstate.withRetryIfBlocking(now.getChildArray(nqn, dstate.tunable)) + val arr = now.getChildArray(nqn, dstate.tunable) if ((arr eq null) || arr.length == 0) throw new InfosetNoSuchChildElementException(now, nqn) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetImpl.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetImpl.scala index 5edb5196e9..6d798b92ba 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetImpl.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetImpl.scala @@ -46,8 +46,7 @@ import org.apache.daffodil.api.metadata.SimpleElementMetadata import org.apache.daffodil.io.DataOutputStream import org.apache.daffodil.io.DirectOrBufferedDataOutputStream import org.apache.daffodil.lib.calendar.DFDLCalendar -import org.apache.daffodil.lib.equality.TypeEqual -import org.apache.daffodil.lib.equality.ViewEqual +import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.exceptions.ThinException import org.apache.daffodil.lib.exceptions.ThrowsSDE diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala index a29163c3c1..1a85916717 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala @@ -22,7 +22,7 @@ import java.math.BigInteger as JBigInteger import java.nio.charset.StandardCharsets import org.apache.daffodil.io.processors.charset.StandardBitsCharsets -import org.apache.daffodil.lib.equality.TypeEqual +import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo import org.apache.daffodil.lib.util.Maybe diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/dfa/Registers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/dfa/Registers.scala index 171012cc14..4cdeca2036 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/dfa/Registers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/dfa/Registers.scala @@ -19,7 +19,7 @@ package org.apache.daffodil.runtime1.processors.dfa import org.apache.daffodil.io.DataInputStream import org.apache.daffodil.io.FormatInfo -import org.apache.daffodil.lib.equality.ViewEqual +import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.util.Pool import org.apache.daffodil.lib.util.Poolable import org.apache.daffodil.runtime1.processors.DelimiterIterator diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/DelimitedUnparsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/DelimitedUnparsers.scala index bbb681444a..eba9b63921 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/DelimitedUnparsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/DelimitedUnparsers.scala @@ -20,7 +20,7 @@ package org.apache.daffodil.unparsers.runtime1 import java.nio.charset.MalformedInputException import java.nio.charset.UnmappableCharacterException -import org.apache.daffodil.lib.equality.ViewEqual +import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.schema.annotation.props.gen.GenerateEscape import org.apache.daffodil.lib.util.Logger import org.apache.daffodil.lib.util.Maybe diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/io/TestInputSourceDataInputStream.scala b/daffodil-core/src/test/scala/org/apache/daffodil/io/TestInputSourceDataInputStream.scala index 967eae726e..8602d387f5 100644 --- a/daffodil-core/src/test/scala/org/apache/daffodil/io/TestInputSourceDataInputStream.scala +++ b/daffodil-core/src/test/scala/org/apache/daffodil/io/TestInputSourceDataInputStream.scala @@ -24,7 +24,7 @@ import java.util.regex.Pattern import org.apache.daffodil.core.util.TestUtils.intercept import org.apache.daffodil.io.processors.charset.StandardBitsCharsets -import org.apache.daffodil.lib.equality.TypeEqual +import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.schema.annotation.props.gen.BitOrder import org.apache.daffodil.lib.schema.annotation.props.gen.ByteOrder import org.apache.daffodil.lib.schema.annotation.props.gen.EncodingErrorPolicy From 2834652d256f7cc2ba378d4c496ea45987b8f299 Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:47:11 -0400 Subject: [PATCH 3/5] Refactor UState, element node creation and choice branch keys UState declared every member, so UStateForSuspension stubbed each one it never uses with a method that fails. The infoset event cursor and TRD stack are now InfosetEventState, and the delimiter stack, escape scheme cache, scratch buffers and child position are DelimiterEscapePositionState. UState delegates to both. UStateMain reads events through InputterEventState, and the clone that resumes a suspension gets NoInfosetEventState and a DelimiterEscapePositionState that copies only the escape scheme and delimiter stacks and shares the scratch buffers of the state it was cloned from. ElementUnparser created, attached and finished the infoset node of every element itself. InfosetTreeState is now the part of UState that the element code works through, and it declares how nodes are made and kept. InfosetFromEvents implements these for the events of an inputter, and UStateMain mixes it in. The moved code and its comments are unchanged. Choice branches are keyed by the name of the element that starts them, so ChoiceBranchEvent is removed. An end event never starts a branch, so it takes the default. Behavior does not change. DAFFODIL-3065 --- .../grammar/primitives/ChoiceCombinator.scala | 8 +- .../primitives/ElementCombinator.scala | 4 +- .../runtime1/ChoiceTermRuntime1Mixin.scala | 23 +- .../runtime1/infoset/ChoiceBranchEvent.scala | 74 --- .../unparsers/InfosetFromEvents.scala | 150 +++++ .../processors/unparsers/UState.scala | 556 ++++++++++++------ .../ChoiceAndOtherVariousUnparsers.scala | 33 +- .../unparsers/runtime1/ElementUnparser.scala | 151 +---- 8 files changed, 568 insertions(+), 431 deletions(-) delete mode 100644 daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/ChoiceBranchEvent.scala create mode 100644 daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromEvents.scala diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ChoiceCombinator.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ChoiceCombinator.scala index b91428e4ca..53be0230bd 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ChoiceCombinator.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ChoiceCombinator.scala @@ -31,7 +31,7 @@ import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.ChoiceLengthKind import org.apache.daffodil.lib.util.MaybeInt import org.apache.daffodil.lib.util.ProperlySerializableMap.* -import org.apache.daffodil.runtime1.infoset.ChoiceBranchEvent +import org.apache.daffodil.lib.xml.NamedQName import org.apache.daffodil.runtime1.processors.RangeBound import org.apache.daffodil.runtime1.processors.parsers.* import org.apache.daffodil.runtime1.processors.unparsers.* @@ -313,8 +313,8 @@ case class ChoiceCombinator(ch: ChoiceTermBase, alternatives: Seq[Gram]) optDefaultUnparser } - val eventUnparserMap = eventRDMap.map { case (cbe, branchTerm) => - (cbe, branchTerm.termContentBody.unparser) + val eventUnparserMap = eventRDMap.map { case (qname, branchTerm) => + (qname, branchTerm.termContentBody.unparser) } val mapValues = eventUnparserMap.map { case (k, v) => v }.filterNot(_.isEmpty) if (mapValues.isEmpty) { @@ -326,7 +326,7 @@ case class ChoiceCombinator(ch: ChoiceTermBase, alternatives: Seq[Gram]) branchForUnparse.get } } else { - val serializableMap: ProperlySerializableMap[ChoiceBranchEvent, Unparser] = + val serializableMap: ProperlySerializableMap[NamedQName, Unparser] = eventUnparserMap.toProperlySerializableMap val cbm = ChoiceBranchMap(serializableMap, branchForUnparse) new ChoiceCombinatorUnparser(ch.modelGroupRuntimeData, cbm, choiceLengthInBits) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala index fb5ee24af9..49bc66883d 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala @@ -109,7 +109,7 @@ class ElementCombinator( if (eAfterValue.isEmpty) Maybe.Nope else Maybe(eAfterValue.unparser) - private lazy val eReptypeUnparser: Maybe[Unparser] = repTypeElementGram.maybeUnparser + private lazy val eRepTypeUnparser: Maybe[Unparser] = repTypeElementGram.maybeUnparser override lazy val unparser: Unparser = { if (context.isOutputValueCalc) { @@ -136,7 +136,7 @@ class ElementCombinator( eBeforeUnparser, eUnparser, eAfterUnparser, - eReptypeUnparser + eRepTypeUnparser ) } else { subComb.unparser diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/ChoiceTermRuntime1Mixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/ChoiceTermRuntime1Mixin.scala index 9aa556a86b..8309b2fad9 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/ChoiceTermRuntime1Mixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/ChoiceTermRuntime1Mixin.scala @@ -25,10 +25,8 @@ import org.apache.daffodil.core.dsom.Term import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.iapi.WarnID import org.apache.daffodil.lib.util.Delay +import org.apache.daffodil.lib.xml.NamedQName import org.apache.daffodil.runtime1.dpath.NodeInfo -import org.apache.daffodil.runtime1.infoset.ChoiceBranchEndEvent -import org.apache.daffodil.runtime1.infoset.ChoiceBranchEvent -import org.apache.daffodil.runtime1.infoset.ChoiceBranchStartEvent import org.apache.daffodil.runtime1.processors.ChoiceDispatchKeyEv import org.apache.daffodil.runtime1.processors.ChoiceRuntimeData @@ -58,7 +56,7 @@ trait ChoiceTermRuntime1Mixin { self: ChoiceTermBase => private lazy val allBranchesClosed = identifyingEventsForAllChoiceBranches.forall { _.isClosed } - final lazy val choiceBranchMap: (Map[ChoiceBranchEvent, Term], Option[Term]) = { + final lazy val choiceBranchMap: (Map[NamedQName, Term], Option[Term]) = { import PossibleNextElements.* @@ -67,7 +65,7 @@ trait ChoiceTermRuntime1Mixin { self: ChoiceTermBase => case t: Term => { val poss = t.identifyingEventsForChoiceBranch poss.pnes.flatMap { case PNE(e, ovr) => - Seq((ChoiceBranchStartEvent(e.namedQName).asInstanceOf[ChoiceBranchEvent], t)) + Seq((e.namedQName, t)) } } case _ => Assert.invariantFailed("must be a term") @@ -103,7 +101,7 @@ trait ChoiceTermRuntime1Mixin { self: ChoiceTermBase => // more than one branch. val noDupes = eventMap.map { - case (event, terms) => { + case (qname, terms) => { Assert.invariant(terms.length > 0) if (terms.length > 1) { if ( @@ -131,7 +129,7 @@ trait ChoiceTermRuntime1Mixin { self: ChoiceTermBase => "Note that elements with dfdl:outputValueCalc cannot be used to distinguish choice branches.\n" + "Note that choice branches with entirely optional content are not allowed.\n" + "The offending choice branches are:\n%s", - event.qname, + qname, terms .map { trd => "%s at %s".format(trd.diagnosticDebugName, trd.locationDescription) @@ -139,20 +137,15 @@ trait ChoiceTermRuntime1Mixin { self: ChoiceTermBase => .mkString("\n") ) } else { - val eventType = event match { - case _: ChoiceBranchEndEvent => "end" - case _: ChoiceBranchStartEvent => "start" - } // there are no element children in any of the branches. SDW( WarnID.MultipleChoiceBranches, - "Multiple choice branches are associated with the %s of element %s.\n" + + "Multiple choice branches are associated with the start of element %s.\n" + "Note that elements with dfdl:outputValueCalc cannot be used to distinguish choice branches.\n" + "Note that choice branches with entirely optional content are not allowed.\n" + "The offending choice branches are:\n%s\n" + "The first branch will be used during unparsing when an infoset ambiguity exists.", - eventType, - event.qname, + qname, terms .map { trd => "%s at %s".format(trd.diagnosticDebugName, trd.locationDescription) @@ -161,7 +154,7 @@ trait ChoiceTermRuntime1Mixin { self: ChoiceTermBase => ) } } - (event, terms(0)) + (qname, terms(0)) } } (noDupes, optDefaultBranch) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/ChoiceBranchEvent.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/ChoiceBranchEvent.scala deleted file mode 100644 index c19e9c1cad..0000000000 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/ChoiceBranchEvent.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.daffodil.runtime1.infoset - -import org.apache.daffodil.lib.exceptions.Assert -import org.apache.daffodil.lib.util.Misc -import org.apache.daffodil.lib.util.UniquenessCache -import org.apache.daffodil.lib.xml.NamedQName - -object ChoiceBranchEvent - extends UniquenessCache[NamedQName, (ChoiceBranchStartEvent, ChoiceBranchEndEvent)] { - - override def apply(nqn: NamedQName) = { - Assert.usage(nqn != null) - super.apply(nqn) - } - - protected def valueFromKey(nqn: NamedQName) = { - Assert.usage(nqn != null) - (new ChoiceBranchStartEvent(nqn), new ChoiceBranchEndEvent(nqn)) - } - - protected def keyFromValue(pair: (ChoiceBranchStartEvent, ChoiceBranchEndEvent)) = { - Assert.usage(pair != null) - Some(pair._1.qname) - } -} - -sealed trait ChoiceBranchEvent extends Serializable { - val qname: NamedQName - - override def toString = Misc.getNameFromClass(this) + "(" + qname + ")" - - override def hashCode = qname.hashCode - -} - -class ChoiceBranchStartEvent(val qname: NamedQName) extends ChoiceBranchEvent { - override def equals(x: Any) = { - x match { - case x: ChoiceBranchStartEvent => x.qname == qname - case _ => false - } - } -} -object ChoiceBranchStartEvent { - def apply(nqn: NamedQName) = ChoiceBranchEvent(nqn)._1 -} -class ChoiceBranchEndEvent(val qname: NamedQName) extends ChoiceBranchEvent { - override def equals(x: Any) = { - x match { - case x: ChoiceBranchEndEvent => x.qname == qname - case _ => false - } - } -} -object ChoiceBranchEndEvent { - def apply(nqn: NamedQName) = ChoiceBranchEvent(nqn)._2 -} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromEvents.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromEvents.scala new file mode 100644 index 0000000000..b262549f9e --- /dev/null +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromEvents.scala @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.runtime1.processors.unparsers + +import org.apache.daffodil.lib.exceptions.Assert +import org.apache.daffodil.lib.util.Maybe.Nope +import org.apache.daffodil.lib.util.Maybe.One +import org.apache.daffodil.runtime1.infoset.DIComplex +import org.apache.daffodil.runtime1.infoset.DIElement +import org.apache.daffodil.runtime1.infoset.DINode +import org.apache.daffodil.runtime1.infoset.DISimple +import org.apache.daffodil.runtime1.infoset.InfosetAccessor +import org.apache.daffodil.runtime1.processors.ElementRuntimeData + +/** + * How unparsing the events of an inputter makes the infoset: it creates the + * node of an element that has no event, attaches each new node to its parent, + * and finishes a node when its end is reached. + */ +trait InfosetFromEvents { self: InfosetTreeState => + + override def getHiddenElement(erd: ElementRuntimeData): DIElement = { + // Since we never get events for elements in hidden contexts, their infoset elements + // will have never been created. This means we need to manually create them + val hiddenElem = if (erd.isComplexType) { + new DIComplex(erd) + } else { + new DISimple(erd) + } + hiddenElem.setHidden() + hiddenElem + } + + override def getOvcElement( + startEvent: InfosetAccessor, + erd: ElementRuntimeData + ): DIElement = { + val e = new DISimple(erd) + // Remove any state that was set by what created this event. Later + // code asserts that OVC elements do not have a value + e.resetValue() + e + } + + override def attachElement(newElem: DIElement): Unit = { + val parentNodeMaybe = currentInfosetNodeMaybe + if (parentNodeMaybe.isDefined) { + val parentComplex = parentNodeMaybe.get.asComplex + Assert.invariant(!parentComplex.isFinal) + if (parentComplex.isNilled) { + // cannot add content to a nilled complex element + UnparseError( + One(newElem.erd.schemaFileLocation), + Nope, + "Nilled complex element %s has content from %s", + parentComplex.erd.namedQName.toExtendedSyntax, + newElem.erd.namedQName.toExtendedSyntax + ) + } + + // We are about to add a child to this complex element. Before we do + // that, if the last child added to this complex is a DIArray, and this + // new child isn't part of that array, that implies that the DIArray + // will have no more children added and should be marked as final, and + // we can attempt to free that array. + val lastChildMaybe = parentComplex.maybeLastChild + if (lastChildMaybe.isDefined) { + val lastChild = lastChildMaybe.get + if (lastChild.isArray && (lastChild.erd ne newElem.erd)) { + lastChild.setFinal() + freeChildIfNoLongerNeeded(parentComplex, parentComplex.numChildren - 1) + } + } + + parentComplex.addChild(newElem, tunable) + } else { + // We do not yet have an infoset element (this new element is the + // root), so add the infoset node to the DIDocument + documentElement.addChild(newElem, tunable) + } + } + + override def finishElement(cur: DINode, erd: ElementRuntimeData): Unit = { + if (cur.isComplex) { + // We are ending a complex element. If the last child of this complex + // is a DIArray, that implies that the array will have no more children + // and should be marked as isFinal. Normally this happens when we add a + // new sibling after an array in attachElement, but in this case there + // is no sibling following the array, so it must be set here. + val lastChild = cur.maybeLastChild + if (lastChild.isDefined && lastChild.get.isArray) { + lastChild.get.setFinal() + freeChildIfNoLongerNeeded(cur, cur.numChildren - 1) + } + } + + // cur is finished, mark it as final and free if possible. Note that we + // need the container and not the parent of the current element to free + // it. This way if this element is in an array, we free this element + // from the array. We also do not set hidden IVC elements as + // final--although we allow hidden IVC elements when unparsing, they + // never get a value so we can't set them as final without breaking + // assertions. Nothing can access hidden IVC elements, so this should + // not break anything + if (!withinHiddenNest || erd.isRepresented) { + cur.setFinal() + } + val curContainer = if (cur.erd.isArray) { + cur.diParent.maybeLastChild.get + } else { + cur.diParent + } + freeChildIfNoLongerNeeded(curContainer, curContainer.numChildren - 1) + markDocumentFinalIfRootEnded() + } + + override def finishOvcElement(cur: DINode): Unit = { + // ovcElem is finished, free it if possible. OVC elements are not allowed in + // arrays, so we can directly get the diParent to get the container DINode + val ovcContainer = cur.diParent + freeChildIfNoLongerNeeded(ovcContainer, ovcContainer.numChildren - 1) + markDocumentFinalIfRootEnded() + } + + protected def markDocumentFinalIfRootEnded(): Unit = { + if (currentInfosetNodeStack.isEmpty) { + // If there is nothing else on the infoset stack after popping off the + // current infoset node, that means we have finished the root element, + // so mark the DIDocument as final + val doc = documentElement + Assert.invariant(!doc.isFinal) + doc.setFinal() + } + } +} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala index d60290ce79..f453439370 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala @@ -54,6 +54,7 @@ import org.apache.daffodil.runtime1.infoset.InfosetInputter import org.apache.daffodil.runtime1.processors.DataLoc import org.apache.daffodil.runtime1.processors.DataProcessor import org.apache.daffodil.runtime1.processors.DelimiterStackUnparseNode +import org.apache.daffodil.runtime1.processors.ElementRuntimeData import org.apache.daffodil.runtime1.processors.EscapeSchemeUnparserHelper import org.apache.daffodil.runtime1.processors.Failure import org.apache.daffodil.runtime1.processors.NonTermRuntimeData @@ -75,9 +76,11 @@ abstract class UState( diagnosticsArg: Seq[api.Diagnostic], dataProcArg: Maybe[DataProcessor], tunable: DaffodilTunables, - areDebugging: Boolean + areDebugging: Boolean, + eventState: InfosetEventState, + delimiterEscapePosition: DelimiterEscapePositionState ) extends ParseOrUnparseState(vbox, diagnosticsArg, dataProcArg, tunable) - with Cursor[InfosetAccessor] + with InfosetTreeState with ThrowsSDE with SavesErrorsAndWarnings { @@ -110,18 +113,18 @@ abstract class UState( /** * Push onto the dynamic TRD context stack */ - def pushTRD(trd: TermRuntimeData): Unit + final def pushTRD(trd: TermRuntimeData): Unit = eventState.pushTRD(trd) /** * Returns the top of the stack if it exists. No state change to stack contents. */ - def maybeTopTRD(): Maybe[TermRuntimeData] + final def maybeTopTRD(): Maybe[TermRuntimeData] = eventState.maybeTopTRD() /** * Pop the dynamic TRD context stack. The popped TRD should be the same as the argument rd. * The popped TRD is returned. */ - def popTRD(trd: TermRuntimeData): TermRuntimeData + final def popTRD(trd: TermRuntimeData): TermRuntimeData = eventState.popTRD(trd) override def toString = { val elt = @@ -138,32 +141,39 @@ abstract class UState( _dataOutputStream = dos } - def currentInfosetNode: DINode - def currentInfosetNodeMaybe: Maybe[DINode] - def escapeSchemeEVCache: MStackOfMaybe[EscapeSchemeUnparserHelper] + final def escapeSchemeEVCache: MStackOfMaybe[EscapeSchemeUnparserHelper] = + delimiterEscapePosition.escapeSchemeEVCache - def withUnparserDataInputStream: LocalStack[StringDataInputStreamForUnparse] - def withByteArrayOutputStream - : LocalStack[(ByteArrayOutputStream, DirectOrBufferedDataOutputStream)] + final def withUnparserDataInputStream: LocalStack[StringDataInputStreamForUnparse] = + delimiterEscapePosition.withUnparserDataInputStream + final def withByteArrayOutputStream + : LocalStack[(ByteArrayOutputStream, DirectOrBufferedDataOutputStream)] = + delimiterEscapePosition.withByteArrayOutputStream - def allTerminatingMarkup: List[DFADelimiter] - def localDelimiters: DelimiterStackUnparseNode - def pushDelimiters(node: DelimiterStackUnparseNode): Unit - def popDelimiters(): Unit + final def allTerminatingMarkup: List[DFADelimiter] = + delimiterEscapePosition.allTerminatingMarkup + final def localDelimiters: DelimiterStackUnparseNode = delimiterEscapePosition.localDelimiters + final def pushDelimiters(node: DelimiterStackUnparseNode): Unit = + delimiterEscapePosition.pushDelimiters(node) + final def popDelimiters(): Unit = delimiterEscapePosition.popDelimiters() - def currentInfosetNodeStack: MStackOfMaybe[DINode] - def arrayIterationIndexStack: MStackOfLong - def occursIndexStack: MStackOfLong - def childIndexStack: MStackOfLong - def groupIndexStack: MStackOfLong - def moveOverOneArrayIterationIndexOnly(): Unit - def moveOverOneOccursIndexOnly(): Unit - def moveOverOneGroupIndexOnly(): Unit - def moveOverOneElementChildOnly(): Unit + final def childIndexStack: MStackOfLong = delimiterEscapePosition.childIndexStack + final def moveOverOneElementChildOnly(): Unit = + delimiterEscapePosition.moveOverOneElementChildOnly() + final override def childPos: Long = delimiterEscapePosition.childPos - def inspectOrError: InfosetAccessor - def advanceOrError: InfosetAccessor - def isInspectArrayEnd: Boolean + final override def advance: Boolean = eventState.advance + final override def advanceAccessor: InfosetAccessor = eventState.advanceAccessor + final override def inspect: Boolean = eventState.inspect + final override def inspectAccessor: InfosetAccessor = eventState.inspectAccessor + + /** + * Use these so if there isn't an event we get a clean diagnostic message saying + * that is what has gone wrong. + */ + final def inspectOrError: InfosetAccessor = eventState.inspectOrError + final def advanceOrError: InfosetAccessor = eventState.advanceOrError + final def isInspectArrayEnd: Boolean = eventState.isInspectArrayEnd override def dataStream = Maybe(getDataOutputStream) @@ -406,9 +416,315 @@ abstract class UState( final val releaseUnneededInfoset: Boolean = !areDebugging && tunable.releaseUnneededInfoset + final def freeChildIfNoLongerNeeded(parent: DINode, index: Int): Unit = + parent.freeChildIfNoLongerNeeded(index, releaseUnneededInfoset) + def delimitedParseResult = Nope } +/** + * The state of the infoset tree as unparsing makes it from events: the event + * cursor, the TRD and node stacks, and the position within the current group, + * array and occurrence. The unparsers that create and finish the nodes of + * elements work through it. + */ +trait InfosetTreeState extends Cursor[InfosetAccessor] { + def tunable: DaffodilTunables + + // How the infoset's nodes are made and kept as unparsing consumes events. + // InfosetFromEvents does it for the events of an inputter. + + // The node of an element in a hidden group, which has no events. + def getHiddenElement(erd: ElementRuntimeData): DIElement + + // The node of an outputValueCalc element whose start event was just consumed. + def getOvcElement(startEvent: InfosetAccessor, erd: ElementRuntimeData): DIElement + + // Adds a node whose start was just reached to the infoset. + def attachElement(newElem: DIElement): Unit + + // Finishes a node whose end was just reached. + def finishElement(cur: DINode, erd: ElementRuntimeData): Unit + def finishOvcElement(cur: DINode): Unit + + def inspectOrError: InfosetAccessor + def advanceOrError: InfosetAccessor + def isInspectArrayEnd: Boolean + + def pushTRD(trd: TermRuntimeData): Unit + def maybeTopTRD(): Maybe[TermRuntimeData] + def popTRD(trd: TermRuntimeData): TermRuntimeData + + def currentInfosetNode: DINode + def currentInfosetNodeMaybe: Maybe[DINode] + def currentInfosetNodeStack: MStackOfMaybe[DINode] + def documentElement: DIDocument + + def arrayIterationIndexStack: MStackOfLong + def occursIndexStack: MStackOfLong + def groupIndexStack: MStackOfLong + def moveOverOneArrayIterationIndexOnly(): Unit + def moveOverOneOccursIndexOnly(): Unit + def moveOverOneGroupIndexOnly(): Unit + def arrayIterationPos: Long + def occursPos: Long + def groupPos: Long + + def withinHiddenNest: Boolean + + def freeChildIfNoLongerNeeded(parent: DINode, index: Int): Unit +} + +/** + * The part of a UState that consumes infoset events from an InfosetInputter: + * the event cursor and the TRD stack. Only the UStates that read the + * infoset events hold a real one. + */ +trait InfosetEventState { + def advance: Boolean + def advanceAccessor: InfosetAccessor + def inspect: Boolean + def inspectAccessor: InfosetAccessor + def inspectOrError: InfosetAccessor + def advanceOrError: InfosetAccessor + def isInspectArrayEnd: Boolean + def pushTRD(trd: TermRuntimeData): Unit + def maybeTopTRD(): Maybe[TermRuntimeData] + def popTRD(trd: TermRuntimeData): TermRuntimeData +} + +/** + * Events read from an InfosetInputter. The purpose names what the caller is + * doing, for the diagnostic when an event is required but none is available. + */ +final class InputterEventState(inputter: InfosetInputter, purpose: String) + extends InfosetEventState { + + override def advance: Boolean = inputter.advance + override def advanceAccessor: InfosetAccessor = inputter.advanceAccessor + override def inspect: Boolean = inputter.inspect + override def inspectAccessor: InfosetAccessor = inputter.inspectAccessor + + override def inspectOrError: InfosetAccessor = { + if (inspect) { + inspectAccessor + } else { + Assert.invariantFailed( + "An InfosetEvent was required for " + purpose + ", but no InfosetEvent was available." + ) + } + } + + override def advanceOrError: InfosetAccessor = { + if (advance) { + advanceAccessor + } else { + Assert.invariantFailed( + "An InfosetEvent was required for " + purpose + ", but no InfosetEvent was available." + ) + } + } + + override def isInspectArrayEnd: Boolean = { + if (!inspect) { + false + } else { + inspectAccessor match { + case e if e.isEnd && e.isArray => true + case _ => false + } + } + } + + override def pushTRD(trd: TermRuntimeData): Unit = inputter.pushTRD(trd) + override def maybeTopTRD(): Maybe[TermRuntimeData] = inputter.maybeTopTRD() + override def popTRD(trd: TermRuntimeData): TermRuntimeData = { + val poppedTRD = inputter.popTRD() + if (poppedTRD ne trd) { + Assert.invariantFailed("TRDs do not match. Expected: " + trd + " got " + poppedTRD) + } + poppedTRD + } +} + +/** + * For a UState that never reads infoset events: a clone made to resume a + * suspension. + */ +object NoInfosetEventState extends InfosetEventState { + private def die = + Assert.invariantFailed("Function should never be needed in UStateForSuspension") + + override def advance: Boolean = die + override def advanceAccessor: InfosetAccessor = die + override def inspect: Boolean = die + override def inspectAccessor: InfosetAccessor = die + override def inspectOrError: InfosetAccessor = die + override def advanceOrError: InfosetAccessor = die + override def isInspectArrayEnd: Boolean = die + override def pushTRD(trd: TermRuntimeData): Unit = die + override def maybeTopTRD(): Maybe[TermRuntimeData] = die + override def popTRD(trd: TermRuntimeData): TermRuntimeData = die +} + +/** + * The part of a UState that only emitting delimited, escaped text uses: the + * delimiter stack, the escape scheme cache, the scratch buffers for measuring + * and escaping text, and the position within the current sequence or choice. + * + * The main unparse owns all of it. The clone that resumes a suspension holds + * only copies of the escape scheme and delimiter stacks, which are never + * pushed onto, and shares the scratch buffers of the state it came from. + */ +final class DelimiterEscapePositionState private ( + tunable: DaffodilTunables, + // The state a suspension's clone came from; Nope for the main unparse. + clonedFrom: Maybe[DelimiterEscapePositionState], + private var escapeSchemeEVCacheMaybe: Maybe[MStackOfMaybe[EscapeSchemeUnparserHelper]], + delimiterStackMaybe: Maybe[MStackOf[DelimiterStackUnparseNode]] +) { + + def escapeSchemeEVCache: MStackOfMaybe[EscapeSchemeUnparserHelper] = { + if (escapeSchemeEVCacheMaybe.isEmpty) { + Assert.invariant(clonedFrom.isEmpty) + escapeSchemeEVCacheMaybe = Maybe(new MStackOfMaybe[EscapeSchemeUnparserHelper](8)) + } + escapeSchemeEVCacheMaybe.get + } + + private lazy val unparserDataInputStream = + new LocalStack[StringDataInputStreamForUnparse](new StringDataInputStreamForUnparse) + + def withUnparserDataInputStream: LocalStack[StringDataInputStreamForUnparse] = { + if (clonedFrom.isEmpty) { + unparserDataInputStream + } else { + clonedFrom.get.withUnparserDataInputStream + } + } + + private lazy val byteArrayOutputStream = + new LocalStack[(ByteArrayOutputStream, DirectOrBufferedDataOutputStream)]( + { + val baos = + new ByteArrayOutputStream() // TODO: PERFORMANCE: Allocates new object. Can reuse one from an onStack/pool via reset() + val dos = DirectOrBufferedDataOutputStream( + baos, + null, + false, + tunable.outputStreamChunkSizeInBytes, + tunable.maxByteArrayOutputStreamBufferSizeInBytes, + tunable.tempFilePath + ) + (baos, dos) + }, + pair => + pair match { + case (baos, dos) => + baos.reset() + dos.resetAllBitPos() + } + ) + + def withByteArrayOutputStream + : LocalStack[(ByteArrayOutputStream, DirectOrBufferedDataOutputStream)] = { + if (clonedFrom.isEmpty) { + byteArrayOutputStream + } else { + clonedFrom.get.withByteArrayOutputStream + } + } + + def pushDelimiters(node: DelimiterStackUnparseNode): Unit = { + Assert.invariant(clonedFrom.isEmpty) + delimiterStackMaybe.get.push(node) + } + + def popDelimiters(): Unit = { + Assert.invariant(clonedFrom.isEmpty) + delimiterStackMaybe.get.pop + } + + def localDelimiters: DelimiterStackUnparseNode = delimiterStackMaybe.get.top + + def allTerminatingMarkup: List[DFADelimiter] = { + delimiterStackMaybe.get.iterator.flatMap { dnode => + dnode.separator ++ dnode.terminator + }.toList + } + + // Sequence and choice unparsers read it to find their current child; build + // tracks position in its own frames instead. A clone has none. + private val childIndexStackMaybe: Maybe[MStackOfLong] = { + if (clonedFrom.isEmpty) { + val stack = MStackOfLong(16) + stack.push(1L) + Maybe(stack) + } else { + Nope + } + } + + def childIndexStack: MStackOfLong = childIndexStackMaybe.get + + def moveOverOneElementChildOnly(): Unit = { + val stack = childIndexStack + stack.setTop(stack.top + 1) + } + + // A clone reports 0, which is read when copying state during debugging. + def childPos: Long = { + if (childIndexStackMaybe.isEmpty) { + 0L + } else { + childIndexStackMaybe.get.top + } + } + + /** + * The surface for a clone that resumes a suspension: it needs only the + * current escape scheme and delimiters, and shares the scratch buffers. + */ + def cloneForSuspension(): DelimiterEscapePositionState = { + val es = + if (escapeSchemeEVCacheMaybe.isDefined && !escapeSchemeEVCacheMaybe.get.isEmpty) { + // If there are any escape schemes, clone the whole MStack, since the + // escape scheme cache logic requires one (only the top is really + // needed, but changing the cache access isn't trivial). Sized to the + // source's depth: nothing pushes onto the clone afterward. + val source = escapeSchemeEVCacheMaybe.get + val esClone = new MStackOfMaybe[EscapeSchemeUnparserHelper](source.length) + esClone.copyFrom(source) + Maybe(esClone) + } else { + Nope + } + val ds = + if (!delimiterStackMaybe.get.isEmpty) { + // If there are any delimiters, clone them all since they may be + // needed for escaping. Sized to the source's depth: push and pop + // both die on this clone, so it never grows past that depth. + val source = delimiterStackMaybe.get + val dsClone = new MStackOf[DelimiterStackUnparseNode](source.length) + dsClone.copyFrom(source) + Maybe(dsClone) + } else { + Nope + } + new DelimiterEscapePositionState(tunable, Maybe(this), es, ds) + } +} + +object DelimiterEscapePositionState { + def apply(tunable: DaffodilTunables): DelimiterEscapePositionState = + new DelimiterEscapePositionState( + tunable, + Nope, + Nope, + Maybe(new MStackOf[DelimiterStackUnparseNode]()) + ) +} + /** * When we create a suspension during unparse, we need to clone the UStateMain * for when the suspension is later resumed. However, we do not need nearly as @@ -424,11 +740,18 @@ final class UStateForSuspension( override val currentInfosetNode: DINode, arrayIterationIndex: Long, occursIndex: Long, - escapeSchemeEVCacheMaybe: Maybe[MStackOfMaybe[EscapeSchemeUnparserHelper]], - delimiterStackMaybe: Maybe[MStackOf[DelimiterStackUnparseNode]], + delimiterEscapePosition: DelimiterEscapePositionState, tunable: DaffodilTunables, areDebugging: Boolean -) extends UState(vbox, mainUState.diagnostics, mainUState.dataProc, tunable, areDebugging) { +) extends UState( + vbox, + mainUState.diagnostics, + mainUState.dataProc, + tunable, + areDebugging, + NoInfosetEventState, + delimiterEscapePosition + ) { _dataOutputStream = dataOutputStream dState.setMode(UnparserBlocking) @@ -444,19 +767,8 @@ final class UStateForSuspension( override def suspensions = mainUState.suspensions - // override def charBufferDataOutputStream = mainUState.charBufferDataOutputStream - override def withUnparserDataInputStream = mainUState.withUnparserDataInputStream - override def withByteArrayOutputStream = mainUState.withByteArrayOutputStream - // $COVERAGE-OFF$ - override def advance: Boolean = die - override def advanceAccessor: InfosetAccessor = die - override def inspect: Boolean = die - override def inspectAccessor: InfosetAccessor = die override def fini(): Unit = die - override def inspectOrError = die - override def advanceOrError = die - override def isInspectArrayEnd = die override def currentInfosetNodeStack = die override def arrayIterationIndexStack = die override def moveOverOneArrayIterationIndexOnly() = die @@ -464,31 +776,17 @@ final class UStateForSuspension( override def moveOverOneOccursIndexOnly() = die override def groupIndexStack = die override def moveOverOneGroupIndexOnly() = die - override def childIndexStack = die - override def moveOverOneElementChildOnly() = die - override def pushDelimiters(node: DelimiterStackUnparseNode) = die - override def popDelimiters() = die + override def getHiddenElement(erd: ElementRuntimeData) = die + override def getOvcElement(startEvent: InfosetAccessor, erd: ElementRuntimeData) = die + override def attachElement(newElem: DIElement) = die + override def finishElement(cur: DINode, erd: ElementRuntimeData) = die + override def finishOvcElement(cur: DINode) = die // $COVERAGE-ON$ override def groupPos = 0 // was die, but this is called when copying state during debugging override def currentInfosetNodeMaybe = Maybe(currentInfosetNode) override def arrayIterationPos = arrayIterationIndex override def occursPos = occursIndex - override def childPos = 0 // was die, but this is called when copying state during debugging. - - override def localDelimiters = delimiterStackMaybe.get.top - override def allTerminatingMarkup = { - delimiterStackMaybe.get.iterator.flatMap { dnode => - dnode.separator ++ dnode.terminator - }.toList - } - - override def escapeSchemeEVCache: MStackOfMaybe[EscapeSchemeUnparserHelper] = - escapeSchemeEVCacheMaybe.get - - override def pushTRD(trd: TermRuntimeData): Unit = die - override def maybeTopTRD() = die - override def popTRD(trd: TermRuntimeData): TermRuntimeData = die override def documentElement = mainUState.documentElement @@ -510,8 +808,18 @@ final class UStateMain private ( diagnosticsArg: Seq[api.Diagnostic], dataProcArg: DataProcessor, tunable: DaffodilTunables, - areDebugging: Boolean -) extends UState(vbox, diagnosticsArg, One(dataProcArg), tunable, areDebugging) { + areDebugging: Boolean, + delimiterEscapePosition: DelimiterEscapePositionState +) extends UState( + vbox, + diagnosticsArg, + One(dataProcArg), + tunable, + areDebugging, + new InputterEventState(inputter, "unparsing"), + delimiterEscapePosition + ) + with InfosetFromEvents { dState.setMode(UnparserBlocking) @@ -531,7 +839,8 @@ final class UStateMain private ( diagnosticsArg, dataProcArg, tunable, - areDebugging + areDebugging, + DelimiterEscapePositionState(tunable) ) setDataOutputStream({ @@ -547,30 +856,6 @@ final class UStateMain private ( }) def cloneForSuspension(suspendedDOS: DirectOrBufferedDataOutputStream): UState = { - val es = - if (!escapeSchemeEVCache.isEmpty) { - // If there are any escape schemes, then we need to clone the whole - // MStack, since the escape scheme cache logic requires an MStack. We - // reallyjust need the top for cloning for suspensions, but that - // requires changes to how the escape schema cache is accessed, which - // isn't a trivial change. - val esClone = new MStackOfMaybe[EscapeSchemeUnparserHelper](escapeSchemeEVCache.length) - esClone.copyFrom(escapeSchemeEVCache) - Maybe(esClone) - } else { - Nope - } - val ds = - if (!delimiterStack.isEmpty) { - // If there are any delimiters, then we need to clone them all since - // they may be needed for escaping - val dsClone = new MStackOf[DelimiterStackUnparseNode](delimiterStack.length) - dsClone.copyFrom(delimiterStack) - Maybe(dsClone) - } else { - Nope - } - val clone = new UStateForSuspension( this, suspendedDOS, @@ -578,8 +863,7 @@ final class UStateMain private ( currentInfosetNodeStack.top.get, // only need the to of the stack, not the whole thing arrayIterationIndexStack.top, // only need the top of the stack, not the whole thing occursIndexStack.top, - es, - ds, + delimiterEscapePosition.cloneForSuspension(), tunable, areDebugging ) @@ -589,71 +873,9 @@ final class UStateMain private ( clone } - override lazy val withUnparserDataInputStream = - new LocalStack[StringDataInputStreamForUnparse](new StringDataInputStreamForUnparse) - override lazy val withByteArrayOutputStream = - new LocalStack[(ByteArrayOutputStream, DirectOrBufferedDataOutputStream)]( - { - val baos = - new ByteArrayOutputStream() // TODO: PERFORMANCE: Allocates new object. Can reuse one from an onStack/pool via reset() - val dos = DirectOrBufferedDataOutputStream( - baos, - null, - false, - tunable.outputStreamChunkSizeInBytes, - tunable.maxByteArrayOutputStreamBufferSizeInBytes, - tunable.tempFilePath - ) - (baos, dos) - }, - pair => - pair match { - case (baos, dos) => - baos.reset() - dos.resetAllBitPos() - } - ) - - override def advance: Boolean = inputter.advance - override def advanceAccessor: InfosetAccessor = inputter.advanceAccessor - override def inspect: Boolean = inputter.inspect - override def inspectAccessor: InfosetAccessor = inputter.inspectAccessor // $COVERAGE-OFF$ // unused, but necessary to meet requirements of Cursor[T] override def fini() = Assert.usageError("Not to be used on UState") // $COVERAGE-ON$ - /** - * Use this so if there isn't an event we get a clean diagnostic message saying - * that is what has gone wrong. - */ - override def inspectOrError = { - if (inspect) - inspectAccessor - else - Assert.invariantFailed( - "An InfosetEvent was required for unparsing, but no InfosetEvent was available." - ) - } - - override def advanceOrError = { - if (advance) - advanceAccessor - else - Assert.invariantFailed( - "An InfosetEvent was required for unparsing, but no InfosetEvent was available." - ) - } - - override def isInspectArrayEnd = { - if (!inspect) false - else { - val p = inspectAccessor - val res = p match { - case e if e.isEnd && e.isArray => true - case _ => false - } - res - } - } def currentInfosetNode: DINode = if (currentInfosetNodeMaybe.isEmpty) null @@ -681,25 +903,6 @@ final class UStateMain private ( override def moveOverOneGroupIndexOnly() = groupIndexStack.setTop(groupIndexStack.top + 1) override def groupPos = groupIndexStack.top - // TODO: it doesn't look anything is actually reading the value of childindex - // stack. Can we get rid of it? - override val childIndexStack = MStackOfLong(16) - childIndexStack.push(1L) - override def moveOverOneElementChildOnly() = childIndexStack.setTop(childIndexStack.top + 1) - override def childPos = childIndexStack.top - - override lazy val escapeSchemeEVCache = new MStackOfMaybe[EscapeSchemeUnparserHelper](8) - - val delimiterStack = new MStackOf[DelimiterStackUnparseNode]() - override def pushDelimiters(node: DelimiterStackUnparseNode) = delimiterStack.push(node) - override def popDelimiters() = delimiterStack.pop - override def localDelimiters = delimiterStack.top - override def allTerminatingMarkup = { - delimiterStack.iterator.flatMap { dnode => - dnode.separator ++ dnode.terminator - }.toList - } - /** * For outputValueCalc we accumulate the suspendables here. * @@ -721,19 +924,6 @@ final class UStateMain private ( def suspensions = suspensionTracker.suspensions - final override def pushTRD(trd: TermRuntimeData) = - inputter.pushTRD(trd) - - final override def maybeTopTRD(): Maybe[TermRuntimeData] = - inputter.maybeTopTRD() - - final override def popTRD(trd: TermRuntimeData) = { - val poppedTRD = inputter.popTRD() - if (poppedTRD ne trd) - Assert.invariantFailed("TRDs do not match. Expected: " + trd + " got " + poppedTRD) - poppedTRD - } - final override def documentElement = inputter.documentElement override def toString = { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ChoiceAndOtherVariousUnparsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ChoiceAndOtherVariousUnparsers.scala index 9cfff9e970..040bea2eb2 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ChoiceAndOtherVariousUnparsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ChoiceAndOtherVariousUnparsers.scala @@ -23,17 +23,26 @@ import org.apache.daffodil.lib.util.Maybe import org.apache.daffodil.lib.util.Maybe.* import org.apache.daffodil.lib.util.MaybeInt import org.apache.daffodil.lib.util.ProperlySerializableMap.* +import org.apache.daffodil.lib.xml.NamedQName import org.apache.daffodil.runtime1.infoset.* import org.apache.daffodil.runtime1.processors.* import org.apache.daffodil.runtime1.processors.unparsers.* +/** + * Maps the name of the element that starts a branch to the unparser for that + * branch. An end event never starts a branch, so it always takes the default. + */ case class ChoiceBranchMap( - lookupTable: ProperlySerializableMap[ChoiceBranchEvent, Unparser], + lookupTable: ProperlySerializableMap[NamedQName, Unparser], unmappedDefault: Option[Unparser] ) extends Serializable { - def get(cbe: ChoiceBranchEvent): Maybe[Unparser] = { - val fromTable = lookupTable.get(cbe) + def get(event: InfosetAccessor): Maybe[Unparser] = { + val fromTable = if (event.isStart) { + lookupTable.get(event.erd.namedQName) + } else { + null + } val res = if (fromTable != null) One(fromTable) else { @@ -92,28 +101,16 @@ class ChoiceCombinatorUnparser( } else { state.pushTRD(mgrd) val event: InfosetAccessor = state.inspectOrError - val key: ChoiceBranchEvent = event match { - // - // The ChoiceBranchStartEvent(...) is not a case class constructor. It is a - // hash-table lookup for a cached value. This avoids constructing these - // objects over and over again. - // - case e if e.isStart && e.isElement => ChoiceBranchStartEvent(e.erd.namedQName) - case e if e.isEnd && e.isElement => ChoiceBranchEndEvent(e.erd.namedQName) - case e if e.isStart && e.isArray => ChoiceBranchStartEvent(e.erd.namedQName) - case e if e.isEnd && e.isArray => ChoiceBranchEndEvent(e.erd.namedQName) - } - - val maybeChildUnparser = choiceBranchMap.get(key) + val maybeChildUnparser = choiceBranchMap.get(event) if (maybeChildUnparser.isEmpty) { UnparseError( One(mgrd.schemaFileLocation), One(state.currentLocation), "Found next element %s, but expected one of %s.", - key.qname.toExtendedSyntax, + event.erd.namedQName.toExtendedSyntax, choiceBranchMap.keys .map { - _.qname.toExtendedSyntax + _.toExtendedSyntax } .mkString(", ") ) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala index c1343654ba..fd5a278b09 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala @@ -23,7 +23,6 @@ import org.apache.daffodil.lib.util.Maybe.* import org.apache.daffodil.lib.util.MaybeULong import org.apache.daffodil.runtime1.dpath.SuspendableExpression import org.apache.daffodil.runtime1.dsom.CompiledExpression -import org.apache.daffodil.runtime1.infoset.DIComplex import org.apache.daffodil.runtime1.infoset.DISimple import org.apache.daffodil.runtime1.infoset.DataValue.DataValuePrimitive import org.apache.daffodil.runtime1.infoset.RetryableException @@ -43,14 +42,14 @@ class ElementUnspecifiedLengthUnparser( eBeforeUnparser: Maybe[Unparser], eUnparser: Maybe[Unparser], eAfterUnparser: Maybe[Unparser], - eReptypeUnparser: Maybe[Unparser] + eRepTypeUnparser: Maybe[Unparser] ) extends ElementUnparserBase( erd, setVarUnparsers, eBeforeUnparser, eUnparser, eAfterUnparser, - eReptypeUnparser + eRepTypeUnparser ) with RegularElementUnparserStartEndStrategy with RepMoveMixin { @@ -123,13 +122,13 @@ sealed abstract class ElementUnparserBase( val eBeforeUnparser: Maybe[Unparser], val eUnparser: Maybe[Unparser], val eAfterUnparser: Maybe[Unparser], - val eReptypeUnparser: Maybe[Unparser] + val eRepTypeUnparser: Maybe[Unparser] ) extends CombinatorUnparser(erd) with RepMoveMixin with ElementUnparserStartEndStrategy { final override def childProcessors = - (eBeforeUnparser.toList ++ eUnparser.toList ++ eAfterUnparser.toList ++ eReptypeUnparser.toList ++ setVarUnparsers.toList).toVector + (eBeforeUnparser.toList ++ eUnparser.toList ++ eAfterUnparser.toList ++ eRepTypeUnparser.toList ++ setVarUnparsers.toList).toVector private val name = erd.name @@ -139,7 +138,7 @@ sealed abstract class ElementUnparserBase( "" + (if (eBeforeUnparser.isDefined) eBeforeUnparser.value.toBriefXML(depthLimit - 1) else "") + - (if (eReptypeUnparser.isDefined) eReptypeUnparser.value.toBriefXML(depthLimit - 1) + (if (eRepTypeUnparser.isDefined) eRepTypeUnparser.value.toBriefXML(depthLimit - 1) else "") + (if (eUnparser.isDefined) eUnparser.value.toBriefXML(depthLimit - 1) else "") + (if (eAfterUnparser.isDefined) eAfterUnparser.value.toBriefXML(depthLimit - 1) @@ -172,8 +171,8 @@ sealed abstract class ElementUnparserBase( } protected def runContentUnparser(state: UState): Unit = { - if (eReptypeUnparser.isDefined) { - eReptypeUnparser.get.unparse1(state) + if (eRepTypeUnparser.isDefined) { + eRepTypeUnparser.get.unparse1(state) } else if (eUnparser.isDefined) eUnparser.get.unparse1(state) } @@ -284,14 +283,14 @@ class ElementSpecifiedLengthUnparser( eBeforeUnparser: Maybe[Unparser], eUnparser: Maybe[Unparser], eAfterUnparser: Maybe[Unparser], - eReptypeUnparser: Maybe[Unparser] + eRepTypeUnparser: Maybe[Unparser] ) extends ElementUnparserBase( context, setVarUnparsers, eBeforeUnparser, eUnparser, eAfterUnparser, - eReptypeUnparser + eRepTypeUnparser ) with RegularElementUnparserStartEndStrategy with ElementSpecifiedLengthMixin { @@ -432,53 +431,11 @@ sealed trait RegularElementUnparserStartEndStrategy extends ElementUnparserStart event.info.element } else { Assert.invariant(state.withinHiddenNest) - // Since we never get events for elements in hidden contexts, their infoset elements - // will have never been created. This means we need to manually create them - val hiddenElem = if (erd.isComplexType) new DIComplex(erd) else new DISimple(erd) - hiddenElem.setHidden() - hiddenElem + state.getHiddenElement(erd) } // now add this new elem to the infoset - val parentNodeMaybe = state.currentInfosetNodeMaybe - if (parentNodeMaybe.isDefined) { - val parentComplex = parentNodeMaybe.get.asComplex - Assert.invariant(!parentComplex.isFinal) - if (parentComplex.isNilled) { - // cannot add content to a nilled complex element - UnparseError( - One(erd.schemaFileLocation), - Nope, - "Nilled complex element %s has content from %s", - parentComplex.erd.namedQName.toExtendedSyntax, - newElem.erd.namedQName.toExtendedSyntax - ) - } - - // We are about to add a child to this complex element. Before we do - // that, if the last child added to this complex is a DIArray, and this - // new child isn't part of that array, that implies that the DIArray - // will have no more children added and should be marked as final, and - // we can attempt to free that array. - val lastChildMaybe = parentComplex.maybeLastChild - if (lastChildMaybe.isDefined) { - val lastChild = lastChildMaybe.get - if (lastChild.isArray && (lastChild.erd ne newElem.erd)) { - lastChild.setFinal() - parentComplex.freeChildIfNoLongerNeeded( - parentComplex.numChildren - 1, - state.releaseUnneededInfoset - ) - } - } - - parentComplex.addChild(newElem, state.tunable) - } else { - // We do not yet have an infoset element (this new element is the - // root), so add the infoset node to the DIDocument - val doc = state.documentElement - doc.addChild(newElem, state.tunable) - } + state.attachElement(newElem) // When the infoset events are being advanced, the currentInfosetNodeStack // is pushing and popping to match the events. This provides the proper @@ -517,44 +474,7 @@ sealed trait RegularElementUnparserStartEndStrategy extends ElementUnparserStart val cur = state.currentInfosetNodeStack.pop.get - if (cur.isComplex) { - // We are ending a complex element. If the last child of this complex - // is a DIArray, that implies that the array will have no more children - // and should be marked as isFinal. Normally this happens when we add a - // new sibling after an array in unparseBegin, but in this case there - // is no sibling following the array, so it must be set here. - val lastChild = cur.maybeLastChild - if (lastChild.isDefined && lastChild.get.isArray) { - lastChild.get.setFinal() - cur.freeChildIfNoLongerNeeded(cur.numChildren - 1, state.releaseUnneededInfoset) - } - } - - // cur is finished, mark it as final and free if possible. Note that we - // need the container and not the parent of the current element to free - // it. This way if this element is in an array, we free this element - // from the array. We also do not set hidden IVC elements as - // final--although we allow hidden IVC elements when unparsing, they - // never get a value so we can't set them as final without breaking - // assertions. Nothing can access hidden IVC elements, so this should - // not break anything - if (!state.withinHiddenNest || erd.isRepresented) cur.setFinal() - val curContainer = - if (cur.erd.isArray) cur.diParent.maybeLastChild.get - else cur.diParent - curContainer.freeChildIfNoLongerNeeded( - curContainer.numChildren - 1, - state.releaseUnneededInfoset - ) - - if (state.currentInfosetNodeStack.isEmpty) { - // If there is nothing else on the infoset stack after popping off the - // current infoset node, that means we have finished the root element, - // so mark the DIDocument as final - val doc = state.documentElement - Assert.invariant(!doc.isFinal) - doc.setFinal() - } + state.finishElement(cur, erd) move(state) @@ -589,11 +509,7 @@ trait OVCStartEndStrategy extends ElementUnparserStartEndStrategy { val endEv = state.advanceOrError // Consume the end event Assert.invariant(endEv.isEnd && endEv.erd == erd) - val e = new DISimple(erd) - // Remove any state that was set by what created this event. Later - // code asserts that OVC elements do not have a value - e.resetValue() - e + state.getOvcElement(startEv, erd) } else { // Event was optional and didn't exist, create a new InfosetElement and add it val e = new DISimple(erd) @@ -601,30 +517,10 @@ trait OVCStartEndStrategy extends ElementUnparserStartEndStrategy { } } else { // Event was hidden and will never exist, create a new InfosetElement and add it - val e = new DISimple(erd) - e.setHidden() - e + state.getHiddenElement(erd) } - // We are about to add a new OVC child to this complex element. Before we - // do that, if the last child added to this complex is a DIArray, that - // implies that the DIArray will have no more children added and should be - // marked as final, and we can attempt to free that array. - val parentNode = state.currentInfosetNode - val parentComplex = parentNode.asComplex - val lastChildMaybe = parentComplex.maybeLastChild - if (lastChildMaybe.isDefined) { - val lastChild = lastChildMaybe.get - if (lastChild.isArray) { - lastChild.setFinal() - parentComplex.freeChildIfNoLongerNeeded( - parentComplex.numChildren - 1, - state.releaseUnneededInfoset - ) - } - } - - parentComplex.addChild(ovcElem, state.tunable) + state.attachElement(ovcElem) state.currentInfosetNodeStack.push(One(ovcElem)) } @@ -632,23 +528,8 @@ trait OVCStartEndStrategy extends ElementUnparserStartEndStrategy { // if an OVC element existed, the start AND end events were consumed in // unparseBegin. No need to advance the cursor here. - // ovcElem is finished, free it if possible. OVC elements are not allowed in - // arrays, so we can directly get the diParent to get the container DINode val ovcElem = state.currentInfosetNodeStack.pop - val ovcContainer = ovcElem.get.diParent - ovcContainer.freeChildIfNoLongerNeeded( - ovcContainer.numChildren - 1, - state.releaseUnneededInfoset - ) - - if (state.currentInfosetNodeStack.isEmpty) { - // If there is nothing else on the infoset stack after popping off the - // current infoset node, that means we have finished the root element, - // so mark the DIDocument as final - val doc = state.documentElement - Assert.invariant(!doc.isFinal) - doc.setFinal() - } + state.finishOvcElement(ovcElem.get) move(state) } From a00881389f5c492bc6ad62f05d3d1c0731b4fee6 Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:46:15 -0400 Subject: [PATCH 4/5] Add a build ahead unparse path driven by an infoset builder tree Unparse can now build the infoset tree ahead of the unparse. A tree of InfosetBuilders, made when the schema is compiled, is stepped by an InfosetBuildCursor that reads the inputter's events and attaches nodes, running up to unparseBuildAheadWindowNodes ahead of the unparse. The unparse reads the built tree as the same start and end events an inputter would produce, through TreeEventState, so it runs the unparsers that event driven unparse runs. A forward reference is resolved against the tree where it exists instead of through a suspension. The tunable infosetBuilderMode selects buildAhead, the default, or eventDriven, the previous behavior, and CI runs the unit tests a second time under eventDriven. Choice branches are keyed by NamedQName, and unparse errors raised by code shared with build carry no data location. When debugging, the infoset commands show only the nodes the unparse has reached. DAFFODIL-3065 --- .github/workflows/main.yml | 8 + .../daffodil/core/grammar/Grammar.scala | 15 + .../daffodil/core/grammar/Production.scala | 14 + .../grammar/primitives/ChoiceCombinator.scala | 40 +- .../DelimiterAndEscapeRelated.scala | 19 + .../primitives/ElementCombinator.scala | 49 +- .../primitives/HiddenGroupCombinator.scala | 3 + .../grammar/primitives/LayeredSequence.scala | 9 + .../primitives/NilEmptyCombinators.scala | 6 + .../grammar/primitives/SequenceChild.scala | 6 + .../primitives/SequenceCombinator.scala | 9 + .../grammar/primitives/SpecifiedLength.scala | 6 + .../core/runtime1/GramRuntime1Mixin.scala | 9 + .../runtime1/SchemaSetRuntime1Mixin.scala | 13 + .../apache/daffodil/core/util/TestUtils.scala | 6 +- .../runtime1/debugger/DaffodilDebugger.scala | 24 +- .../runtime1/infoset/InfosetBuilder.scala | 473 ++++++ .../runtime1/infoset/InfosetWalker.scala | 37 +- .../runtime1/processors/DataProcessor.scala | 231 ++- .../processors/SchemaSetRuntimeData.scala | 3 + .../parsers/SequenceChildBases.scala | 10 +- .../unparsers/InfosetBuildState.scala | 135 ++ .../unparsers/InfosetFromEvents.scala | 3 +- .../unparsers/InfosetFromTree.scala | 67 + .../processors/unparsers/TreeEventState.scala | 392 +++++ .../processors/unparsers/UState.scala | 151 +- .../unparsers/UStateMainForBuildAhead.scala | 54 + .../processors/unparsers/Unparser.scala | 5 + .../unparsers/runtime1/ElementUnparser.scala | 42 +- .../runtime1/LayeredSequenceUnparser.scala | 3 + .../runtime1/SeparatedSequenceUnparsers.scala | 10 +- .../runtime1/SequenceChildUnparsers.scala | 31 +- .../UnseparatedSequenceUnparsers.scala | 6 +- .../TestBuildAheadDataProcessor.scala | 253 +++ .../unparsers/InfosetBuildTestFixture.scala | 105 ++ .../unparsers/TestInfosetBuildCursor.scala | 419 +++++ .../org/apache/daffodil/xsd/dafext.xsd | 30 + .../daffodil/propGen/TunableGenerator.scala | 61 +- .../daffodil/cliTest/TestCLIDebugger.scala | 54 + .../apache/daffodil/unparser/buildAhead.tdml | 1436 +++++++++++++++++ .../daffodil/unparser/TestBuildAhead.scala | 67 + 41 files changed, 4163 insertions(+), 151 deletions(-) create mode 100644 daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetBuilder.scala create mode 100644 daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildState.scala create mode 100644 daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromTree.scala create mode 100644 daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/TreeEventState.scala create mode 100644 daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UStateMainForBuildAhead.scala create mode 100644 daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/TestBuildAheadDataProcessor.scala create mode 100644 daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildTestFixture.scala create mode 100644 daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/unparsers/TestInfosetBuildCursor.scala create mode 100644 daffodil-test/src/test/resources/org/apache/daffodil/unparser/buildAhead.tdml create mode 100644 daffodil-test/src/test/scala/org/apache/daffodil/unparser/TestBuildAhead.scala diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 16bad4ca51..d7bbf4b576 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -197,6 +197,14 @@ jobs: - name: Run Unit Tests run: $SBT test + # Build ahead is the default, so run the suite once more event driven to + # keep that unparse path tested. + - name: Run Unit Tests (event-driven unparse) + if: matrix.os == 'ubuntu-22.04' && matrix.java_version == '17' + env: + DAFFODIL_TDML_TUNABLES: infosetBuilderMode=eventDriven + run: $SBT test + - name: Run Integration Tests run: $SBT daffodil-test-integration/test diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/Grammar.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/Grammar.scala index 47f1f3740f..ded35b7704 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/Grammar.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/Grammar.scala @@ -21,6 +21,8 @@ import org.apache.daffodil.core.compiler.ForParser import org.apache.daffodil.core.compiler.ForUnparser import org.apache.daffodil.core.dsom.* import org.apache.daffodil.lib.exceptions.Assert +import org.apache.daffodil.runtime1.infoset.InfosetBuilder +import org.apache.daffodil.runtime1.infoset.SeqCompInfosetBuilder import org.apache.daffodil.runtime1.processors.parsers.AssertExpressionEvaluationParser import org.apache.daffodil.runtime1.processors.parsers.NadaParser import org.apache.daffodil.runtime1.processors.parsers.SeqCompParser @@ -124,6 +126,19 @@ class SeqComp private (context: SchemaComponent, children: Seq[Gram]) else if (unparserChildren.length == 1) unparserChildren.head else new SeqCompUnparser(context.runtimeData, unparserChildren.toArray) } + + // Only the (usually at most one) child(ren) that actually build infoset + // content contribute here; children that build no infoset events + // (delimiters, padding, etc.) simply have no builder to collect. + lazy val builderChildren: Array[InfosetBuilder] = { + children + .filter(x => !x.isEmpty && (x.forWhat != ForParser)) + .map(_.builder) + .filterNot(_.isEmpty) + .toArray + } + + final override lazy val builder: InfosetBuilder = SeqCompInfosetBuilder(builderChildren) } object EmptyGram extends Gram(null) { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/Production.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/Production.scala index 833b1f71f2..aa108b6099 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/Production.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/Production.scala @@ -22,6 +22,8 @@ import org.apache.daffodil.core.compiler.ForUnparser import org.apache.daffodil.core.compiler.ParserOrUnparser import org.apache.daffodil.core.dsom.SchemaComponent import org.apache.daffodil.lib.util.Logger +import org.apache.daffodil.runtime1.infoset.InfosetBuilder +import org.apache.daffodil.runtime1.infoset.NadaInfosetBuilder import org.apache.daffodil.runtime1.processors.parsers.NadaParser import org.apache.daffodil.unparsers.runtime1.NadaUnparser @@ -107,4 +109,16 @@ final class Prod( else unp } + + final override lazy val builder: InfosetBuilder = { + if (gram.isEmpty) { + NadaInfosetBuilder + } else { + (forWhat, gram.forWhat) match { + case (ForParser, _) => NadaInfosetBuilder + case (_, ForParser) => NadaInfosetBuilder + case _ => gram.builder + } + } + } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ChoiceCombinator.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ChoiceCombinator.scala index 53be0230bd..fb79fc89a6 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ChoiceCombinator.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ChoiceCombinator.scala @@ -29,10 +29,17 @@ import org.apache.daffodil.lib.cookers.ChoiceBranchKeyCooker import org.apache.daffodil.lib.cookers.IntRangeCooker import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.ChoiceLengthKind +import org.apache.daffodil.lib.util.Maybe +import org.apache.daffodil.lib.util.Maybe.Nope +import org.apache.daffodil.lib.util.Maybe.One import org.apache.daffodil.lib.util.MaybeInt import org.apache.daffodil.lib.util.ProperlySerializableMap.* import org.apache.daffodil.lib.xml.NamedQName +import org.apache.daffodil.runtime1.infoset.ChoiceInfosetBuilder +import org.apache.daffodil.runtime1.infoset.InfosetBuilder +import org.apache.daffodil.runtime1.infoset.NadaInfosetBuilder import org.apache.daffodil.runtime1.processors.RangeBound +import org.apache.daffodil.runtime1.processors.TermRuntimeData import org.apache.daffodil.runtime1.processors.parsers.* import org.apache.daffodil.runtime1.processors.unparsers.* import org.apache.daffodil.unparsers.runtime1.* @@ -266,8 +273,15 @@ case class ChoiceCombinator(ch: ChoiceTermBase, alternatives: Seq[Gram]) } } + private lazy val eventUnparserMap = ch.choiceBranchMap._1.map { case (qname, branchTerm) => + (qname, branchTerm.termContentBody.unparser) + } + + private lazy val hasEventBranchUnparser: Boolean = + eventUnparserMap.exists { case (_, branchUnparser) => !branchUnparser.isEmpty } + override lazy val unparser: Unparser = { - val (eventRDMap, optDefaultBranch) = ch.choiceBranchMap + val optDefaultBranch = ch.choiceBranchMap._2 /* * Since it's impossible to know the hiddenness for terms at this level (unless * they're a hiddenGroupRef), we always attempt to find a defaultable unparser. @@ -313,11 +327,7 @@ case class ChoiceCombinator(ch: ChoiceTermBase, alternatives: Seq[Gram]) optDefaultUnparser } - val eventUnparserMap = eventRDMap.map { case (qname, branchTerm) => - (qname, branchTerm.termContentBody.unparser) - } - val mapValues = eventUnparserMap.map { case (k, v) => v }.filterNot(_.isEmpty) - if (mapValues.isEmpty) { + if (!hasEventBranchUnparser) { if (branchForUnparse.isEmpty) { new NadaUnparser(null) } else { @@ -332,4 +342,22 @@ case class ChoiceCombinator(ch: ChoiceTermBase, alternatives: Seq[Gram]) new ChoiceCombinatorUnparser(ch.modelGroupRuntimeData, cbm, choiceLengthInBits) } } + + override lazy val builder: InfosetBuilder = { + val (eventRDMap, optDefaultBranch) = ch.choiceBranchMap + + if (!hasEventBranchUnparser && optDefaultBranch.isEmpty) { + NadaInfosetBuilder + } else { + val branchMap: Map[NamedQName, (TermRuntimeData, InfosetBuilder)] = + eventRDMap.map { case (qname, branchTerm) => + (qname, (branchTerm.termRuntimeData, branchTerm.termContentBody.builder)) + } + val defaultBranch: Maybe[(TermRuntimeData, InfosetBuilder)] = optDefaultBranch match { + case Some(term) => One((term.termRuntimeData, term.termContentBody.builder)) + case None => Nope + } + new ChoiceInfosetBuilder(ch.modelGroupRuntimeData, branchMap, defaultBranch) + } + } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/DelimiterAndEscapeRelated.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/DelimiterAndEscapeRelated.scala index 6bf946a91d..a17900ab78 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/DelimiterAndEscapeRelated.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/DelimiterAndEscapeRelated.scala @@ -21,9 +21,11 @@ import org.apache.daffodil.core.dsom.* import org.apache.daffodil.core.grammar.Gram import org.apache.daffodil.core.grammar.Terminal import org.apache.daffodil.lib.exceptions.Assert +import org.apache.daffodil.lib.util.Maybe import org.apache.daffodil.lib.util.Maybe.* import org.apache.daffodil.lib.util.Misc import org.apache.daffodil.lib.xml.XMLUtils +import org.apache.daffodil.runtime1.infoset.InfosetBuilder import org.apache.daffodil.runtime1.processors.parsers.{ Parser as DaffodilParser, * } import org.apache.daffodil.runtime1.processors.unparsers.Unparser as DaffodilUnparser import org.apache.daffodil.unparsers.runtime1.* @@ -55,6 +57,10 @@ case class DelimiterStackCombinatorSequence(sq: SequenceTermBase, body: Gram) override lazy val unparser: DaffodilUnparser = new DelimiterStackUnparser(uInit, uSep, uTerm, sq.termRuntimeData, body.unparser) + + // Delimiters build no infoset events; the builder tree skips straight to + // whatever this sequence's body itself builds, if anything. + override lazy val builder: InfosetBuilder = body.builder } case class DelimiterStackCombinatorChoice(ch: ChoiceTermBase, body: Gram) @@ -77,6 +83,10 @@ case class DelimiterStackCombinatorChoice(ch: ChoiceTermBase, body: Gram) override lazy val unparser: DaffodilUnparser = new DelimiterStackUnparser(uInit, None, uTerm, ch.termRuntimeData, body.unparser) + + // Delimiters build no infoset events; the builder tree skips straight to + // whatever this choice's body itself builds, if anything. + override lazy val builder: InfosetBuilder = body.builder } case class DelimiterStackCombinatorElement(e: ElementBase, body: Gram) @@ -111,6 +121,10 @@ case class DelimiterStackCombinatorElement(e: ElementBase, body: Gram) if (u.isEmpty) u else new DelimiterStackUnparser(uInit, None, uTerm, e.termRuntimeData, u) } + + // Delimiters build no infoset events; the builder tree skips straight to + // whatever this element's body itself builds, if anything. + override lazy val builder: InfosetBuilder = body.builder } case class DynamicEscapeSchemeCombinatorElement(e: ElementBase, body: Gram) @@ -135,4 +149,9 @@ case class DynamicEscapeSchemeCombinatorElement(e: ElementBase, body: Gram) if (u.isEmpty || schemeUnparseIsConstant) u else new DynamicEscapeSchemeUnparser(schemeUnparseOpt.get, e.termRuntimeData, u) } + + // The escape scheme only governs delimiter matching in written bytes; the + // builder tree skips straight to whatever this element's body itself + // builds, if anything. + override lazy val builder: InfosetBuilder = body.builder } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala index 49bc66883d..6c5ecc6e19 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/ElementCombinator.scala @@ -28,6 +28,9 @@ import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind import org.apache.daffodil.lib.schema.annotation.props.gen.Representation import org.apache.daffodil.lib.schema.annotation.props.gen.TestKind import org.apache.daffodil.lib.util.Maybe +import org.apache.daffodil.runtime1.infoset.ElementInfosetBuilder +import org.apache.daffodil.runtime1.infoset.InfosetBuilder +import org.apache.daffodil.runtime1.infoset.NadaInfosetBuilder import org.apache.daffodil.runtime1.processors.parsers.CaptureEndOfContentLengthParser import org.apache.daffodil.runtime1.processors.parsers.CaptureEndOfValueLengthParser import org.apache.daffodil.runtime1.processors.parsers.CaptureStartOfContentLengthParser @@ -44,6 +47,7 @@ import org.apache.daffodil.unparsers.runtime1.CaptureStartOfValueLengthUnparser import org.apache.daffodil.unparsers.runtime1.ElementOVCSpecifiedLengthUnparser import org.apache.daffodil.unparsers.runtime1.ElementOVCUnspecifiedLengthUnparser import org.apache.daffodil.unparsers.runtime1.ElementSpecifiedLengthUnparser +import org.apache.daffodil.unparsers.runtime1.ElementUnparserBase import org.apache.daffodil.unparsers.runtime1.ElementUnparserInputValueCalc import org.apache.daffodil.unparsers.runtime1.ElementUnspecifiedLengthUnparser import org.apache.daffodil.unparsers.runtime1.ElementUnusedUnparser @@ -111,6 +115,12 @@ class ElementCombinator( private lazy val eRepTypeUnparser: Maybe[Unparser] = repTypeElementGram.maybeUnparser + private lazy val isSpecifiedLength: Boolean = + (context.lengthKind._eq_(LengthKind.Explicit)) || + (context.isSimpleType && + (context.lengthKind._eq_(LengthKind.Implicit)) && + (context.impliedRepresentation._eq_(Representation.Text))) + override lazy val unparser: Unparser = { if (context.isOutputValueCalc) { new ElementOVCSpecifiedLengthUnparser( @@ -122,12 +132,7 @@ class ElementCombinator( eAfterUnparser, context.ovcCompiledExpression ) - } else if ( - (context.lengthKind._eq_(LengthKind.Explicit)) || - (context.isSimpleType && - (context.lengthKind._eq_(LengthKind.Implicit)) && - (context.impliedRepresentation._eq_(Representation.Text))) - ) { + } else if (isSpecifiedLength) { new ElementSpecifiedLengthUnparser( context.erd, @@ -143,6 +148,26 @@ class ElementCombinator( } } + private lazy val eBuilder: InfosetBuilder = { + if (eValue.isEmpty) { + NadaInfosetBuilder + } else { + eValue.builder + } + } + private lazy val eRepTypeBuilder: InfosetBuilder = repTypeElementGram.builder + + // Shares the memoized unparser above for unparseBegin/unparseEnd, so + // build and the unparse see identical node-creation behavior. + override lazy val builder: InfosetBuilder = { + if (context.isOutputValueCalc || isSpecifiedLength) { + val eu = unparser.asInstanceOf[ElementUnparserBase] + val contentBuilder = eRepTypeBuilder.orElse(eBuilder) + new ElementInfosetBuilder(context.erd, eu, contentBuilder) + } else { + subComb.builder + } + } } case class ElementUnused(ctxt: ElementBase) @@ -374,6 +399,14 @@ class ElementParseAndUnspecifiedLength( new ElementUnparserInputValueCalc(context.erd, uSetVar) } } + + // Shares the memoized unparser above for unparseBegin/unparseEnd, so + // build and the unparse see identical nilled/OVC/IVC node-creation behavior. + override lazy val builder: InfosetBuilder = { + val eu = unparser.asInstanceOf[ElementUnparserBase] + val contentBuilder = eRepTypeBuilder.orElse(eBuilder) + new ElementInfosetBuilder(context.erd, eu, contentBuilder) + } } abstract class ElementCombinatorBase( @@ -449,4 +482,8 @@ abstract class ElementCombinatorBase( def unparser: Unparser + lazy val eBuilder: InfosetBuilder = eGram.builder + + lazy val eRepTypeBuilder: InfosetBuilder = repTypeElementGram.builder + } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/HiddenGroupCombinator.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/HiddenGroupCombinator.scala index a993c98cfe..06b31c841b 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/HiddenGroupCombinator.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/HiddenGroupCombinator.scala @@ -20,6 +20,8 @@ package org.apache.daffodil.core.grammar.primitives import org.apache.daffodil.core.dsom.ModelGroup import org.apache.daffodil.core.grammar.Gram import org.apache.daffodil.core.grammar.Terminal +import org.apache.daffodil.runtime1.infoset.HiddenGroupInfosetBuilder +import org.apache.daffodil.runtime1.infoset.InfosetBuilder import org.apache.daffodil.runtime1.processors.parsers.HiddenGroupCombinatorParser import org.apache.daffodil.runtime1.processors.parsers.Parser import org.apache.daffodil.runtime1.processors.unparsers.Unparser @@ -34,4 +36,5 @@ final class HiddenGroupCombinator(ctxt: ModelGroup, body: Gram) override lazy val unparser: Unparser = new HiddenGroupCombinatorUnparser(ctxt.modelGroupRuntimeData, body.unparser) + override lazy val builder: InfosetBuilder = HiddenGroupInfosetBuilder(body.builder) } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/LayeredSequence.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/LayeredSequence.scala index 2b75da3f4b..3157ce71f2 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/LayeredSequence.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/LayeredSequence.scala @@ -21,6 +21,8 @@ import org.apache.daffodil.core.dsom.* import org.apache.daffodil.core.grammar.Terminal import org.apache.daffodil.core.layers.LayerSchemaCompiler import org.apache.daffodil.lib.util.Misc +import org.apache.daffodil.runtime1.infoset.InfosetBuilder +import org.apache.daffodil.runtime1.infoset.SequenceInfosetBuilder import org.apache.daffodil.runtime1.processors.parsers.LayeredSequenceParser import org.apache.daffodil.runtime1.processors.parsers.Parser as DaffodilParser import org.apache.daffodil.runtime1.processors.unparsers.Unparser as DaffodilUnparser @@ -47,4 +49,11 @@ case class LayeredSequence(sq: SequenceGroupTermBase, bodyTerm: SequenceChild) override lazy val unparser: DaffodilUnparser = new LayeredSequenceUnparser(srd, bodyUnparser) + + // The layer transform builds no infoset events, but this is still a one-child + // sequence position: it must push/pop bodyTerm's TRD and advance the + // group index like any sequence child, or next-element resolution on + // the shared InfosetInputter breaks. + override lazy val builder: InfosetBuilder = + SequenceInfosetBuilder(Array(bodyTerm.sequenceChildBuildInfo)) } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/NilEmptyCombinators.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/NilEmptyCombinators.scala index 1ed7a027bc..3ea97248bc 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/NilEmptyCombinators.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/NilEmptyCombinators.scala @@ -21,6 +21,8 @@ import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.grammar.Gram import org.apache.daffodil.core.grammar.Terminal import org.apache.daffodil.lib.exceptions.Assert +import org.apache.daffodil.runtime1.infoset.InfosetBuilder +import org.apache.daffodil.runtime1.infoset.NilOrContentInfosetBuilder import org.apache.daffodil.runtime1.processors.parsers.ComplexNilOrContentParser import org.apache.daffodil.runtime1.processors.parsers.SimpleNilOrValueParser import org.apache.daffodil.unparsers.runtime1.ComplexNilOrContentUnparser @@ -59,4 +61,8 @@ case class ComplexNilOrContent(ctxt: ElementBase, nilGram: Gram, contentGram: Gr override lazy val unparser = ComplexNilOrContentUnparser(ctxt.erd, nilUnparser, contentUnparser) + // A nilled complex element has no children to build; nilled-ness is only + // known once the node exists, so this needs a real runtime check, not a + // static pass-through. + override lazy val builder: InfosetBuilder = NilOrContentInfosetBuilder(contentGram.builder) } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SequenceChild.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SequenceChild.scala index 725eff5363..418b7eb373 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SequenceChild.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SequenceChild.scala @@ -25,6 +25,8 @@ import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind import org.apache.daffodil.lib.schema.annotation.props.gen.OccursCountKind import org.apache.daffodil.lib.schema.annotation.props.gen.Representation import org.apache.daffodil.runtime1.dpath.NodeInfo +import org.apache.daffodil.runtime1.infoset.InfosetBuilder +import org.apache.daffodil.runtime1.infoset.SequenceChildInfosetBuildInfo import org.apache.daffodil.runtime1.processors.parsers.* import org.apache.daffodil.unparsers.runtime1.* @@ -69,6 +71,7 @@ abstract class SequenceChild(protected val sq: SequenceTermBase, child: Term, gr protected lazy val childParser = child.termContentBody.parser protected lazy val childUnparser = child.termContentBody.unparser + protected lazy val childBuilder: InfosetBuilder = child.termContentBody.builder final override lazy val parser = sequenceChildParser final override lazy val unparser = sequenceChildUnparser @@ -82,6 +85,9 @@ abstract class SequenceChild(protected val sq: SequenceTermBase, child: Term, gr final lazy val optSequenceChildUnparser: Option[SequenceChildUnparser] = if (childUnparser.isEmpty) None else Some(unparser) + final lazy val sequenceChildBuildInfo: SequenceChildInfosetBuildInfo = + SequenceChildInfosetBuildInfo(unparser, childBuilder) + /** * There's only parse result helpers here, so let's abbreviate */ diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SequenceCombinator.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SequenceCombinator.scala index aab310aa2c..6dc8c8e269 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SequenceCombinator.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SequenceCombinator.scala @@ -24,6 +24,9 @@ import org.apache.daffodil.lib.schema.annotation.props.SeparatorSuppressionPolic import org.apache.daffodil.lib.util.Maybe import org.apache.daffodil.lib.util.MaybeInt import org.apache.daffodil.lib.util.Misc +import org.apache.daffodil.runtime1.infoset.InfosetBuilder +import org.apache.daffodil.runtime1.infoset.SequenceChildInfosetBuildInfo +import org.apache.daffodil.runtime1.infoset.SequenceInfosetBuilder import org.apache.daffodil.runtime1.processors.parsers.* import org.apache.daffodil.runtime1.processors.unparsers.* import org.apache.daffodil.unparsers.runtime1.{ Separated as SeparatedUnparser, * } @@ -124,6 +127,9 @@ class OrderedSequence(sq: SequenceTermBase, sequenceChildrenArg: Seq[SequenceChi } } } + + override lazy val builder: InfosetBuilder = + SequenceInfosetBuilder(sequenceChildren.map { _.sequenceChildBuildInfo }) } class UnorderedSequence( @@ -220,4 +226,7 @@ class UnorderedSequence( } } } + + override lazy val builder: InfosetBuilder = + SequenceInfosetBuilder(sequenceChildren.map { _.sequenceChildBuildInfo }) } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala index 0ae3e6a4d9..c5b758b0aa 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala @@ -25,6 +25,7 @@ import org.apache.daffodil.core.grammar.Terminal import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits import org.apache.daffodil.runtime1.dpath.NodeInfo.PrimType +import org.apache.daffodil.runtime1.infoset.InfosetBuilder import org.apache.daffodil.runtime1.processors.parsers.* import org.apache.daffodil.runtime1.processors.unparsers.* import org.apache.daffodil.unparsers.runtime1.* @@ -44,6 +45,11 @@ abstract class SpecifiedLengthCombinatorBase(val e: ElementBase, eGramArg: => Gr u } + // None of the length-kind wrapping below (explicit/implicit/prefixed + // lengths, pattern matching) creates infoset nodes; the builder tree skips + // straight to whatever the wrapped element content itself builds. + override lazy val builder: InfosetBuilder = eGram.builder + def kind: String def toBriefXML(depthLimit: Int = -1): String = { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/GramRuntime1Mixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/GramRuntime1Mixin.scala index 54b45c8148..b7e52b1ad1 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/GramRuntime1Mixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/GramRuntime1Mixin.scala @@ -20,6 +20,8 @@ package org.apache.daffodil.core.runtime1 import org.apache.daffodil.core.grammar.Gram import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.util.Maybe +import org.apache.daffodil.runtime1.infoset.InfosetBuilder +import org.apache.daffodil.runtime1.infoset.NadaInfosetBuilder import org.apache.daffodil.runtime1.processors.parsers.Parser import org.apache.daffodil.runtime1.processors.unparsers.Unparser @@ -59,4 +61,11 @@ trait GramRuntime1Mixin { self: Gram => else Maybe(u) } } + + /** + * Provides this Gram's node in the dedicated InfosetBuilder tree that parallels + * the Unparser tree. Most Grams create or select no infoset content and + * inherit this Nada default; only those that do override it. + */ + def builder: InfosetBuilder = NadaInfosetBuilder } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/SchemaSetRuntime1Mixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/SchemaSetRuntime1Mixin.scala index 3e9c5634fe..989629ee51 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/SchemaSetRuntime1Mixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/SchemaSetRuntime1Mixin.scala @@ -22,6 +22,8 @@ import org.apache.daffodil.core.dsom.SequenceTermBase import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.util.Logger import org.apache.daffodil.runtime1.iapi.DFDL +import org.apache.daffodil.runtime1.infoset.InfosetBuilder +import org.apache.daffodil.runtime1.infoset.NadaInfosetBuilder import org.apache.daffodil.runtime1.layers.LayerRuntimeCompiler import org.apache.daffodil.runtime1.layers.LayerRuntimeData import org.apache.daffodil.runtime1.processors.DataProcessor @@ -61,6 +63,16 @@ trait SchemaSetRuntime1Mixin { unp }.value + // Built for every unparse-capable compile, whatever infosetBuilderMode says, + // so the tunable can be changed on a compiled DataProcessor. + lazy val builder: InfosetBuilder = LV(Symbol("builder")) { + if (generateUnparser) { + root.document.builder + } else { + NadaInfosetBuilder + } + }.value + private lazy val layerRuntimeCompiler = new LayerRuntimeCompiler private lazy val allLayers: Seq[LayerRuntimeData] = LV(Symbol("allLayers")) { @@ -88,6 +100,7 @@ trait SchemaSetRuntime1Mixin { new SchemaSetRuntimeData( parser, unparser, + builder, root.elementRuntimeData, variableMap, allLayers, diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/util/TestUtils.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/util/TestUtils.scala index bc70863fe3..5b885956d8 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/util/TestUtils.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/util/TestUtils.scala @@ -118,9 +118,11 @@ object TestUtils { testSchema: scala.xml.Elem, infosetXML: Node, unparseTo: String, - areTracing: Boolean = false + areTracing: Boolean = false, + tunables: Map[String, String] = Map.empty ): java.util.List[api.Diagnostic] = { - val compiler = Compiler().withTunable("allowExternalPathExpressions", "true") + val compiler = + Compiler().withTunable("allowExternalPathExpressions", "true").withTunables(tunables) val pf = compiler.compileNode(testSchema) if (pf.isError) throwDiagnostics(pf.getDiagnostics) var u = saveAndReload(pf.onPath("/").asInstanceOf[DataProcessor]) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/debugger/DaffodilDebugger.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/debugger/DaffodilDebugger.scala index c5d53cdfc7..d1bc022582 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/debugger/DaffodilDebugger.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/debugger/DaffodilDebugger.scala @@ -49,6 +49,7 @@ import org.apache.daffodil.runtime1.processors.* import org.apache.daffodil.runtime1.processors.parsers.* import org.apache.daffodil.runtime1.processors.unparsers.UState import org.apache.daffodil.runtime1.processors.unparsers.UStateForSuspension +import org.apache.daffodil.runtime1.processors.unparsers.UStateMainForBuildAhead import org.apache.daffodil.runtime1.processors.unparsers.Unparser case class DebuggerExitException() extends UnsuppressableException("Debugger exit") @@ -468,22 +469,29 @@ class DaffodilDebugger( } } - private def infosetToString(ie: InfosetElement): String = { + private def infosetToString(ie: InfosetElement, state: ParseOrUnparseState): String = { val bos = new java.io.ByteArrayOutputStream() val xml = new XMLTextInfosetOutputter(bos, pretty = true, minimal = true) + // When unparsing builds the infoset ahead, show only the part the unparse + // has reached, as an event-driven unparse would have built it by now. + val reached = state match { + case buildAhead: UStateMainForBuildAhead => buildAhead.reachedChildCounts() + case _ => null + } val iw = StreamingInfosetWalker( ie.asInstanceOf[DIElement], xml, walkHidden = !DebuggerConfig.removeHidden, ignoreBlocks = true, - releaseUnneededInfoset = false + releaseUnneededInfoset = false, + visibleChildCounts = reached ) iw.walk(lastWalk = true) bos.toString("UTF-8") } - private def debugPrettyPrintXML(ie: InfosetElement): Unit = { - val infosetString = infosetToString(ie) + private def debugPrettyPrintXML(ie: InfosetElement, state: ParseOrUnparseState): Unit = { + val infosetString = infosetToString(ie, state) debugPrintln(infosetString) } @@ -1123,11 +1131,11 @@ class DaffodilDebugger( debugPrintln(_) } res match { - case ie: InfosetElement => debugPrettyPrintXML(ie) + case ie: InfosetElement => debugPrettyPrintXML(ie, state) case nodeSeq: Seq[Any] => nodeSeq.foreach { a => a match { - case ie: InfosetElement => debugPrettyPrintXML(ie) + case ie: InfosetElement => debugPrettyPrintXML(ie, state) case _ => debugPrintln(a) } } @@ -1164,7 +1172,7 @@ class DaffodilDebugger( // // Displays the empty element since it has no value. // - debugPrettyPrintXML(nd.diElement) + debugPrettyPrintXML(nd.diElement, state) state.suppressDiagnosticAndSucceed(r) } case _ => throw r @@ -1810,7 +1818,7 @@ class DaffodilDebugger( debugPrintln("No Infoset", " ") } case _ => { - val infosetString = infosetToString(node) + val infosetString = infosetToString(node, state) val lines = infosetString.split("\r?\n") val dropCount = diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetBuilder.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetBuilder.scala new file mode 100644 index 0000000000..c2f89afb6e --- /dev/null +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetBuilder.scala @@ -0,0 +1,473 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.runtime1.infoset + +import org.apache.daffodil.lib.exceptions.Assert +import org.apache.daffodil.lib.util.MStackOf +import org.apache.daffodil.lib.util.Maybe +import org.apache.daffodil.lib.util.Maybe.Nope +import org.apache.daffodil.lib.util.Maybe.One +import org.apache.daffodil.lib.xml.NamedQName +import org.apache.daffodil.runtime1.processors.ElementRuntimeData +import org.apache.daffodil.runtime1.processors.ModelGroupRuntimeData +import org.apache.daffodil.runtime1.processors.TermRuntimeData +import org.apache.daffodil.runtime1.processors.unparsers.InfosetBuildState +import org.apache.daffodil.runtime1.processors.unparsers.InfosetTreeState +import org.apache.daffodil.runtime1.processors.unparsers.UnparseError +import org.apache.daffodil.unparsers.runtime1.ElementUnparserBase +import org.apache.daffodil.unparsers.runtime1.RepeatingChildUnparser +import org.apache.daffodil.unparsers.runtime1.SequenceChildUnparser + +/** + * A node in a much smaller, dedicated tree of InfosetBuilders that parallels the + * full Unparser tree: only Grams that actually create or select infoset + * content (elements, sequences, choices, hidden groups) contribute one, so + * building the infoset never has to dispatch through the many wrapper + * unparsers that build no infoset events (delimiters, escape schemes, + * layers, padding, specified-length) that sit between them in the + * Unparser tree. + * + * An InfosetBuilder is immutable compiled-schema state shared by every unparse. The + * per-unparse position lives in the InfosetBuildFrame it creates, on an InfosetBuildCursor's + * explicit stack, so building can stop after any step and continue later + * without holding a thread or a JVM call stack. + */ +trait InfosetBuilder extends Serializable { + def newFrame(): InfosetBuildFrame + + // True only for NadaInfosetBuilder, which contributes nothing to the tree. + def isEmpty: Boolean = false + + // This builder if it builds anything, else the other one. + final def orElse(other: InfosetBuilder): InfosetBuilder = if (!isEmpty) this else other +} + +/** + * One InfosetBuilder's in-progress state for one unparse. `step` performs one + * transition and must either push exactly one child frame onto the cursor + * (this frame is stepped again once that child pops) or pop itself from the + * cursor to signal it is complete. + */ +abstract class InfosetBuildFrame { + def step(cursor: InfosetBuildCursor): Unit +} + +/** + * The explicit stack of InfosetBuildFrames that stands in for the call stack of a + * recursive build. `advance` runs it until the lead window is full, so a + * caller that needs more infoset tree can pull it forward directly. Driven + * against an `InfosetBuildState`. + */ +final class InfosetBuildCursor( + root: InfosetBuilder, + val buildState: InfosetBuildState +) { + def state: InfosetTreeState = buildState + + private val stack = new MStackOf[InfosetBuildFrame](64) + + push(root.newFrame()) + + def push(frame: InfosetBuildFrame): Unit = stack.push(frame) + + def pop(): Unit = stack.pop + + def isFinished: Boolean = stack.isEmpty + + /** + * Steps until the lead window is full or building completes, so it takes a + * single step when the window is already full. With lastAdvance it ignores + * the window and runs until building completes. A failure propagates and + * ends the unparse, so the cursor is not used again. + */ + def advance(lastAdvance: Boolean = false): Unit = { + while (!stack.isEmpty) { + stack.top.step(this) + if (!lastAdvance && buildState.leadExceedsBuildAheadLimit) { + return + } + } + } +} + +/** + * Builder for a Gram that creates or selects no infoset content. A builder + * with nothing to build collapses into this one, and a sequence drops its + * children that have it, so a parent only holds one where it needs a value + * in that slot, such as a choice branch with no content, which the branch map + * still needs an entry for. A parent that holds one recognizes it by isEmpty + * and never builds it. + */ +object NadaInfosetBuilder extends InfosetBuilder { + override def isEmpty = true + + override def toString = "Nada" + + override def newFrame(): InfosetBuildFrame = + Assert.abort("NadaInfosetBuilders are all supposed to optimize out!") +} + +/** + * Builds each of several sibling Grams' content in order. Used only where a + * `~` composition has more than one child that actually builds infoset + * content; the common case of at most one such child never needs this. + */ +final private class SeqCompInfosetBuilder(children: Array[InfosetBuilder]) + extends InfosetBuilder { + override def newFrame(): InfosetBuildFrame = new InfosetBuildFrame { + private var i = 0 + override def step(cursor: InfosetBuildCursor): Unit = { + if (i < children.length) { + val child = children(i) + i += 1 + cursor.push(child.newFrame()) + } else { + cursor.pop() + } + } + } +} + +object SeqCompInfosetBuilder { + def apply(children: Array[InfosetBuilder]): InfosetBuilder = { + if (children.isEmpty) { + NadaInfosetBuilder + } else if (children.length == 1) { + children.head + } else { + new SeqCompInfosetBuilder(children) + } + } +} + +/** + * Builds one element's infoset node and, for complex types, builds + * descendant nodes via contentBuilder. The element unparser's + * unparseBegin/unparseEnd are the same element-kind-specific + * (plain/nillable/OVC/etc.) node-creation logic unparse() itself uses, + * including the bounded-lookahead lead-counter hookup and the deferred + * simple-value finalization; only the "what does this element contain" + * step is redirected to the builder tree instead of back into the + * unparser tree. + */ +final class ElementInfosetBuilder( + erd: ElementRuntimeData, + elementUnparser: ElementUnparserBase, + contentBuilder: InfosetBuilder +) extends InfosetBuilder { + + override def newFrame(): InfosetBuildFrame = new InfosetBuildFrame { + private var contentPushed = false + + override def step(cursor: InfosetBuildCursor): Unit = { + val state = cursor.state + if (!contentPushed) { + elementUnparser.unparseBegin(state) + if (erd.isComplexType) { + state.pushTRD(erd.optComplexTypeModelGroupRuntimeData.get) + if (!contentBuilder.isEmpty) { + contentPushed = true + cursor.push(contentBuilder.newFrame()) + return + } + } + } + if (erd.isComplexType) { + state.popTRD(erd.optComplexTypeModelGroupRuntimeData.get) + } + elementUnparser.unparseEnd(state) + cursor.pop() + } + } +} + +/** + * Pairs a sequence child's existing occurs-count/array bookkeeping (reused + * as-is from the Unparser tree, since it is cheap, pure state bookkeeping + * unrelated to the tree-walking overhead this InfosetBuilder tree exists to avoid) + * with that same child's own InfosetBuilder, which SequenceInfosetBuilder builds + * instead of the child's full Unparser. + */ +final case class SequenceChildInfosetBuildInfo( + childUnparser: SequenceChildUnparser, + childBuilder: InfosetBuilder +) + +/** + * Where a sequence's frame is in its walk over the children. Start is before + * the first step. NextChild is between children. AfterScalar and + * AfterOccurrence are right after a child's frame popped. InArray is an array + * or optional loop between occurrences. + */ +private enum SequenceBuildPhase { + case Start, NextChild, AfterScalar, InArray, AfterOccurrence +} + +/** + * Builds an entire sequence's children, scalar and array/optional alike, + * each through its own InfosetBuilder. + */ +final private class SequenceInfosetBuilder(children: Array[SequenceChildInfosetBuildInfo]) + extends InfosetBuilder { + + override def newFrame(): InfosetBuildFrame = new SequenceInfosetBuildFrame + + private final class SequenceInfosetBuildFrame extends InfosetBuildFrame { + import SequenceBuildPhase.* + + private var phase: SequenceBuildPhase = Start + private var index = 0 + // children(index), read once per child and used by every later phase. + private var current: SequenceChildInfosetBuildInfo = null + private var rep: RepeatingChildUnparser = null + private var numOccurrences = 0 + private var maxReps = 0L + + override def step(cursor: InfosetBuildCursor): Unit = { + val state = cursor.state + phase match { + case Start => { + state.groupIndexStack.push(1L) + phase = NextChild + nextChild(cursor, state) + } + case NextChild => nextChild(cursor, state) + case AfterScalar => { + current.childUnparser.trd match { + case erd: ElementRuntimeData if !erd.isRepresented => // ok, skip group advance + case _ => state.moveOverOneGroupIndexOnly() + } + finishChild(state) + } + case AfterOccurrence => { + numOccurrences += 1 + state.moveOverOneArrayIterationIndexOnly() + state.moveOverOneOccursIndexOnly() + state.moveOverOneGroupIndexOnly() + phase = InArray + } + case InArray => { + if (rep.shouldDoUnparser(rep, state)) { + phase = AfterOccurrence + cursor.push(current.childBuilder.newFrame()) + } else { + rep.checkFinalOccursCountBetweenMinAndMaxOccurs( + state, + rep, + numOccurrences, + maxReps, + state.arrayIterationPos - 1 + ) + rep.consumeEndArrayEvent(rep.erd, state) + finishRepeating(state) + } + } + } + } + + private def nextChild(cursor: InfosetBuildCursor, state: InfosetTreeState): Unit = { + if (index == children.length) { + state.groupIndexStack.pop() + cursor.pop() + } else { + current = children(index) + val cu = current.childUnparser + state.pushTRD(cu.trd) + cu match { + case r: RepeatingChildUnparser => { + rep = r + state.arrayIterationIndexStack.push(1L) + state.occursIndexStack.push(1L) + numOccurrences = 0 + maxReps = r.maxRepeatsConst + + Assert.invariant(state.inspect, "No event for building.") + val ev = state.inspectAccessor + if (ev.erd eq r.erd) { + r.startArrayOrOptional(state) + phase = InArray + } else { + r.checkFinalOccursCountBetweenMinAndMaxOccurs( + state, + r, + numOccurrences, + maxReps, + 0 + ) + finishRepeating(state) + } + } + case _ => { + phase = AfterScalar + cursor.push(current.childBuilder.newFrame()) + } + } + } + } + + private def finishRepeating(state: InfosetTreeState): Unit = { + state.arrayIterationIndexStack.pop() + state.occursIndexStack.pop() + rep = null + finishChild(state) + } + + private def finishChild(state: InfosetTreeState): Unit = { + state.popTRD(children(index).childUnparser.trd) + index += 1 + phase = NextChild + } + } +} + +object SequenceInfosetBuilder { + def apply(children: Array[SequenceChildInfosetBuildInfo]): InfosetBuilder = { + val nonEmptyChildren = children.filterNot(_.childBuilder.isEmpty) + if (nonEmptyChildren.isEmpty) { + NadaInfosetBuilder + } else { + new SequenceInfosetBuilder(nonEmptyChildren) + } + } +} + +/** + * Builds just the one structurally-present branch of a choice, resolved + * from the next infoset event, through that branch's own InfosetBuilder. + */ +final class ChoiceInfosetBuilder( + mgrd: ModelGroupRuntimeData, + branchMap: Map[NamedQName, (TermRuntimeData, InfosetBuilder)], + defaultBranch: Maybe[(TermRuntimeData, InfosetBuilder)] +) extends InfosetBuilder { + + private def resolveBranch(state: InfosetTreeState): (TermRuntimeData, InfosetBuilder) = { + if (state.withinHiddenNest) { + defaultBranch.get + } else { + state.pushTRD(mgrd) + val event = state.inspectOrError + // An end event never starts a branch, so it always takes the default. + val fromTable = if (event.isStart) { + branchMap.get(event.erd.namedQName) + } else { + None + } + val resolved = if (fromTable.isDefined) { + fromTable + } else { + defaultBranch.toOption + } + if (resolved.isEmpty) { + UnparseError( + One(mgrd.schemaFileLocation), + Nope, + "Found next element %s, but expected one of %s.", + event.erd.namedQName.toExtendedSyntax, + branchMap.keys.map { _.toExtendedSyntax }.mkString(", ") + ) + } + state.popTRD(mgrd) + resolved.get + } + } + + override def newFrame(): InfosetBuildFrame = new InfosetBuildFrame { + private var branchTRD: TermRuntimeData = null + + override def step(cursor: InfosetBuildCursor): Unit = { + val state = cursor.state + if (branchTRD == null) { + val (trd, builder) = resolveBranch(state) + branchTRD = trd + state.pushTRD(trd) + if (!builder.isEmpty) { + cursor.push(builder.newFrame()) + } + } else { + state.popTRD(branchTRD) + cursor.pop() + } + } + } +} + +/** + * Builds the body of a hidden group. withinHiddenNest must stay maintained + * during build too: it is what tells a hidden element's unparseBegin/ + * unparseEnd to manufacture a node instead of consuming an event that will + * never exist. + */ +final private class HiddenGroupInfosetBuilder(bodyBuilder: InfosetBuilder) + extends InfosetBuilder { + override def newFrame(): InfosetBuildFrame = new InfosetBuildFrame { + private var bodyPushed = false + + override def step(cursor: InfosetBuildCursor): Unit = { + if (!bodyPushed) { + bodyPushed = true + cursor.state.incrementHiddenDef() + cursor.push(bodyBuilder.newFrame()) + } else { + cursor.state.decrementHiddenDef() + cursor.pop() + } + } + } +} + +object HiddenGroupInfosetBuilder { + def apply(bodyBuilder: InfosetBuilder): InfosetBuilder = { + if (bodyBuilder.isEmpty) { + NadaInfosetBuilder + } else { + new HiddenGroupInfosetBuilder(bodyBuilder) + } + } +} + +/** + * A nilled complex element has no children to build; nilled-ness is only + * known once the node exists, so this checks it at build time rather than + * resolving statically to either branch. + */ +final private class NilOrContentInfosetBuilder(contentBuilder: InfosetBuilder) + extends InfosetBuilder { + override def newFrame(): InfosetBuildFrame = new InfosetBuildFrame { + private var contentPushed = false + + override def step(cursor: InfosetBuildCursor): Unit = { + if (!contentPushed && !cursor.state.currentInfosetNode.asComplex.isNilled) { + contentPushed = true + cursor.push(contentBuilder.newFrame()) + } else { + cursor.pop() + } + } + } +} + +object NilOrContentInfosetBuilder { + def apply(contentBuilder: InfosetBuilder): InfosetBuilder = { + if (contentBuilder.isEmpty) { + NadaInfosetBuilder + } else { + new NilOrContentInfosetBuilder(contentBuilder) + } + } +} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetWalker.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetWalker.scala index b2d36dc92c..2ab29cc6a3 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetWalker.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetWalker.scala @@ -150,6 +150,13 @@ object StreamingInfosetWalker { * (e.g. due to an unresolved point of uncertainty) and increases the * number of walk() calls to skip before trying again. This defines the * maximum number of skipped calls, even as that number increases. + * + * @param visibleChildCounts + * + * When not null, the number of children of a container that the walk may + * visit, for a container that has an entry; any other container is walked + * in full. This should only be used while debugging, to leave out nodes + * that exist in the infoset but that unparsing has not reached. */ def apply( root: DIElement, @@ -158,7 +165,8 @@ object StreamingInfosetWalker { ignoreBlocks: Boolean, releaseUnneededInfoset: Boolean, walkSkipMin: Int = 32, - walkSkipMax: Int = 2048 + walkSkipMax: Int = 2048, + visibleChildCounts: java.util.Map[DINode, Integer] = null ): StreamingInfosetWalker = { // Determine the container of the root node and the index in which it @@ -187,7 +195,8 @@ object StreamingInfosetWalker { ignoreBlocks, releaseUnneededInfoset, walkSkipMin, - walkSkipMax + walkSkipMax, + visibleChildCounts ) } @@ -252,6 +261,12 @@ object StreamingInfosetWalker { * being blocked for removal (e.g. due to an unresolved point of uncertainty) * and increases the number of walk() calls to skip before trying again. This * defines the maximum number of skiped calls, even as this number increases. + * + * @param visibleChildCounts + * + * When not null, the number of children of a container that the walk may + * visit, for a container that has an entry; any other container is walked + * in full. This should only be used while debugging. */ class StreamingInfosetWalker private ( startingContainerNode: DINode, @@ -261,7 +276,8 @@ class StreamingInfosetWalker private ( ignoreBlocks: Boolean, releaseUnneededInfoset: Boolean, walkSkipMin: Int, - walkSkipMax: Int + walkSkipMax: Int, + visibleChildCounts: java.util.Map[DINode, Integer] ) extends InfosetWalker { /** @@ -563,12 +579,25 @@ class StreamingInfosetWalker private ( finished = true } + private def visibleChildCount(containerNode: DINode): Int = { + if (visibleChildCounts eq null) { + containerNode.numChildren + } else { + val count = visibleChildCounts.get(containerNode) + if (count eq null) { + containerNode.numChildren + } else { + count.intValue + } + } + } + /** * Output start/end events for DIComplex/DIArray/DISimple, and mutate state * so we are looking at the next node in the infoset. */ private def infosetWalkerStepMove(containerNode: DINode, containerIndex: Int): Unit = { - if (containerIndex < containerNode.numChildren) { + if (containerIndex < visibleChildCount(containerNode)) { // This block means we need to create a start event for the element // at containerIndex. Once we create that event we // need to mutate the state of the InfosetWalker so that the next time we diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala index 3b16d3680e..50ab43a300 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala @@ -39,6 +39,7 @@ import org.apache.daffodil.api.validation.ValidatorInitializationException import org.apache.daffodil.api.validation.Validators import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.iapi.DaffodilTunables +import org.apache.daffodil.lib.iapi.InfosetBuilderMode import org.apache.daffodil.lib.iapi.WithDiagnostics import org.apache.daffodil.runtime1.dsom.* import org.apache.daffodil.runtime1.iapi.DFDL @@ -61,6 +62,7 @@ import org.apache.daffodil.lib.util.ThreadSafePool import org.apache.daffodil.runtime1.events.MultipleEventHandler import org.apache.daffodil.runtime1.externalvars.ExternalVariablesLoader import org.apache.daffodil.runtime1.infoset.DIElement +import org.apache.daffodil.runtime1.infoset.InfosetBuildCursor import org.apache.daffodil.runtime1.infoset.InfosetException import org.apache.daffodil.runtime1.infoset.InfosetInputter import org.apache.daffodil.runtime1.infoset.TeeInfosetOutputter @@ -68,6 +70,10 @@ import org.apache.daffodil.runtime1.infoset.XMLTextInfosetOutputter import org.apache.daffodil.runtime1.processors.parsers.PState import org.apache.daffodil.runtime1.processors.parsers.ParseError import org.apache.daffodil.runtime1.processors.parsers.Parser +import org.apache.daffodil.runtime1.processors.unparsers.ChildNotBuiltException +import org.apache.daffodil.runtime1.processors.unparsers.InfosetBuildState +import org.apache.daffodil.runtime1.processors.unparsers.NotUnparsableUnparser +import org.apache.daffodil.runtime1.processors.unparsers.TreeEventState import org.apache.daffodil.runtime1.processors.unparsers.UState import org.apache.daffodil.runtime1.processors.unparsers.UnparseError @@ -458,6 +464,189 @@ class DataProcessor( } def unparse(actualInputter: api.infoset.InfosetInputter, outStream: java.io.OutputStream) = { + // A NotUnparsableUnparser (dfdl:parseUnparsePolicy="parseOnly") has no + // builders to run ahead, and unparseEventDriven already gives the correct + // diagnostic for it, so it never builds ahead. + val canBuildAhead = !ssrd.unparser.isInstanceOf[NotUnparsableUnparser] + if ((tunables.infosetBuilderMode eq InfosetBuilderMode.BuildAhead) && canBuildAhead) { + unparseBuildAhead(actualInputter, outStream) + } else { + unparseEventDriven(actualInputter, outStream) + } + } + + /** + * Shared by unparseBuildAhead and unparseEventDriven's top-level + * catch blocks: maps an exception caught during unparsing to a failed + * `state` plus its `unparseResult`, or rethrows if it's not one of the + * known unparse-error categories. + */ + private def unparseErrorResult(state: UState, t: Throwable): UnparseResult = t match { + case ue: UnparseError => { + state.addUnparseError(ue) + state.unparseResult + } + case procErr: ProcessingError => { + state.setFailed(procErr.toUnparseError) + state.unparseResult + } + case sde: SchemaDefinitionError => { + // A SDE was detected at runtime (perhaps due to a runtime-valued property like byteOrder or encoding) + // These are fatal, and there's no notion of backtracking them, so they propagate to top level + // here. + state.setFailed(sde) + state.unparseResult + } + case sdefw: SchemaDefinitionErrorFromWarning => { + state.setFailed(sdefw) + state.unparseResult + } + case e: ErrorAlreadyHandled => { + state.setFailed(e.th) + state.unparseResult + } + case e: TunableLimitExceededError => { + state.setFailed(e) + state.unparseResult + } + case se: org.xml.sax.SAXException => { + state.setFailed(new UnparseError(None, None, se)) + state.unparseResult + } + case e: scala.xml.parsing.FatalError => { + state.setFailed(new UnparseError(None, None, e)) + state.unparseResult + } + case ie: InfosetException => { + state.setFailed(new UnparseError(None, None, ie)) + state.unparseResult + } + case th: Throwable => throw th + } + + /** + * Unparses with a build pass running ahead of the unparse (gated on + * `infosetBuilderMode`). An `InfosetBuildCursor` over `InfosetBuildState` + * builds the infoset tree from the inputter's events, and the unparse + * reads that tree back as events, advancing the cursor whenever it needs a + * node that does not exist yet. + */ + private def unparseBuildAhead( + actualInputter: api.infoset.InfosetInputter, + outStream: java.io.OutputStream + ): UnparseResult = { + val rootUnparser = ssrd.unparser + + val inputter = new InfosetInputter(actualInputter) + + // Build side. The root element always has a builder: it is exactly the + // case that gets ElementInfosetBuilder wrapped around it, regardless of + // schema content. + val infosetBuildState = new InfosetBuildState(inputter, tunables) + val cursor = new InfosetBuildCursor(ssrd.builder, infosetBuildState) + + // Unparse side. The tree events are the events of the tree build is making, + // so the unparsers read it as they read an inputter's. The state holds the + // output stream, which must be cleaned up. + val treeEvents = new TreeEventState( + cursor, + !areDebugging && tunables.releaseUnneededInfoset + ) + val unparseState = + UState.createInitialUStateForBuildAhead( + outStream, + this, + inputter, + areDebugging, + treeEvents + ) + + try { + inputter.initialize(ssrd.elementRuntimeData, tunables) + + if (areDebugging) { + Assert.invariant(optDebugger.isDefined) + addEventHandler(debugger) + } + if (areDebugging) { + unparseState.notifyDebugging(true) + } + // The root TRD is on the stack of the tree events, as it is + // on an inputter's when it is initialized. + unparseState.pushTRD(ssrd.elementRuntimeData) + init(unparseState, rootUnparser) + // Forces evaluation of non-constant defineVariable defaults; the unparse + // side reads and writes variables, so it needs this on its own copy. + unparseState.initializeVariables() + unparseState.getDataOutputStream.setPriorBitOrder( + ssrd.elementRuntimeData.defaultBitOrder + ) + + try { + rootUnparser.unparse1(unparseState) + unparseState.popTRD(rootUnparser.context.asInstanceOf[TermRuntimeData]) + } catch { + // A genuine deadlock (if any) surfaces via the final + // evalSuspensions(isFinal = true) below. + case _: ChildNotBuiltException => + } + + // The unparse only ever advances build as far as it needs, so build may + // still have its trailing end events left to consume. + cursor.advance(lastAdvance = true) + + // Build's stacks must end up balanced and the inputter must have nothing + // left unconsumed. + infosetBuildState.popTRD(rootUnparser.context.asInstanceOf[TermRuntimeData]) + Assert.invariant(infosetBuildState.arrayIterationIndexStack.length == 1) + Assert.invariant(infosetBuildState.occursIndexStack.length == 1) + Assert.invariant(infosetBuildState.groupIndexStack.length == 1) + Assert.invariant(infosetBuildState.currentInfosetNodeMaybe.isEmpty) + Assert.invariant(infosetBuildState.maybeTopTRD().isEmpty) + val remainingEvent = infosetBuildState.advanceMaybe + if (remainingEvent.isDefined) { + UnparseError( + Nope, + Nope, + "Expected no remaining events, but received %s.", + remainingEvent.get + ) + } + + // The final suspension sweep runs BEFORE the stack-depth invariants: one + // tripping first could mask the real SuspensionDeadlockException diagnostic + // this ordering exists to surface. + unparseState.setProcessor(rootUnparser) + unparseState.evalSuspensions(isFinal = true) + Assert.invariant(unparseState.arrayIterationIndexStack.length == 1) + Assert.invariant(unparseState.occursIndexStack.length == 1) + Assert.invariant(unparseState.groupIndexStack.length == 1) + Assert.invariant(unparseState.childIndexStack.length == 1) + Assert.invariant(unparseState.currentInfosetNodeMaybe.isEmpty) + Assert.invariant(unparseState.escapeSchemeEVCache.isEmpty) + Assert.invariant(unparseState.maybeTopTRD().isEmpty) + Assert.invariant(!unparseState.withinHiddenNest) + Assert.invariant(!unparseState.getDataOutputStream.isFinished) + try { + unparseState.getDataOutputStream.setFinished(unparseState) + } catch { + case boc: BitOrderChangeException => + unparseState.SDE(boc) + case fio: FileIOException => + unparseState.SDE(fio) + } + unparseState.unparseResult + } catch { + case t: Throwable => unparseErrorResult(unparseState, t) + } finally { + unparseState.getDataOutputStream.cleanUp() + } + } + + private def unparseEventDriven( + actualInputter: api.infoset.InfosetInputter, + outStream: java.io.OutputStream + ) = { val inputter = new InfosetInputter(actualInputter) val unparserState = UState.createInitialUState(outStream, this, inputter, areDebugging) @@ -477,47 +666,7 @@ class DataProcessor( unparserState.evalSuspensions(isFinal = true) unparserState.unparseResult } catch { - case ue: UnparseError => { - unparserState.addUnparseError(ue) - unparserState.unparseResult - } - case procErr: ProcessingError => { - val x = procErr - unparserState.setFailed(x.toUnparseError) - unparserState.unparseResult - } - case sde: SchemaDefinitionError => { - // A SDE was detected at runtime (perhaps due to a runtime-valued property like byteOrder or encoding) - // These are fatal, and there's no notion of backtracking them, so they propagate to top level - // here. - unparserState.setFailed(sde) - unparserState.unparseResult - } - case sdefw: SchemaDefinitionErrorFromWarning => { - unparserState.setFailed(sdefw) - unparserState.unparseResult - } - case e: ErrorAlreadyHandled => { - unparserState.setFailed(e.th) - unparserState.unparseResult - } - case e: TunableLimitExceededError => { - unparserState.setFailed(e) - unparserState.unparseResult - } - case se: org.xml.sax.SAXException => { - unparserState.setFailed(new UnparseError(None, None, se)) - unparserState.unparseResult - } - case e: scala.xml.parsing.FatalError => { - unparserState.setFailed(new UnparseError(None, None, e)) - unparserState.unparseResult - } - case ie: InfosetException => { - unparserState.setFailed(new UnparseError(None, None, ie)) - unparserState.unparseResult - } - case th: Throwable => throw th + case t: Throwable => unparseErrorResult(unparserState, t) } finally { unparserState.getDataOutputStream.cleanUp() } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/SchemaSetRuntimeData.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/SchemaSetRuntimeData.scala index e830aead7b..ae93ebde92 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/SchemaSetRuntimeData.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/SchemaSetRuntimeData.scala @@ -18,6 +18,7 @@ package org.apache.daffodil.runtime1.processors import org.apache.daffodil.lib.exceptions.ThrowsSDE +import org.apache.daffodil.runtime1.infoset.InfosetBuilder import org.apache.daffodil.runtime1.layers.LayerRuntimeCompiler import org.apache.daffodil.runtime1.layers.LayerRuntimeData import org.apache.daffodil.runtime1.layers.LayerVarsRuntime @@ -27,6 +28,8 @@ import org.apache.daffodil.runtime1.processors.unparsers.Unparser final class SchemaSetRuntimeData( val parser: Parser, val unparser: Unparser, + /** Nada only when the schema was compiled without an unparser. */ + val builder: InfosetBuilder, val elementRuntimeData: ElementRuntimeData, /* * The original variables determined by the schema compiler. diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SequenceChildBases.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SequenceChildBases.scala index fc079317c0..865db6879c 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SequenceChildBases.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SequenceChildBases.scala @@ -494,7 +494,7 @@ trait MinMaxRepeatsMixin { final val ock = erd.maybeOccursCountKind.get - private val minRepeats_ = { + final val minRepeatsConst: Long = { val mr = if (ock eq OccursCountKind.Parsed) 0 else erd.minOccurs @@ -506,7 +506,7 @@ trait MinMaxRepeatsMixin { * For example, when occursCountKind is parsed, then minRepeats is 0, regardless * of the value of minOccurs. */ - def minRepeats(state: ParseOrUnparseState): Long = minRepeats_ + def minRepeats(state: ParseOrUnparseState): Long = minRepeatsConst /** * True if the loop has a finite upper bound on number of iterations. @@ -514,7 +514,7 @@ trait MinMaxRepeatsMixin { * for speculative parsing cases, it's not OCK parsed, or OCK implicit with * maxOccurs unbounded. */ - private val maxRepeats_ = { + final val maxRepeatsConst: Long = { if (ock eq OccursCountKind.Parsed) Long.MaxValue else if (erd.maxOccurs == -1) Long.MaxValue else erd.maxOccurs @@ -525,9 +525,9 @@ trait MinMaxRepeatsMixin { * For example, when occursCountKind is parsed, then maxRepeats is -1 (meaning unbounded) * regardless of the value of maxOccurs. */ - def maxRepeats(state: ParseOrUnparseState): Long = maxRepeats_ + def maxRepeats(state: ParseOrUnparseState): Long = maxRepeatsConst - private val isBoundedMax_ = maxRepeats_ < Long.MaxValue + private val isBoundedMax_ = maxRepeatsConst < Long.MaxValue def isBoundedMax: Boolean = isBoundedMax_ diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildState.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildState.scala new file mode 100644 index 0000000000..8885dab038 --- /dev/null +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildState.scala @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.runtime1.processors.unparsers + +import org.apache.daffodil.lib.exceptions.Assert +import org.apache.daffodil.lib.iapi.DaffodilTunables +import org.apache.daffodil.lib.util.MStackOfMaybe +import org.apache.daffodil.lib.util.Maybe +import org.apache.daffodil.lib.util.Maybe.Nope +import org.apache.daffodil.runtime1.infoset.DIDocument +import org.apache.daffodil.runtime1.infoset.DIElement +import org.apache.daffodil.runtime1.infoset.DINode +import org.apache.daffodil.runtime1.infoset.InfosetAccessor +import org.apache.daffodil.runtime1.infoset.InfosetInputter +import org.apache.daffodil.runtime1.processors.ElementRuntimeData +import org.apache.daffodil.runtime1.processors.TermRuntimeData + +/** + * The "build" side of the build and unparse split: it walks the infoset + * events from an actual `InfosetInputter` and builds the infoset tree ahead + * of the unparse. It needs only the tree state, so it is not a `UState`: + * it has no output stream, variables or debugger state, and build never + * writes content. + * + * Used only when the `infosetBuilderMode` tunable is buildAhead when unparsing; + * otherwise only `UStateMain` is constructed. + */ +final class InfosetBuildState( + private val inputter: InfosetInputter, + override val tunable: DaffodilTunables +) extends InfosetTreeState + with TraversalIndexStacks + with InfosetFromEvents { + + // Build never frees a node, so finishing one only marks it final. A simple + // node stays open for the unparse, which gives it its value. + override def finishElement(cur: DINode, erd: ElementRuntimeData): Unit = { + if (cur.isComplex) { + val lastChild = cur.maybeLastChild + if (lastChild.isDefined && lastChild.get.isArray) { + lastChild.get.setFinal() + } + if (!withinHiddenNest || erd.isRepresented) { + cur.setFinal() + } + } + markDocumentFinalIfRootEnded() + } + + override def finishOvcElement(cur: DINode): Unit = markDocumentFinalIfRootEnded() + + private val eventState: InfosetEventState = new InputterEventState(inputter, "building") + + override def advance: Boolean = eventState.advance + override def advanceAccessor: InfosetAccessor = eventState.advanceAccessor + override def inspect: Boolean = eventState.inspect + override def inspectAccessor: InfosetAccessor = eventState.inspectAccessor + override def fini(): Unit = Assert.usageError("Not to be used on InfosetBuildState") + override def inspectOrError: InfosetAccessor = eventState.inspectOrError + override def advanceOrError: InfosetAccessor = eventState.advanceOrError + override def isInspectArrayEnd: Boolean = eventState.isInspectArrayEnd + + override def pushTRD(trd: TermRuntimeData): Unit = eventState.pushTRD(trd) + override def maybeTopTRD(): Maybe[TermRuntimeData] = eventState.maybeTopTRD() + override def popTRD(trd: TermRuntimeData): TermRuntimeData = eventState.popTRD(trd) + + override def documentElement: DIDocument = inputter.documentElement + + override val currentInfosetNodeStack = new MStackOfMaybe[DINode] + + override def currentInfosetNode: DINode = { + if (currentInfosetNodeMaybe.isEmpty) { + null + } else { + currentInfosetNodeMaybe.get + } + } + + override def currentInfosetNodeMaybe: Maybe[DINode] = { + if (currentInfosetNodeStack.isEmpty) { + Nope + } else { + currentInfosetNodeStack.top + } + } + + // Build tracks child position in its own frames, never in a stack. + override def moveOverOneElementChildOnly(): Unit = () + + private var hiddenDepth = 0 + override def incrementHiddenDef(): Unit = hiddenDepth += 1 + override def decrementHiddenDef(): Unit = hiddenDepth -= 1 + override def withinHiddenNest: Boolean = hiddenDepth > 0 + + // Build runs ahead of the unparse, which frees each node once it is done + // with it, so build leaves freeing to the unparse. + override def freeChildIfNoLongerNeeded(parent: DINode, index: Int): Unit = () + + // The lead is how far build is ahead of the unparse: the nodes build has + // constructed that the unparse has not yet finished. Build counts a node when + // it joins the tree, which keeps an ancestor's count ahead of its + // descendants', and the unparse uncounts it when it finishes the node. + private var lead: Long = 0 + + override def attachElement(newElem: DIElement): Unit = { + super.attachElement(newElem) + lead += 1 + } + + def decrementLead(): Unit = { + lead -= 1 + Assert.invariant(lead >= 0) + } + + def currentLead: Long = lead + + // Build stops advancing once the lead exceeds the build ahead limit, which + // bounds how far ahead of the unparse it may run. + def leadExceedsBuildAheadLimit: Boolean = lead > tunable.unparseBuildAheadWindowNodes +} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromEvents.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromEvents.scala index b262549f9e..6c8460b80c 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromEvents.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromEvents.scala @@ -30,7 +30,8 @@ import org.apache.daffodil.runtime1.processors.ElementRuntimeData /** * How unparsing the events of an inputter makes the infoset: it creates the * node of an element that has no event, attaches each new node to its parent, - * and finishes a node when its end is reached. + * and finishes a node when its end is reached. A state that unparses a tree + * that was already built overrides these. */ trait InfosetFromEvents { self: InfosetTreeState => diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromTree.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromTree.scala new file mode 100644 index 0000000000..00b161cd3d --- /dev/null +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetFromTree.scala @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.runtime1.processors.unparsers + +import org.apache.daffodil.runtime1.infoset.DIElement +import org.apache.daffodil.runtime1.infoset.DINode +import org.apache.daffodil.runtime1.infoset.InfosetAccessor +import org.apache.daffodil.runtime1.processors.ElementRuntimeData + +/** + * How unparsing a tree that build made ahead of it finds the infoset nodes, in place of InfosetFromEvents: build already made, attached + * and finished each node, so this takes each node from the tree, and + * finishing one only leaves what build has counted for it. + */ +trait InfosetFromTree { self: InfosetTreeState => + + protected def treeEvents: TreeEventState + + // Hidden elements have no events, but build already created this one. + override def getHiddenElement(erd: ElementRuntimeData): DIElement = + treeEvents.takeExistingHiddenChild(currentInfosetNode, erd) + + // Build already made this element and removed what created its event. + override def getOvcElement( + startEvent: InfosetAccessor, + erd: ElementRuntimeData + ): DIElement = + startEvent.info.element + + // Build already attached it. + override def attachElement(newElem: DIElement): Unit = () + + override def finishElement(cur: DINode, erd: ElementRuntimeData): Unit = + finishExistingNode(cur) + + override def finishOvcElement(cur: DINode): Unit = finishExistingNode(cur) + + /** + * Build attached the node and marked it final, apart from a simple node + * still waiting for its value, and the event state frees it once its end + * event is out. Unparsing the node is done, so it leaves build's lead. + */ + private def finishExistingNode(cur: DINode): Unit = { + if (cur.isSimple && !cur.isFinal && cur.asSimple.hasValue) { + cur.setFinal() + } + treeEvents.decrementLead() + if (withinHiddenNest) { + treeEvents.freeExistingHiddenChild() + } + } +} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/TreeEventState.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/TreeEventState.scala new file mode 100644 index 0000000000..8d31186dd7 --- /dev/null +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/TreeEventState.scala @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.runtime1.processors.unparsers + +import scala.annotation.tailrec + +import org.apache.daffodil.lib.exceptions.Assert +import org.apache.daffodil.lib.util.MStackOf +import org.apache.daffodil.lib.util.MStackOfInt +import org.apache.daffodil.lib.util.Maybe +import org.apache.daffodil.lib.util.Maybe.Nope +import org.apache.daffodil.lib.util.Maybe.One +import org.apache.daffodil.runtime1.infoset.DIArray +import org.apache.daffodil.runtime1.infoset.DIDocument +import org.apache.daffodil.runtime1.infoset.DIElement +import org.apache.daffodil.runtime1.infoset.DINode +import org.apache.daffodil.runtime1.infoset.DISimple +import org.apache.daffodil.runtime1.infoset.Info +import org.apache.daffodil.runtime1.infoset.InfosetAccessor +import org.apache.daffodil.runtime1.infoset.InfosetBuildCursor +import org.apache.daffodil.runtime1.infoset.InfosetEventKind +import org.apache.daffodil.runtime1.processors.ElementRuntimeData +import org.apache.daffodil.runtime1.processors.TermRuntimeData + +/** + * The infoset events of the tree that build is constructing, so the + * unparsers consume them exactly as they consume the events of an + * InfosetInputter. A node build has not produced yet is waited for by + * advancing the build cursor. Hidden nodes get no events, as + * in the event stream the tree was built from. + * + * The walk is an explicit stack of the containers being visited, so it can + * stop after any event and resume later. + */ +final class TreeEventState( + cursor: InfosetBuildCursor, + releaseUnneededInfoset: Boolean +) extends InfosetEventState { + + // The walk's stack of containers being visited, innermost last. It is held + // in parallel arrays, so visiting a node allocates nothing. + private var containerStack = new Array[DINode](16) + + // For the container at the same position of the stack, the index of the next + // of its children to visit. + private var nextChildIndexes = new Array[Int](16) + + // For the container at the same position of the stack, what kind of node it + // is, found once when the container was first reached. + private var containerKinds = new Array[NodeKind](16) + + // How many containers are on the stack. + private var stackDepth = 0 + + // The simple node whose start event was emitted and whose end event is next. + // A simple node has no children to visit, so it never goes on the stack. + private var pendingEnd: DINode = null + + // The document may not exist yet when this is constructed, so the walk + // starts at the first event asked for. + private var started = false + + // What a node is, found once when it is first reached, so the walk does not + // test its type again for each of its events. + private def kindOf(node: DINode): NodeKind = { + node match { + case _: DISimple => NodeKind.Simple + case _: DIArray => NodeKind.Array + case _: DIDocument => NodeKind.Document + case _ => NodeKind.Complex + } + } + + private def pushContainer(node: DINode, kind: NodeKind): Unit = { + if (stackDepth == containerStack.length) { + containerStack = java.util.Arrays.copyOf(containerStack, stackDepth * 2) + nextChildIndexes = java.util.Arrays.copyOf(nextChildIndexes, stackDepth * 2) + containerKinds = java.util.Arrays.copyOf(containerKinds, stackDepth * 2) + } + containerStack(stackDepth) = node + nextChildIndexes(stackDepth) = 0 + containerKinds(stackDepth) = kind + stackDepth += 1 + } + + private def popContainer(): Unit = { + stackDepth -= 1 + containerStack(stackDepth) = null + } + + // Two accessors trade roles: one holds the event that was computed and not + // yet consumed, the other holds the event consumed last. Consuming an event + // swaps them instead of copying the event. + private var inspected = InfosetAccessor() + private var advanced = InfosetAccessor() + private var hasInspectedEvent = false + + private val trdStack = new MStackOf[TermRuntimeData]() + + // Makes the next event the inspected one, if there is one. + private def fill(): Boolean = { + if (!hasInspectedEvent) { + if (!started) { + started = true + pushContainer(cursor.buildState.documentElement, NodeKind.Document) + } + hasInspectedEvent = computeNext() + } + hasInspectedEvent + } + + // Sets the inspected accessor to the next event. False if there are no more. + @tailrec + private def computeNext(): Boolean = { + if (pendingEnd ne null) { + val ended = pendingEnd + pendingEnd = null + setEvent(ended, NodeKind.Simple, isStart = false) + freeEndedChild() + true + } else if (stackDepth == 0) { + false + } else { + val innermost = stackDepth - 1 + val currentNode = containerStack(innermost) + val currentKind = containerKinds(innermost) + if (childExistsOrFinal(currentNode, nextChildIndexes(innermost))) { + // A node ahead of this walk that is already freed was a hidden one, + // which finished before the walk got here. + val nextChild = currentNode.child(nextChildIndexes(innermost)) + nextChildIndexes(innermost) += 1 + if (nextChild eq null) { + computeNext() + } else { + val nextKind = kindOf(nextChild) + if (isHidden(nextChild, nextKind)) { + computeNext() + } else { + setEvent(nextChild, nextKind, isStart = true) + if (nextKind eq NodeKind.Simple) { + pendingEnd = nextChild + } else { + pushContainer(nextChild, nextKind) + } + true + } + } + } else if (currentKind eq NodeKind.Document) { + popContainer() + computeNext() + } else { + emitEnd(currentNode, currentKind) + true + } + } + } + + // Advances build until the child exists, or the parent is final with no + // child there. The unparse only needs the child to exist, so it knows which + // branch or occurrence it is on, not for it to have a value. Build having + // finished with the child still missing is a stall. + private def childExistsOrFinal(parent: DINode, index: Int): Boolean = { + while (index >= parent.numChildren) { + if (parent.isFinal) { + return false + } + if (cursor.isFinished) { + throw new ChildNotBuiltException + } + cursor.advance() + } + true + } + + /** + * For each container the unparse is inside, how many of its children the + * unparse has reached, so a debugger can show the infoset as an event-driven + * unparse would have built it by now. A container the unparse is not inside + * is fully reached or not reached at all, so it has no entry. A node whose + * start event is computed but not yet consumed is left out. + */ + def reachedChildCounts(): java.util.Map[DINode, Integer] = { + val counts = new java.util.IdentityHashMap[DINode, Integer] + var i = 0 + while (i < stackDepth) { + counts.put(containerStack(i), nextChildIndexes(i)) + i += 1 + } + if (hasInspectedEvent && inspected.isStart) { + if (pendingEnd ne null) { + // The started node is simple, so it is not on the stack. + counts.put(containerStack(stackDepth - 1), nextChildIndexes(stackDepth - 1) - 1) + } else if (stackDepth >= 2) { + counts.remove(containerStack(stackDepth - 1)) + counts.put(containerStack(stackDepth - 2), nextChildIndexes(stackDepth - 2) - 1) + } + } + counts + } + + // The unparse is done with a node, so build is no longer ahead of it by one. + def decrementLead(): Unit = cursor.buildState.decrementLead() + + // Once a node's end event is out, the node is no longer needed in its + // parent, whose frame is now on top and whose next index is just past it. + private def freeEndedChild(): Unit = { + val parent = stackDepth - 1 + containerStack(parent).freeChildIfNoLongerNeeded( + nextChildIndexes(parent) - 1, + releaseUnneededInfoset + ) + } + + private def isHidden(node: DINode, kind: NodeKind): Boolean = { + if (kind eq NodeKind.Array) { + node.numChildren > 0 && node.isHidden + } else { + node.isHidden + } + } + + private def emitEnd(endedNode: DINode, kind: NodeKind): Unit = { + setEvent(endedNode, kind, isStart = false) + popContainer() + freeEndedChild() + } + + private def setEvent(node: DINode, kind: NodeKind, isStart: Boolean): Unit = { + if (kind eq NodeKind.Array) { + inspected.kind = if (isStart) { + InfosetEventKind.StartArray + } else { + InfosetEventKind.EndArray + } + inspected.info = Info(node.erd) + } else { + inspected.kind = if (isStart) { + InfosetEventKind.StartElement + } else { + InfosetEventKind.EndElement + } + inspected.info = Info(node.asInstanceOf[DIElement]) + } + } + + // The child index in each container where the search for its next hidden + // child resumes. Hidden nodes are asked for in the order they appear in the tree. + private lazy val hiddenCursors = new java.util.IdentityHashMap[DINode, Integer](8) + + // The container and index of each hidden node taken for unparsing and not yet freed. + private lazy val handedOutContainers = new MStackOf[DINode](8) + private lazy val handedOutIndexes = MStackOfInt(8) + + def takeExistingHiddenChild(parent: DINode, erd: ElementRuntimeData): DIElement = { + val container = if (erd.isArray) { + parent.child(findNextHiddenIndex(parent, erd, isArray = true)) + } else { + parent + } + val index = findNextHiddenIndex(container, erd, isArray = false) + handedOutContainers.push(container) + handedOutIndexes.push(index) + container.child(index).asInstanceOf[DIElement] + } + + def freeExistingHiddenChild(): Unit = { + val container = handedOutContainers.pop + container.freeChildIfNoLongerNeeded(handedOutIndexes.pop(), releaseUnneededInfoset) + } + + // The cursor stays on a hidden array while its occurrences are handed out one + // at a time, and moves past the array when a different hidden node is asked for. + private def findNextHiddenIndex( + container: DINode, + erd: ElementRuntimeData, + isArray: Boolean + ): Int = { + val cursor = hiddenCursors.get(container) + var index = if (cursor eq null) { + 0 + } else { + cursor.intValue + } + var found = false + while (!found) { + Assert.invariant(childExistsOrFinal(container, index)) + // A hidden node is always ready; a node already freed is not hidden. + val child = container.child(index) + if ((child ne null) && isHidden(child, kindOf(child)) && (child.erd eq erd)) { + found = true + } else { + index += 1 + } + } + val nextCursor = if (isArray) { + index + } else { + index + 1 + } + hiddenCursors.put(container, nextCursor) + index + } + + override def advance: Boolean = { + if (fill()) { + val consumed = inspected + inspected = advanced + advanced = consumed + hasInspectedEvent = false + true + } else { + false + } + } + + override def advanceAccessor: InfosetAccessor = advanced + + override def inspect: Boolean = fill() + + override def inspectAccessor: InfosetAccessor = inspected + + override def inspectOrError: InfosetAccessor = { + if (inspect) { + inspectAccessor + } else { + Assert.invariantFailed( + "An InfosetEvent was required for unparsing, but no InfosetEvent was available." + ) + } + } + + override def advanceOrError: InfosetAccessor = { + if (advance) { + advanceAccessor + } else { + Assert.invariantFailed( + "An InfosetEvent was required for unparsing, but no InfosetEvent was available." + ) + } + } + + override def isInspectArrayEnd: Boolean = inspect && inspected.isEnd && inspected.isArray + + override def pushTRD(trd: TermRuntimeData): Unit = trdStack.push(trd) + + override def maybeTopTRD(): Maybe[TermRuntimeData] = { + if (trdStack.isEmpty) { + Nope + } else { + One(trdStack.top) + } + } + + override def popTRD(trd: TermRuntimeData): TermRuntimeData = { + val popped = trdStack.pop + if (popped ne trd) { + Assert.invariantFailed("TRDs do not match. Expected: " + trd + " got " + popped) + } + popped + } +} + +/** + * What a node the walk visits is, which decides the events it gets. A complex + * node is a complex element other than the document. + */ +private[unparsers] enum NodeKind { + case Simple, Complex, Array, Document +} + +/** + * Thrown when the unparse needs a child that does not exist after build has + * finished. Caught only by the top-level driver, which still runs its normal + * finalization (the genuine, diagnostic-producing final suspension drain) + * rather than treating this as the final outcome itself. + */ +final class ChildNotBuiltException extends Exception diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala index f453439370..02b7f780b0 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala @@ -76,7 +76,7 @@ abstract class UState( diagnosticsArg: Seq[api.Diagnostic], dataProcArg: Maybe[DataProcessor], tunable: DaffodilTunables, - areDebugging: Boolean, + val areDebugging: Boolean, eventState: InfosetEventState, delimiterEscapePosition: DelimiterEscapePositionState ) extends ParseOrUnparseState(vbox, diagnosticsArg, dataProcArg, tunable) @@ -141,6 +141,8 @@ abstract class UState( _dataOutputStream = dos } + def currentInfosetNode: DINode + def currentInfosetNodeMaybe: Maybe[DINode] final def escapeSchemeEVCache: MStackOfMaybe[EscapeSchemeUnparserHelper] = delimiterEscapePosition.escapeSchemeEVCache @@ -162,6 +164,27 @@ abstract class UState( delimiterEscapePosition.moveOverOneElementChildOnly() final override def childPos: Long = delimiterEscapePosition.childPos + def currentInfosetNodeStack: MStackOfMaybe[DINode] + def arrayIterationIndexStack: MStackOfLong + def occursIndexStack: MStackOfLong + def groupIndexStack: MStackOfLong + def moveOverOneArrayIterationIndexOnly(): Unit + def moveOverOneOccursIndexOnly(): Unit + def moveOverOneGroupIndexOnly(): Unit + + // A dfdl:occursIndex() expression in an occurrence's own content reads + // arrayIterationIndexStack/occursIndexStack's top, which must track the + // occurrence currently being unparsed; the build pass pushes the same pair + // directly around each array/optional occurrence group. + final def pushOccurrenceIndices(): Unit = { + arrayIterationIndexStack.push(1L) + occursIndexStack.push(1L) + } + final def popOccurrenceIndices(): Unit = { + arrayIterationIndexStack.pop() + occursIndexStack.pop() + } + final override def advance: Boolean = eventState.advance final override def advanceAccessor: InfosetAccessor = eventState.advanceAccessor final override def inspect: Boolean = eventState.inspect @@ -414,25 +437,32 @@ abstract class UState( def documentElement: DIDocument - final val releaseUnneededInfoset: Boolean = !areDebugging && tunable.releaseUnneededInfoset + // Whether a node is freed once it is done with. + private[unparsers] def releaseUnneededInfoset: Boolean final def freeChildIfNoLongerNeeded(parent: DINode, index: Int): Unit = parent.freeChildIfNoLongerNeeded(index, releaseUnneededInfoset) def delimitedParseResult = Nope + + // Retries the suspensions that can be resolved now. + def runSuspensions(): Unit + } /** - * The state of the infoset tree as unparsing makes it from events: the event - * cursor, the TRD and node stacks, and the position within the current group, - * array and occurrence. The unparsers that create and finish the nodes of - * elements work through it. + * The state of the infoset tree as unparsing builds it: the event cursor, the + * TRD and node stacks, and the position within the current group, array and + * occurrence. Both an event-driven unparse (UState) and the build ahead build + * (InfosetBuildState) walk the infoset through it; the build has no output + * stream, variables or debugger state, so it is not a UState. */ trait InfosetTreeState extends Cursor[InfosetAccessor] { def tunable: DaffodilTunables // How the infoset's nodes are made and kept as unparsing consumes events. - // InfosetFromEvents does it for the events of an inputter. + // InfosetFromEvents does it for the events of an inputter; a state that + // unparses a tree that was already built overrides these. // The node of an element in a hidden group, which has no events. def getHiddenElement(erd: ElementRuntimeData): DIElement @@ -471,7 +501,15 @@ trait InfosetTreeState extends Cursor[InfosetAccessor] { def groupPos: Long def withinHiddenNest: Boolean + def incrementHiddenDef(): Unit + def decrementHiddenDef(): Unit + def moveOverOneElementChildOnly(): Unit + + // unparseBegin and unparseEnd free children, and build runs them too. Build + // runs ahead of the unparse, which may already have freed the node, so build + // must not free: it would find a null slot or free a node the unparse has + // not read yet. def freeChildIfNoLongerNeeded(parent: DINode, index: Int): Unit } @@ -753,6 +791,10 @@ final class UStateForSuspension( delimiterEscapePosition ) { + // Follows the main state this suspension state was cloned from. + override private[unparsers] def releaseUnneededInfoset: Boolean = + mainUState.releaseUnneededInfoset + _dataOutputStream = dataOutputStream dState.setMode(UnparserBlocking) dState.setCurrentNode(thisElement.asInstanceOf[DINode]) @@ -781,6 +823,7 @@ final class UStateForSuspension( override def attachElement(newElem: DIElement) = die override def finishElement(cur: DINode, erd: ElementRuntimeData) = die override def finishOvcElement(cur: DINode) = die + override def runSuspensions() = die // $COVERAGE-ON$ override def groupPos = 0 // was die, but this is called when copying state during debugging @@ -801,7 +844,35 @@ final class UStateForSuspension( } } -final class UStateMain private ( +/** + * Stack-backed array-iteration/occurs/group/child index tracking, shared + * by UStateMain and InfosetBuildState: each stack starts seeded with 1L, and + * moveOverOne*Only bumps its top by one as navigation advances. + * UStateForSuspension needs none of this (it stubs the stacks to die and + * tracks arrayIterationPos/occursPos as plain frozen Longs instead), so + * this lives in a mixin rather than directly on UState. + */ +trait TraversalIndexStacks { self: InfosetTreeState => + override val arrayIterationIndexStack = MStackOfLong(16) + arrayIterationIndexStack.push(1L) + override def moveOverOneArrayIterationIndexOnly(): Unit = + arrayIterationIndexStack.setTop(arrayIterationIndexStack.top + 1) + override def arrayIterationPos = arrayIterationIndexStack.top + + override val occursIndexStack = MStackOfLong(16) + occursIndexStack.push(1L) + override def moveOverOneOccursIndexOnly(): Unit = + occursIndexStack.setTop(occursIndexStack.top + 1) + override def occursPos = occursIndexStack.top + + override val groupIndexStack = MStackOfLong() + groupIndexStack.push(1L) + override def moveOverOneGroupIndexOnly(): Unit = + groupIndexStack.setTop(groupIndexStack.top + 1) + override def groupPos = groupIndexStack.top +} + +class UStateMain private[unparsers] ( private val inputter: InfosetInputter, outStream: java.io.OutputStream, vbox: VariableBox, @@ -809,18 +880,25 @@ final class UStateMain private ( dataProcArg: DataProcessor, tunable: DaffodilTunables, areDebugging: Boolean, - delimiterEscapePosition: DelimiterEscapePositionState + mainDelimiterEscapePosition: DelimiterEscapePositionState, + eventState: InfosetEventState ) extends UState( vbox, diagnosticsArg, One(dataProcArg), tunable, areDebugging, - new InputterEventState(inputter, "unparsing"), - delimiterEscapePosition + eventState, + mainDelimiterEscapePosition ) + with TraversalIndexStacks with InfosetFromEvents { + override def runSuspensions(): Unit = evalSuspensions(isFinal = false) + + private[unparsers] final val releaseUnneededInfoset: Boolean = + !areDebugging && tunable.releaseUnneededInfoset + dState.setMode(UnparserBlocking) def this( @@ -830,7 +908,8 @@ final class UStateMain private ( diagnosticsArg: Seq[api.Diagnostic], dataProcArg: DataProcessor, tunable: DaffodilTunables, - areDebugging: Boolean + areDebugging: Boolean, + eventState: InfosetEventState ) = this( inputter, @@ -840,7 +919,8 @@ final class UStateMain private ( dataProcArg, tunable, areDebugging, - DelimiterEscapePositionState(tunable) + DelimiterEscapePositionState(tunable), + eventState ) setDataOutputStream({ @@ -863,7 +943,7 @@ final class UStateMain private ( currentInfosetNodeStack.top.get, // only need the to of the stack, not the whole thing arrayIterationIndexStack.top, // only need the top of the stack, not the whole thing occursIndexStack.top, - delimiterEscapePosition.cloneForSuspension(), + mainDelimiterEscapePosition.cloneForSuspension(), tunable, areDebugging ) @@ -887,22 +967,6 @@ final class UStateMain private ( override val currentInfosetNodeStack = new MStackOfMaybe[DINode](16) - override val arrayIterationIndexStack = MStackOfLong(16) - arrayIterationIndexStack.push(1L) - override def moveOverOneArrayIterationIndexOnly() = - arrayIterationIndexStack.setTop(arrayIterationIndexStack.top + 1) - override def arrayIterationPos = arrayIterationIndexStack.top - - override val occursIndexStack = MStackOfLong(16) - occursIndexStack.push(1L) - override def moveOverOneOccursIndexOnly() = occursIndexStack.setTop(occursIndexStack.top + 1) - override def occursPos = occursIndexStack.top - - override val groupIndexStack = MStackOfLong() - groupIndexStack.push(1L) - override def moveOverOneGroupIndexOnly() = groupIndexStack.setTop(groupIndexStack.top + 1) - override def groupPos = groupIndexStack.top - /** * For outputValueCalc we accumulate the suspendables here. * @@ -958,8 +1022,33 @@ object UState { diagnostics, dataProc.asInstanceOf[DataProcessor], dataProc.tunables, - areDebugging + areDebugging, + new InputterEventState(inputter, "unparsing") ) newState } + + /** + * For the unparse of a tree that is being built, which reads it as events. + * The inputter still owns the infoset document the events refer to. + */ + def createInitialUStateForBuildAhead( + outStream: java.io.OutputStream, + dataProc: DFDL.DataProcessor, + inputter: InfosetInputter, + areDebugging: Boolean, + treeEvents: TreeEventState + ): UStateMainForBuildAhead = { + val variables = dataProc.variableMap.copy() + new UStateMainForBuildAhead( + inputter, + outStream, + variables, + Nil, + dataProc.asInstanceOf[DataProcessor], + dataProc.tunables, + areDebugging, + treeEvents + ) + } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UStateMainForBuildAhead.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UStateMainForBuildAhead.scala new file mode 100644 index 0000000000..ed97fbdadb --- /dev/null +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UStateMainForBuildAhead.scala @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.runtime1.processors.unparsers + +import org.apache.daffodil.lib.iapi.DaffodilTunables +import org.apache.daffodil.lib.iapi as api +import org.apache.daffodil.runtime1.infoset.DINode +import org.apache.daffodil.runtime1.infoset.InfosetInputter +import org.apache.daffodil.runtime1.processors.DataProcessor +import org.apache.daffodil.runtime1.processors.VariableMap + +/** + * The UState for unparsing a tree that build made ahead of it, by reading + * the tree as events and taking each node from the tree. + */ +final class UStateMainForBuildAhead private[unparsers] ( + inputter: InfosetInputter, + outStream: java.io.OutputStream, + vmap: VariableMap, + diagnosticsArg: Seq[api.Diagnostic], + dataProcArg: DataProcessor, + tunable: DaffodilTunables, + areDebugging: Boolean, + override protected val treeEvents: TreeEventState +) extends UStateMain( + inputter, + outStream, + vmap, + diagnosticsArg, + dataProcArg, + tunable, + areDebugging, + treeEvents + ) + with InfosetFromTree { + + // How far the unparse of the built tree has reached, for the debugger. + def reachedChildCounts(): java.util.Map[DINode, Integer] = treeEvents.reachedChildCounts() +} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/Unparser.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/Unparser.scala index 592544ff0d..7c7f750350 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/Unparser.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/Unparser.scala @@ -79,6 +79,11 @@ sealed trait Unparser extends Processor { UnparseError(One(context.schemaFileLocation), One(ustate.currentLocation), s, args*) } + // Code shared with build has no data location to report. + def UE(state: InfosetTreeState, s: String, args: Any*) = { + UnparseError(One(context.schemaFileLocation), Nope, s, args*) + } + } /** diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala index fd5a278b09..101ad34887 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala @@ -59,8 +59,8 @@ class ElementUnspecifiedLengthUnparser( } sealed trait RepMoveMixin { - def move(start: UState): Unit = { - start.childIndexStack.setTop(start.childIndexStack.top + 1) + def move(start: InfosetTreeState): Unit = { + start.moveOverOneElementChildOnly() } } @@ -83,8 +83,8 @@ class ElementUnparserInputValueCalc(erd: ElementRuntimeData, setVarUnparsers: Ar * Move over in the element children, but not in the group. * This avoids separators for this IVC element. */ - override def move(state: UState): Unit = { - state.childIndexStack.setTop(state.childIndexStack.top + 1) + override def move(state: InfosetTreeState): Unit = { + state.moveOverOneElementChildOnly() } } @@ -206,6 +206,8 @@ sealed abstract class ElementUnparserBase( unparseEnd(state) + retrySuspensionsAfterEnd(state) + if (state.dataProc.isDefined) state.dataProc.value.endElement(state, this) } @@ -380,16 +382,20 @@ sealed trait ElementUnparserStartEndStrategy { * Consumes the required infoset events and changes context so that the * element's DIElement node is the context element. */ - protected def unparseBegin(state: UState): Unit + def unparseBegin(state: InfosetTreeState): Unit /** * Restores prior context. Consumes end-element event. */ - protected def unparseEnd(state: UState): Unit + def unparseEnd(state: InfosetTreeState): Unit protected def captureRuntimeValuedExpressionValues(ustate: UState): Unit - protected def move(start: UState): Unit + // Only an unparse creates suspensions, so retrying them is not part of + // unparseEnd, which build runs too. + protected def retrySuspensionsAfterEnd(ustate: UState): Unit + + protected def move(start: InfosetTreeState): Unit protected def erd: ElementRuntimeData @@ -402,7 +408,7 @@ sealed trait RegularElementUnparserStartEndStrategy extends ElementUnparserStart * Consumes the required infoset events and changes context so that the * element's DIElement node is the context element. */ - final override protected def unparseBegin(state: UState): Unit = { + final override def unparseBegin(state: InfosetTreeState): Unit = { if (erd.isQuasiElement) { // Quasi elements are used for RepType and PrefixedLength, and have no corresponding // events in the infoset inputter. The parent parser will push a DIElement for us to @@ -422,7 +428,7 @@ sealed trait RegularElementUnparserStartEndStrategy extends ElementUnparserStart // this indicates that the incoming infoset (as events) doesn't match the schema UnparseError( Nope, - One(state.currentLocation), + Nope, "Expected element start event for %s, but received %s.", erd.namedQName.toExtendedSyntax, event @@ -447,7 +453,7 @@ sealed trait RegularElementUnparserStartEndStrategy extends ElementUnparserStart /** * Restores prior context. Consumes end-element event. */ - final override protected def unparseEnd(state: UState): Unit = { + final override def unparseEnd(state: InfosetTreeState): Unit = { if (erd.isQuasiElement) { // Quasi elements are used for TypeValueCalc, and have no corresponding events in the infoset inputter // The parent parser will handle pushing and poping the Infoset, so we do not need to do anything here. @@ -464,7 +470,7 @@ sealed trait RegularElementUnparserStartEndStrategy extends ElementUnparserStart // this indicates that the incoming infoset (as events) doesn't match the schema UnparseError( Nope, - One(state.currentLocation), + Nope, "Expected element end event for %s, but received %s.", erd.namedQName.toExtendedSyntax, event @@ -473,12 +479,15 @@ sealed trait RegularElementUnparserStartEndStrategy extends ElementUnparserStart } val cur = state.currentInfosetNodeStack.pop.get - state.finishElement(cur, erd) move(state) + } + } - state.asInstanceOf[UStateMain].evalSuspensions(isFinal = false) + final override protected def retrySuspensionsAfterEnd(ustate: UState): Unit = { + if (!erd.isQuasiElement) { + ustate.runSuspensions() } } @@ -493,7 +502,7 @@ trait OVCStartEndStrategy extends ElementUnparserStartEndStrategy { /** * For OVC, the behavior w.r.t. consuming infoset events is different. */ - protected final override def unparseBegin(state: UState): Unit = { + final override def unparseBegin(state: InfosetTreeState): Unit = { val ovcElem = if (!state.withinHiddenNest) { // outputValueCalc elements are optional in the infoset. If the next event @@ -524,16 +533,17 @@ trait OVCStartEndStrategy extends ElementUnparserStartEndStrategy { state.currentInfosetNodeStack.push(One(ovcElem)) } - protected final override def unparseEnd(state: UState): Unit = { + final override def unparseEnd(state: InfosetTreeState): Unit = { // if an OVC element existed, the start AND end events were consumed in // unparseBegin. No need to advance the cursor here. - val ovcElem = state.currentInfosetNodeStack.pop state.finishOvcElement(ovcElem.get) move(state) } + final override protected def retrySuspensionsAfterEnd(ustate: UState): Unit = {} + // For OVC, or for a target length expression, // // If we delayed evaluating the expressions, some variables might not be read diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/LayeredSequenceUnparser.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/LayeredSequenceUnparser.scala index 3856c0b272..2bd9f19895 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/LayeredSequenceUnparser.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/LayeredSequenceUnparser.scala @@ -95,6 +95,9 @@ class LayeredSequenceUnparser( // layer stack is potentially still needed, so // nothing can be cleaned up at this point. } catch { + // A signal from waiting on build, not a failure of the layer; rewrapping + // it would hide the stall diagnostic. + case e: ChildNotBuiltException => throw e case t: Throwable if (layerDriver ne null) => layerDriver.handleThrowable(t) case t: Throwable => LayerDriver.handleThrowableWithoutLayer(t) // otherwise we have no layer driver, so we were unable to load the layer. diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SeparatedSequenceUnparsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SeparatedSequenceUnparsers.scala index 7c451f1ddb..eb5941fcc7 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SeparatedSequenceUnparsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SeparatedSequenceUnparsers.scala @@ -108,8 +108,8 @@ class RepOrderedSeparatedSequenceChildUnparser( ) extends RepeatingChildUnparser(childUnparser, srd, erd) with Separated { - override def checkArrayPosAgainstMaxOccurs(state: UState) = - state.arrayIterationPos <= maxRepeats(state) + override def checkArrayPosAgainstMaxOccurs(state: InfosetTreeState) = + state.arrayIterationPos <= maxRepeatsConst } class OrderedSeparatedSequenceUnparser( @@ -319,7 +319,7 @@ class OrderedSeparatedSequenceUnparser( state.occursIndexStack.push(1L) val erd = unparser.erd var numOccurrences = 0 - val maxReps = unparser.maxRepeats(state) + val maxReps = unparser.maxRepeatsConst // // The number of occurrances we unparse is always exactly driven // by the number of infoset events for the repeating/optional element. @@ -565,7 +565,7 @@ class OrderedSeparatedSequenceUnparser( unparser.isPositional && unparser.isBoundedMax && (!unparser.isDeclaredLast || !unparser.isPotentiallyTrailing) ) { - val maxReps = unparser.maxRepeats(state) + val maxReps = unparser.maxRepeatsConst while (numOccurrences < maxReps) { unparseOneWithSuppression( unparser, @@ -613,7 +613,7 @@ class OrderedSeparatedSequenceUnparser( Assert.invariant(erd.isRepresented) // arrays/optionals cannot have inputValueCalc var numOccurrences = 0 - val maxReps = unparser.maxRepeats(state) + val maxReps = unparser.maxRepeatsConst Assert.invariant(state.inspect) val ev = state.inspectAccessor diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SequenceChildUnparsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SequenceChildUnparsers.scala index b1e7ac96fb..009c499c53 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SequenceChildUnparsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SequenceChildUnparsers.scala @@ -82,7 +82,7 @@ abstract class RepeatingChildUnparser( * Sets up for the start of an array/optional. For true array, pulls an event, which must be a start-array * event. */ - final def startArrayOrOptional(state: UState): Unit = { + final def startArrayOrOptional(state: InfosetTreeState): Unit = { val ev = state.inspectAccessor if (ev.erd.isArray) { // only pull start array event for a true array, not an optional. @@ -98,6 +98,21 @@ abstract class RepeatingChildUnparser( * Validates array dimensions if validation has been requested. */ final def endArrayOrOptional(currentArrayERD: ElementRuntimeData, state: UState): Unit = { + consumeEndArrayEvent(currentArrayERD, state) + + // State could be Success or Failure here. + endArray(state, state.occursPos - 1) + } + + /** + * The part of ending an array/optional that only consumes the end-array + * event, which build shares with unparse. Array validation needs the full + * UState, so it stays in endArrayOrOptional. + */ + final def consumeEndArrayEvent( + currentArrayERD: ElementRuntimeData, + state: InfosetTreeState + ): Unit = { if (currentArrayERD.isArray) { // only pull end array event for a true array, not an optional val event = state.advanceOrError @@ -110,9 +125,6 @@ abstract class RepeatingChildUnparser( ) } } - - // State could be Success or Failure here. - endArray(state, state.occursPos - 1) } /** @@ -125,7 +137,10 @@ abstract class RepeatingChildUnparser( * If the event is not a start, it must be an endArray for the enclosing complex element, and * the answer is false. */ - final def shouldDoUnparser(unparser: RepeatingChildUnparser, state: UState): Boolean = { + final def shouldDoUnparser( + unparser: RepeatingChildUnparser, + state: InfosetTreeState + ): Boolean = { val childRD = unparser.trd val res = childRD match { @@ -176,7 +191,7 @@ abstract class RepeatingChildUnparser( * bound occurrences with maxOccurs. * */ - def checkArrayPosAgainstMaxOccurs(state: UState): Boolean + def checkArrayPosAgainstMaxOccurs(state: InfosetTreeState): Boolean /** * For OccursCountKind 'implicit', we need to check for arrayPos in range @@ -189,7 +204,7 @@ abstract class RepeatingChildUnparser( * 'unbounded', then maxReps will be Long.MaxValue. */ def checkFinalOccursCountBetweenMinAndMaxOccurs( - state: UState, + state: InfosetTreeState, unparser: RepeatingChildUnparser, numOccurrences: Int, maxReps: Long, @@ -197,7 +212,7 @@ abstract class RepeatingChildUnparser( ): Unit = { import OccursCountKind.* - val minReps = unparser.minRepeats(state) + val minReps = unparser.minRepeatsConst val ev = state.inspectAccessor val erd = unparser.erd diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/UnseparatedSequenceUnparsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/UnseparatedSequenceUnparsers.scala index 8b4488a884..2662868611 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/UnseparatedSequenceUnparsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/UnseparatedSequenceUnparsers.scala @@ -45,9 +45,9 @@ class RepOrderedUnseparatedSequenceChildUnparser( ) extends RepeatingChildUnparser(childUnparser, srd, erd) with Unseparated { - override def checkArrayPosAgainstMaxOccurs(state: UState): Boolean = { + override def checkArrayPosAgainstMaxOccurs(state: InfosetTreeState): Boolean = { if (ock eq OccursCountKind.Implicit) - state.arrayIterationPos <= maxRepeats(state) + state.arrayIterationPos <= maxRepeatsConst else true } @@ -105,7 +105,7 @@ class OrderedUnseparatedSequenceUnparser( state.occursIndexStack.push(1L) val erd = unparser.erd var numOccurrences = 0 - val maxReps = unparser.maxRepeats(state) + val maxReps = unparser.maxRepeatsConst // // The number of occurrances we unparse is always exactly driven diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/TestBuildAheadDataProcessor.scala b/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/TestBuildAheadDataProcessor.scala new file mode 100644 index 0000000000..4a9ba36b2c --- /dev/null +++ b/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/TestBuildAheadDataProcessor.scala @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.runtime1.processors + +import java.io.ByteArrayOutputStream +import scala.xml.Node + +import org.apache.daffodil.api +import org.apache.daffodil.core.compiler.Compiler +import org.apache.daffodil.lib.util.SchemaUtils +import org.apache.daffodil.lib.xml.XMLUtils +import org.apache.daffodil.runtime1.infoset.ScalaXMLInfosetInputter + +import org.junit.Assert.* +import org.junit.Test + +/** + * Checks that a compiled schema always carries an infoset builder, that the + * infosetBuilderMode tunable can be changed on a compiled DataProcessor, and how + * DataProcessor.unparse behaves when it cannot start. Whether unparse output is the same in both + * modes is covered by buildAhead.tdml, run with DAFFODIL_TDML_TUNABLES set to + * each value of infosetBuilderMode. + */ +class TestBuildAheadDataProcessor { + + val example = XMLUtils.EXAMPLE_NAMESPACE + + private def unparseToBytes(dp: DataProcessor, infosetXML: Node): Array[Byte] = { + val out = new ByteArrayOutputStream() + val res = dp.unparse(new ScalaXMLInfosetInputter(infosetXML), out) + assertFalse(res.getDiagnostics.toString, res.isError) + out.toByteArray + } + + // A setup failure (inputter never produces StartDocument) must yield + // a failed UnparseResult, not an NPE from a null error-path state. + @Test def testMalformedInfosetInputterGetsCleanErrorNotNPE(): Unit = { + // Needs build ahead in use (the tunable on), or DataProcessor.unparse takes the + // event-driven path and unparseBuildAhead (the method under test) would never run. + val sch = SchemaUtils.dfdlTestSchema( + , + , + + + + + + + + , + elementFormDefault = "unqualified" + ) + val dp = Compiler() + .withTunable("infosetBuilderMode", "buildAhead") + .compileNode(sch) + .onPath("/") + .asInstanceOf[DataProcessor] + + // hasNext() = false immediately means initialize()'s + // "!delegate.hasNext" check fires straight away, before any actual + // infoset event is produced; exactly the "never starts with + // StartDocument" failure this guards. + val neverStartsInputter = new api.infoset.InfosetInputter { + override def getEventType() = null + override def getLocalName() = null + override def getNamespaceURI() = null + override def getSimpleText( + primType: org.apache.daffodil.runtime1.dpath.NodeInfo.Kind, + runtimeProperties: java.util.Map[String, String] + ) = null + override def isNilled(): java.lang.Boolean = null + override def hasNext() = false + override def next(): Unit = () + override def fini(): Unit = () + } + + val out = new ByteArrayOutputStream() + val res = dp.unparse(neverStartsInputter, out) + + assertTrue("expected a failed UnparseResult, not a successful one", res.isError) + assertTrue( + res.getDiagnostics.get(0).getMessage.contains("does not start with StartDocument") + ) + } + + // A schema with no OVC at all still gets a builder. + @Test def testSchemaWithNoOVCAtAllStillGetsBuilder(): Unit = { + val sch = SchemaUtils.dfdlTestSchema( + , + , + + + + + + + , + elementFormDefault = "unqualified" + ) + val dp = Compiler() + .withTunable("infosetBuilderMode", "buildAhead") + .compileNode(sch) + .onPath("/") + .asInstanceOf[DataProcessor] + assertFalse(dp.ssrd.builder.isEmpty) + } + + // A schema where every OVC is resolvable-without-writing gets a builder: + // the common case. + @Test def testSchemaWithOnlyResolvableOVCGetsBuilder(): Unit = { + val sch = SchemaUtils.dfdlTestSchema( + , + , + + + + + + + + , + elementFormDefault = "unqualified" + ) + val dp = Compiler() + .withTunable("infosetBuilderMode", "buildAhead") + .compileNode(sch) + .onPath("/") + .asInstanceOf[DataProcessor] + assertFalse(dp.ssrd.builder.isEmpty) + } + + // The builder exists even when the tunable is off at compile time, and the + // same compiled processor unparses identically with the tunable switched + // either way afterward. + @Test def testTunableCanChangeAfterCompile(): Unit = { + val sch = SchemaUtils.dfdlTestSchema( + , + , + + + + + + + + , + elementFormDefault = "unqualified" + ) + val dp = Compiler() + .withTunable("infosetBuilderMode", "eventDriven") + .compileNode(sch) + .onPath("/") + .asInstanceOf[DataProcessor] + assertFalse(dp.ssrd.builder.isEmpty) + + val infoset = 7 + val eventDriven = unparseToBytes(dp, infoset) + val buildAhead = unparseToBytes( + dp.copy(tunables = dp.tunables.withTunable("infosetBuilderMode", "buildAhead")), + infoset + ) + assertArrayEquals(eventDriven, buildAhead) + assertArrayEquals( + eventDriven, + unparseToBytes( + dp.copy(tunables = dp.tunables.withTunable("infosetBuilderMode", "eventDriven")), + infoset + ) + ) + } + + // Confirms the builder survives a save/reload round trip. + @Test def testBuilderSurvivesSaveReload(): Unit = { + val sch = SchemaUtils.dfdlTestSchema( + , + , + + + + + + + + , + elementFormDefault = "unqualified" + ) + val dp = Compiler() + .withTunable("infosetBuilderMode", "buildAhead") + .compileNode(sch) + .onPath("/") + .asInstanceOf[DataProcessor] + assertFalse(dp.ssrd.builder.isEmpty) + + val os = new ByteArrayOutputStream() + dp.save(java.nio.channels.Channels.newChannel(os)) + val reloadedDp = Compiler() + .reload(new java.io.ByteArrayInputStream(os.toByteArray)) + .asInstanceOf[DataProcessor] + assertFalse(reloadedDp.ssrd.builder.isEmpty) + + val infoset = xyz + assertArrayEquals( + unparseToBytes(dp, infoset), + unparseToBytes(reloadedDp, infoset) + ) + } +} diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildTestFixture.scala b/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildTestFixture.scala new file mode 100644 index 0000000000..9f9df8db12 --- /dev/null +++ b/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildTestFixture.scala @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.runtime1.processors.unparsers + +import java.io.ByteArrayOutputStream +import scala.jdk.CollectionConverters.* +import scala.xml.Node + +import org.apache.daffodil.api +import org.apache.daffodil.core.compiler.Compiler +import org.apache.daffodil.runtime1.infoset.InfosetBuildCursor +import org.apache.daffodil.runtime1.infoset.InfosetInputter +import org.apache.daffodil.runtime1.infoset.ScalaXMLInfosetInputter +import org.apache.daffodil.runtime1.processors.DataProcessor +import org.apache.daffodil.runtime1.processors.TermRuntimeData + +/** + * Shared helpers for tests of the infoset build cursor and build state. + */ +object InfosetBuildTestFixture { + private def throwDiagnostics(ds: java.util.List[api.Diagnostic]): Nothing = + throw new Exception(ds.asScala.map(_.getMessage()).mkString("\n")) + + /** + * Compiles testSchema with the given tunables and returns the resulting + * DataProcessor without a saveAndReload round-trip, since the tests build + * state directly off the live object. + */ + def compileForUnparse( + testSchema: Node, + tunables: Map[String, String] = Map.empty + ): DataProcessor = { + val pf = Compiler().withTunables(tunables).compileNode(testSchema) + if (pf.isError) throwDiagnostics(pf.getDiagnostics) + val dp = pf.onPath("/").asInstanceOf[DataProcessor] + if (dp.isError) throwDiagnostics(dp.getDiagnostics) + dp + } + + /** + * Builds a fresh InfosetInputter walking infosetXML against dp, already + * initialized with the root TRD pushed. + */ + def newInitializedInputter(infosetXML: Node, dp: DataProcessor): InfosetInputter = { + val inputter = new InfosetInputter(new ScalaXMLInfosetInputter(infosetXML)) + inputter.initialize(dp.ssrd.elementRuntimeData, dp.tunables) + inputter + } + + /** + * Unparses infosetXML event driven, then again by building the infoset ahead + * and unparsing the built tree. Returns (eventDrivenBytes, buildAheadBytes) for + * the caller to assert equality on. + */ + def getEventDrivenAndBuildAheadBytes( + dp: DataProcessor, + infosetXML: Node + ): (Array[Byte], Array[Byte]) = { + val eventDrivenOut = new ByteArrayOutputStream() + val eventDrivenResult = dp.unparse(new ScalaXMLInfosetInputter(infosetXML), eventDrivenOut) + if (eventDrivenResult.isError) throwDiagnostics(eventDrivenResult.getDiagnostics) + + val buildAheadOut = new ByteArrayOutputStream() + val buildAheadInputter = newInitializedInputter(infosetXML, dp) + val cursor = new InfosetBuildCursor( + dp.ssrd.builder, + new InfosetBuildState(buildAheadInputter, dp.tunables) + ) + val buildAheadState = UState.createInitialUStateForBuildAhead( + buildAheadOut, + dp, + buildAheadInputter, + false, + new TreeEventState(cursor, false) + ) + buildAheadState.getDataOutputStream.setPriorBitOrder( + dp.ssrd.elementRuntimeData.defaultBitOrder + ) + + val rootUnparser = dp.ssrd.unparser + buildAheadState.pushTRD(dp.ssrd.elementRuntimeData) + rootUnparser.unparse1(buildAheadState) + buildAheadState.popTRD(rootUnparser.context.asInstanceOf[TermRuntimeData]) + cursor.advance(lastAdvance = true) + buildAheadState.evalSuspensions(isFinal = true) + buildAheadState.getDataOutputStream.setFinished(buildAheadState) + + (eventDrivenOut.toByteArray, buildAheadOut.toByteArray) + } +} diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/unparsers/TestInfosetBuildCursor.scala b/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/unparsers/TestInfosetBuildCursor.scala new file mode 100644 index 0000000000..5a3fddeaf0 --- /dev/null +++ b/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/processors/unparsers/TestInfosetBuildCursor.scala @@ -0,0 +1,419 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.runtime1.processors.unparsers + +import java.io.ByteArrayOutputStream +import java.nio.charset.StandardCharsets +import scala.xml.Node + +import org.apache.daffodil.lib.util.SchemaUtils +import org.apache.daffodil.lib.xml.XMLUtils +import org.apache.daffodil.runtime1.infoset.DIArray +import org.apache.daffodil.runtime1.infoset.InfosetBuildCursor +import org.apache.daffodil.runtime1.infoset.StreamingInfosetWalker +import org.apache.daffodil.runtime1.infoset.XMLTextInfosetOutputter +import org.apache.daffodil.runtime1.processors.TermRuntimeData + +import org.junit.Assert.* +import org.junit.Test + +/** + * Tests the infoset build cursor and build state against the unparse that + * reads the built tree as events: the event sequence build surfaces, the lead + * counter, the build ahead window, and the build ahead unparse matching an + * event-driven unparse for scalar, array and choice content. + */ +class TestInfosetBuildCursor { + + val example = XMLUtils.EXAMPLE_NAMESPACE + + // A separated sequence of three scalars. + private val separatedRowSchema = SchemaUtils.dfdlTestSchema( + , + , + + + + + + + + + , + elementFormDefault = "unqualified" + ) + + private val separatedRowInfoset = + + Alice + 30 + Boston + + + // A scalar, an array and a choice in one separated sequence. + private val arrayChoiceSchema = SchemaUtils.dfdlTestSchema( + , + , + + + + + + + + + + + + , + elementFormDefault = "unqualified" + ) + + // Three "item" occurrences exercise the array loop; typeB (not typeA) + // exercises actual choice resolution. lengthKind=delimited (not explicit) + // avoids double-firing CaptureStartOfContentLengthUnparser's non-idempotent + // marker, since a tree reused from a completed unparse runs it again. + private val arrayChoiceInfoset = + +
H
+ a + b + c + X + + + /** + * A build side, an InfosetBuildCursor over an InfosetBuildState, and an + * unparse side that reads the tree the build side makes. + */ + private final class BuildAheadRun(sch: Node, infoset: Node, buildAheadLimit: Long = 100) { + val dp = InfosetBuildTestFixture.compileForUnparse( + sch, + Map( + "releaseUnneededInfoset" -> "false", + "infosetBuilderMode" -> "buildAhead", + "unparseBuildAheadWindowNodes" -> buildAheadLimit.toString + ) + ) + val buildInputter = InfosetBuildTestFixture.newInitializedInputter(infoset, dp) + val buildState = new InfosetBuildState(buildInputter, dp.tunables) + val cursor = new InfosetBuildCursor(dp.ssrd.builder, buildState) + + def buildAll(): Unit = cursor.advance(lastAdvance = true) + + def rootNode = buildInputter.documentElement.child(0).asComplex + + // Unparses the tree build produced, pulling build forward as the unparse + // needs it, and returns the output. + def unparseBuiltTree(): String = { + val out = new ByteArrayOutputStream() + val state = UState.createInitialUStateForBuildAhead( + out, + dp, + buildInputter, + false, + new TreeEventState(cursor, false) + ) + state.getDataOutputStream.setPriorBitOrder(dp.ssrd.elementRuntimeData.defaultBitOrder) + + val rootUnparser = dp.ssrd.unparser + state.pushTRD(dp.ssrd.elementRuntimeData) + rootUnparser.unparse1(state) + state.popTRD(rootUnparser.context.asInstanceOf[TermRuntimeData]) + // Build may still have trailing end events to consume, and a speculative + // separator is written via a suspension that must drain before the DOS + // is finalized. + buildAll() + state.evalSuspensions(isFinal = true) + state.getDataOutputStream.setFinished(state) + new String(out.toByteArray, StandardCharsets.US_ASCII) + } + } + + // A tree built outside an event-driven unparse, compared against that unparse. + private def assertBuildAheadMatchesEventDriven(sch: Node, infoset: Node): Array[Byte] = { + // Event-driven on purpose: the tunable would otherwise replace it. + val dp = InfosetBuildTestFixture.compileForUnparse( + sch, + Map("releaseUnneededInfoset" -> "false", "infosetBuilderMode" -> "eventDriven") + ) + val (eventDrivenBytes, buildAheadBytes) = + InfosetBuildTestFixture.getEventDrivenAndBuildAheadBytes(dp, infoset) + assertArrayEquals(eventDrivenBytes, buildAheadBytes) + eventDrivenBytes + } + + @Test def testBuildStateSurfacesCorrectEventSequence(): Unit = { + val run = new BuildAheadRun(separatedRowSchema, separatedRowInfoset) + + // Drives through the actual InfosetBuilder frames rather than hand-driven + // advance() calls, since next-element resolution depends on the TRD + // push/pop the element frame performs. This schema's separator never + // reaches InfosetBuildState: the InfosetBuilder tree skips the + // delimiter-stack wrapper. + run.buildAll() + + assertEquals(4L, run.buildState.currentLead) // row, name, age, city + + assertEquals(3, run.rootNode.numChildren) + assertEquals("name", run.rootNode.child(0).erd.name) + assertEquals("age", run.rootNode.child(1).erd.name) + assertEquals("city", run.rootNode.child(2).erd.name) + } + + @Test def testLeadCounterIncrementsOnBuildAndDecrementsOnUnparse(): Unit = { + // Fixed dfdl:length is safe here because this tree comes from + // InfosetBuildState, which never runs content-unparsing (including + // CaptureStartOfContentLengthUnparser). + val sch = SchemaUtils.dfdlTestSchema( + , + , + + + + + + + + + , + elementFormDefault = "unqualified" + ) + + val run = new BuildAheadRun(sch, separatedRowInfoset) + + assertEquals(0L, run.buildState.currentLead) + run.buildAll() + // row itself, name, age, city = 4 elements total, each incrementing once + // via unparseBegin's actual hookup. + assertEquals(4L, run.buildState.currentLead) + + // The unparse reads the same already-built tree, decrementing the lead + // counter as it goes. + assertEquals("Alice30Boston", run.unparseBuiltTree()) + + // The unparse decremented once per element too, so the counter is back to + // 0: build and the unparse agree on how many nodes exist. + assertEquals(0L, run.buildState.currentLead) + } + + // With a small buildAheadLimit, one advance() leaves the lead counter just + // past it, and the unparse is still correct across the refills the unparse + // triggers while it runs. + @Test def testBuildStopsAtBuildAheadLimit(): Unit = { + val numItems = 40 + val buildAheadLimit = 3L + + val sch = SchemaUtils.dfdlTestSchema( + , + , + + + + + + + , + elementFormDefault = "unqualified" + ) + + val items = (0 until numItems).map(i => {s"i$i"}) + val infoset = + + {items} + + val expectedBytes = (0 until numItems).map(i => s"i$i").mkString(",") + + val run = new BuildAheadRun(sch, infoset, buildAheadLimit) + + run.cursor.advance() + + // numItems + 1 (row + all items) is what currentLead would equal here if + // advance() ignored the build ahead limit. It counts one node per element, so + // it stops at the first node past the limit. + assertFalse("expected build to stop with more left to build", run.cursor.isFinished) + assertEquals( + s"expected build to stop as soon as the lead passed buildAheadLimit=$buildAheadLimit " + + s"(numItems=$numItems)", + buildAheadLimit + 1, + run.buildState.currentLead + ) + + // The unparse pulls the rest of the tree forward as it needs it; the + // output must be byte-for-byte correct despite having been built across + // many separate advance() calls rather than a single one-shot build pass. + assertEquals(expectedBytes, run.unparseBuiltTree()) + } + + @Test def testBuildAheadMatchesEventDriven(): Unit = { + assertBuildAheadMatchesEventDriven(separatedRowSchema, separatedRowInfoset) + } + + @Test def testArrayAndChoiceBuildAheadMatchesEventDriven(): Unit = { + val eventDrivenBytes = + assertBuildAheadMatchesEventDriven(arrayChoiceSchema, arrayChoiceInfoset) + assertEquals("H,a,b,c,X", new String(eventDrivenBytes, StandardCharsets.US_ASCII)) + } + + // Drives InfosetBuildState directly, then feeds its tree to the unparse + // (end-to-end build-then-unparse). + @Test def testStandaloneBuildStateNavigatesArrayChoiceSeparator(): Unit = { + val run = new BuildAheadRun(arrayChoiceSchema, arrayChoiceInfoset) + + // The cursor builds the whole tree from the inputter, including the array + // and choice content. + run.buildAll() + + // row, header, item x3, typeB = 6 elements total. + assertEquals(6L, run.buildState.currentLead) + + assertEquals(3, run.rootNode.numChildren) + assertEquals("header", run.rootNode.child(0).erd.name) + assertEquals("item", run.rootNode.child(1).erd.name) + assertEquals(3, run.rootNode.child(1).asInstanceOf[DIArray].numChildren) + assertEquals("typeB", run.rootNode.child(2).erd.name) + + // Unparsing the tree InfosetBuildState just constructed confirms it's a + // usable, fully-built tree, not just a navigation exercise. + assertEquals("H,a,b,c,X", run.unparseBuiltTree()) + } + + // Each event as "start element name", "end array name" and so on. + private def eventDescriptions(events: InfosetEventState): List[String] = { + val descriptions = List.newBuilder[String] + while (events.advance) { + val event = events.advanceAccessor + val position = if (event.isStart) { + "start" + } else { + "end" + } + val kind = if (event.isElement) { + "element" + } else { + "array" + } + descriptions += position + " " + kind + " " + event.erd.name + } + descriptions.result() + } + + @Test def testTreeEventsForScalars(): Unit = { + val run = new BuildAheadRun(separatedRowSchema, separatedRowInfoset) + val treeEvents = new TreeEventState(run.cursor, true) + assertEquals( + List( + "start element row", + "start element name", + "end element name", + "start element age", + "end element age", + "start element city", + "end element city", + "end element row" + ), + eventDescriptions(treeEvents) + ) + } + + private val arrayChoiceEvents = List( + "start element row", + "start element header", + "end element header", + "start array item", + "start element item", + "end element item", + "start element item", + "end element item", + "start element item", + "end element item", + "end array item", + "start element typeB", + "end element typeB", + "end element row" + ) + + @Test def testTreeEventsForArrayAndChoice(): Unit = { + val run = new BuildAheadRun(arrayChoiceSchema, arrayChoiceInfoset) + val treeEvents = new TreeEventState(run.cursor, true) + assertEquals(arrayChoiceEvents, eventDescriptions(treeEvents)) + } + + @Test def testTreeEventsPullBuildOnlyAsFarAsNeeded(): Unit = { + val run = new BuildAheadRun(arrayChoiceSchema, arrayChoiceInfoset, buildAheadLimit = 1) + val treeEvents = new TreeEventState(run.cursor, true) + assertTrue(treeEvents.advance) + assertEquals("row", treeEvents.advanceAccessor.erd.name) + // Only the root has been needed so far, so build has not run to the end. + assertFalse(run.cursor.isFinished) + assertEquals(arrayChoiceEvents.tail, eventDescriptions(treeEvents)) + assertTrue(run.cursor.isFinished) + } + + // The infoset as a debugger shows it: the whole built tree, limited to what + // the unparse has reached. + private def reachedInfoset(run: BuildAheadRun, treeEvents: TreeEventState): String = { + val out = new ByteArrayOutputStream() + val xml = new XMLTextInfosetOutputter(out, pretty = false, minimal = true) + StreamingInfosetWalker( + run.buildInputter.documentElement, + xml, + walkHidden = false, + ignoreBlocks = true, + releaseUnneededInfoset = false, + visibleChildCounts = treeEvents.reachedChildCounts() + ).walk(lastWalk = true) + out.toString("UTF-8") + } + + @Test def testReachedChildCountsLimitTheDebuggerInfoset(): Unit = { + val run = new BuildAheadRun(separatedRowSchema, separatedRowInfoset) + run.buildAll() + val treeEvents = new TreeEventState(run.cursor, false) + + // start row, start name, end name + assertTrue(treeEvents.advance) + assertTrue(treeEvents.advance) + assertTrue(treeEvents.advance) + val afterName = reachedInfoset(run, treeEvents) + assertTrue(afterName, afterName.contains("Alice")) + assertFalse(afterName, afterName.contains("30")) + + // The start of age is computed but not consumed, so age is not reached. + assertTrue(treeEvents.inspect) + assertFalse(reachedInfoset(run, treeEvents).contains("30")) + + assertTrue(treeEvents.advance) + val afterAgeStart = reachedInfoset(run, treeEvents) + assertTrue(afterAgeStart, afterAgeStart.contains("30")) + assertFalse(afterAgeStart, afterAgeStart.contains("Boston")) + } +} diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd index db5568846a..b91f2922f3 100644 --- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd +++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd @@ -534,6 +534,29 @@ + + + + How unparsing gets the infoset. Values are: + - buildAhead: a build pass runs ahead of an unparse pass over the same + infoset tree, resolving forward references against the tree directly + instead of via Suspensions where possible. How far build may run + ahead of unparse is bounded by unparseBuildAheadWindowNodes. + - eventDriven: the events of the infoset inputter are unparsed as they + arrive, with no tree built ahead. + + + + + + + Only consulted when infosetBuilderMode is buildAhead. The maximum + number of infoset nodes build may construct ahead of unparse before + build pauses to let unparse catch up. Bounds peak memory use to + roughly this many infoset nodes rather than the whole infoset. + + + @@ -710,6 +733,13 @@ + + + + + + + diff --git a/daffodil-propgen/src/main/scala/org/apache/daffodil/propGen/TunableGenerator.scala b/daffodil-propgen/src/main/scala/org/apache/daffodil/propGen/TunableGenerator.scala index d6632308b8..84c3e505ac 100644 --- a/daffodil-propgen/src/main/scala/org/apache/daffodil/propGen/TunableGenerator.scala +++ b/daffodil-propgen/src/main/scala/org/apache/daffodil/propGen/TunableGenerator.scala @@ -79,7 +79,8 @@ class TunableGenerator(schemaRootConfig: scala.xml.Node, schemaRootExt: scala.xm | if (configOpt.isDefined) { | val loader = new DaffodilXMLLoader() | val node = loader.load(URISchemaSource(Paths.get(configPath).toFile, configOpt.get), Some(XMLUtils.dafextURI)) - | tunablesMap(node) + | val optTunablesNode = (node \ "tunables").headOption + | optTunablesNode.map(tunablesMap(_)).getOrElse(Map.empty) | } else { | Map.empty | } @@ -98,15 +99,17 @@ class TunableGenerator(schemaRootConfig: scala.xml.Node, schemaRootExt: scala.xm | tunables.foldLeft(this) { case (dafTuns, (tunable, value)) => dafTuns.withTunable(tunable, value) } | } | + | // One large match over every tunable name would exceed the JVM's + | // 64KB-per-method bytecode limit as tunables accumulate over time, so + | // this is split into a chain of smaller private methods instead, each + | // falling through to the next on no match. | def withTunable(tunable: String, value: String): DaffodilTunables = { - | tunable match { + | withTunablePart0(tunable, value) + | } + | """.trim.stripMargin val bottom = """ - | case _ => throw new IllegalArgumentException("Unknown tunable: " + tunable) - | } - | } - | | private def throwInvalidTunableValue(tunable: String, value: String) = { | throw new IllegalArgumentException("Invalid value for tunable " + tunable + ": " + value) | } @@ -114,6 +117,11 @@ class TunableGenerator(schemaRootConfig: scala.xml.Node, schemaRootExt: scala.xm |} """.trim.stripMargin + // How many tunables' worth of case-match bytecode go into each + // withTunablePartN method; small enough to leave headroom against the + // JVM's 64KB-per-method limit as more tunables are added over time. + private val tunablesPerPart = 8 + val tunablesRoot = (schemaRootConfig \ "element").find(_ \@ "name" == "tunables").get val tunableNodes = tunablesRoot \\ "all" \ "element" @@ -165,15 +173,38 @@ class TunableGenerator(schemaRootConfig: scala.xml.Node, schemaRootExt: scala.xm .map(_.scalaDefinition) .mkString(" ", ",\n ", ")") - val conversionString = - tunables - .map { tunable => - tunable.scalaConversion - .split("\n") - .filter(_.trim.length > 0) - .mkString(" ", "\n ", "") + // Split across several withTunablePartN methods rather than one large + // match: each chunk's case bodies, then a fallthrough to the next + // chunk's method, or to the final "unknown tunable" error on the last. + val tunableChunks = tunables.grouped(tunablesPerPart).toIndexedSeq + val numParts = tunableChunks.length + val partsString = + tunableChunks.zipWithIndex + .map { case (chunk, idx) => + val casesString = + chunk + .map { tunable => + tunable.scalaConversion + .split("\n") + .filter(_.trim.length > 0) + .mkString(" ", "\n ", "") + } + .mkString("\n") + val fallthrough = + if (idx == numParts - 1) + """ case _ => throw new IllegalArgumentException("Unknown tunable: " + tunable)""" + else + s" case _ => withTunablePart${idx + 1}(tunable, value)" + s""" + | private def withTunablePart${idx}(tunable: String, value: String): DaffodilTunables = { + | tunable match { + |${casesString} + |${fallthrough} + | } + | } + """.trim.stripMargin } - .mkString("\n") + .mkString("\n\n") w.write(top) w.write("\n") @@ -181,7 +212,7 @@ class TunableGenerator(schemaRootConfig: scala.xml.Node, schemaRootExt: scala.xm w.write("\n") w.write(middle) w.write("\n") - w.write(conversionString) + w.write(partsString) w.write("\n") w.write(bottom) w.write("\n") diff --git a/daffodil-test-integration/src/test/scala/org/apache/daffodil/cliTest/TestCLIDebugger.scala b/daffodil-test-integration/src/test/scala/org/apache/daffodil/cliTest/TestCLIDebugger.scala index 626dc019a0..27b1518249 100644 --- a/daffodil-test-integration/src/test/scala/org/apache/daffodil/cliTest/TestCLIDebugger.scala +++ b/daffodil-test-integration/src/test/scala/org/apache/daffodil/cliTest/TestCLIDebugger.scala @@ -24,7 +24,10 @@ import org.apache.daffodil.cli.Main.ExitCode import org.apache.daffodil.cli.cliTest.Util.* import org.apache.daffodil.core.util.TestUtils.intercept +import net.sf.expectit.matcher.Matchers.eof import net.sf.expectit.matcher.Matchers.regexp +import org.junit.Assert.assertTrue +import org.junit.Assert.fail import org.junit.Test /** @@ -1325,4 +1328,55 @@ class TestCLIDebugger { }(ExitCode.Success) } + /** + * Tracing an unparse must step through the same bit positions whether or not + * the build-ahead path is used. The + * trace also shows the infoset, data and diff at each step; those differ + * in small ways on the build ahead path (the child and group indexes, and + * nodes built one step ahead), so they are not compared. + */ + @Test def test_CLI_Tdml_Trace_buildAheadUnparseMatchesEventDriven(): Unit = { + val tdml = path( + "daffodil-test/src/test/resources/org/apache/daffodil/unparser/buildAhead.tdml" + ) + + def steps(buildAhead: Boolean): Seq[String] = { + val mode = if (buildAhead) { + "buildAhead" + } else { + "eventDriven" + } + val tunables = Map("DAFFODIL_TDML_TUNABLES" -> s"infosetBuilderMode=$mode") + var transcript = "" + runCLI( + args"test -t $tdml nviScopedVariableWithValueLengthOVC", + fork = true, + envs = envs ++ tunables + ) { cli => + transcript = cli.expect(eof()).getInput + }(ExitCode.Success) + transcript.linesIterator + .filter { line => + line.startsWith("bitPosition:") || line.startsWith("-----") + } + .map(_.replaceAll("@[0-9a-f]+", "")) + .toSeq + } + + val eventDriven = steps(buildAhead = false) + val buildAhead = steps(buildAhead = true) + assertTrue("expected a trace of steps", eventDriven.exists(_.startsWith("bitPosition:"))) + val firstDifference = eventDriven.zipAll(buildAhead, "", "").indexWhere { + case (a, b) => a != b + } + if (firstDifference >= 0) { + def around(lines: Seq[String]) = + lines.slice(firstDifference - 3, firstDifference + 3).mkString("\n ") + fail( + s"first difference at line $firstDifference of ${eventDriven.length} event-driven and " + + s"${buildAhead.length} build ahead lines\n event-driven:\n ${around(eventDriven)}" + + s"\n build ahead:\n ${around(buildAhead)}" + ) + } + } } diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/unparser/buildAhead.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/unparser/buildAhead.tdml new file mode 100644 index 0000000000..d19735a99c --- /dev/null +++ b/daffodil-test/src/test/resources/org/apache/daffodil/unparser/buildAhead.tdml @@ -0,0 +1,1436 @@ + + + + + + + + 3 + + + + + + 2 + 1 + 1 + + + + + + 2 + + + + + + -1 + + + 2 + + + + + + + + + + + + + + + + + + + + 005 + + + 006005 + + + + + + + + + + + + + + + + + + + + + + + + + +
H
+ a + b + c + X + + + + H,a,b,c,X + + + + + + + + + + + + + + + + + + + + + a + b + + + + a,b + + + + + + + + + + + + + + + + + + + + + + + + + + 3 + + + [hello,3 + + + + + + + + + + + + + + + + + + + + + + a + b + c + + + + + 61 0A 62 0A 63 0A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + + + first_defaultable1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + 1 + + + + first_defaultable01 + + + + + + + + + + + + + + + + + + + + + i0 + i1 + i2 + i3 + i4 + i5 + i6 + i7 + i8 + i9 + i10 + i11 + + + + i0,i1,i2,i3,i4,i5,i6,i7,i8,i9,i10,i11 + + + + + + + + + + + + + + + + + + + + + + + + + + + B + I1 + I2 + A + + + + B,I1I2,A + + + + + + + + + + + + + + + + + + + + + + XY + + + + 58 59 20 20 20 + + + + + + + + + + + + + + + + + + + + + + + + + + V + + + HV + + + + + + + + + + + + + + + + + + + + + + + A + B + + + + [AB] + + + + + + + + + + + + + + + + + + + [] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + V + + + V + + + + + + + + + + + + + + + + + + + + + + + + + + one, two + three + + + + one#, two,three + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ABCDEFGH + Q + + + + ABCDEFGHQ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ABCDEFGHIJ + Q + + + + + exceeded fixed layer length of 8 + + + + + + + + + + + + + + + + + + + + + + 005 + + + 007006005 + + + + + + + + + + + + + + + + + + + + + + 1 + 2 + 3 + + + + + 00 00 00 01 00 00 00 02 00 00 00 03 + + + + + + + + + + + + + + + + + + + + + + + + 5 + hello + + + + 05hello + + + + + + + + + + + + + + + + + + + + + + + + hello + + + 05hello + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hi + bye + + + + + 06hi,bye + + + + + + + + + + + + + + + + + + + + + 1 + 2 + + + + 122 + + + + + + + + + + + + + + + + + + + + + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + + + + 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789100 + + + + + + + + + + + + + + + + + + + + + + + + + y + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 2 + 3 + 4 + + true + + + + 1,2,3,4 + + + + + + + + + + + + + + + + + + + + + + + + + + + value0 + value1 + value2 + value3 + value4 + value5 + value6 + value7 + + + + 6 |value06 |value16 |value26 |value36 |value46 |value56 |value66 |value7 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 00value0 + 01value1 + 02value2 + 03value3 + 04value4 + 05value5 + 06value6 + 07value7 + + + + 0 |6 |value01 |7 |value12 |8 |value23 |9 |value34 |10 |value45 |11 |value56 |12 |value67 |13 |value7 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 42 + hello + + + + 42|47 |hello + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 00 + 01 + 02 + 03 + 04 + 05 + 06 + 07 + + + + + 30 20 31 20 32 20 33 20 34 20 35 20 36 20 37 20 2D 31 20 20 + + + + + + + + + + + + + + + + + + + + + + + + 5 + hello + + + + 05|hello + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 3 + xyz + + + + 03|xyz + + + + + + + + + + + + + + + + + + + + xyz + + + 03xyz + + + + + + + + + + + + + + + + + + + + + + + 005 + xyz + + + + 00600503xyz + + + diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/unparser/TestBuildAhead.scala b/daffodil-test/src/test/scala/org/apache/daffodil/unparser/TestBuildAhead.scala new file mode 100644 index 0000000000..15830e92de --- /dev/null +++ b/daffodil-test/src/test/scala/org/apache/daffodil/unparser/TestBuildAhead.scala @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.unparser + +import org.apache.daffodil.junit.tdml.TdmlSuite +import org.apache.daffodil.junit.tdml.TdmlTests + +import org.junit.Test + +object TestBuildAhead extends TdmlSuite { + val tdmlResource = "/org/apache/daffodil/unparser/buildAhead.tdml" +} + +class TestBuildAhead extends TdmlTests { + val tdmlSuite = TestBuildAhead + + @Test def ovcSuspension = test + @Test def arrayChoiceSeparator = test + @Test def absentTrailingOptionalSuppressesSeparator = test + @Test def hiddenChoice = test + @Test def trailingArrayPostfixSeparator = test + @Test def choiceBranchWithAbsentOptionalElement = test + @Test def choiceBranchWithPresentOptionalElement = test + @Test def manyOccurrenceArrayWithSmallBuildAheadLimit = test + @Test def nestedBareSequence = test + @Test def fixedLengthChoicePadding = test + @Test def hiddenGroup = test + @Test def initiatorTerminator = test + @Test def nillableComplexWithEmptyContentNotNilled = test + @Test def nillableComplexWithEmptyContentNilled = test + @Test def hiddenGroupWithEmptyBody = test + @Test def escapeScheme = test + @Test def layeredSequence = test + @Test def layeredSequenceLengthMismatchError = test + @Test def nestedOvcSuspensions = test + @Test def binaryIntArray = test + @Test def variableLengthExpression = test + @Test def prefixedLength = test + @Test def prefixedLengthComplexContent = test + @Test def ovcCountOfPrecedingArray = test + @Test def ovcCountOfPrecedingArrayAfterBuildFullyFinishes = test + @Test def dynamicTerminatorReferencingArrayCount = test + @Test def ivcExistsOverNestedArray = test + @Test def manyValueLengthForwardReferencesThrottled = test + @Test def nviScopedVariableWithValueLengthOVC = test + @Test def singleNviScopedVariableWithValueLengthOVC = test + @Test def nviScopedSetVariableWithNoForwardReference = test + @Test def delimitedVariableLengthExpression = test + @Test def delimitedComplexVariableLengthExpression = test + @Test def purelyContentLengthOVC = test + @Test def mixedResolvableAndContentLengthOVC = test +} From 27e57a34be54d710cc2afeaefbb529053e98817a Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:21:05 -0400 Subject: [PATCH 5/5] fixup! Add a build ahead unparse path driven by an infoset builder tree --- .../runtime1/processors/DataProcessor.scala | 11 +--- .../unparsers/InfosetBuildState.scala | 4 ++ .../processors/unparsers/TreeEventState.scala | 16 +++--- .../processors/unparsers/UState.scala | 35 ++++++++++++ .../unparsers/UStateMainForBuildAhead.scala | 54 ------------------- .../processors/unparsers/Unparser.scala | 7 +-- .../unparsers/runtime1/ElementUnparser.scala | 4 +- .../runtime1/LayeredSequenceUnparser.scala | 3 -- .../apache/daffodil/unparser/buildAhead.tdml | 37 +++++++++++++ .../daffodil/unparser/TestBuildAhead.scala | 1 + 10 files changed, 89 insertions(+), 83 deletions(-) delete mode 100644 daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UStateMainForBuildAhead.scala diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala index 50ab43a300..650317832f 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala @@ -70,7 +70,6 @@ import org.apache.daffodil.runtime1.infoset.XMLTextInfosetOutputter import org.apache.daffodil.runtime1.processors.parsers.PState import org.apache.daffodil.runtime1.processors.parsers.ParseError import org.apache.daffodil.runtime1.processors.parsers.Parser -import org.apache.daffodil.runtime1.processors.unparsers.ChildNotBuiltException import org.apache.daffodil.runtime1.processors.unparsers.InfosetBuildState import org.apache.daffodil.runtime1.processors.unparsers.NotUnparsableUnparser import org.apache.daffodil.runtime1.processors.unparsers.TreeEventState @@ -582,14 +581,8 @@ class DataProcessor( ssrd.elementRuntimeData.defaultBitOrder ) - try { - rootUnparser.unparse1(unparseState) - unparseState.popTRD(rootUnparser.context.asInstanceOf[TermRuntimeData]) - } catch { - // A genuine deadlock (if any) surfaces via the final - // evalSuspensions(isFinal = true) below. - case _: ChildNotBuiltException => - } + rootUnparser.unparse1(unparseState) + unparseState.popTRD(rootUnparser.context.asInstanceOf[TermRuntimeData]) // The unparse only ever advances build as far as it needs, so build may // still have its trailing end events left to consume. diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildState.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildState.scala index 8885dab038..818d26028d 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildState.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/InfosetBuildState.scala @@ -17,6 +17,7 @@ package org.apache.daffodil.runtime1.processors.unparsers +import org.apache.daffodil.api.DataLocation import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.iapi.DaffodilTunables import org.apache.daffodil.lib.util.MStackOfMaybe @@ -81,6 +82,9 @@ final class InfosetBuildState( override def documentElement: DIDocument = inputter.documentElement + // The build runs ahead of the output, so it has no data location. + override def maybeCurrentLocation: Maybe[DataLocation] = Nope + override val currentInfosetNodeStack = new MStackOfMaybe[DINode] override def currentInfosetNode: DINode = { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/TreeEventState.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/TreeEventState.scala index 8d31186dd7..1abc27b5cc 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/TreeEventState.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/TreeEventState.scala @@ -180,7 +180,13 @@ final class TreeEventState( return false } if (cursor.isFinished) { - throw new ChildNotBuiltException + UnparseError( + Nope, + Nope, + "Expected child %s of %s, but the build finished without it.", + index + 1, + parent.erd.namedQName.toExtendedSyntax + ) } cursor.advance() } @@ -382,11 +388,3 @@ final class TreeEventState( private[unparsers] enum NodeKind { case Simple, Complex, Array, Document } - -/** - * Thrown when the unparse needs a child that does not exist after build has - * finished. Caught only by the top-level driver, which still runs its normal - * finalization (the genuine, diagnostic-producing final suspension drain) - * rather than treating this as the final outcome itself. - */ -final class ChildNotBuiltException extends Exception diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala index 02b7f780b0..376092cd66 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UState.scala @@ -239,6 +239,8 @@ abstract class UState( new DataLoc(bitPos1b, bitLimit1b, Left(getDataOutputStream), mrd) } + final override def maybeCurrentLocation: Maybe[DataLocation] = One(currentLocation) + lazy val unparseResult = new UnparseResult(dataProc.get, this) def bitPos0b = if (getDataOutputStream.maybeAbsBitPos0b.isDefined) @@ -460,6 +462,10 @@ abstract class UState( trait InfosetTreeState extends Cursor[InfosetAccessor] { def tunable: DaffodilTunables + // Where in the data an error raised through this state happened. A state + // with no output has none to report. + def maybeCurrentLocation: Maybe[DataLocation] + // How the infoset's nodes are made and kept as unparsing consumes events. // InfosetFromEvents does it for the events of an inputter; a state that // unparses a tree that was already built overrides these. @@ -999,6 +1005,35 @@ class UStateMain private[unparsers] ( } } +/** + * The UState for unparsing a tree that build made ahead of it, by reading + * the tree as events and taking each node from the tree. + */ +final class UStateMainForBuildAhead private[unparsers] ( + inputter: InfosetInputter, + outStream: java.io.OutputStream, + vmap: VariableMap, + diagnosticsArg: Seq[api.Diagnostic], + dataProcArg: DataProcessor, + tunable: DaffodilTunables, + areDebugging: Boolean, + override protected val treeEvents: TreeEventState +) extends UStateMain( + inputter, + outStream, + vmap, + diagnosticsArg, + dataProcArg, + tunable, + areDebugging, + treeEvents + ) + with InfosetFromTree { + + // How far the unparse of the built tree has reached, for the debugger. + def reachedChildCounts(): java.util.Map[DINode, Integer] = treeEvents.reachedChildCounts() +} + object UState { def createInitialUState( diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UStateMainForBuildAhead.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UStateMainForBuildAhead.scala deleted file mode 100644 index ed97fbdadb..0000000000 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/UStateMainForBuildAhead.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.daffodil.runtime1.processors.unparsers - -import org.apache.daffodil.lib.iapi.DaffodilTunables -import org.apache.daffodil.lib.iapi as api -import org.apache.daffodil.runtime1.infoset.DINode -import org.apache.daffodil.runtime1.infoset.InfosetInputter -import org.apache.daffodil.runtime1.processors.DataProcessor -import org.apache.daffodil.runtime1.processors.VariableMap - -/** - * The UState for unparsing a tree that build made ahead of it, by reading - * the tree as events and taking each node from the tree. - */ -final class UStateMainForBuildAhead private[unparsers] ( - inputter: InfosetInputter, - outStream: java.io.OutputStream, - vmap: VariableMap, - diagnosticsArg: Seq[api.Diagnostic], - dataProcArg: DataProcessor, - tunable: DaffodilTunables, - areDebugging: Boolean, - override protected val treeEvents: TreeEventState -) extends UStateMain( - inputter, - outStream, - vmap, - diagnosticsArg, - dataProcArg, - tunable, - areDebugging, - treeEvents - ) - with InfosetFromTree { - - // How far the unparse of the built tree has reached, for the debugger. - def reachedChildCounts(): java.util.Map[DINode, Integer] = treeEvents.reachedChildCounts() -} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/Unparser.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/Unparser.scala index 7c7f750350..f1294df7d0 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/Unparser.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/unparsers/Unparser.scala @@ -75,13 +75,8 @@ sealed trait Unparser extends Processor { ustate.setMaybeProcessor(savedProc) } - def UE(ustate: UState, s: String, args: Any*) = { - UnparseError(One(context.schemaFileLocation), One(ustate.currentLocation), s, args*) - } - - // Code shared with build has no data location to report. def UE(state: InfosetTreeState, s: String, args: Any*) = { - UnparseError(One(context.schemaFileLocation), Nope, s, args*) + UnparseError(One(context.schemaFileLocation), state.maybeCurrentLocation, s, args*) } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala index 101ad34887..bf2dadeb21 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/ElementUnparser.scala @@ -428,7 +428,7 @@ sealed trait RegularElementUnparserStartEndStrategy extends ElementUnparserStart // this indicates that the incoming infoset (as events) doesn't match the schema UnparseError( Nope, - Nope, + state.maybeCurrentLocation, "Expected element start event for %s, but received %s.", erd.namedQName.toExtendedSyntax, event @@ -470,7 +470,7 @@ sealed trait RegularElementUnparserStartEndStrategy extends ElementUnparserStart // this indicates that the incoming infoset (as events) doesn't match the schema UnparseError( Nope, - Nope, + state.maybeCurrentLocation, "Expected element end event for %s, but received %s.", erd.namedQName.toExtendedSyntax, event diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/LayeredSequenceUnparser.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/LayeredSequenceUnparser.scala index 2bd9f19895..3856c0b272 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/LayeredSequenceUnparser.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/LayeredSequenceUnparser.scala @@ -95,9 +95,6 @@ class LayeredSequenceUnparser( // layer stack is potentially still needed, so // nothing can be cleaned up at this point. } catch { - // A signal from waiting on build, not a failure of the layer; rewrapping - // it would hide the stall diagnostic. - case e: ChildNotBuiltException => throw e case t: Throwable if (layerDriver ne null) => layerDriver.handleThrowable(t) case t: Throwable => LayerDriver.handleThrowableWithoutLayer(t) // otherwise we have no layer driver, so we were unable to load the layer. diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/unparser/buildAhead.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/unparser/buildAhead.tdml index d19735a99c..e5f5d06239 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/unparser/buildAhead.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/unparser/buildAhead.tdml @@ -28,6 +28,12 @@ xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext" xmlns:ex="http://example.com"> + + + eventDriven + + + 3 @@ -1433,4 +1439,35 @@ 00600503xyz + + + + + + + + + + + + + + + + + + 001002 + + + + Expected element start event for + Data location was preceding byte + + + diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/unparser/TestBuildAhead.scala b/daffodil-test/src/test/scala/org/apache/daffodil/unparser/TestBuildAhead.scala index 15830e92de..1ea2a80b9f 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/unparser/TestBuildAhead.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/unparser/TestBuildAhead.scala @@ -64,4 +64,5 @@ class TestBuildAhead extends TdmlTests { @Test def delimitedComplexVariableLengthExpression = test @Test def purelyContentLengthOVC = test @Test def mixedResolvableAndContentLengthOVC = test + @Test def eventDrivenInfosetMismatchReportsDataLocation = test }