diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..d4f94d52 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,8 @@ +## Checklist +Before you mark your PR as ready for review, please ensure you have completed the following. + +- [ ] Added unit tests for new functionality +- [ ] Properly documented any new code +- [ ] Updated Sphinx documentation in `docs/` and checked for updated configuration/runtime options +- [ ] Added a jupyter notebook tutorial in `docs/source/tutorials` for any new functionality +- [ ] Verified local docs build with `sphinx-build -W source/ build` diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml new file mode 100644 index 00000000..68de807b --- /dev/null +++ b/.github/workflows/build-docs.yaml @@ -0,0 +1,29 @@ +name: Build docs + +on: + pull_request: + branches: + - main + +jobs: + build: + name: MegaLinter + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up Pandoc + uses: pandoc/actions/setup@v1 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Install dependencies + run: | + python --version + python -m pip install --upgrade pip + pip install -e .[dev] + - name: "Build docs" + working-directory: "docs" + run: | + sphinx-build -W source/ build diff --git a/.github/workflows/ci-tests-drafts.yaml b/.github/workflows/ci-tests-drafts.yaml index 7f1dfe80..42d839d0 100644 --- a/.github/workflows/ci-tests-drafts.yaml +++ b/.github/workflows/ci-tests-drafts.yaml @@ -25,7 +25,7 @@ jobs: python --version python -m pip install --upgrade pip pip install -e . - pip install -e .[test] + pip install -e .[dev] pip install pytest pytest-cov - name: Register Jupyter Kernel run: | diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index d74f335f..c9c8aac1 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -30,7 +30,7 @@ jobs: python --version python -m pip install --upgrade pip pip install -e . - pip install -e .[test] + pip install -e .[dev] pip install pytest pytest-cov - name: Register Jupyter Kernel run: | diff --git a/README.md b/README.md index 30f0d675..4528a991 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,8 @@ the inputs and outputs of the system under test, supported by mathematical found enable causal inference. Each causal test case targets the causal effect of a specific intervention on the system under test--that is, a deliberate modification to the input configuration expected to produce a corresponding change in one or more outputs. -![Causal Testing Workflow](images/schematic-dark.png#gh-dark-mode-only) -![Causal Testing Workflow](images/schematic.png#gh-light-mode-only) +![Causal Testing Workflow](docs/source/_static/images/testing-workflow-dark.png#gh-dark-mode-only) +![Causal Testing Workflow](docs/source/_static/images/testing-workflow.png#gh-light-mode-only) ## Installation diff --git a/causal_testing/__main__.py b/causal_testing/__main__.py index 76f9e6be..e7eb0436 100644 --- a/causal_testing/__main__.py +++ b/causal_testing/__main__.py @@ -115,8 +115,11 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace: parser_discover.add_argument( "-t", "--technique", - help="The name of the technique to use. Currently supported are 'HillClimberDiscovery' and 'NSGADiscovery'", - required=True, + default="HillClimberDiscovery", + help=( + "The name of the technique to use. Currently supported are 'HillClimberDiscovery' and 'NSGADiscovery'. " + "Defaults to HillClimberDiscovery." + ), ) parser_discover.add_argument( "-V", diff --git a/causal_testing/discovery/nsga_discovery.py b/causal_testing/discovery/nsga_discovery.py index 3cb99222..646d20fb 100644 --- a/causal_testing/discovery/nsga_discovery.py +++ b/causal_testing/discovery/nsga_discovery.py @@ -39,7 +39,7 @@ def binary_string_to_causal_dag(self, individual: np.array) -> CausalDAG: Converts a binary string representation of a causal DAG to a CausalDAG object. :param individual: Bitstring of the same length as `possible_edges` such that 1 at position `i` represents - possible_edges[i] being an edge in the graph and 0 represents it not being. + possible_edges[i] being an edge in the graph and 0 represents it not being. :returns: The converted CausalDAG instance. """ causal_dag = CausalDAG() diff --git a/causal_testing/estimation/abstract_regression_estimator.py b/causal_testing/estimation/abstract_regression_estimator.py index cee528b0..fbc58759 100644 --- a/causal_testing/estimation/abstract_regression_estimator.py +++ b/causal_testing/estimation/abstract_regression_estimator.py @@ -64,6 +64,7 @@ def __init__( def _get_adjusted_variables(self, tree: ast.AST) -> set[str]: """ Recursively return variables in an AST. + :returns: Set of all variables not used as part of a function. """ if isinstance(tree, ast.Name) and tree.id != self.treatment_variable: @@ -108,7 +109,9 @@ def _setup_covariates(self, df: pd.DataFrame) -> pd.Series: Parse the formula and set up the covariates from the design matrix so we can use them in the statsmodels array API. This allows us to only parse the formula once, rather than using the formula API, which parses it every time the regression model is fit, which can be a lot if using causal test adequacy. + :param df: The data to use. + :returns: The data and the covariate columns. """ _, covariate_data = dmatrices(self.formula, df, return_type="dataframe") @@ -154,9 +157,10 @@ def treatment_columns(self, model: RegressionResultsWrapper) -> list[str]: This is a workaround for statsmodels mangling the names of categorical variables to include the values. :param model: The fitted model from which to extract the variable names. + :returns: A list of the feature names in the model that represent the treatment. Normally this will just be - [treatment_name], but for categorical treatments, you'll have - [treatment_name[value_1], treatment_name[value_2]]. + [treatment_name], but for categorical treatments, you'll have + [treatment_name[value_1], treatment_name[value_2]]. """ return [ param @@ -170,7 +174,7 @@ def _predict(self, df) -> pd.DataFrame: :param df: The data to use. :param: adjustment_config: The values of the adjustment variables to use. - :return: The estimated outcome under control and treatment, with confidence intervals in the form of a + :returns: The estimated outcome under control and treatment, with confidence intervals in the form of a dataframe with columns "predicted", "se", "ci_lower", and "ci_upper". """ model = self.fit_model(df) diff --git a/causal_testing/testing/causal_test_case.py b/causal_testing/testing/causal_test_case.py index fa6e4c90..ec1de7b8 100644 --- a/causal_testing/testing/causal_test_case.py +++ b/causal_testing/testing/causal_test_case.py @@ -20,6 +20,7 @@ class CausalTestCase: variables, a CausalTestCase stores the values of these variables. Also the outcome variable and value are specified. The goal of a CausalTestCase is to test whether the intervention made to the control via the treatment causes the model-under-test to produce the expected change. + :param base_test_case: A BaseTestCase object consisting of a treatment variable, outcome variable and effect :param expected_causal_effect: The expected causal effect (Positive, Negative, No Effect). :param effect_measure: A string which denotes the type of estimate to return. @@ -70,10 +71,11 @@ def measure_adequacy( ) -> DataAdequacy: """ Calculate the adequacy measurement, and populate the data_adequacy field. + :param df: The original dataset to use. :param bootstrap_size: The number of bootstrap samples to use. (Defaults to 100) :param group_by: For IPCWEstimator - the "id" column to ensure that entire individuals are sampled rather than - random rows. + random rows. """ results = [] outcomes = [] @@ -130,8 +132,7 @@ def execute_test( :param suppress_estimation_errors: Set to True to suppress estimation errors. (Defaults to False) :param bootstrap_size: The number of bootstrap samples to use. (Defaults to 100) :param group_by: For IPCWEstimator - the "id" column to ensure that entire individuals are sampled rather than - random rows. - :return causal_test_result: A CausalTestResult for the executed causal test case. + random rows. """ if not self.skip: try: @@ -161,7 +162,8 @@ def estimate_effect(self, df: pd.DataFrame) -> CausalTestResult: Execute a causal test case and return the causal test result. :param df: The data to use. - :return causal_test_result: A CausalTestResult for the executed causal test case. + + :returns: A CausalTestResult for the executed causal test case. """ if self.query: df = df.query(self.query) diff --git a/docs/source/_static/images/.gitignore b/docs/source/_static/images/.gitignore new file mode 100644 index 00000000..3eec47da --- /dev/null +++ b/docs/source/_static/images/.gitignore @@ -0,0 +1,3 @@ +*.aux +*.log +*.pdf diff --git a/docs/source/_static/images/discovery-workflow.png b/docs/source/_static/images/discovery-workflow.png new file mode 100644 index 00000000..bb147370 Binary files /dev/null and b/docs/source/_static/images/discovery-workflow.png differ diff --git a/docs/source/_static/images/discovery-workflow.tex b/docs/source/_static/images/discovery-workflow.tex new file mode 100644 index 00000000..9a645d77 --- /dev/null +++ b/docs/source/_static/images/discovery-workflow.tex @@ -0,0 +1,114 @@ +\documentclass{standalone} + +\usepackage{tikz} +\usetikzlibrary{arrows.meta,positioning,shapes,calc,fit,overlay-beamer-styles, backgrounds} +\usepackage{dsfont,pifont} +\newcommand*{\expe}{\mathds{E}} +\usepackage{amsmath} +\usepackage{booktabs} +\usepackage{fontawesome7} + +\usepackage[default]{FiraSans} +\usepackage[mathrm=sym]{unicode-math} +\setmathfont{Fira Math} + +\newcommand{\indep}{\perp \!\!\! \perp} + +\begin{document} +\tikzset{ + node/.style={circle, draw, minimum size=3ex, inner sep=0.2}, + edge/.style={-{Stealth[length=3mm]}}, +} + +\newcommand{\cmark}{\ding{51}}% +\newcommand{\xmark}{\ding{55}}% + +\begin{tikzpicture}[background rectangle/.style={fill=none}, show background rectangle, color=black] + + % Data + \begin{scope}[name prefix=data-, local bounding box=test-data] + \node[draw=none, rectangle] (title) {Test Data}; + \node[anchor=north] (table) at (title.south) { + \begin{tabular}{rrrrrr} + \toprule + $X_1$ & $X_2$ & $I$ & $Y_1$ & $Y_2$ & $Y_3$ \\ + \midrule + 1.2 & ``UK'' & 0.3 & 7.8 & 4 & True \\ + 3.2 & ``UK'' & 0.1 & 7.6 & 8 & False \\ + \multicolumn{6}{c}{$\vdots$} \\ + \bottomrule + \end{tabular} + }; + \node[draw, rectangle] [fit=(title) (table)] {}; + \end{scope} + + + % ci + \begin{scope}[name prefix=ci-, local bounding box=ci, anchor=west, shift={($(data-test-data.east) + (1, 0)$)}] + \node[draw=none, rectangle, shift={(0, -0.05)}] (title) at ({(0, 0)} |- data-title) {Causal Discovery}; + \begin{scope}[shift={($(title.south) + (-0.65, -0.3)$)}, local bounding box=brain] + \path[draw,line width=0.025cm] (0.9871, -0.2624) circle (0.066cm); + \path[draw,line width=0.025cm] (0.8736, -0.5912) circle (0.066cm); + \path[draw,line width=0.025cm] (0.8597, -0.8965) circle (0.066cm); + \path[draw,line width=0.025cm] (1.0417, -1.1239) circle (0.066cm); + \path[draw,line width=0.025cm] (0.4807, -1.1496) circle (0.066cm); + \path[draw,line width=0.025cm] (0.4566, -0.876) circle (0.066cm); + \path[draw,line width=0.025cm] (0.2702, -0.5893) circle (0.066cm); + \path[draw,line width=0.025cm] (0.4498, -0.313) circle (0.066cm); + + \path[draw,line width=0.025cm,miter limit=4.0] (0.5183, 0) -- (0.1888, -0.1903) -- (0.1888, -0.4236) -- (0, -0.5325) -- (0, -0.8925) -- (0.1592, -0.9844) -- (0.1592, -1.2186) -- (0.5134, -1.4231) -- (0.6931, -1.3338) -- (0.8729, -1.4231) -- (1.227, -1.2186) -- (1.227, -0.9844) -- (1.3862, -0.8925) -- (1.3862, -0.5325) -- (1.1975, -0.4236) -- (1.1975, -0.1903) -- (0.868, 0) -- (0.6931, -0.101) -- cycle; + \path[draw,line width=0.025cm,miter limit=4.0] (0.6931, -1.3338) -- (0.6931, -0.101); + \path[draw,line width=0.025cm] (0.3839, -0.313) -- (0.1888, -0.313); + \path[draw,line width=0.025cm] (0.2702, -0.5234) -- (0.2702, -0.313); + \path[draw,line width=0.025cm] (0.4567, -0.81) -- (0.4567, -0.5785) -- (0.6931, -0.5785); + \path[draw,line width=0.025cm] (0.4148, -1.1496) -- (0.2428, -1.1496) -- (0.2428, -0.8899) -- (0, -0.8899); + \path[draw,line width=0.025cm] (0.9871, -0.3283) -- (0.9871, -0.4203) -- (0.6931, -0.4203); + \path[draw,line width=0.025cm] (0.8736, -0.5253) -- (0.8736, -0.4203); + \path[draw,line width=0.025cm] (0.9256, -0.8965) -- (1.1381, -0.8965) -- (1.1381, -0.7111) -- (1.3862, -0.7111); + \path[draw,line width=0.025cm] (0.9758, -1.1239) -- (0.8728, -1.1239) -- (0.8728, -1.423); + \end{scope} + + \coordinate (bot) at ({(0, 0)} |- data-table.south); + + \node[draw, rectangle] [fit=(title) (brain) (bot)] {}; + \end{scope} + + % Causal DAG + \begin{scope}[name prefix=dag-, anchor=west, shift={($(ci-ci.east) + (1.275, 0)$)}] + \node[draw=none, rectangle, shift={(0, -0.05)}] (title) at ({(0, 0)} |- data-title) {Causal DAG}; + + % Shift origin to title's bottom edge, then push down by 1cm + \begin{scope}[anchor=north, shift={([yshift=-1.5cm]title.south)}, local bounding box=dag] + \node[node] (x1) at (-1, 0) {$X_1$}; + \node[node] (x2) at (-1, 1.4) {$X_2$}; + \node[node] (i) at (0, 0.7) {$I$}; + \node[node] (y1) at (1,0) {$Y_{1}$}; + \node[node] (y2) at (1,0.7) {$Y_2$}; + \node[node] (y3) at (1,1.4) {$Y_3$}; + + \draw[edge] (x1) to (i); + \draw[edge] (x2) to (i); + \draw[edge] (i) to (y1); + \draw[edge] (i) to (y2); + \draw[edge] (i) to (y3); + \draw[edge] (x1) to (y1); + \draw[edge] (x2) to (y3); + \end{scope} + + \node[draw=none, rectangle] (nodes) [fit=(x1) (x2) (y1) (y2) (y3) (i)] {}; + + % DAG outline + \coordinate (top) at ({(0, 0)} |- data-title.north); + \coordinate (bot) at ({(0, 0)} |- data-table.south); + \node[draw, rectangle] (dag) [fit=(dag-title) (dag) (top) (bot)] {}; + \end{scope} + + + + + + %Information flow + \draw[edge,dashed] (data-test-data) -- (ci-ci); + \draw[edge, dashed] (ci-ci) -- (dag-dag); +\end{tikzpicture} +\end{document} diff --git a/docs/source/_static/images/testing-workflow-dark.png b/docs/source/_static/images/testing-workflow-dark.png new file mode 100644 index 00000000..dbb0eb9c Binary files /dev/null and b/docs/source/_static/images/testing-workflow-dark.png differ diff --git a/docs/source/_static/images/testing-workflow.png b/docs/source/_static/images/testing-workflow.png new file mode 100644 index 00000000..c2ad6282 Binary files /dev/null and b/docs/source/_static/images/testing-workflow.png differ diff --git a/docs/source/_static/images/testing-workflow.tex b/docs/source/_static/images/testing-workflow.tex new file mode 100644 index 00000000..6e3e8fd5 --- /dev/null +++ b/docs/source/_static/images/testing-workflow.tex @@ -0,0 +1,127 @@ +\documentclass{standalone} + +\usepackage{tikz} +\usetikzlibrary{arrows.meta,positioning,shapes,calc,fit,overlay-beamer-styles, backgrounds} +\usepackage{dsfont,pifont} +\newcommand*{\expe}{\mathds{E}} +\usepackage{amsmath} +\usepackage{booktabs} +\usepackage{fontawesome7} + +\usepackage[default]{FiraSans} +\usepackage[mathrm=sym]{unicode-math} +\setmathfont{Fira Math} + +\newcommand{\indep}{\perp \!\!\! \perp} + +\begin{document} +\tikzset{ + node/.style={circle, draw, minimum size=3ex, inner sep=0.2}, + edge/.style={-{Stealth[length=3mm]}}, +} + +\newcommand{\cmark}{\ding{51}}% +\newcommand{\xmark}{\ding{55}}% + +\begin{tikzpicture}[background rectangle/.style={fill=none}, show background rectangle, color=black] + + % Test Case + \begin{scope}[name prefix=test-, local bounding box=test-case] + \node[draw=none, rectangle, anchor=north] (title) at (0, 0) {Causal Test Cases}; + \node[anchor=north,align=center] (tuple) at (title.south) {$I \to_{?} Y_3$\hspace{5mm}$X_1 \indep_? X_2$}; + \node[draw, rectangle] [fit=(title) (tuple)] {}; + \end{scope} + + % ci + \begin{scope}[name prefix=ci-, local bounding box=ci, shift={($(test-test-case.east) + (1.2, 0)$)}] + \node[draw=none, rectangle, anchor=south west] (title) {Causal Inference}; + \node[draw=none, rectangle, anchor=north, align=center] (brain) at (title.south) {\faIcon{hexagon-nodes-bolt}}; + + \coordinate (top) at ({(0, 0)} |- test-title.north); + \coordinate (bot) at ({(0, 0)} |- test-tuple.south); + + \node[draw, rectangle] [fit=(title) (brain) (top) (bot)] {}; + \end{scope} + + % Estimate + \begin{scope}[name prefix=estimate-, local bounding box=estimate, shift={($(ci-ci.east)+(1, 0)$)}] + \node[draw=none, rectangle, anchor=south west] (title) {Causal Estimate}; + \node[anchor=north] (table) at (title.south) {\faIcon{chart-line}}; + \coordinate (top) at ({(0, 0)} |- test-title.north); + \coordinate (bot) at ({(0, 0)} |- test-tuple.south); + \node[draw, rectangle] [fit=(title) (table) (top) (bot)] {}; + \end{scope} + + % Oracle + \begin{scope}[name prefix=oracle-, local bounding box=test-oracle, shift={($(estimate-estimate.east) + (1, 0)$)}] + \node[draw=none, rectangle, anchor=south west] (title) {Test Oracle}; + \node[draw=none, rectangle, anchor=north] (scale) at (title.south) {\faIcon{scale-balanced}}; + + \coordinate (top) at ({(0, 0)} |- test-title.north); + \coordinate (bot) at ({(0, 0)} |- test-tuple.south); + \node[draw, rectangle] [fit=(title) (scale) (top) (bot)] {}; + \end{scope} + + % Outcome + \begin{scope}[name prefix=outcome-, local bounding box=test-outcome, shift={($(oracle-test-oracle.east) + (1, 0)$)}] + \node[draw=none, rectangle, anchor=south west] (title) at (0,0) {Test Outcomes}; + \node[draw=none, anchor=north] (ok) at (title.south) {\cmark ~ \xmark}; + + \coordinate (top) at ({(0, 0)} |- test-title.north); + \coordinate (bot) at ({(0, 0)} |- test-tuple.south); + \node[draw, rectangle] (test-outcome) [fit=(outcome-title) (outcome-ok) (top) (bot)] {}; + \end{scope} + + + % Causal DAG + \begin{scope}[name prefix=dag-, shift={(0, 2)}] + \node[node] (x1) at (-1, 0) {$X_1$}; + \node[node] (x2) at (-1, 1.4) {$X_2$}; + \node[node] (i) at (0, 0.7) {$I$}; + \node[node] (y1) at (1,0) {$Y_{1}$}; + \node[node] (y2) at (1,0.7) {$Y_2$}; + \node[node] (y3) at (1,1.4) {$Y_3$}; + + \draw[edge] (x1) to (i); + \draw[edge] (x2) to (i); + \draw[edge] (i) to (y1); + \draw[edge] (i) to (y2); + \draw[edge] (i) to (y3); + \draw[edge] (x1) to (y1); + \draw[edge] (x2) to (y3); + \node[draw=none, rectangle] (nodes) [fit=(x1) (x2) (y1) (y2) (y3) (i)] {}; + \node[draw=none, rectangle, anchor=south] (title) at (nodes.north) {Causal DAG}; + \end{scope} + % DAG outline + \node[draw, rectangle] (dag) [fit=(dag-nodes) (dag-title) (dag-title)] {}; + + % Data + \begin{scope}[name prefix=data-, local bounding box=test-data, shift={($(dag) + (5, 1.12)$)}] + \node[draw=none, rectangle] (title) {Test Data}; + \node[anchor=north] (table) at (title.south) { + \begin{tabular}{rrrrrr} + \toprule + $X_1$ & $X_2$ & $I$ & $Y_1$ & $Y_2$ & $Y_3$ \\ + \midrule + 1.2 & ``UK'' & 0.3 & 7.8 & 4 & True \\ + 3.2 & ``UK'' & 0.1 & 7.6 & 8 & False \\ + \multicolumn{6}{c}{$\vdots$} \\ + \bottomrule + \end{tabular} + }; + \node[draw, rectangle] [fit=(title) (table)] {}; + \end{scope} + + + %Information flow + \draw[edge, dashed] (dag.290) -- (ci-ci.160); + \draw[edge, dashed] (dag) -- node [left] {\texttt{generate}} (test-test-case.north); + \draw[edge, dashed] (test-test-case) -- node [below, shift={(-0.12, 0)}] {\texttt{test}} (ci-ci); + + \draw[edge, dashed] (data-test-data.south) -- (data-test-data |- ci-ci.north); + \draw[edge, dashed] (ci-ci) -- (estimate-estimate); + + \draw[edge, dashed] (estimate-estimate) -- (oracle-test-oracle.west |- estimate-estimate); + \draw[edge, dashed] (oracle-test-oracle.east |- outcome-test-outcome) -- (outcome-test-outcome); +\end{tikzpicture} +\end{document} diff --git a/docs/source/background.rst b/docs/source/background.rst deleted file mode 100644 index 33f5104e..00000000 --- a/docs/source/background.rst +++ /dev/null @@ -1,109 +0,0 @@ -Background -===================================== - - -.. raw:: html - - - -.. raw:: html - - - -.. container:: zoom-container - - .. figure:: ../../images/schematic.png - :class: zoomable-image - :alt: Schematic diagram of the Causal Testing Framework - :align: center - - **Figure:** Schematic diagram of the Causal Testing Framework. - This figure illustrates the modular architecture and data flow between key components. - -.. raw:: html - -
- -The Causal Testing Framework primarily consists of the following 3 components: 1) Modelling Scenario, 2) Causal Graph, and 2) Causal Test Case. - -#. - :doc:`Causal Graph <../modules/causal_specification>`\ : To apply graphical causal inference techniques for testing, we need a *specification*. - For this, we must specify the variables that are relevant to the modelling scenario of interest, and any constraints over them. - We must also specify the expected causal relationships between the variables as a directed acyclic graph (DAG). - - -#. - :doc:`Causal Tests <../modules/causal_testing>`\ : With a causal specification in hand, we can now design a series of test cases that interrogate the causal relationships of interest in the scenario-under-test. - Informally, a causal test consists of an input configuration, an intervention which is applied to the input, and the expected *causal effect* of that intervention on some output. - In other words, a causal test case states the expected causal effect of a particular intervention made to an input configuration. - For each modelling scenario, the user should create a set of causal tests. - Once a causal test case has been defined, it can be evaluated as follows: - - a. Using the causal specification, identify an estimand for the effect of the intervention on the output of interest, where the *estimand* represents a statistical procedure capable of estimating the causal effect of the intervention on the output. - #. Apply a statistical estimator (e.g. ``linear regression``) to the data to obtain a point estimate for the causal effect. - Depending on the estimator used, confidence intervals may also be obtained at a specified significance level, e.g. 0.05 corresponds to 95% confidence intervals (optional). - The :doc:`Estimators Overview <../modules/estimators>` contains a list of the various estimators we support. - #. Return the casual test result including a point estimate and 95% confidence intervals, usually quantifying the average treatment effect (ATE). - #. Compare the estimated causal effect to the expected causal effect specified in the causal test case. - The test passes if the two match, and fails otherwise. - -For more information on each of these components, follow the links above to their respective module description pages. diff --git a/docs/source/conf.py b/docs/source/conf.py index 7f81d7d1..94126a7c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -33,6 +33,8 @@ "sphinx.ext.autosummary", ] +autosectionlabel_prefix_document = True + autosummary_generate = True autosummary_imported_members = False @@ -78,7 +80,7 @@ html_theme = "sphinx_rtd_theme" # Static files such as CSS or images -html_static_path = ["_static", os.path.abspath("../../images")] +html_static_path = ["_static"] # Custom CSS html_css_files = ["css/custom.css"] diff --git a/docs/source/dafni.rst b/docs/source/dafni.rst new file mode 100644 index 00000000..608be752 --- /dev/null +++ b/docs/source/dafni.rst @@ -0,0 +1,43 @@ +Using the CTF on DAFNI +====================== + +The Causal Testing Framework is also available to run on `DAFNI `_, allowing you to generate causal tests and evaluate causal effects from your input data and DAGs without installing the framework locally. This lets you integrate CTF into workflows with other models or datasets easily. + +Data tab +-------- + +- Upload the required input files as a dataset. Typically, this includes: + + - ``dag.dot`` – the directed acyclic graph defining causal relationships between variables. + - ``runtime_data.csv`` – the CSV file containing runtime input data. + - ``causal_tests.json`` – optional; if provided, the framework will run tests directly. Otherwise, tests will be generated automatically. + + **Note:** Input files must remain in the ``data/inputs`` structure; this is required by the workflow. + +Workflow tab +------------- + +- Select the CTF workflow. +- In the Parameter sets section, click **Create**. +- In the page that opens: + + - Select the model in the workflow (typically ``causal-testing-framework``). + - Complete the sections at the bottom: + + - **Parameters:** Set or confirm environment variables from the ``.env`` file (e.g., ``EXECUTION_MODE``, ``CAUSAL_TESTS``, ``CAUSAL_TEST_RESULTS``). These control whether tests are generated or executed, the filenames, estimator, effect type, and other runtime options. + - **Datasets:** Click the icon and select the dataset containing your input files (``dag.dot``, ``runtime_data.csv``, ``causal_tests.json``). All input files will be placed in the required ``data/inputs`` directory when running the workflow. + +- Unselect the model if needed, click **Continue**, and complete any required metadata such as the name of the parameter set. + +Execute the workflow +---------------------- + +- Click **Execute workflow with parameter set**. +- If successful, the workflow will either generate ``causal_tests.json`` (if not provided) or run the tests and create ``causal_test_results.json`` in ``data/outputs``. +- After completion, you can view the results in the **Data tab** as a new output dataset. + +Customisation and chaining +-------------------------- + +- You can create additional workflows to customise input parameters, filenames, or estimators. +- Multiple CTF workflows can also be chained to run sequential analyses or to integrate with other models and datasets, combining results for more complex causal testing scenarios. diff --git a/docs/source/glossary.rst b/docs/source/glossary.rst index 6a6632ae..27ce98f2 100644 --- a/docs/source/glossary.rst +++ b/docs/source/glossary.rst @@ -5,53 +5,82 @@ Glossary .. glossary:: - Causal inference - Causal inference (:abbr:`CI (Causal Inference)`) is a family of statistical techniques designed to quantify and establish **causal** relationships in data. In contrast to purely statistical techniques that are driven by associations in data, CI incorporates knowledge about the data-generating mechanisms behind relationships in data to derive causal conclusions. + Adjustment + The process of controlling, or "taking into account", variables other than the treatment and outcome in order to calculate an unbiassed effect estimate. - Causal DAG - A Directed Acyclic Graph depicting the direct causal relationships between variables, in which an edge ``X -> Y`` indicates that ``X`` directly causes ``Y``. That is, there exists an intervention on ``X`` which brings about a change in ``Y``. + Adjustment Set + A set of variables that must be controlled or "taken into account" to calculate an unbiassed effect estimate. - **Treatment Variable** - The changed variable of interest (``X``). + Causal inference + Causal inference (:abbr:`CI (Causal Inference)`) is a family of statistical techniques designed to quantify and establish **causal** relationships in data. + In contrast to purely statistical techniques that are driven by associations in data, CI incorporates knowledge about the data-generating mechanisms behind relationships in data to derive causal conclusions. - **Outcome Variable** - The observed variable of interest (``Y``). + Causal DAG + A Directed Acyclic Graph depicting the direct causal relationships between variables, in which an edge ``X -> Y`` indicates that ``X`` directly causes ``Y``. + That is, there exists an intervention on ``X`` which brings about a change in ``Y``. - Causal Test Case - Formally, a causal test case is a 4-tuple ``(M, X, delta, Y)`` that captures the expected causal effect, Y, of an intervention, ``delta``, made to an input valuation, ``X``, on some model outcome in the context of modelling scenario ``M``. - Simply put, causal tests are essentially `metamorphic tests `_ that are executed using statistical causal inference. + Causal Test Case + A causal test case asserts an expected causal effect on an :term:`outcome variable` that results from an :term:`intervention` (change) on a :term:`treatment variable` `X`. + See `this paper ` for a formal definition and extended explanation. + Causal test cases can be as simple as `X has a causal effect on Y` or as complex as `Y should triple when we change X from 3 to 4 while Z is held constant at 8`. - Directed acyclic graph - DAG - A directed acyclic graph (:abbr:`DAG (Directed Acyclic Graph)`) is a graphical representation used in causal inference to model and visualize relationships between variables. - In a DAG, nodes represent variables, and directed edges between nodes indicate causal relationships, with the absence of cycles ensuring acyclicity. + Confidence Intervals + The range of values that are likely to contain the "true" causal effect value, with respect to a given significance level. + For example, if we estimate the 95% confidence intervals, this can be interpreted as meaning that if the same data generation procedure were repeated 100 times from the same underlying population, approximately 95 of the resulting intervals would be expected to contain the true value. + There are also alternative interpretations in the literature, which interested readers are invited to investigate in their own time. - Estimate Type - The effect measure to use, typically ATE, CATE, Risk Ratio, or Odds Ratio. + DAG + Directed acyclic graph + A directed acyclic graph (:abbr:`DAG (Directed Acyclic Graph)`) is a graphical representation used in causal inference to model and visualize relationships between variables. + In a DAG, nodes represent variables, and directed edges between nodes indicate causal relationships, with the absence of cycles ensuring acyclicity. - **ATE** - **Average treatment effect** (:abbr:`ATE (Average Treatment Effect)`): The additive difference in the outcome between the control and treatment populations. + Effect Measure + Effect Measures + The effect measure to use, typically ATE, CATE, Risk Ratio, or Odds Ratio. - **CATE** - **Conditional ATE** (:abbr:`CATE (Conditional Average Treatment Effect)`): The additive difference in the outcome between the control and treatment populations across different strata of the population. + ATE + Average Treatment Effect + The additive difference in the outcome between the control and treatment populations. - **Risk Ratio** - The multiplicative difference in the outcome between the control and treatment populations. + CATE + Conditional Average Treatment Effect + The additive difference in the outcome between the control and treatment populations across different strata of the population. - **Odds Ratio** - The ratio of the odds of A in the presence of B and the odds of A in the absence of B. + Risk Ratio + The multiplicative difference in the outcome between the control and treatment populations. - Intervention - An intervention ``delta : X -> X'`` is a function which manipulates the values of a subset of input valuations. + Odds Ratio + The ratio of the odds of A in the presence of B and the odds of A in the absence of B. - Minimal Adjustment Set - The smallest set of variables which must be controlled, or "adjusted for", to produce an unbiased estimate of causal effect. + Identification + The process of analysing a causal DAG to determine the variables which should be *adjusted for* in order to calculate an unbiassed causal effect of a treatment variable X on an outcome variable Y. + Interested readers can find a more technical definition `here `_. - Scenario - A modelling scenario ``M`` is a pair ``(X, C)`` where ``X`` is a non-strict subset of the model's input variables and ``C`` is a set of constraints over valuations of ``C``, which may be empty. + Inestimable + When a test case is evaluated with insufficient data to calculate a causal effect estimate at all, the test will return an *inestimable* outcome (rather than pass or fail). + This is typically an indication that the data has violated the :doc:`positivity ` assumption, fundamental for causal inference. - Scenario Execution - A software execution satisfying a given modelling scenario. + Intervention + An intervention ``delta : X -> X'`` is a function which manipulates the values of a subset of input valuations. - Test Oracle - A test oracle determines whether the observed outcome is correct. In our framework, this is whether the expected causal effect matches the estimated causal effect. + Minimal Adjustment Set + The smallest set of variables which must be controlled, or "adjusted for", to produce an unbiased estimate of causal effect. + + Outcome Variable + The variable in a :term:`causal test case` that is being observed. + This is also referred to as the "dependent variable" in some fields. + + Potential Outcome + When we run a system under a particular configuration, we can observe one of several possible (of *potential*) outcomes. + Interested readers can find a more technical definition `here `_. + + Test Oracle + A test oracle determines whether the observed outcome is correct. In our framework, this is whether the expected causal effect matches the estimated causal effect. + + Test Adequacy + A measurement of how well a given system has been tested. + While all metrics are approximate, the common goal is that a better score should indicate a lower probability of failure. + + Treatment Variable + The variable in a :term:`causal test case` that is being changed. + This is also referred to as the "independent variable" in some fields. diff --git a/docs/source/index.rst b/docs/source/index.rst index e0536a98..e9590670 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,43 +7,102 @@ Welcome to the Causal Testing Framework Motivation ---------- -A common problem in computer science is to develop robust and reliable software systems that can perform correctly under various input configurations and maintain consistency across complex, physical scenarios. However, software systems, and more specifically computational models, can be difficult to test: they may contain hundreds of parameters, making testing all possible inputs computationally infeasible; some models may be inherently non-deterministic, producing different outputs for the same inputs due to randomness; or there may exist hidden causal relationships between input-output pairs, causing errors that only appear under specific combinations of input configurations. +From predicting the weather to simulating disease transmission, scientific software plays an increasingly pivotal role in developing scientific understanding that informs our everyday lives. +However, they are also some of the most difficult software systems to properly test. +They have large, complex input spaces, are computationally expensive to run, often rely on stochastic black-box components, and are applied in exploratory contexts where the expected outcomes are not known. +From a practical standpoint, the time and effort that can be dedicated to testing is often limited, especially in an academic context, making it especially important to maximise the efficiency of the limited number of test runs we are able to perform. -The Framework -------------- +The Causal Testing Framework has two main workflows: -The Causal Testing Framework is composed of a :term:`causal inference`-driven architecture designed for functional black-box testing. -It leverages graphical causal inference (CI) techniques to specify and evaluate software behaviour from a black-box perspective. -Within this framework, causal directed acyclic graphs (DAGs) are used to represent the expected cause–effect relationships between -the inputs and outputs of the system under test, supported by mathematical foundations for designing statistical procedures that -enable causal inference. Each causal test case targets the causal effect of a specific intervention on the system under test--that is, -a deliberate modification to the input configuration expected to produce a corresponding change in one or more outputs. + - **Causal Testing** involves specifying the expected causal relationships and testing that the data conforms to this. + - **Causal Discovery** involves infering the causal effects from the data and checking that the model is reasonable. + +Causal Testing +-------------- + + .. figure:: _static/images/testing-workflow.png + :alt: Schematic diagram of the Causal Testing Workflow. + :align: center + + **Figure:** Schematic diagram of the Causal Testing Workflow. + +The Causal Testing Framework uses graphical :term:`causal inference` to specify and validate software behaviour by estimating the causal effects between variables. +This requires three main components: + +#. + :doc:`Causal Graph <../modules/causal_dag>`\ : This specifies the expected causal relationships between the variables in the form of a directed acyclic graph (DAG). + The nodes in your DAG represent variables in your system, and edges between the variables represent the "flow of causality" such that an edge from X to Y represents the value of Y being caused (i.e. directly affected) by the value of X. + +#. + :doc:`Test Data <../modules/test_data>`\ : This is the data that will be used to estimate the causal effects between variables and evaluate your causal test cases. + This takes the form of a table in which columns represent the variables in your DAG and each row represents a run of the system. + +#. + :doc:`Causal Tests <../modules/causal_tests>`\ : Each causal test case validates that the causal effect between the *treatment* and *outcome* variable that can be estimated from the :doc:`test data <../modules/test_data>` is as expected. + The most basic causal test case simply validates the presence or absence of a causal effect. + The Causal Testing Framework can automatically generate a suite of such tests from the causal DAG alone. + You can then customise and refine these tests to suite your needs. + +An example of this workflow can be seen in our :doc:`tutorials `\. + +You do not need to be familiar with causal inference to use the causal testing framework, since the technical parts can all be handled automatically "under the hood". +Throughout this documentation, we will gently and informally introduce the various concepts that are necessary to understand and use the framework. +Interested readers are invited to check out the following additional resources for a more detailed introduction, as well as our `paper `_ on causal testing. + +* `The Book of Why `_ provides a "pop science" introduction and motivation to causal inference. +* `Causal Inference in Statistics: A Primer `_ provides a slightly more technical, yet still mostly approachable introduction. +* `What if `_ provides a more technical introduction with formal definitions and examples for those with a statistical background. +* `Causality: Models, Reasoning, and Inference `_ provides a highly technical introduction with formal definitions, proofs, and mathematical details. + +Causal Discovery +---------------- + +.. figure:: _static/images/discovery-workflow.png + :alt: Schematic diagram of the Causal Discovery Workflow. + :align: center + + **Figure:** Schematic diagram of the Causal Discovery Workflow. + +An alternative, more analytical approach involves using the Causal Testing Framework to automatically "discover" the causal relationships between variables from execution data. +This approach is well suited to users who do not have a concrete notion of the expected causal relationships between variables upfront, or are looking to gain an understanding of how an unfamiliar system works, +Note that we do not recommend using discovered DAGs for Causal Testing without careful manual inspection, since such graphs will trivially lead to passing test cases. -If you have any questions about our framework, you can also reach us by `email `__. .. toctree:: - :hidden: - :caption: Home -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Introduction + :maxdepth: 1 + :hidden: + + installation - background - installation - tutorials +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Quick Start + quick_start/causal_testing + quick_start/causal_discovery .. toctree:: :hidden: :maxdepth: 1 :caption: Module Descriptions - /modules/causal_specification - /modules/estimators - /modules/custom_estimators - /modules/causal_testing - /modules/discovery + /modules/causal_dag + /modules/test_data + /modules/causal_tests + /modules/causal_estimate + /modules/test_oracle + /modules/test_adequacy + /modules/causal_discovery + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Tutorials + + tutorials/vaccinating_elderly/vaccinating_elderly_tutorial + tutorials/poisson_line_process/poisson_line_process_tutorial + tutorials/visualising_causal_test_results/visualise_causal_test_results .. toctree:: :maxdepth: 2 @@ -53,6 +112,13 @@ If you have any questions about our framework, you can also reach us by `email < /autoapi/index +.. toctree:: + :maxdepth: 1 + :hidden: + :caption: DAFNI integration + + dafni + .. toctree:: :hidden: :maxdepth: 1 diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 0667a06b..2df644d4 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -1,22 +1,23 @@ -Getting Started +Installation ================ -Installation ------------------ * We currently support Python versions 3.11, 3.12, 3.13, and 3.14. -* The Causal Testing Framework can be installed through `conda-forge`_ (recommended), the `Python Package Index (PyPI)`_, or directly from source (recommended for contributors). +* The Causal Testing Framework can be installed through `conda-forge`_ (recommended), the `Python Package Index (PyPI)`_, or directly from `source`_ (recommended for contributors). .. _conda-forge: https://anaconda.org/conda-forge/causal-testing-framework -.. _Python Package Index (PyPI): https://pypi.org/project/causal-testing-framework/ +.. _Python Package Index (PyPI): https://pypi.org/project/causal-testing-framework +.. _source: https://github.com/CITCOM-project/CausalTestingFramework .. note:: We recommend you use a 64-bit OS (standard in most modern machines) as we have had reports of the installation crashing on legacy 32-bit Debian systems. Method 1: Installing via conda-forge (Recommended) -................................................... +-------------------------------------------------- -**We recommend using conda or mamba for installation**, as they provide better dependency management and environment isolation, particularly for scientific computing workflows. +**We recommend using conda or mamba for installation**, as they provide better dependency management and environment isolation. +Some of our dependencies (most notably scipy) can be tricky to install on Windows as they require certain system dependencies. +Anaconda alleviates most of these installation issues. First, create a new conda environment with a supported Python version, e.g:: @@ -37,7 +38,7 @@ Install :code:`causal-testing-framework`:: Method 2: Installing via pip -.............................. +---------------------------- If you prefer using pip or need the development packages, you can install from PyPI. @@ -53,7 +54,7 @@ If you also want to install the framework with (optional) development packages/t Method 3: Installing via Source (For Developers/Contributors) -............................................................... +------------------------------------------------------------- If you're planning to contribute to the project or need an editable installation for development, you can install directly from source:: @@ -82,66 +83,19 @@ To also install developer tools:: pip install -e .[dev] Verifying Your Installation ------------------------------ - +--------------------------- After installation, verify that the framework is installed correctly in your environment:: python -c "import causal_testing; print(causal_testing.__version__)" Next Steps ------------ - -* Check out the :doc:`tutorials` to learn how to use the framework. -* Read about :doc:`modules/causal_specification` to understand causal specifications and :doc:`modules/causal_testing` for the end-to-end causal testing process. -* Run the command for guidance on how to generate your causal tests directly from your input DAG:: - - causal-testing generate --help - -* and the command on guidance on how to execute your causal tests:: - - causal-testing test --help - - -Using the CTF on DAFNI -====================== - -The Causal Testing Framework is also available to run on `DAFNI `_, allowing you to generate causal tests and evaluate causal effects from your input data and DAGs without installing the framework locally. This lets you integrate CTF into workflows with other models or datasets easily. - -Data tab --------- - -- Upload the required input files as a dataset. Typically, this includes: - - - ``dag.dot`` – the directed acyclic graph defining causal relationships between variables. - - ``runtime_data.csv`` – the CSV file containing runtime input data. - - ``causal_tests.json`` – optional; if provided, the framework will run tests directly. Otherwise, tests will be generated automatically. - - **Note:** Input files must remain in the ``data/inputs`` structure; this is required by the workflow. - -Workflow tab -------------- - -- Select the CTF workflow. -- In the Parameter sets section, click **Create**. -- In the page that opens: - - - Select the model in the workflow (typically ``causal-testing-framework``). - - Complete the sections at the bottom: - - - **Parameters:** Set or confirm environment variables from the ``.env`` file (e.g., ``EXECUTION_MODE``, ``CAUSAL_TESTS``, ``CAUSAL_TEST_RESULTS``). These control whether tests are generated or executed, the filenames, estimator, effect type, and other runtime options. - - **Datasets:** Click the icon and select the dataset containing your input files (``dag.dot``, ``runtime_data.csv``, ``causal_tests.json``). All input files will be placed in the required ``data/inputs`` directory when running the workflow. - -- Unselect the model if needed, click **Continue**, and complete any required metadata such as the name of the parameter set. - -Execute the workflow ----------------------- - -- Click **Execute workflow with parameter set**. -- If successful, the workflow will either generate ``causal_tests.json`` (if not provided) or run the tests and create ``causal_test_results.json`` in ``data/outputs``. -- After completion, you can view the results in the **Data tab** as a new output dataset. +---------- -Customisation and chaining --------------------------- +* Read the quick start guides for :doc:`quick_start/causal_testing` and :doc:`quick_start/causal_discovery`. +* Check out tutorials to learn how to use the framework. +* Run the following commands for guidance on the various commands and parameter option :: -- You can create additional workflows to customise input parameters, filenames, or estimators. -- Multiple CTF workflows can also be chained to run sequential analyses or to integrate with other models and datasets, combining results for more complex causal testing scenarios. + causal-testing --help # To see the available commands + causal-testing generate --help # To see how to generate your causal tests from your DAG + causal-testing test --help # To see how to execute your causal tests + causal-testing discover --help # To see how to discover causal structures from your data diff --git a/docs/source/modules/causal_dag.rst b/docs/source/modules/causal_dag.rst new file mode 100644 index 00000000..46eefd75 --- /dev/null +++ b/docs/source/modules/causal_dag.rst @@ -0,0 +1,43 @@ +Causal DAG +========== + +As in traditional software testing, the specification defines the expected behaviour of the system. +In the Causal Testing Framework, this specification takes the form of a directed acyclic graph (DAG) that sets out the expected causal relationships between variables in a system. +To do this, we use the `DOT language `_, which provides an intuitive text-based representation of DAGs in which an edge from node :code:`X` to :code:`Y` is specified as :code:`X -> Y;`. + +As an example, consider the DAG for the :doc:`vaccinating the elderly tutorial <../tutorials/vaccinating_elderly/vaccinating_elderly_tutorial>`. +This scenario has two inputs `vaccine` and `max_doses` and three outputs `cum_vaccinations`, `cum_vaccinated`, and `cum_infections`. +We do not expect `max_doses` to have a causal effect on any of the outputs since this remains constant throughout modelling scenario. + +.. image:: ../../../examples/covasim_/vaccinating_elderly/dag.png + :alt: Causal DAG of the vaccinating the elderly modelling scenario + +.. literalinclude:: ../../../examples/covasim_/vaccinating_elderly/dag.dot + :language: graphviz + :caption: **Figure:** Example Causal DAG for the vaccinating the elderly example. + + +Specifying Causal DAGs +---------------------- + +Unfortunately, there is very little universally applicable guidance for specifying a DAG, since the expected relationships will vary greatly between systems. +However, if you are in doubt as to whether an edge should be included between a pair of variables, the `advice from the causal inference community `_ is to include the edge. +Is a stronger assumption to exclude an edge (which signifies a known independence between two variables) than to include one (which signifies the possibility of a causal effect). +While you can use our :doc:`causal discovery ` tools to infer a DAG from a dataset, you should not use this for causal testing without careful examination and validation of the inferred relationships to ensure that they are sensible and meaningful. + +Causal Identification +--------------------- + +A key step in the evaluation of causal test cases is :term:`identification`. +The Causal Testing Framework carries out this process entirely automatically, so you do not need to know the technicalities of this to perform causal testing. +Interested readers can find a technical explanation `here `_, but the process essentially involves inspecting the causal relationships in the DAG and picking out variables which should be *adjusted for* in order to calculate an unbiassed causal effect. + +The intuition is embedded in the old adage "correlation does not necessarily imply causation". +A real-world example of this is that we would not expect the number of fans sold on any particular day to have a direct causal effect on the number of ice creams sold --- there is nothing about owning a fan that would inherently compel anyone to buy ice cream. +However, there will be a *correlation* between the two, since both are more likely to be sold when it is hot outside. +Therefore, to test the absence of causality, we must take temperature into account when estimating the causal effect. + +In the causal testing framework, the causal DAG not only serves as the specification, but also to perform this identification. +The key benefit of this is that it allows us to use pre-existing datasets that were not specially curated for testing while still maintaining trustworthy test outcomes. +Therefore, it is important that all relevant variables are recorded in the DAG, even if they are missing from the data or cannot be meaningfully recorded. +While this may occasionally require :doc:`special estimation techniques <../modules/causal_estimate>` to adjust for unobserved variables, and may occasionally result in untestable relationships, it prevents biassed causal effect estimates from leading to untrustworthy test outcomes. diff --git a/docs/source/modules/discovery.rst b/docs/source/modules/causal_discovery.rst similarity index 77% rename from docs/source/modules/discovery.rst rename to docs/source/modules/causal_discovery.rst index ff9d4126..6a9770e6 100644 --- a/docs/source/modules/discovery.rst +++ b/docs/source/modules/causal_discovery.rst @@ -1,33 +1,32 @@ -================ Causal Discovery ================ -The Causal Discovery tool generates a directed acyclic graph (DAG) that represents the causal relationships between -variables in your input dataset(s). This generated DAG can then serve as the foundational causal specification for +The :doc:`causal discovery tool <../quick_start/causal_discovery>` generates a directed acyclic graph (DAG) that represents the causal relationships between +variables in your input dataset(s). This generated DAG can then serve as the foundational causal specification for your causal model. .. note:: - Automated causal discovery is a starting point. The resulting DAG must always be manually inspected to ensure it + Automated causal discovery is a starting point. The resulting DAG must always be manually inspected to ensure it is a valid representation of your system. Configuration Options --------------------- The tool supports various configurations to tailor the discovery process to your specific dataset and domain knowledge: -* **Technique:** - Choose the causal discovery algorithm to use. Currently supported techniques are HillClimb and - NSGA. You can add custom techniques by implementing abstract_discovery.py and adding the new technique to your +* **Technique:** + Choose the causal discovery algorithm to use. Currently supported techniques are HillClimb and + NSGA. You can add custom techniques by implementing abstract_discovery.py and adding the new technique to your endpoints in pyproject.toml. - For additional control on the search process, you can specify the following optional parameters: + For additional control on the search process, you can specify the following optional parameters: - * For both techniques, you can specify the ``max_iterations`` and ``random_seed`` parameters. As well as domain + * For both techniques, you can specify the ``max_iterations`` and ``random_seed`` parameters. As well as domain knowledge constraints (see below). - * For the HillClimbing technique, you can specify the ``max_iterations_without_improvement`` parameter. Where - max_iterations_without_improvement is the number of iterations after an improvement is found, before the algorithm - widens its search space to avoid getting stuck in a local minima. - + * For the HillClimbing technique, you can specify the ``max_iterations_without_improvement`` parameter. Where + max_iterations_without_improvement is the number of iterations after an improvement is found, before the algorithm + widens its search space to avoid getting stuck in a local minima. + * If you are using the NSGA technique, you can specify the ``population_size`` and ``num_parents_mating`` parameters. Parameter defaults: @@ -37,11 +36,11 @@ The tool supports various configurations to tailor the discovery process to your | ``num_parents_mating``: 2 | ``random_seed``: 0 -* **Domain Knowledge (Edge Constraints)** You can explicitly include or exclude specific edges in the output DAG using +* **Domain Knowledge (Edge Constraints)** You can explicitly include or exclude specific edges in the output DAG using dot files. Regular expressions (regex) are supported. - **Example:** Consider the DAG for the `vaccinating the elderly - `_ + **Example:** Consider the DAG for the `vaccinating the elderly + `_ modelling scenario. This scenario has two inputs: ``vaccine`` and ``max_doses``. .. container:: zoom-container @@ -50,33 +49,33 @@ The tool supports various configurations to tailor the discovery process to your :class: zoomable-image :alt: Causal DAG of the vaccinating the elderly modelling scenario - * **Excluding Edges:** If domain knowledge dictates that ``max_doses`` has no causal effect on any outputs, you + * **Excluding Edges:** If domain knowledge dictates that ``max_doses`` has no causal effect on any outputs, you can specify ``max_doses -> ".*"`` and ``".*" -> max_doses`` in the *exclude edges* dot file. - * **Including Edges:** If it is known that ``vaccine`` directly affects all three outputs, you can specify + * **Including Edges:** If it is known that ``vaccine`` directly affects all three outputs, you can specify ``vaccine -> "cum_.*"`` in the *include edges* dot file. - * **Specifying Variables:** You can specify the variables to include in the discovery process using the + * **Specifying Variables:** You can specify the variables to include in the discovery process using the ``--variables`` argument. If not specified, all variables in the input dataset(s) will be considered. -* **Examples:** To generate a DAG using the HillClimb technique with a maximum of 500 iterations, with 25 iterations +* **Examples:** To generate a DAG using the HillClimb technique with a maximum of 500 iterations, with 25 iterations without improvement, a random seed of 63, you can use the following command: .. code-block:: bash - + causal-testing discover \ --technique HillClimberDiscovery \ --data-paths /test_data1.csv /test_data2.csv \ --output /tmp/resultant_dag.dot \ - --technique-kwargs max_iterations=500 max_iterations_without_improvement=25 \ + --technique-kwargs max_iterations=500 max_iterations_without_improvement=25 \ random_seed=63 - Or to generate a DAG using the NSGA technique with a population size of 10, a num_parents_mating of 3, and + Or to generate a DAG using the NSGA technique with a population size of 10, a num_parents_mating of 3, and specified included and excluded edges, you can use the following command: .. code-block:: bash - + causal-testing discover \ --technique NSGADiscovery \ --data-paths /test_data1.csv \ --output /tmp/resultant_dag.dot \ --include-edges /include_edges.dot \ --exclude-edges /exclude_edges.dot \ - --technique-kwargs population_size=10 num_parents_mating=3 \ No newline at end of file + --technique-kwargs population_size=10 num_parents_mating=3 diff --git a/docs/source/modules/causal_estimate.rst b/docs/source/modules/causal_estimate.rst new file mode 100644 index 00000000..1a322837 --- /dev/null +++ b/docs/source/modules/causal_estimate.rst @@ -0,0 +1,160 @@ +Causal Estimate +=============== + +This page provides an overview on how to choose the most appropriate estimator for your workflow. +When using the :code:`generate` command to generate causal tests from a causal DAG, the estimator used is chosen based on the datatype of your outcome variable: + +* Linear regression is used for numerical variables +* Logistic regression is used for boolean variables +* Multinomial regression is used for categorical variables + +In general, you won't need to change this. +However, if you have variables that are not recorded in your data, you may need to use the *instrumental variable estimator*. +This uses the concept of `instrumental variables `_ to :term:`adjust for ` variables without needing to know their values. +To change the estimator, you will need change the name of the estimator in the JSON representation of the test cases produced by the :code:`generate` command. +Depending on which estimator you are using, you may also need to add values for different parameters. +See below for details. + +Another useful customisation option for regression estimators is to modify the formula used for estimation. +By default, the linear, logistic, and multinomial regression estimators all use the formula :code:`Y ~ X + Z1 + Z2 + ...`, where :code:`Y` is the :term:`outcome variable`, :code:`X` is the :term:`treatment variable`, and :code:`Z1`, :code:`Z2`, etc. are the variables in the :term:`adjustment set`. +However, if you know in advance that your causal relationship is non-linear, it may be wise to refine this equation somewhat. +This is mostly quite intuitive, but does have a few quirks. +See the `patsy documentation `_ for an explanation of the available operators. +An example can be seen in our :doc:`tutorials <../tutorials/poisson_line_process/poisson_line_process_tutorial>`. + + +LinearRegressionEstimator +------------------------- + +**Recommended use:** For continuous numerical outcomes (e.g. the number of people who are vaccinated). + +.. autoclass:: causal_testing.estimation.linear_regression_estimator.LinearRegressionEstimator + :members: + :exclude-members: from_formula, regressor + :undoc-members: + :show-inheritance: + :noindex: + +LogisticRegressionEstimator +--------------------------- + +**Recommended use:** For binary outcomes (yes/no, true/false, success/failure). + +.. autoclass:: causal_testing.estimation.logistic_regression_estimator.LogisticRegressionEstimator + :members: + :exclude-members: from_formula, regressor + :undoc-members: + :show-inheritance: + :noindex: + +MultinomialRegressionEstimator +------------------------------ + +**Recommended use:** For categorical outcomes (e.g. colurs: Red, Green, Blue). + +.. autoclass:: causal_testing.estimation.multinomial_regression_estimator.MultinomialRegressionEstimator + :members: + :exclude-members: from_formula, regressor + :undoc-members: + :show-inheritance: + :noindex: + +InstrumentalVariableEstimator +----------------------------- + +**Recommended use:** When dealing with unmeasured confounding using instrumental variables. + +.. autoclass:: causal_testing.estimation.instrumental_variable_estimator.InstrumentalVariableEstimator + :members: + :undoc-members: + :show-inheritance: + :noindex: + +IPCWEstimator +------------- + +**Recommended use:** For handling missing data or selection bias using inverse probability of censoring weighting (e.g. time-varying data). + +.. autoclass:: causal_testing.estimation.ipcw_estimator.IPCWEstimator + :members: + :undoc-members: + :show-inheritance: + :noindex: + +ExperimentalEstimator +--------------------- + +**Recommended use:** For randomised controlled trials or experimental data where treatment assignment is randomised. + Directly runs the system under test multiple times with different configurations (e.g. you need to collect new data by executing your system multiple times). + +.. autoclass:: causal_testing.estimation.experimental_estimator.ExperimentalEstimator + :members: + :undoc-members: + :show-inheritance: + :noindex: + +Custom Estimators +----------------- + +If the above estimators are not sufficient for your needs, you can implement your own custom estimator by extending the :code:`Estimator` class and implementing the abstract :code:`add_modelling_assumptions` method and the estimation method for the causal effect measure you wish to calculate. +For example, if you wished to estimate the :term:`ATE` using the empirical mean of the recorded outcome under the control and treatment values, you would need to implement a method called :code:`estimate_ate`. +If you wished to estimate the risk ratio, you would need to call your method :code:`estimate_risk_ratio`. +The code for the :code:`EmpiricalMeanEstimator` is shown below. + +.. code-block:: python + + import pandas as pd + from causal_testing.estimation.abstract_estimator import Estimator + from causal_testing.estimation.effect_estimate import EffectEstimate + + class EmpiricalMeanEstimator(Estimator): + """ + Custom estimator class to estimate the causal effect based on the empirical mean. + """ + + def add_modelling_assumptions(self): + """ + Add modelling assumptions to the estimator. This is a list of strings which list the modelling assumptions that + must hold if the resulting causal inference is to be considered valid. + """ + self.modelling_assumptions += "The data must contain runs with the exact configuration of interest." + + def estimate_ate(self, df: pd.DataFrame) -> EffectEstimate: + """Estimate the outcomes under control and treatment. + :param df: The data to use. + :return: The empirical average treatment effect. + """ + + control_results = df.where(df[self.treatment_variable] == self.control_value)[self.outcome_variable].dropna() + treatment_results = df.where(df[self.treatment_variable] == self.treatment_value)[ + self.outcome_variable + ].dropna() + + def ate(sample1, sample2): + return sample1.mean() - sample2.mean() + + bootstraps = bootstrap((treatment_results, control_results), ate, confidence_level=self.alpha) + return EffectEstimate( + type="ate", + value=ate(treatment_results, control_results), + ci_low=bootstraps.confidence_interval.low, + ci_high=bootstraps.confidence_interval.high, + ) + +Once you have implemented your estimator, you will need to register it as an extra entry point in your project's :code:`pyproject.toml` file so that the Causal Testing Framework can find it. +For example, if you had defined your :code:`EmpiricalMeanEstimator` class in a module called :code:`empirical_mean_estimator` in a folder called :code:`custom_estimators`, you would register it as follows. +You will also need to reinstall your project, e.g. with :code:`pip install -e .` each time you add a new estimator to your :code:`pyproject.toml`. +You do not need to reinstall each time you edit your project for source code edits. + + +.. code-block:: ini + + [project.entry-points."estimators"] + CustomFlakefighter = "custom_estimators.empirical_mean_estimator:EmpiricalMeanEstimator" + +Of course, for this to work, your module needs to be discoverable on your python path. +That is, you should be able to execute :code:`from custom_estimators.empirical_mean_estimator import EmpiricalMeanEstimator` successfully from within the current working directory. + +You can also add your custom estimator to causal test cases specified in JSON. +To do so, you can simply set the :code:`estimator` property to the name of your estimator class and the :code:`estimate_type` property to the name of your causal effect measure. +In the above :code:`EmpiricalMeanEstimator` example, :code:`estimator` would be set to :code:`"EmpiricalMeanEstimator"` and :code:`estimate_type` would be set to :code:`"ate"`. diff --git a/docs/source/modules/causal_specification.rst b/docs/source/modules/causal_specification.rst deleted file mode 100644 index 4db13b02..00000000 --- a/docs/source/modules/causal_specification.rst +++ /dev/null @@ -1,121 +0,0 @@ - -Causal Specification -===================== - -As in traditional software testing, the specification defines the expected behaviour of the system. -In causal testing, this is made up of two components: the modelling scenario and the causal graph. -These components are then used to design statistical experiments that can answer causal questions about the system-under-test. - -1. Modelling Scenario ---------------------- - -- In causal testing, our units of interest are specific usage **scenarios** of the system-under-test. - For example, when testing an epidemiological computational model, one scenario could focus on the simulation of the spread of a virus through a population. - For this scenario, we may then test how a number of interventions should **cause** some outputs to change e.g. vaccinations should reduce the total number of deaths. - -- Each scenario is defined as a series of constraints placed over a set of input variables. - A constraint is simply a mapping from an input variable to a specific value or distribution that characterises the scenario in question. - For example, a scenario simulating the spread of a virus would likely place constraints on the location, population demographics, and who is vaccinated. - -- Requirements for this scenario should describe how a particular intervention - (e.g.changing the number of people, changing who is vaccinated, etc.) is expected to cause a particular outcome (number of infections, deaths, R0, etc.) to change. - The way these requirements are expressed is up to the user, however, it is essential that they focus on the expected effect of an intervention. - -2. Causal Graph ---------------- - -To isolate the causal effect of the defined interventions, the user needs to express the anticipated cause-effect relationships amongst the inputs and outputs involved in the scenario. -This is done using a directed acyclic graph (DAG) in which nodes represent variables in the system and edges represent causal effects. -In order to apply CI techniques, we need to capture causality amongst the inputs and outputs in the scenario-under-test. -Therefore, for each scenario, the user must define a causal DAG. - -As an example, consider the DAG shown below for the `vaccinating the elderly example. `_ -This modelling scenario has two inputs `vaccine` and `max_doses` and three outputs `cum_vaccinations`, `cum_vaccinated`, and `cum_infections`. -We do not expect `max_doses` to have a causal effect on any of the outputs since this remains constant throughout modelling scenario. - -.. container:: zoom-container - - .. image:: ../../../examples/covasim_/vaccinating_elderly/dag.png - :class: zoomable-image - :alt: Causal DAG of the vaccinating the elderly modelling scenario - -.. literalinclude:: ../../../examples/covasim_/vaccinating_elderly/dag.dot - :language: graphviz - :caption: **Figure:** Example Causal DAG for the vaccinating the elderly example. - -.. raw:: html - - - -.. raw:: html - - - -.. note:: - - Unfortunately, there is no universally applicable guidance or algorithm that can be followed to create a causal DAG, but there are three general requirements that should be satisfied: - - 1. The DAG must contain all inputs and outputs involved in the scenario-under-test. - - 2. If there are any other variables which are not directly involved but are expected to have a causal relationship with the variables in the scenario-under-test, these should also be added to the graph. For example, the size of the room might be partially caused by the simulated location (house styles, average wealth, etc.), in which case location should be added to the DAG with an edge to room size and any other variables it is deemed to influence. - - 3. If in doubt, add an edge. It’s a stronger assumption to exclude an edge (X and Y are independent) than to include one (X has some potentially negligible causal effect on Y). diff --git a/docs/source/modules/causal_testing.rst b/docs/source/modules/causal_testing.rst deleted file mode 100644 index 3d4e4e6d..00000000 --- a/docs/source/modules/causal_testing.rst +++ /dev/null @@ -1,164 +0,0 @@ -Causal Testing -============== - -A ``causal test`` or ``causal test case`` is the expected change in an outcome that applying an intervention to the input should cause. -In this context, an intervention is simply a function which manipulates the input configuration of the scenario-under-test in a way that is expected to cause a change to some outcome. -Programmatically, the data structure of causal tests can either be a ``.json`` file or hard-coded (e.g. our :doc:`tutorials <../tutorials>` contain examples of how to -encode your causal tests). Moreover, by ``causal testing`` we refer to the overall process and execution of using the ``modelling scenario``, ``causal graph``, and ``causal test case(s)``, including statistical estimators, -to determine whether each test case passes or fails relative to the test oracle. - -Getting Started ---------------- - -To perform causal testing, you need 3 key ingredients: - -1. **Precisely-specified causal test cases** - Define what you want to test with clear treatment and outcome variables (e.g. *"Does wearing a mask reduce infection rates by 10%?"*). - -2. **Data covering the range of parameter values you're interested in** - Ensure your dataset includes observations across the conditions you want to compare (e.g. runs both with and without precautions). - -3. **A correctly specified causal DAG that includes all relevant variables** - Your DAG should capture all the relevant causal relationships in your system, including the variables you can't measure. - -.. note:: - This framework is designed to be practical and usable. It leverages the causal DAG to automatically identify which variables require adjustment to obtain unbiased causal estimates, then applies your chosen statistical estimator to compute the causal effects. - -.. tip:: - **Handling unmeasured confounding:** If instrumental variable methods cannot be applied due to unobserved confounding, you may need to simplify the DAG by removing certain variables. However, doing so can introduce bias into the estimated causal effects. - - .. caution:: - Removing unmeasured confounders from your DAG reduces validity — your estimates may no longer represent the full causal effect. Proceed carefully and document which potential confounders were excluded from the analysis. - -Example: Testing Virus Spread in a Classroom ---------------------------------------------- - -In the following sections, we describe the end-to-end process of ``causal testing`` for a hypothetical epidemiological computational model in which we are testing the model within a classroom scenario. -In particular, suppose we're interested in how various precautions, such as hand-washing and mask-wearing, can prevent the spread of a virus within a classroom. - -1. Modelling Scenario ---------------------- - -For our modelling scenario, suppose we define the scenario with the following constraints: - -* ``n_people ~ Uniform(20, 30)`` (There are between 20 and 30 people in the classroom). -* ``environment = Grid(x,y ~ Uniform(20, 40))`` (The classroom is square grid of between 20x20 and 40x40 units). -* ``n_infected_t0 = 1`` (One person is infected initially). -* ``precaution = None`` (No precautions taken). - We also specify the output we are interested in as ``n_infected_t5``\ , the number of people infected after five days of daily one hour lessons. - - -2. Causal Graph ----------------- - -Then, we create a simple causal directed acyclic graph (DAG), which represents causality amongst these variables: - -.. raw:: html - - - -.. raw:: html - - - -.. container:: zoom-container - - .. figure:: /_static/images/example_dag.png - :class: zoomable-image - :alt: Schematic diagram of the Causal Testing Framework - :align: center - - **Figure:** Pictorial representation of the Causal DAG in this example. - -.. code-block:: - - digraph CausalDAG { - n_people -> n_infected_t5 - environment -> n_infected_t5 - n_people -> environment - n_infected_t0 -> n_infected_t5 - environment -> precaution - precaution -> n_infected_t5 - } - - - -3. Causal Test Cases --------------------- - -We then define a number of causal test cases to apply to the scenario-under-test. For example, supposing we expect mask wearing and hand washing to have a preventative effect: - -* ``mask_wearing_test = (X={precaution = None}, \Delta = {precaution = Mask}, Y = {-20% < n_infected_t5 < -10% })`` (Mask wearing is expected to result in between 10% and 20% fewer infections). -* ``hand_washing_test = (X={precaution = None}, \Delta = {precaution = Hand Washing}, Y = {-40% < n_infected_t5 < -25%})`` (Hand washing is expected to result in between 25% and 40% fewer infections). - -- To run these test cases experimentally, we need to execute both ``X`` and ``\Delta(X)`` - that is, with and without the interventions. Since the only difference between these test cases is the intervention, we can conclude that the observed difference in ``n_infected_t5`` was caused by the interventions. While this is the simplest approach, it can be extremely inefficient at scale, particularly when dealing with complex software such as computational models. - -- To run these test cases observationally, we need *valid* observational data for the scenario-under-test. This means we can only use executions with between 20 and 30 people, a square environment of size betwen 20x20 and 40x40, and where a single person was initially infected. In addition, this data must contain executions both with and without the intervention. Next, we need to identify any sources of bias in this data and determine a procedure to counteract them. This is achieved automatically using graphical causal inference techniques that identify a set of variables that can be adjusted to obtain a causal estimate. Finally, for any categorical biasing variables, we need to make sure we have executions corresponding to each category otherwise we have a positivity violation (i.e. missing data). In the worst case, this at least guides the user to an area of the system-under-test that should be executed. - -4. Causal Testing ------------------ - -- After obtaining suitable test data, we can now apply causal inference. First, as described above, we use our causal graph to identify a set of adjustment variables that mitigate all bias in the data. Next, we use statistical models to adjust for these variables (implementing the statistical procedure necessary to isolate the causal effect) and obtain the desired causal estimate. Depending on the statistical model used, we can also generate 95% confidence intervals (or confidence intervals at any confidence level for that matter). - -- In our example, the causal DAG tells us it is necessary to adjust for ``environment`` in order to obtain the causal effect of ``precaution`` on ``n_infected_t5``. Supposing the relationship is linear, we can employ a linear regression model of the form ``n_infected_t5 ~ p0*precaution + p1*environment`` to carry out this adjustment. If we use experimental data, only a single environment is used by design and therefore the adjustment has no impact. However, if we use observational data, the environment may vary and therefore this adjustment will look at the causal effect within different environments and then provide a weighted average, which turns out to be the partial coefficient ``p0``. - -5. Test Oracle Procedure -------------------------- - -- After conducting causal inference, all that remains is to ascertain whether the causal effect is expected or not. In our example, this is simply a case of checking whether the causal effect on ``n_infected_t5`` falls within the specified range. However, in the future, we may wish to implement more complex oracles. diff --git a/docs/source/modules/causal_tests.rst b/docs/source/modules/causal_tests.rst new file mode 100644 index 00000000..727425af --- /dev/null +++ b/docs/source/modules/causal_tests.rst @@ -0,0 +1,33 @@ +Causal Test Cases +================= + +A :term:`causal test case` asserts the expected change in an :term:`outcome variable` that applying an :term:`intervention` to the :term:`treatment variable` should cause. +Causal test cases can be as simple as `X has a causal effect on Y` or as complex as `Y should triple when we change X from 3 to 4 while Z is held constant at 8`. +Interested readers can find a formal definition and extended explanation in `this paper `. + +A key difference between causal testing and traditional testing is that, with causal testing, the test cases are completely separate entities from the data used to evaluate them. +In traditional testing, the two are almost synonymous. +For those already familiar with traditional testing techniques, this can be very difficult to conceptualise. +However, it is worth taking the time to fully appreciate this separation and the advantages it affords, the main such advantage being that it allows the same data (set of system executions) to be re-used to evaluate *multiple* causal tests. + +Components of a test case +------------------------- + +Causal test cases have three main components. + +1. + The :doc:`estimator ` defines the form of the causal relationship under test and how the causal effect will be estimated. + This is where the treatment and outcome variables are recorded, as well as any variables that must be :term:`adjusted ` for to calculate an unbiassed effect estimate. + +2. + The :term:`effect measure` is the causal effect that will be estimated. + One such metric is the :term:`average treatment effect` is the additive difference in the :term:`outcome variable` that we expect to observe as a result of our :term:`intervention`. + Another such metric is the :term:`risk ratio`, which is the multiplicative difference. + +3. + The :doc:`expected effect ` is the value of the :term:`effect measure` that we expect to see. + For example, we may expect our intervention to cause our outcome variable to decrease by 3, or become 4 times larger. + Alternatively, we not be able to specify the causal effect so precisely and so could validate that the effect measure is positive or negative, or even just that there is some change or no change. + +In addition to these three main components, you can also give each test case a name, for easy recognition. +Test cases can also be individually skipped, which can be useful if particular test cases are known to be problematic in some way, for example taking a long time to run. diff --git a/docs/source/modules/custom_estimators.rst b/docs/source/modules/custom_estimators.rst deleted file mode 100644 index 54d470fa..00000000 --- a/docs/source/modules/custom_estimators.rst +++ /dev/null @@ -1,65 +0,0 @@ -Custom Estimators -================= - -If the supported :ref:`estimators` are not sufficient for your needs, you can implement your own custom estimator by extending the :code:`Estimator` class and implementing the abstract :code:`add_modelling_assumptions` method and the estimation method for the causal effect measure you wish to calculate. -For example, if you wished to estimate the ATE using the empirical mean of the recorded outcome under the control and treatment values, you would need to implement a method called :code:`estimate_ate`. -If you wished to estimate the risk ratio, you would need to call your method :code:`estimate_risk_ratio`. -The code for the :code:`EmpiricalMeanEstimator` is shown below. - -.. code-block:: python - - from causal_testing.estimation.abstract_estimator import Estimator - from scipy.stats import bootstrap - - class EmpiricalMeanEstimator(Estimator): - """ - Custom estimator class to estimate the causal effect based on the empirical mean. - """ - - def add_modelling_assumptions(self): - """ - Add modelling assumptions to the estimator. This is a list of strings which list the modelling assumptions that - must hold if the resulting causal inference is to be considered valid. - """ - self.modelling_assumptions += "The data must contain runs with the exact configuration of interest." - - def estimate_ate(self) -> EffectEstimate: - """Estimate the outcomes under control and treatment. - :return: The empirical average treatment effect. - """ - treatment_variable = self.base_test_case.treatment_variable.name - outcome_variable = self.base_test_case.outcome_variable.name - - control_results = self.df.where(self.df[treatment_variable] == self.control_value)[outcome_variable].dropna() - treatment_results = self.df.where(self.df[treatment_variable] == self.treatment_value)[ - outcome_variable - ].dropna() - - def risk_ratio(sample1, sample2): - return sample1.mean() - sample2.mean() - - bootstraps = bootstrap((treatment_results, control_results), risk_ratio, confidence_level=self.alpha) - return EffectEstimate( - type="risk_ratio", - value=risk_ratio(treatment_results, control_results), - ci_low=bootstraps.confidence_interval.low, - ci_high=bootstraps.confidence_interval.high, - ) - -Once you have implemented your estimator, you will need to register it as an extra entry point in your project's :code:`pyproject.toml` file so that the Causal Testing Framework can find it. -For example, if you had defined your :code:`EmpiricalMeanEstimator` class in a module called :code:`empirical_mean_estimator` in a folder called :code:`custom_estimators`, you would register it as follows. -You will also need to reinstall your project, e.g. with :code:`pip install -e .` each time you add a new estimator to your :code:`pyproject.toml`. -You do not need to reinstall each time you edit your project for source code edits. - - -.. code-block:: ini - - [project.entry-points."estimators"] - CustomFlakefighter = "custom_estimators.empirical_mean_estimator:EmpiricalMeanEstimator" - -Of course, for this to work, your module needs to be discoverable on your python path. -That is, you should be able to execute :code:`from custom_estimators.empirical_mean_estimator import EmpiricalMeanEstimator` successfully from within the current working directory. - -You can also add your custom estimator to causal test cases specified in JSON. -To do so, you can simply set the :code:`estimator` property to the name of your estimator class and the :code:`estimate_type` property to the name of your causal effect measure. -In the above :code:`EmpiricalMeanEstimator` example, :code:`estimator` would be set to :code:`"EmpiricalMeanEstimator"` and :code:`estimate_type` would be set to :code:`"ate"`. diff --git a/docs/source/modules/estimators.rst b/docs/source/modules/estimators.rst deleted file mode 100644 index 20c1a26e..00000000 --- a/docs/source/modules/estimators.rst +++ /dev/null @@ -1,90 +0,0 @@ -.. _estimators: - -Estimators Overview -=================== - -This page provides an overview on how to choose the most appropriate estimator for your workflow. - - -LinearRegressionEstimator -~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Recommended use:** For continuous numerical outcomes (e.g. the number of people who are vaccinated). - -.. autoclass:: causal_testing.estimation.linear_regression_estimator.LinearRegressionEstimator - :members: - :exclude-members: from_formula, regressor - :undoc-members: - :show-inheritance: - :noindex: - -LogisticRegressionEstimator -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Recommended use:** For binary outcomes (yes/no, true/false, success/failure). - -.. autoclass:: causal_testing.estimation.logistic_regression_estimator.LogisticRegressionEstimator - :members: - :exclude-members: from_formula, regressor - :undoc-members: - :show-inheritance: - :noindex: - -MultinomialRegressionEstimator -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Recommended use:** For categorical outcomes (e.g. colurs: Red, Green, Blue). - -.. autoclass:: causal_testing.estimation.multinomial_regression_estimator.MultinomialRegressionEstimator - :members: - :exclude-members: from_formula, regressor - :undoc-members: - :show-inheritance: - :noindex: - -CubicSplineRegressionEstimator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Recommended use:** For continuous outcomes with non-linear relationships or changes in behaviour. -Useful when the relationship between treatment and outcome cannot be captured by a linear model. - -.. autoclass:: causal_testing.estimation.cubic_spline_estimator.CubicSplineRegressionEstimator - :members: - :undoc-members: - :show-inheritance: - :noindex: - - -InstrumentalVariableEstimator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Recommended use:** When dealing with unmeasured confounding using instrumental variables. - -.. autoclass:: causal_testing.estimation.instrumental_variable_estimator.InstrumentalVariableEstimator - :members: - :undoc-members: - :show-inheritance: - :noindex: - -IPCWEstimator -~~~~~~~~~~~~~ - -**Recommended use:** For handling missing data or selection bias using inverse probability of censoring weighting (e.g. time-varying data). - -.. autoclass:: causal_testing.estimation.ipcw_estimator.IPCWEstimator - :members: - :undoc-members: - :show-inheritance: - :noindex: - -ExperimentalEstimator -~~~~~~~~~~~~~~~~~~~~~ - -**Recommended use:** For randomised controlled trials or experimental data where treatment assignment is randomised. - Directly runs the system under test multiple times with different configurations (e.g. you need to collect new data by executing your system multiple times). - -.. autoclass:: causal_testing.estimation.experimental_estimator.ExperimentalEstimator - :members: - :undoc-members: - :show-inheritance: - :noindex: diff --git a/docs/source/modules/test_adequacy.rst b/docs/source/modules/test_adequacy.rst new file mode 100644 index 00000000..7baed46e --- /dev/null +++ b/docs/source/modules/test_adequacy.rst @@ -0,0 +1,40 @@ +Causal Test Adequacy +==================== + +As with all testing techniques, the question of "How do I know when I can stop testing?" is a tricky one to answer. +In once sense, this is actually *easier* to answer for causal testing than in the traditional context, since the causal DAG provides an exhaustive list of all of the causal relationships (and independences) that should be checked. +However, determining whether your data leads to *accurate* causal effect estimates and *trustworthy* test outcomes is another matter. + +Intuitively, we might say that the :term:`confidence intervals` associated with a given causal effect estimate should be able to give us some insight into this since wider intervals indicate a higher level of uncertainty associated with the estimate. +However, many systems are extremely stochastic and can produce outputs that vary by orders of magnitude, even for repeated runs of the same input configuration. +For such systems, the confidence intervals will always be wide, no matter how much data is collected. +What we really need to know to determine test adequacy is whether collecting additional data would change the causal effect estimate or, critically, the outcome of the test case. + +To tackle this problem, we have produced a dedicated causal test adequacy metric. +Because of the complex nature of the test adequacy question in this context, the metric itself has a slightly more nuanced interpretation than a simple percentage. +The full technical details can be found in `this paper `_. +An informal explanation is as follows. + +The fundamental question we want to answer is "will collecting additional data change the causal effect estimate?" or, in other words "is our causal effect estimate stable?". +To investigate this, we repeatedly resample the data and re-evaluate the test case, first calculating the causal effect estimate and then checking that it is as expected. +From the resampled causal effect estimates, we calculate the `kurtosis `_, which measures the "tailedness" of the distribution of estimates. +The basic idea here is that adequate test data will lead to a normal distribution of estimates. + +Interpretation +-------------- + +Kurtosis values close to zero represent adequate test data. +This represents a *stable* causal effect estimate and a trustworthy test outcome. +Kurtosis values less than zero also represent stable causal effect estimates, but these can be thought of as "too stable". +In other words, we don't have enough data to have fully observed the stochasticity of the system under test. +Kurtosis values larger than zero represent unstable causal effect estimates. +That is, the estimates are highly dependent on individual data points, meaning that the causal test outcomes are unlikely to be trustworthy. +While, the acceptable kurtosis values will vary between systems and applications, a general rule would be that kurtosis values between 0 and 1 are mostly acceptable, with negative values being generally undesirable. + +Configuration Options +--------------------- + +Since causal test adequacy involves repeatedly resampling the data and re-executing the test case, the main configuration option here is the number of times the data is resampled. +To obtain the most accurate estimate, more samples is always better, but this can become extremely computationally expensive, since the process effectively involves repeatedly executing the test suite, so for *n* bootstraps, the causal test adequacy calculation will take around *n* times the runtime of the test suite. +The default value is 100 samples, which we do not recommend going below. +If your test suite is very fast to run, we recommend running 1000 samples or even more. diff --git a/docs/source/modules/test_data.rst b/docs/source/modules/test_data.rst new file mode 100644 index 00000000..e2fe9c42 --- /dev/null +++ b/docs/source/modules/test_data.rst @@ -0,0 +1,30 @@ +Test Data +========= + +A key difference between causal testing and traditional testing is that causal testing operates statistically over *multiple runs* of a system. +This means that the process of collecting test data is completely separate from the process of evaluating test cases. +Furthermore, the process of causal :term:`identification`, enables pre-existing data to be used rather than having to collate a bespoke set of system runs without risking untrustworthy test outcomes due to biassed datasets. +The benefit of this is a potentially huge saving in computational cost. + +The causal inference techniques that underpin the Causal Testing Framework make `three key assumptions `_ about the data. +In practice, any test data that records all relevant variables and achieves a good coverage of the input space should satisfy these assumptions. + +1. **No unobserved confounding:** This means that all relevant variables have been recorded. +As discussed in the section on :doc:`causal graphs `\ , it is important to record all relevant causal relationships in the causal DAG to facilitate causal :term:`identification`. +The Causal Testing Framework provides :ref:`special estimation techniques ` that can adjust for unobserved variables in certain circumstances. + +2. **Consistency:** Formally, this means that the observed output of a run of the system under a particular configuration matches the :term:`potential outcome` under that configuration. +Intuitively, this should be trivially true (especially for software systems) if the causal test case and the causal DAG are both well-specified. +For example, when examining the relationship between a person's weight and their risk of heart attach, the risk (and thus the potential outcomes) associated with different weight loss interventions may be very different. +In this example, we could add a node to our causal DAG which represents which (if any) weight loss interventions a person has undergone in order to facilitate proper :term:`identification` and :term:`adjustment`. + +3. **Positivity:** Formally, this means that the probability of all relevant treatment values is non-zero. +Intuitively, this just means that the dataset needs to achieve good coverage of the input space. +This is of particular importance for binary and categorical inputs when several variables need to be adjusted for, as a lack of data can make it impossible to estimate a causal effect. + +How much data is enough? +------------------------ + +Where code coverage is a common :term:`test adequacy` metric for traditional testing techniques, it has been shown to be a `poor metric `_ for the kinds of systems that causal testing is intended to test. +As with any statistical technique, the trustworthiness of causal test results is entirely dependent on the data used to evaluate them, but it can be difficult to tell whether the data you have is sufficient. +Fortunately, our :doc:`causal test adequacy ` measurement can give an indication. diff --git a/docs/source/modules/test_oracle.rst b/docs/source/modules/test_oracle.rst new file mode 100644 index 00000000..9b5b57c0 --- /dev/null +++ b/docs/source/modules/test_oracle.rst @@ -0,0 +1,80 @@ +Test Oracle +=========== + +As in traditional testing, the `oracle `_ is a procedure used to determine whether the observed behaviour is actually correct. +In causal testing, this represents checking that the causal effect estimated from the test data is what was expected. +The Causal Testing Framework supports several causal effects by default. +The most basic oracle procedure is to simply validate the presence or absence of a causal effect. +This requires nothing more than the edges of the causal DAG. +If you know the direction of a causal relationship (positive or negative), you can add a little more precision to your causal tests. +If you know precisely what the causal effect should be, you can check for a particular value, within a specified tolerance. + +SomeEffect +---------- + +**Recommended use:** For validating the presence of a causal effect between two variables. +For additive :term:`effect measures` such as :term:`ATE`, :term:`CATE`, this involves checking that the :term:`confidence intervals` associated with a causal effect estimate do not contain zero. +For multiplicative :term:`effect measures` such as :term:`risk ratio` this involves checking that the :term:`confidence intervals` associated with a causal effect estimate do not contain one. + +.. autoclass:: causal_testing.testing.causal_effect.SomeEffect + :members: + :undoc-members: + :show-inheritance: + :noindex: + +NoEffect +---------- + +**Recommended use:** For validating the absence of a causal effect between two variables. +For additive :term:`effect measures` such as :term:`ATE`, :term:`CATE`, this involves checking that the :term:`confidence intervals` associated with a causal effect estimate contain zero. +For multiplicative :term:`effect measures` such as :term:`risk ratio` this involves checking that the :term:`confidence intervals` associated with a causal effect estimate contain one. + +.. autoclass:: causal_testing.testing.causal_effect.NoEffect + :members: + :undoc-members: + :show-inheritance: + :noindex: + +Positive +---------- + +**Recommended use:** For validating a positive causal effect. +For additive :term:`effect measures` such as :term:`ATE`, :term:`CATE`, this involves checking that the :term:`confidence intervals` associated with a causal effect estimate are both above zero. +For multiplicative :term:`effect measures` such as :term:`risk ratio` this involves checking that the :term:`confidence intervals` associated with a causal effect estimate are both above one. + +.. autoclass:: causal_testing.testing.causal_effect.Positive + :members: + :undoc-members: + :show-inheritance: + :noindex: + +Negative +---------- + +**Recommended use:** For validating a negative causal effect. +For additive :term:`effect measures` such as :term:`ATE`, :term:`CATE`, this involves checking that the :term:`confidence intervals` associated with a causal effect estimate are both below zero. +For multiplicative :term:`effect measures` such as :term:`risk ratio` this involves checking that the :term:`confidence intervals` associated with a causal effect estimate are both below one. + +.. autoclass:: causal_testing.testing.causal_effect.Negative + :members: + :undoc-members: + :show-inheritance: + :noindex: + +ExactValue +---------- + +**Recommended use:** For specifying a precise value for the expected causal effect. +Here, you can also specify arithmetic tolerance, categorical tollerance (the minimum proportion of categories that must exhibit the expected effect for the test to pass), and confidence interval limits. + +.. autoclass:: causal_testing.testing.causal_effect.ExactValue + :members: + :undoc-members: + :show-inheritance: + :noindex: + +Custom Causal Effects +--------------------- + +As with :ref:`custom estimators `, you can also implement your own custom causal effects if the options above are not sufficient for your needs. +To do this, you can extend the :code:`CausalEffect` class and implement your own :code:`apply` method that takes an :doc:`effect estimate <../autoapi/causal_testing/estimation/effect_estimate/index>` and returns boolean :code:`True` if the test should pass and :code:`False` otherwise. diff --git a/docs/source/quick_start/causal_discovery.rst b/docs/source/quick_start/causal_discovery.rst new file mode 100644 index 00000000..4f765f1b --- /dev/null +++ b/docs/source/quick_start/causal_discovery.rst @@ -0,0 +1,46 @@ +Causal Discovery +================ + +This quick start guide shows how to carry out causal discovery using the commandline interface. +The result will be a causal DAG that shows the causal relationships within the system. + +Step 1: Prepare the Data +------------------------ + +This is the exact same process as step 3 in the :doc:`causal testing quick start guide `. +To perform causal discovery, you will need to collect form a table of values in which each column will correspond to a variable in your DAG, and each row represents a valuation of those variables for a single run of the system. +This need not be collected specially --- you can easily use pre-existing data if you have it. + +Step 2: Run the Causal Discovery +-------------------------------- + +The data is all you need to run causal discovery. +Assuming your data is saved in a single CSV file called :code:`data.csv`, you can run causal discovery with the following command:: + + causal-testing discover --data paths data.csv --output dag.dot + +This will create a `textual representation `_ of the inferred DAG in a file called :code:`dag.dot` in the same format used for causal testing. +There are various configuration options available here, including the causal discovery technique used, and options to provide known relationships and independences. +For full details, run :code:`causal-testing discover --help`. + +.. note:: + There may be more than one DAG that explains a given dataset. + For best results, we recommend generating multiple DAGs with different random seeds and examining commonalities between them. + +.. warning:: + Causal discovery should **not** be seen as an easy way to create specifications for causal testing! + While causal discovery can be very to understand the causal relationships between variables in a system, it critical to check that the inferred relationships are **sensible and meaningful**. + +Step 3: Evaluate your DAG +------------------------- + +The construction of a DAG is an iterative process. +As with any data-driven technique, the output of causal discovery is entirely dependent on the data that the algorithm is given. +If the dataset is small, the resulting DAG may be overfitted to the dataset, meaning the corresponding test outcomes are highly dependent on a few individual points. +To help mitigate this risk, we provide an evaluation function to help you investigate this:: + + causal-testing evaluate --data paths data.csv --dag-path dag.dot --output output.csv + +This will iteratively resample the data and evaluate causal tests to check the presence and absence of the causal relationships that the DAG specifies. +The result will be a CSV file saved to :code:`output.csv` which gives confidence intervals for the number of passing, failing, and :term:`inestimable` tests, as well as the results with the full dataset. +Narrower confidence intervals indicate that the DAG gives *stable* test outcomes, and so is less likely to be overfitted to the dataset. diff --git a/docs/source/quick_start/causal_testing.rst b/docs/source/quick_start/causal_testing.rst new file mode 100644 index 00000000..ac5cc5c4 --- /dev/null +++ b/docs/source/quick_start/causal_testing.rst @@ -0,0 +1,72 @@ +Causal Testing +============== + +This quick start guide shows how to carry set up and run causal tests using the command line interface. +This is the simplest way to interact with the Causal Testing Framework as you do not need to write any code. +The result will be a JSON file containing a set of pass/fail test outcomes for each of the specified causal relationships that you can then use to explore the system. + +Step 1: Prepare the DAG +----------------------- + +The first step is to specify the expected causal relationships between your variables using a directed acyclic graph (DAG). +To do this, we use the `DOT language `_, which provides an intuitive text-based representation of DAGs. +The syntax is very lightweight: an edge from node :code:`X` to :code:`Y` is specified as :code:`X -> Y;`. +If you would prefer to use a visual editor, you can use `Dagitty `_ and copy the *Model code*. + +A simple example is shown below. +The first line specifies that the graph is a :code:`digraph` (directed graph), and names it :code:`expected_relationships`. +The next three lines list the variables :code:`X`, :code:`Y`, and :code:`Z`. +The next line lists a single edge :code:`X -> Y`, indicating that :code:`X` should cause :code:`Y`. +:code:`Z` has no incoming or outgoing edges, so should be independent of both :code:`X` and :code:`Y`. + +.. code-block:: graphviz + + digraph expected_relationships { + X; + Y; + Z; + + X -> Y; + } + +Step 2: Prepare the Causal Test Cases +------------------------------------- + +Having prepared the causal DAG, you can then use the Causal Testing Framework automatically convert the specified causal relationships to test cases. +Each DAG implicitly encodes two types of causal relationship: causal dependences (i.e. the edges of the DAG) and causal *in*\ dependences (i.e. the non-edges) of the graph. +Assuming you have saved your DAG in a text file called :code:`dag.dot` in the current working directory, you can generate the corresponding causal test cases using the following command from your command shell:: + + causal-testing generate --dag-path dag.dot -output tests.json + +This will output a JSON file containing the causal test cases to :code:`tests.json`. +While these test cases are executable "out of the box", they can be fully customised to suit your needs. +To do this, you can either manually edit :code:`tests.json` (be careful as your changes will be overwritten if you regenerate the test cases), or by providing additional configuration options to the :code:`generate` command above (run :code:`causal-testing generate --help` to see the full list). + +Step 3: Prepare the Test Data +----------------------------- + +Causal test cases are evaluated statistically with respect to a *set* of system executions. +This set of executions is specified as a table of values in which each column corresponds to a variable in your DAG, and each row represents a valuation of those variables for a single run of the system. +For an example, check out our interactive :doc:`tutorial <../tutorials/vaccinating_elderly/vaccinating_elderly_tutorial>`. + +A major strength of the Causal Testing Framework is that the specification of the expected causal relationships is completely separate from the collection of test data, so you can evaluate the same tests on multiple different datasets with very little additional effort. +This means that, if you already have some data from previous runs of the system, you can get to testing straight away without needing to run system under test again. +The framework supports `several file formats `_, including CSV, excel, parquet, and even HTML. + +Step 4: Evaluate the Test Cases +------------------------------- + +We now have everything we need to evaluate the causal tests: the DAG, the data, and the test cases themselves. +Assuming your data is saved in a single CSV file called :code:`data.csv`, you can execute your causal tests with the following command:: + + causal-testing test --dag dag.dot --data-paths data.csv --test-config tests.json --output test_results.json + +This will execute your causal test cases and produce a file called :code:`test_results.json` that will contain your causal test results. +There are various configuration options at this stage. +Run :code:`causal-testing test --help` to see them all. + +.. note:: + In traditional testing, when a test case fails, this means that there must be a problem with either the system or the test case. + Because causal testing is a statistical technique, test outcomes depend on the data they are evaluated with. + If you have insufficient data to calculate a reliable causal effect estimate, tests may fail even for fault-free systems. + Check out our :doc:`../modules/test_data` and :doc:`../modules/test_adequacy` pages for more information. diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst deleted file mode 100644 index d66c5287..00000000 --- a/docs/source/tutorials.rst +++ /dev/null @@ -1,13 +0,0 @@ -Tutorials -====================== - -The following tutorials demonstrate how to use the Causal Testing Framework. If you're new -to the framework, we recommend you start with the `Testing a Software System Tutorial `_. -All tutorials follow a step-by-step demonstration of how to correctly interact with the causal testing framework's components. - -.. toctree:: - :maxdepth: 1 - - tutorials/vaccinating_elderly/vaccinating_elderly_tutorial - tutorials/poisson_line_process/poisson_line_process_tutorial - tutorials/visualising_causal_test_results/visualise_causal_test_results diff --git a/images/.gitignore b/images/.gitignore deleted file mode 100644 index 794db9c4..00000000 --- a/images/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -schematic.* -!schematic.tex -!schematic.png diff --git a/images/schematic-dark.png b/images/schematic-dark.png deleted file mode 100644 index 1be68423..00000000 Binary files a/images/schematic-dark.png and /dev/null differ diff --git a/images/schematic.png b/images/schematic.png deleted file mode 100644 index c1445da4..00000000 Binary files a/images/schematic.png and /dev/null differ diff --git a/images/schematic.tex b/images/schematic.tex deleted file mode 100644 index 7ad9b7c8..00000000 --- a/images/schematic.tex +++ /dev/null @@ -1,154 +0,0 @@ -\documentclass{standalone} - -\usepackage{tikz} -\usetikzlibrary{arrows,positioning,shapes,calc,fit,overlay-beamer-styles, backgrounds} -\usepackage{dsfont,pifont} -\newcommand*{\expe}{\mathds{E}} -\usepackage{amsmath} -\usepackage{booktabs} - -\usepackage[default]{FiraSans} -\usepackage[mathrm=sym]{unicode-math} -\setmathfont{Fira Math} - -\begin{document} -\tikzset{ - node/.style={circle, draw, minimum size=3ex, inner sep=0.2}, - edge/.style={->,> = latex'}, -} - -\newcommand{\cmark}{\ding{51}}% -\newcommand{\xmark}{\ding{55}}% - -\begin{tikzpicture}[background rectangle/.style={fill=none}, show background rectangle, color=black] - - % Test Case - \begin{scope}[name prefix=test-, local bounding box=test-case] - \node[draw=none, rectangle, anchor=north] (title) at (0, 0) {Causal Test Case}; - \node[anchor=north] (tuple) at (title.south) {$(X=i, \Delta=\text{increase}, Y=y_1)$}; - \node[draw, rectangle] [fit=(title) (tuple)] {}; - \end{scope} - - % Estimand - \begin{scope}[name prefix=estimand-, local bounding box=estimand, anchor=south, shift={($(test-test-case.east |- test-tuple.south) + (1, 0)$)}] - \node[anchor=south west] (eqn) at (0,0) { - $\Delta Y=\expe{[I=0 | X_1]} - \expe{[I=1 | X_1]} $ - }; - \node[draw=none, rectangle, anchor=south] (title) at (eqn.north) {Statistical Estimand}; - \node[draw, rectangle] [fit=(estimand-title) (estimand-eqn)] {}; - \end{scope} - - % Estimate - \begin{scope}[name prefix=estimate-, local bounding box=estimate, shift={($(estimand-estimand.east)+(1, 0)$)}] - \node[draw=none, rectangle, anchor=south west] (title) at (0, 0) {Causal Estimate}; - \node[anchor=north] (table) at (title.south) { - $\Delta Y=5$ - }; - \coordinate (top) at ({(0, 0)} |- test-title.north); - \coordinate (bot) at ({(0, 0)} |- estimand-eqn.south); - \node[draw, rectangle] [fit=(title) (table) (top) (bot)] {}; - \end{scope} - - % Oracle - \begin{scope}[name prefix=oracle-, local bounding box=test-oracle, shift={($(estimate-estimate.east) + (1.54, -0.4)$)}] - \begin{scope}[shift={(0,0)}, local bounding box=brain, scale=1.2] - \begin{scope}[shift={(-7.6932,3.5256)}, local bounding box=brain] - \path[draw,line width=0.025cm] (8.162, -2.8955) circle (0.066cm); - \path[draw,line width=0.025cm] (8.0485, -3.2243) circle (0.066cm); - \path[draw,line width=0.025cm] (8.0346, -3.5296) circle (0.066cm); - \path[draw,line width=0.025cm] (8.2166, -3.757) circle (0.066cm); - \path[draw,line width=0.025cm] (7.6556, -3.7827) circle (0.066cm); - \path[draw,line width=0.025cm] (7.6315, -3.5091) circle (0.066cm); - \path[draw,line width=0.025cm] (7.4451, -3.2224) circle (0.066cm); - \path[draw,line width=0.025cm] (7.6247, -2.9461) circle (0.066cm); - - \path[draw,line width=0.025cm,miter limit=4.0] (7.6932, -2.6331) -- (7.3637, -2.8234) -- (7.3637, -3.0567) -- (7.1749, -3.1656) -- (7.1749, -3.5256) -- (7.3341, -3.6175) -- (7.3341, -3.8517) -- (7.6883, -4.0562) -- (7.868, -3.9669) -- (8.0478, -4.0562) -- (8.4019, -3.8517) -- (8.4019, -3.6175) -- (8.5611, -3.5256) -- (8.5611, -3.1656) -- (8.3724, -3.0567) -- (8.3724, -2.8234) -- (8.0429, -2.6331) -- (7.868, -2.7341) -- cycle; - \path[draw,line width=0.025cm,miter limit=4.0] (7.868, -3.9669) -- (7.868, -2.7341); - \path[draw,line width=0.025cm] (7.5588, -2.9461) -- (7.3637, -2.9461); - \path[draw,line width=0.025cm] (7.4451, -3.1565) -- (7.4451, -2.9461); - \path[draw,line width=0.025cm] (7.6316, -3.4431) -- (7.6316, -3.2116) -- (7.868, -3.2116); - \path[draw,line width=0.025cm] (7.5897, -3.7827) -- (7.4177, -3.7827) -- (7.4177, -3.523) -- (7.1749, -3.523); - \path[draw,line width=0.025cm] (8.162, -2.9614) -- (8.162, -3.0534) -- (7.868, -3.0534); - \path[draw,line width=0.025cm] (8.0485, -3.1584) -- (8.0485, -3.0534); - \path[draw,line width=0.025cm] (8.1005, -3.5296) -- (8.313, -3.5296) -- (8.313, -3.3442) -- (8.5611, -3.3442); - \path[draw,line width=0.025cm] (8.1507, -3.757) -- (8.0477, -3.757) -- (8.0477, -4.0561); - \end{scope} - \end{scope} - \node[draw=none, rectangle, anchor=south] (title) at (brain.north) {Test Oracle}; - - \node[draw, rectangle] [fit=(title) (brain)] {}; - \end{scope} - - % Outcome - \begin{scope}[name prefix=outcome-, local bounding box=test-outcome, shift={($(oracle-brain.east |- estimate-estimate.east) + (1, 0)$)}] - \node[draw=none, rectangle, anchor=south west] (title) at (0,0) {Test Outcomes}; - \node[draw=none, anchor=north] (ok) at (title.south) {\cmark ~ \xmark}; - - \coordinate (top) at ({(0, 0)} |- test-title.north); - \coordinate (bot) at ({(0, 0)} |- estimand-eqn.south); - \node[draw, rectangle] (test-outcome) [fit=(outcome-title) (outcome-ok) (top) (bot)] {}; - \end{scope} - - - % Causal DAG - \begin{scope}[name prefix=dag-, shift={($(estimand-estimand.north) + (0, 2)$)}] - \node[node] (x1) at (-1, 0) {$X_1$}; - \node[node] (x2) at (-1, 1.4) {$X_2$}; - \node[node] (i) at (0, 0.7) {$I$}; - \node[node] (y1) at (1,0) {$Y_{1}$}; - \node[node] (y2) at (1,0.7) {$Y_2$}; - \node[node] (y3) at (1,1.4) {$Y_3$}; - - \draw[edge] (x1) to (i); - \draw[edge] (x2) to (i); - \draw[edge] (i) to (y1); - \draw[edge] (i) to (y2); - \draw[edge] (i) to (y3); - \draw[edge] (x1) to (y1); - \draw[edge] (x2) to (y3); - \node[draw=none, rectangle] (nodes) [fit=(x1) (x2) (y1) (y2) (y3) (i)] {}; - \node[draw=none, rectangle, anchor=south] (title) at (nodes.north) {Causal DAG}; - \end{scope} - - % Scenario - \begin{scope}[name prefix=scenario-, shift={($(estimand-estimand.south) + (0, -2)$)}] - \node[draw=none, rectangle] (title) at (0, 0) {Modelling Scenario}; - \node[anchor=north] (constraints) at (title.south) {$\{ x_1 < 5, x_2 = \text{``UK''} \}$}; - \end{scope} - \node[draw, rectangle] (scenario) [fit=(scenario-title) (scenario-constraints)] {}; - - % Data - \begin{scope}[name prefix=data-, local bounding box=test-data] - \node[draw=none, rectangle] (title) at (estimate-estimate |- dag-title) {Test Data}; - \node[anchor=north] (table) at (title.south) { - \begin{tabular}{rrrrrr} - \toprule - $X_1$ & $X_2$ & $I$ & $Y_1$ & $Y_2$ & $Y_3$ \\ - \midrule - 1.2 & ``UK'' & 0.3 & 7.8 & 4 & 100 \\ - 3.2 & ``UK'' & 0.1 & 7.6 & 8 & 95 \\ - \multicolumn{6}{c}{$\vdots$} \\ - \bottomrule - \end{tabular} - }; - \node[draw, rectangle] [fit=(title) (table)] {}; - \end{scope} - - % DAG outline - \node[draw, rectangle] (dag) [fit=(dag-nodes) (dag-title) (dag-title |- data-table.south)] {}; - - %Information flow - \draw[edge, dashed] (dag) -- (estimand-estimand.north); - \draw[edge, dashed] (test-test-case) -- (estimand-estimand); - - \draw[edge, dashed] (scenario.north) -- (estimand-estimand); - \draw[edge, dashed] (scenario.north) -- ([yshift=7.3mm]scenario.north) -- ([yshift=7.3mm]scenario.north -| test-test-case) -- (test-test-case); - \draw[edge, dashed] (scenario.north) -- ([yshift=7.3mm]scenario.north) -- ([yshift=7.3mm]scenario.north -| estimate-estimate) -- (estimate-estimate); - - \draw[edge, dashed] (data-test-data.south) -- (estimate-estimate.north); - \draw[edge, dashed] (estimand-estimand) -- (estimate-estimate); - - \draw[edge, dashed] (estimate-estimate) -- (oracle-test-oracle.west |- estimate-estimate); - \draw[edge, dashed] (oracle-test-oracle.east |- outcome-test-outcome) -- (outcome-test-outcome); -\end{tikzpicture} -\end{document} diff --git a/paper/paper.md b/paper/paper.md index 7781ec4f..c78bdfeb 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -69,7 +69,7 @@ The user may also refine tests to validate the nature of a particular relationsh Next, the user supplies a set of runtime data in the form of a table with each column representing a variable and rows containing the value of each variable for a particular run of the software. Finally, the CTF automatically validates the causal properties by using the causal DAG to identify a statistical estimand [@pearl2009causality] (essentially a set of features in the data which must be controlled for), calculate a causal effect estimate from the supplied data, and validating this against the expected causal relationship. -![Causal Testing workflow.\label{fig:schematic}](../images/schematic.png) +![Causal Testing workflow.\label{fig:schematic}](../docs/_static/images/schematic.png) ## Test Adequacy Because the properties being tested are completely separate from the data used to validate them, traditional coverage-based metrics are not appropriate here. diff --git a/pyproject.toml b/pyproject.toml index f773ba49..c33ced14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,8 +51,6 @@ dev = [ "pandoc", "pre-commit", "tox", -] -test = [ "nbclient", "nbformat", "ipykernel",