diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 8c24b79..0000000 --- a/.dockerignore +++ /dev/null @@ -1,25 +0,0 @@ -# Git -.git -.github -.githooks -.gitignore - -# Local environment and secrets -.venv -.env - -# Python cache and test cache -__pycache__ -.pytest_cache -.ruff_cache -.mypy_cache -.coverage -htmlcov - -# Build artifacts -dist -build -*.egg-info - -# Project docs not needed for the test image -docs \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f87759..0b53865 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,9 +4,33 @@ Thank you for your interest in contributing to ARGUS. ARGUS is a Python-based market analytics project focused on clean data workflows, reliable code, useful metrics and future AI-assisted monitoring. -The project is still growing, so contributions should be small, focused and easy to review. You do not need to be an expert to contribute, but your changes should be understandable, reliable and related to the current project direction. +This project is still growing, so contributions should help the project become more stable, understandable and useful step by step. -Good starting points are issues labeled `good first issue`. These issues are usually smaller, easier to review and better suited for getting familiar with the project. +> [!IMPORTANT] +> ARGUS values reliability, clear communication and long-term skill building. +> Contributions should improve the project without creating unnecessary complexity. + +--- + +## Project Mindset + +ARGUS is not only about adding features quickly. + +The project is built around: + +- clean Python code +- understandable architecture +- reliable tests +- useful documentation +- careful data handling +- practical analytics +- continuous learning + +Good contributions should make the project easier to use, test, maintain or extend. + +--- + +## What You Can Contribute Helpful contributions include: @@ -14,14 +38,17 @@ Helpful contributions include: - tests - documentation improvements - small refactorings +- validation improvements - analytics metrics +- chart improvements - data-source clients -- UI or chart improvements -- CI/CD and tooling improvements -- architecture or research notes +- CI/CD improvements +- issue clarification +- architecture notes +- examples and usage instructions -> [!IMPORTANT] -> Please keep changes focused and avoid adding unnecessary complexity. +> [!NOTE] +> Large features should usually start with an issue or short discussion before implementation. --- @@ -49,45 +76,46 @@ Bad examples: --- -## Contribution Expectations +## Development Setup -Contributors are expected to keep changes focused, understandable and related to the issue or task. +Clone the repository: -Please: +```bash +git clone https://github.com/BytecodeBrewer/argus.git +cd argus +``` -- explain your changes clearly -- be open to review feedback -- improve your contribution step by step after feedback -- avoid unrelated rewrites -- respect the existing architecture unless there is a clear reason to change it -- do not add scripts that automatically run `git add`, `git commit`, `git push` or create pull requests unless this was discussed first +Create a virtual environment: -A contribution may be declined or delayed if it: +```bash +python -m venv .venv +``` -- does not fit the current roadmap -- adds too much complexity too early -- breaks existing functionality -- lacks necessary checks or documentation -- duplicates existing work -- bypasses the repository workflow +Activate it. ---- +On Windows PowerShell: -## Branch Workflow +```powershell +.venv\Scripts\Activate.ps1 +``` -For issue-based work, create your branch from the related GitHub issue when possible. +On macOS/Linux: -GitHub may suggest a branch name based on the issue title. You can shorten it if the generated name is too long. +```bash +source .venv/bin/activate +``` -Good branch names are focused and describe the task: +Install the project with development dependencies: -```text -43-research-forecasting-approach -33-add-yfinance-client -40-improve-test-coverage +```bash +pip install -e ".[dev]" ``` -If you create the branch manually, use: +--- + +## Branch Workflow + +Create a new branch for your work: ```bash git checkout -b @@ -96,30 +124,24 @@ git checkout -b Example: ```bash -git checkout -b 43-research-forecasting-approach +git checkout -b 12-add-volatility-metric ``` +Use focused branch names that describe the work. + --- ## Commit Expectations Commits should be small, understandable and related to the current task. -ARGUS uses a conventional commit style with an issue reference: - -```text -type(#issue): short description -``` - Good commit messages: ```text -docs(#43): research first forecasting approach -feat(#33): add yfinance historical data client -test(#40): add tests for conversion service -fix(#33): handle empty historical data response -refactor(#34): split metric helpers -ci(#10): update commit message workflow +Add rolling volatility metric +Fix currency validation edge case +Update README setup instructions +Add tests for trend metrics ``` Avoid unclear messages: @@ -133,25 +155,21 @@ final ``` > [!TIP] -> A good commit tells future readers what changed and which issue it belongs to. +> A good commit tells future readers what changed and why it belongs to the task. --- -## Checks +## Testing -Before opening a pull request, run the project checks: +Before opening a pull request, run the test suite: ```bash pytest -ruff check . -ruff format --check . ``` -These checks verify that tests pass, code style is valid and formatting is consistent. - -A pull request should not be marked as ready for review if checks are failing without explanation. +A pull request should not be opened as ready for review if tests are failing without explanation. -If a check fails and you are unsure why, mention it clearly in the pull request. +If a test fails and you do not know why, mention it clearly in the pull request. > [!IMPORTANT] > CI checks must pass before a pull request can be merged. @@ -160,23 +178,65 @@ If a check fails and you are unsure why, mention it clearly in the pull request. ## Pull Request Expectations -Pull requests should target `develop` unless the maintainer explicitly says otherwise. +A good pull request should include: -Do not open feature, research or documentation pull requests directly against `main`. -The `main` branch is reserved for stable/release-ready changes. +- a clear title +- a short explanation of what changed +- a link to the related issue if available +- notes about tests +- screenshots for UI changes if useful +- a short explanation of any trade-offs -Please use the pull request template and fill it out clearly. +Pull requests should be focused and reviewable. -The template helps reviewers understand: +Before opening a pull request, run: -- what changed -- which issue is related -- whether tests were run -- whether documentation or screenshots are needed -- if there are any notes or trade-offs +```bash +pytest +ruff check . +ruff format --check . +``` + +--- + +## Reliability Expectations + +Contributors are expected to work reliably. -Do not bypass the pull request template or replace it with an unrelated auto-generated description. -It makes reviewing harder and may delay the merge. +This means: + +- do not submit random or unfinished code without context +- do not ignore failing tests +- do not introduce secrets, API keys or local machine paths +- do not rewrite unrelated parts of the project without discussion +- communicate if you are unsure +- keep changes understandable for future contributors +- respect the existing architecture unless there is a clear reason to change it + +Reliability does not mean knowing everything already. + +It means being honest, careful and consistent. + +--- + +## Learning Mindset + +ARGUS welcomes contributors who want to improve their technical skills. + +You do not need to be an expert to contribute. + +Helpful behavior includes: + +- asking clear questions +- explaining your reasoning +- being open to review feedback +- improving your code after feedback +- learning from tests, errors and architecture discussions +- documenting what you learned when it helps others + +> [!NOTE] +> This project values skill growth. +> A thoughtful small contribution is better than a large unclear one. --- @@ -205,14 +265,25 @@ For analytics code: ## Secrets and API Keys -Never commit secrets, API keys, tokens, passwords, `.env` files or local config files with private data. +Never commit secrets. + +Do not commit: -Use a local `.env` file for secrets: +- API keys +- tokens +- passwords +- `.env` files +- local config files with private data + +Use a local `.env` file for secrets. ```env -EXCHANGE_RATE_API_KEY=your_api_key_here +api_key=your_api_key_here ``` +> [!WARNING] +> If you accidentally commit a secret, revoke it immediately and inform the maintainer. + --- ## Documentation @@ -228,4 +299,44 @@ Useful documentation includes: - data-source assumptions - troubleshooting notes +Repository-level files such as `README.md`, `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md` and `LICENSE` belong in the repository root. + Technical notes, research and deeper explanations belong in `docs/`. + +--- + +## Communication + +Please communicate respectfully and constructively. + +When giving feedback: + +- focus on the code or idea, not the person +- explain the reason behind suggestions +- be specific +- stay open to alternatives + +When receiving feedback: + +- assume good intent +- ask questions if something is unclear +- improve the contribution step by step + +All contributors are expected to follow the project’s Code of Conduct. + +--- + +## Maintainer Notes + +The maintainer may ask for changes before merging a pull request. + +A contribution may be declined if it: + +- does not fit the current roadmap +- adds too much complexity too early +- breaks existing functionality +- lacks necessary tests +- duplicates existing work +- does not follow the project’s quality expectations + +This helps keep ARGUS stable, learnable and maintainable. \ No newline at end of file diff --git a/README.md b/README.md index 8723389..607f0d0 100644 --- a/README.md +++ b/README.md @@ -120,18 +120,11 @@ README.md - Tkinter - pytest -### Current data sources +### Current data source - ExchangeRate API for live currency conversion - yfinance for historical market-data retrieval and analytics -### Storage - -- DuckDB — local analytical storage for normalized historical market data - ->[!Note] -> See docs/storage.md for details. - --- ## Planned / Future Tech Stack @@ -145,32 +138,46 @@ Planned or likely future technologies include: - Frankfurter API for historical FX data - possible additional market-data APIs later +### Data processing + +- pandas +- NumPy +- possibly Polars later for larger datasets + ### Storage - PostgreSQL +- DuckDB +- Parquet +- optional cloud storage ### Visualization and UI +- matplotlib +- Plotly - NiceGUI -- Django ### DevOps and deployment +- GitHub Actions +- Docker - Docker Compose -- Travis CI +- cloud deployment later ### Cloud and data engineering -- Azure +- Azure, GCP or AWS depending on project direction - scheduled ingestion -- agentic Workflows -- Blob Storage -- scaled analysis +- data quality checks +- reporting pipelines ### AI and agentic workflows - LLM-assisted summaries - RAG over stored reports or notes +- agentic data checks +- anomaly monitoring +- human-in-the-loop signal review > [!CAUTION] > AI and agentic features are future-stage ideas. @@ -192,7 +199,6 @@ Recommended for development: - VS Code - a virtual environment - pytest -- Docker, if you want to run tests in an isolated container environment > [!NOTE] > Runtime dependencies are managed through `pyproject.toml`. @@ -248,7 +254,7 @@ pip install -e ".[dev]" ## API Key Setup -ARGUS uses the ExchangeRate API for live currency conversion. Historical analytics currently use yfinance and do not require an additional API key. +ARGUS currently uses the ExchangeRate API for live currency conversion. ### 1. Create an API key @@ -278,7 +284,7 @@ The `.env` file must stay local and should never be committed. --- -## Running ARGUS Locally +## Running ARGUS Start the current Tkinter GUI: @@ -288,22 +294,6 @@ python -m argus.main This starts the local ARGUS prototype with calculator, currency conversion and basic analytics views. -## Running Argus in Docker - -ARGUS includes a minimal Docker setup for running the test suite in an isolated container environment. - -Build the Docker image: - -```bash -docker build -t argus . -``` - -Run ARGUS in a container: - -```bash -docker run --rm argus -``` - ### Legacy CLI / Debug Interface The legacy CLI is still available for quick local checks and debugging: @@ -320,7 +310,7 @@ python src/legacy/debug_main.py ## Running Tests -Run the test suite locally: +Run the test suite: ```bash pytest @@ -356,4 +346,4 @@ Current focus: - add stronger market metrics - expand pandas-based analytics workflows - improve dashboard usefulness without adding unnecessary chart noise -- document metric definitions, assumptions and data-source behavior +- document metric definitions, assumptions and data-source behavior \ No newline at end of file diff --git a/argus_probe.duckdb b/argus_probe.duckdb deleted file mode 100644 index 9c9ef34..0000000 Binary files a/argus_probe.duckdb and /dev/null differ diff --git a/dockerfile b/dockerfile deleted file mode 100644 index 1dce2e8..0000000 --- a/dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM python:3.11-slim - -WORKDIR /app - -COPY pyproject.toml README.md ./ -COPY src/ ./src/ -COPY tests/ ./tests/ - -RUN python -m pip install --upgrade pip \ - && pip install -e ".[dev]" - -CMD ["pytest"] \ No newline at end of file diff --git a/docs/databases-and-storage.md b/docs/databases-and-storage.md new file mode 100644 index 0000000..a4356be --- /dev/null +++ b/docs/databases-and-storage.md @@ -0,0 +1,193 @@ +# ARGUS Storage Research + +## Intro + +It was previously a research what ARGUS should store and which database/storage approach fits the project. +Now it's a simple documentation about how the storage and domain logic looks like and should develop in the next sprints. + +ARGUS is moving from live API requests and in-memory analytics toward real data workflows. +The first storage decision should support local market analytics, SQL practice and future dashboard features without adding unnecessary infrastructure too early. + +--- + +## First Storage Approach + +DuckDB should be the first storage technology for ARGUS. + +Reason: + +* ARGUS currently needs local analytical storage, not a full server database +* DuckDB fits historical time-series analysis well +* it supports SQL-based analytics without requiring a database server +* it works well with Python and notebook-based exploration +* it keeps the first storage implementation manageable +* it can later be replaced or complemented by PostgreSQL if ARGUS becomes more product-like + +The first storage implementation should focus on: + +* historical market data +* cleaned OHLCV-ready price data +* source information +* instruments that ARGUS can analyze + +PostgreSQL and SQLGate become more relevant later. + +For the first DuckDB phase, the goal is to build a clean local analytics workflow. + +--- + +## Developer Interaction Workflow + +ARGUS should use a practical developer workflow for DuckDB. + +The goal is to make the database easy to inspect, explore and validate before logic is moved into production code. + +### Notebook Exploration + +Notebooks should be the main exploration layer. + +They are useful for: + +* opening the DuckDB database +* testing SQL queries +* validating imported data +* comparing SQL results with pandas calculations +* exploring metric logic +* documenting research assumptions + +This workflow is especially useful before turning queries into reusable project code. + +Notebook exploration should be preferred over a GUI database tool in the first phase. + +### DuckDB CLI + +The DuckDB CLI should be used for quick database inspection. + +It is useful for: + +* checking available tables +* running small SQL queries +* validating stored records +* debugging the local database file + +The CLI is not the main research environment, but it is useful as a fast inspection tool. + +--- + +## First Data Model Direction + +The first data model should support FX data now and broader market data later. + +ARGUS should not use a narrow `date | value` table as the main market-data model. + +That would work for simple exchange rates, but it would become limiting once ARGUS adds stocks, ETFs, indices or broader market APIs. + +The first model focuses on two primary metadata entities, supported by standard operational schemas and communication interfaces: + +```text +DataSource +Instrument +``` + +### DataSource + +Stores where data came from. + +Current operational fields: + +```text +name +provider_kind +requires_api_key (defaults to False) +``` + +### Instrument + +Stores what ARGUS can analyze. +Current operational fields: + +```text +symbol +name +asset_class +currency (optional) +exchange (optional) +base_currency (optional) +quote_currency (optional) +``` + +### Internal Operational Models + +To support decoupled runtime communication and execution pipelines, ARGUS utilizes specialized Python dataclasses. These structures orchestrate active data flows between data-fetching clients and services. + +**MarketDataRequest** +Encapsulates parameters passed to fetching engines and downstream query handlers. + +* `source`: The targeted `DataSource` instance +* `instrument`: The targeted `Instrument` instance +* `timeframe`: Granularity of requested bars (like `"1d"`, `"1h"`). +* `start`: Temporal lower bound (`datetime.date`) +* `end`: Temporal upper bound (`datetime.date`) + +**MarketDataResponse** +Encapsulates runtime data payloads, structural enforcement, and pipeline feedback. + +* `source`: The originating `DataSource` instance +* `instrument`: The corresponding `Instrument` instance +* `bars`: A `pandas.DataFrame` holding the time-series payload +* `message`: Pipeline feedback or error context (defaults to `""`) + +### The Price Bar Schema (Data Uniformity) + +Crucially, **Price Bars are not treated as a standalone domain metadata model or database entity.** Instead, the Price Bar specification acts purely as an internal **uniformity schema** to normalize incoming time-series data across disparate third-party APIs. + +All responses must be mapped into this standard schema layout within the `bars` DataFrame. + +#### Provider Mapping Layer + +Third-party structures are converted to this unified standard upon ingest. For example, `yfinance` fields map to the uniform schema via `YFINANCE_PRICE_BAR_MAPPING`: + +* `Date` $\rightarrow$ `timestamp` +* `Open` $\rightarrow$ `open` +* `High` $\rightarrow$ `high` +* `Low` $\rightarrow$ `low` +* `Close` $\rightarrow$ `close` +* `Adj Close` $\rightarrow$ `adjusted_close` +* `Volume` $\rightarrow$ `volume` + +--- + +## Future Direction + +Later sprints can expand the storage layer step by step. + +Possible later additions: + +| Future Area | Possible Additions | +| --- | --- | +| Better source mapping | source-specific symbols, provider metadata | +| Watchlists | user-selected instruments | +| Reports | generated report metadata and history | +| Macro data | FRED indicators and observations | +| Paper trading | simulated orders, positions and portfolio history | +| Server architecture | PostgreSQL | +| SQL tooling | SQLGate with PostgreSQL | +| Cloud direction | managed PostgreSQL or cloud storage | + +SQLGate should be kept for a later PostgreSQL phase. + +It becomes useful when ARGUS moves toward: + +* server-based storage +* stronger database management +* richer metadata +* more stable application state +* user-facing features +* report history +* cloud-ready architecture + +Additional metadata such as documentation links, terms links or provider governance fields can also become useful later. + +For the first DuckDB phase, these details should stay in research documentation instead of the database schema. + +--- diff --git a/docs/forecast_research.md b/docs/forecast_research.md deleted file mode 100644 index b0cc9b7..0000000 --- a/docs/forecast_research.md +++ /dev/null @@ -1,50 +0,0 @@ -# Research: First Forecasting Approach for Market Time Series - -## 1. Realistic First Prediction Task for ARGUS - -A realistic first prediction task for ARGUS is **next-day exchange-rate movement** or **trend direction**. Predicting the exact next value (point forecast) is generally much harder and often less useful for trading/signal workflows than predicting the direction of the movement (up/down). A directional classification task serves as a simple, actionable signal for basic workflows. - -## 2. Baseline Methods to Implement First - -Before jumping into complex models, the following baselines should be implemented to evaluate the added value of any machine learning model: - -- **Naive last-value forecast**: The prediction for the next period is exactly the value from the current period. This is surprisingly hard to beat in random walk-like financial time series. -- **Moving average forecast**: A simple rolling average to predict the next value or determine trend direction. -- **Simple linear regression**: To capture basic linear trends over a given historical window. - -## 3. Libraries: NumPy, pandas, or scikit-learn? - -The first implementation should use **pandas** and **scikit-learn**: - -- **pandas**: Excellent for time-series manipulation, rolling windows, lagging features, and handling missing data. -- **scikit-learn**: Offers robust implementations of simple models (e.g., Linear Regression, Logistic Regression for direction) and provides standardized metrics and cross-validation tools designed for time series (e.g., `TimeSeriesSplit`). - -## 4. Evaluation Metrics - -For the initial approaches, we should focus on: - -- **Directional accuracy**: The percentage of times the model correctly predicts the direction of the price movement (up vs down). This is often more relevant than magnitude errors. -- **MAE (Mean Absolute Error)**: If point forecasting is used, MAE is more robust to outliers than RMSE and provides a linear penalty for errors. -- **RMSE (Root Mean Squared Error)**: Useful to penalize larger errors more heavily, but should be secondary to directional accuracy for basic signal generation. - -## 5. Why is LSTM not the first implementation step? - -LSTMs are highly complex, require a large amount of well-structured data to train effectively without overfitting, and are notoriously difficult to tune. For financial time series, which suffer from low signal-to-noise ratios, an LSTM is likely to overfit the training data or collapse to predicting the last known value. Starting with an LSTM obscures whether the underlying data has any predictive power and sets a high barrier for debugging and infrastructure. - -## 6. Prerequisites for an LSTM Ticket - -Before considering LSTMs or other deep learning approaches, the following must be established: - -- A reliable data ingestion and preprocessing pipeline. -- Established baseline performance metrics (e.g., a naive model and a linear regression model) to compare against. -- Sufficient historical data size. -- A robust backtesting and cross-validation framework to ensure the LSTM isn't just memorizing data or overfitting. -- Hardware/infrastructure to support longer training times and hyperparameter tuning. - -## 7. Recommended First Implementation Approach - -**Recommendation**: Start with **directional trend prediction** (predicting whether the next value is higher or lower than the current value) using a simple **Logistic Regression** model via **scikit-learn**. - -- Use **pandas** to create basic lagged features (e.g., previous returns, moving averages). -- Evaluate using **directional accuracy**. -- Compare performance strictly against a **naive momentum** (predicting the trend continues) or **majority-class** baseline. diff --git a/docs/research-data-sources.md b/docs/research-data-sources similarity index 100% rename from docs/research-data-sources.md rename to docs/research-data-sources diff --git a/docs/research-databases-and-storage.md b/docs/research-databases-and-storage.md deleted file mode 100644 index 484cd61..0000000 --- a/docs/research-databases-and-storage.md +++ /dev/null @@ -1,388 +0,0 @@ -# ARGUS Storage Research - -## Goal - -Research what ARGUS should store and which database/storage approach fits the project. - -ARGUS is moving from live API requests and in-memory analytics toward real data workflows. -The first storage decision should support local market analytics, SQL practice and future dashboard features without adding unnecessary infrastructure too early. - ---- - -## Storage Use Cases - -ARGUS should eventually store different kinds of data, but not all of them need to be implemented at once. - -Relevant storage use cases are: - -* historical exchange rates -* cleaned historical market data -* source information -* instruments that ARGUS can analyze -* later watchlists -* later generated reports -* later macroeconomic data -* later paper-trading history - -The first implementation should focus on historical market data and the basic entities needed to query it. - ---- - -## Storage Candidates - -ARGUS should compare storage options based on the current project phase. - -The project currently needs local analytical storage, not a full server or cloud database. - -### DuckDB - -DuckDB is a local analytical database. - -It is a strong fit for ARGUS because it supports SQL-based analytics without requiring a database server. - -Useful for: - -* historical market data -* local time-series analysis -* SQL practice -* Python-based analytics -* notebook-based exploration -* dashboard data preparation - -Limitations: - -* not a server database -* less suitable for multi-user product features later - ---- - -### SQLite - -SQLite is a simple local database. - -It is strong for small app storage and simple persistence. - -Useful for: - -* settings -* small app-state data -* simple local tables -* later watchlists -* lightweight metadata - -Limitations: - -* less analytics-focused than DuckDB -* not ideal as the main storage layer for historical market data -* better for app-state than analytical time-series queries - ---- - -### PostgreSQL - -PostgreSQL is a server-based relational database. - -It is a strong long-term option when ARGUS becomes more product-like. - -Useful for: - -* server-based storage -* user-facing features -* report history -* watchlists -* paper-trading history -* richer metadata -* cloud-ready architecture -* SQLGate usage later - -Limitations: - -* more setup than needed right now -* requires server or Docker setup -* adds infrastructure complexity too early - -Fit for ARGUS: - -PostgreSQL should be introduced later when ARGUS moves toward a server-based or cloud-ready architecture. - ---- - -## Local, Server and Cloud Options - -| Option | Meaning | Fit Now | Fit Later | -| --- | --- | ---: | ---: | -| Local storage | Database runs locally inside or next to the project | High | High | -| Server database | Database runs as a separate service, for example PostgreSQL | Medium | High | -| Cloud storage/database | Managed storage or database in the cloud | Low | High | - -ARGUS should start with local storage. - -Reason: - -* simpler setup -* easier learning curve -* good fit for a Python analytics project -* no cloud or server infrastructure required yet -* enough for historical data, metrics and dashboard development - -Server and cloud storage should come later when ARGUS has stronger product features such as reports, user state, paper-trading history or deployment needs. - ---- - -## Recommended First Storage Approach - -DuckDB should be the first storage technology for ARGUS. - -Reason: - -* ARGUS currently needs local analytical storage, not a full server database -* DuckDB fits historical time-series analysis well -* it supports SQL-based analytics without requiring a database server -* it works well with Python and notebook-based exploration -* it keeps the first storage implementation manageable -* it can later be replaced or complemented by PostgreSQL if ARGUS becomes more product-like - -The first storage implementation should focus on: - -* historical market data -* cleaned OHLCV-ready price data -* source information -* instruments that ARGUS can analyze - -PostgreSQL and SQLGate become more relevant later. - -For the first DuckDB phase, the goal is to build a clean local analytics workflow. - ---- - -## Developer Interaction Workflow - -ARGUS should use a practical developer workflow for DuckDB. - -The goal is to make the database easy to inspect, explore and validate before logic is moved into production code. - -### Notebook Exploration - -Notebooks should be the main exploration layer. - -They are useful for: - -* opening the DuckDB database -* testing SQL queries -* validating imported data -* comparing SQL results with pandas calculations -* exploring metric logic -* documenting research assumptions - -This workflow is especially useful before turning queries into reusable project code. - -Notebook exploration should be preferred over a GUI database tool in the first phase. - -### DuckDB CLI - -The DuckDB CLI should be used for quick database inspection. - -It is useful for: - -* checking available tables -* running small SQL queries -* validating stored records -* debugging the local database file - -The CLI is not the main research environment, but it is useful as a fast inspection tool. - -A GUI tool such as DBeaver can be tested if needed, but it should stay optional. - ---- - -## First Data Model Direction - -The first data model should support FX data now and broader market data later. - -ARGUS should not use a narrow `date | value` table as the main market-data model. - -That would work for simple exchange rates, but it would become limiting once ARGUS adds stocks, ETFs, indices or broader market APIs. - -The first model should focus on three related entities: - -```text -data_sources -instruments -price_bars -``` - -> [!NOTE] -> The fields below describe the future database-oriented structure. -> Technical fields such as `id`, `instrument_id`, `source_id`, `created_at` and `updated_at` are expected to appear in the database layer. -> Internal Python models may reference related objects directly, for example `source` and `instrument`, before database IDs exist. - -### data_sources - -Stores where data came from. - -Recommended first database fields: - -```text -id -name -provider_kind -requires_api_key -created_at -updated_at -``` - -Example internal/source records: - -| name | provider_kind | requires_api_key | -| ---------------- | ------------- | ---------------: | -| ExchangeRate API | fx_rates | true | -| yfinance | market_prices | false | -| FRED | macro_data | true | - -### instruments - -Stores what ARGUS can analyze. - -Examples: - -* EUR/USD -* AAPL -* SPY -* S&P 500 -* BTC-USD - -Recommended first database fields: - -```text -id -symbol -name -asset_class -currency -exchange -base_currency -quote_currency -created_at -updated_at -``` - -Example instrument records: - -| symbol | name | asset_class | currency | exchange | base_currency | quote_currency | -| ------- | ---------------- | ----------- | -------- | --------- | ------------- | -------------- | -| EUR/USD | Euro / US Dollar | fx | null | null | EUR | USD | -| AAPL | Apple Inc. | stock | USD | NASDAQ | null | null | -| SPY | SPDR S&P 500 ETF | etf | USD | NYSE Arca | null | null | - -### price_bars - -Stores historical market data in an OHLCV-ready structure. - -Recommended first database fields: - -```text -id -instrument_id -source_id -timestamp -timeframe -open -high -low -close -adjusted_close -volume -created_at -updated_at -``` - -FX-style exchange-rate data can be represented as a price bar by storing the rate in `close`. - -The other OHLCV fields can stay empty until ARGUS uses data sources that provide them. - -Example price bar records shown with joined source and instrument information for readability: - -| source | instrument | timestamp | timeframe | open | high | low | close | adjusted_close | volume | -| -------- | ---------- | ---------- | --------- | -----: | -----: | -----: | -----: | -------------: | -------: | -| yfinance | EUR/USD | 2024-01-02 | 1d | null | null | null | 1.095 | null | null | -| yfinance | AAPL | 2024-01-02 | 1d | 187.15 | 188.44 | 183.89 | 185.64 | 184.25 | 50200000 | - ---- - -## Recommended First Implementation Step - -The first storage implementation should not be tied to one specific data provider. - -ARGUS currently works with an existing ExchangeRate API client and evaluates broader market data through yfinance. -Frankfurter may be added later as a stronger FX-oriented historical data source. - -The storage layer should therefore focus on a normalized internal market-data format instead of depending on one API response structure. - -Recommended first step: - -```text -active data client -→ normalize into instruments and price_bars -→ store in DuckDB -→ query with SQL -→ use results for analytics and charts -``` - ---- - -## Future Direction - -Later sprints can expand the storage layer step by step. - -Possible later additions: - -| Future Area | Possible Additions | -| --- | --- | -| Better source mapping | source-specific symbols, provider metadata | -| Watchlists | user-selected instruments | -| Reports | generated report metadata and history | -| Macro data | FRED indicators and observations | -| Paper trading | simulated orders, positions and portfolio history | -| Server architecture | PostgreSQL | -| SQL tooling | SQLGate with PostgreSQL | -| Cloud direction | managed PostgreSQL or cloud storage | - -SQLGate should be kept for a later PostgreSQL phase. - -It becomes useful when ARGUS moves toward: - -* server-based storage -* stronger database management -* richer metadata -* more stable application state -* user-facing features -* report history -* cloud-ready architecture - -Additional metadata such as documentation links, terms links or provider governance fields can also become useful later. - -For the first DuckDB phase, these details should stay in research documentation instead of the database schema. - ---- - -## Final Recommendation - -ARGUS should start with DuckDB as the first local analytics storage layer. - -DuckDB fits the current phase best because ARGUS needs local analytical SQL workflows, not a full server database yet. - -The first implementation should store historical market data in an OHLCV-ready structure. - -The recommended first data model is: - -```text -data_sources -instruments -price_bars -``` - -Notebook exploration should be the main developer workflow before SQL logic is moved into application code. - -The DuckDB CLI can be used for quick inspection. - -PostgreSQL and SQLGate should be introduced later when ARGUS moves toward a more product-like or cloud-based architecture. diff --git a/docs/roadmap.md b/docs/roadmap.md index c637327..85706af 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -27,121 +27,92 @@ Scope: Outcome: Sprint 1 established the local ARGUS foundation with package structure, GUI prototype, analytics prototype, tests, documentation, CI, Dependabot and governance files. -### Sprint 2 — Reporting & Market Analytics Foundation +### Sprint 2 — Market Analytics & Data Source Expansion **Status:** In progress -Move ARGUS from a simple FX-focused prototype toward a first usable market analytics and reporting tool. +Move from simple FX conversion toward broader market analytics. -**Scope:** - -- Add stronger market analytics metrics: +Scope: +- Add stronger market metrics: - cumulative return - strongest / weakest day - rolling volatility - - basic performance analytics - - basic risk analytics -- Add or improve real market data support: - + - performance analytics + - risk analytics +- Extend the current dashboard without adding unnecessary chart noise +- Add or evaluate new data clients: + - Frankfurter for historical FX data - yfinance for broader market data - - existing FX conversion remains available where useful +- Replace or reduce dependency on the current ExchangeRate API where needed - Improve pandas-based analysis workflows -- Introduce local storage for historical market data -- Add report generation and export -- Add a first simple prediction feature -- Introduce NiceGUI as the next GUI direction -- Extend the current dashboard with real market analytics -- Add tests for metric calculations, data transformations and storage behavior -- Improve CI/CD with first deployment or release automation steps - -**Outcome:** - -ARGUS becomes a basic market analytics and reporting tool. -Users can fetch market data, store it locally, calculate metrics, generate a first report and view results through a first modern dashboard. +- Add tests for metric calculations and data transformations +- Document metric definitions, assumptions and chart behavior ---- +Outcome: +ARGUS becomes a basic market analytics tool, not only a converter. -### Sprint 3 — Advanced Local Analytics & Product Quality +### Sprint 3 — Storage, Web-Ready UI & Data Architecture **Status:** Planned -Expand the local ARGUS application into a stronger analytics product with better data handling, UI structure, predictions and quality checks. - -**Scope:** - -- Extend the local storage layer -- Add a first local ETL workflow -- Improve the NiceGUI dashboard structure and usability -- Explore how NiceGUI can later interact with a more modern frontend stack such as Django, React or Node.js-based services -- Keep Tkinter as legacy/prototype unless it is no longer useful -- Add more metrics, instruments and prediction features -- Improve report templates and report structure -- Introduce first LLM-based summaries for generated reports -- Add first performance tests -- Introduce Snyk or another dependency/security scanning workflow -- Improve code quality, test coverage and maintainability - -**Outcome:** +Prepare ARGUS for persistent data workflows and a stronger product interface. -ARGUS becomes a more scalable local analytics application. -It can process more instruments, produce better reports, provide first automated summaries and offer more reliable insight into market data. - ---- +Scope: -### Sprint 4 — Extended Analysis & Cloud-Ready Foundation +- Add local storage layer: + - PostgreSQL, DuckDB, SQLite or Parquet depending on use case +- Store historical market data +- Separate ingestion, transformation, analytics and presentation layers more clearly +- Start NiceGUI as the main web-ready UI direction +- Keep Tkinter as legacy/prototype unless still useful +- Keep CLI as internal/debug interface only +- Add clearer architecture documentation +- Prepare the project for larger data workflows and external contributors -**Status:** Planned - -Prepare ARGUS for deeper analysis, cloud interaction and future portfolio-assistant workflows while keeping the local product usable and transparent. +Outcome: +ARGUS has a clearer data architecture and starts moving from local prototype toward a scalable analytics application. -**Scope:** +### Sprint 4 — Cloud, Pipelines & Portfolio-Grade Data Engineering -- Add Docker Compose for a more complete local development setup -- Introduce a first Azure connection, focused on simple storage or artifact exchange -- Improve the LLM workflow -- Introduce a first RAG-ready structure for reports, notes, documentation and stored analysis artifacts -- Add data quality checks -- Improve caching and efficient storage of market data -- Add more export options for users -- Add more metrics and better metadata visualization -- Improve transparency around data sources, generated reports and analysis assumptions -- Prepare clear interfaces for future cloud and assistant workflows +**Status:** Future -**Outcome:** +Turn ARGUS into a stronger end-to-end data engineering project. -ARGUS becomes ready to interact with a future cloud layer. -The application can produce clearer, more transparent market analysis and prepares the foundation for retrieval-based workflows, stronger automation and future ARGUS Core integration. +Scope: ---- +- Docker / Docker Compose +- Scheduled data ingestion +- Cloud storage or cloud database +- CI/CD improvements +- Data quality checks +- Basic pipeline orchestration +- Reporting layer +- Architecture diagram +- Deployment documentation -### Sprint 5 — Cloud Interaction & Agentic Monitoring Foundation +Target workflow: -**Status:** Planned +```text +API → Ingestion → Storage → Transformation → Analysis → Visualization → CI/CD +``` -Start the first cloud-connected ARGUS workflows and introduce the foundation for monitoring, agentic checks and strategy-support features. +### Sprint 5 — AI-Assisted Research & Agentic Monitoring -**Scope:** +**Status:** Future vision -- Add first cloud workflows that extend local analysis -- Connect local ARGUS workflows with the first cloud-side services -- Extend RAG over stored market notes, reports, documentation and analysis artifacts -- Add agentic checks for: +Add AI support only after the data, storage, service and reporting layers are stable. - - data quality - - anomalies - - recurring market scans - - report consistency -- Add first human-in-the-loop review workflows for signals or strategy ideas -- Add automated monitoring workflows -- Prepare the first foundations for: +Scope: - - paper trading - - backtesting - - controlled strategy evaluation - - future portfolio-assistant workflows +- LLM-assisted report summaries +- Explanation of unusual movements +- RAG over stored market notes, reports or documentation +- Agentic checks for data quality, anomalies and recurring market scans +- Human-in-the-loop signal review +- Automated monitoring workflows -**Outcome:** +Outcome: -ARGUS and the first cloud-side services begin to interact. -ARGUS becomes useful not only as an analytics and reporting tool, but also as the first foundation for monitoring, strategy evaluation and controlled market-research workflows. +ARGUS starts behaving like its name: a system that continuously watches market data, evaluates it and helps generate useful signals. diff --git a/docs/storage.md b/docs/storage.md deleted file mode 100644 index 27ce106..0000000 --- a/docs/storage.md +++ /dev/null @@ -1,154 +0,0 @@ -# ARGUS Storage Layer - -ARGUS uses DuckDB as the local storage layer for normalized market data. - -The storage layer stores ARGUS-internal market data structures and provides reusable historical data for analytics, charts, dashboards and reports. - -The storage design follows the direction described in [`docs/research-databases-and-storage.md`](research-databases-and-storage.md). - -## Storage Workflow - -ARGUS uses a storage-first workflow for historical market data. - -```text -User / GUI / Analytics request - ↓ -Market data service - ↓ -Check DuckDB storage - ↓ -If data exists: - read stored data - return it for analytics, charts or reports - -If data is missing: - fetch data from a client/API - normalize the response into ARGUS-internal data - return the normalized data - save the normalized data in DuckDB -``` - -DuckDB is used to avoid unnecessary repeated API calls and to make historical market data reusable across analytics, dashboard and reporting workflows. - -Fresh API data can be used immediately after normalization and is also persisted so future requests can use the local storage layer first. - -## Schema Overview - -The first storage schema is based on three related entities: - -```text -data_sources -instruments -price_bars -``` - -### `data_sources` - -Stores where market data came from. - -Examples: - -```text -yfinance -ExchangeRate API -Frankfurter -FRED -``` - -Each source describes a provider or API that can deliver market, FX or macro data. - -### `instruments` - -Stores what ARGUS can analyze. - -Examples: - -```text -EUR/USD -AAPL -SPY -BTC-USD -``` - -An instrument represents the internal ARGUS identity of an asset, currency pair, ETF, index or other market object. - -Provider-specific symbols should be normalized before storage. For example: - -```text -yfinance provider symbol: EURUSD=X -ARGUS instrument symbol: EUR/USD -``` - -### `price_bars` - -Stores historical time-series values in an OHLCV-ready structure. - -A price bar belongs to: - -```text -one data source -one instrument -one timestamp -one timeframe -``` - -FX rates are stored as `close` values. - -For simple FX data, the remaining OHLCV fields can stay empty. For broader market data, the same structure can store open, high, low, close, adjusted close and volume values. - -The combination of source, instrument, timestamp and timeframe identifies a unique stored price bar. - -## Internal Models and Storage - -ARGUS uses internal domain models before data is stored: - -```text -DataSource -Instrument -PriceBar -``` - -These models describe the meaning of the data inside ARGUS. - -The storage layer translates these internal models into DuckDB tables: - -```text -DataSource -> data_sources -Instrument -> instruments -PriceBar -> price_bars -``` - -In Python, a `PriceBar` references a `DataSource` and an `Instrument`. - -In DuckDB, this relationship is stored through IDs: - -```text -price_bars.source_id -> data_sources.id -price_bars.instrument_id -> instruments.id -``` - -This keeps the database normalized while still allowing ARGUS to work with meaningful internal models in Python. - -## Reading Stored Data - -Stored price bars can be read by: - -```text -source -instrument -start date -end date -``` - -The storage layer joins `price_bars`, `data_sources` and `instruments` so that stored IDs become readable market data again. - -Read operations return tabular data that can be used by: - -```text -analytics -charts -dashboards -reports -``` - -This allows ARGUS to process stored historical data without depending on raw API response structures. diff --git a/src/argus/analytics/charts/trend_chart.py b/src/argus/analytics/charts/trend_chart.py index 1293361..db5dacb 100644 --- a/src/argus/analytics/charts/trend_chart.py +++ b/src/argus/analytics/charts/trend_chart.py @@ -1,8 +1,8 @@ import matplotlib.pyplot as plt -from argus.services.timeseries_service import prepare_trend_analysis +import pandas as pd -def create_trendchart(curr_symbol: str, start: str, end: str, interval: str): +def create_trendchart(df: pd.DataFrame, min_max_rates: dict): """ Create a trend chart for exchange-rate analysis. @@ -30,10 +30,6 @@ def create_trendchart(curr_symbol: str, start: str, end: str, interval: str): Minimum and maximum exchange-rate values are marked with scatter points and annotations. """ - result = prepare_trend_analysis(curr_symbol, start, end, interval) - if result is None: - return None - df, min_max_rates = result min_date = min_max_rates["min_date"][0] min_rate = min_max_rates["min_rate"][0] max_date = min_max_rates["max_date"][0] diff --git a/src/argus/clients/exchangerate_client.py b/src/argus/clients/exchangerate_client.py index d899718..8ea5e89 100644 --- a/src/argus/clients/exchangerate_client.py +++ b/src/argus/clients/exchangerate_client.py @@ -24,16 +24,6 @@ def get_rates(curr1: str, curr2: str): resp.raise_for_status() payload = resp.json() - if payload["result"] == "success": - data["result"] = "success" - data["conversion_rate"] = payload["conversion_rate"] - return data - else: - data["result"] = "error" - data["error_type"] = payload.get("error_type") - check_error(data["error_type"]) - return None - except req.exceptions.Timeout: print("API hat zu lange gebraucht.") return None @@ -51,6 +41,16 @@ def get_rates(curr1: str, curr2: str): print("Unerwartete API-Antwortstruktur.") return None + if payload.get("result") == "success": + data["result"] = "success" + data["conversion_rate"] = payload.get("conversion_rate") + return data + else: + data["result"] = "error" + data["error_type"] = payload.get("error_type") + check_error(data["error_type"]) + return None + def check_error(err_type: str) -> None: """ diff --git a/src/argus/clients/yfinance_client.py b/src/argus/clients/yfinance_client.py index de86041..468b87b 100644 --- a/src/argus/clients/yfinance_client.py +++ b/src/argus/clients/yfinance_client.py @@ -1,43 +1,48 @@ import yfinance as yf -import logging - - -def get_timeseries(curr_symbol, start, end, interval): - """ - Fetch historical exchange-rate time series data from Yahoo Finance. - - Args: - curr_symbol (str): Currency symbol used by Yahoo Finance, for example - "EURUSD=X". - start (str): Start date of the requested time range in YYYY-MM-DD format. - end (str): End date of the requested time range in YYYY-MM-DD format. - interval (str): Data interval supported by Yahoo Finance, for example - "1d", "1h", or "15m". - - Returns: - pandas.DataFrame | None: A DataFrame containing the columns ``date`` and - ``rate`` if data was successfully fetched. Returns ``None`` if the - request fails, returns no data, or an exception occurs. - """ +import pandas as pd +from argus.domain.internal_models import ( + MarketDataRequest, + PRICE_BAR_COLUMNS, + YFINANCE_PRICE_BAR_MAPPING, +) + + +def get_timeseries(request: MarketDataRequest) -> pd.DataFrame: + start = str(request.start) + end = str(request.end) + timeframe = request.timeframe + curr_pair = ( + f"{request.instrument.base_currency}{request.instrument.quote_currency}=X" + ) + try: - yf_logger = logging.getLogger("yfinance") - yf_logger.disabled = True - data = yf.download( - tickers=curr_symbol, + raw_resp = yf.download( + tickers=curr_pair, start=start, end=end, - interval=interval, + interval=timeframe, multi_level_index=False, progress=False, ) - yf_logger.disabled = False - if data is None: - return None - if data.empty: - return None - data = data.reset_index() - data = data[["Date", "Close"]] - data = data.rename(columns={"Date": "date", "Close": "rate"}) - return data except Exception: - return None + raise ConnectionError("Network error or connection timeout") + + if raw_resp is None: + raise ConnectionError("Yahoo Finance API returned an invalid response") + + if ( + raw_resp.empty + or "Close" not in raw_resp.columns + or raw_resp["Close"].dropna().empty + ): + raise ValueError("Quote not found or no data available for symbol") + + resp = normalize_yfinance_bars(raw_resp) + return resp + + +def normalize_yfinance_bars(raw_df: pd.DataFrame) -> pd.DataFrame: + df = raw_df.copy() + df = df.reset_index() + df = df.rename(columns=YFINANCE_PRICE_BAR_MAPPING) + return df[list(PRICE_BAR_COLUMNS)] diff --git a/src/argus/domain/internal_models.py b/src/argus/domain/internal_models.py index 3b7630e..0d64796 100644 --- a/src/argus/domain/internal_models.py +++ b/src/argus/domain/internal_models.py @@ -1,5 +1,6 @@ from dataclasses import dataclass from datetime import date +import pandas as pd @dataclass @@ -20,15 +21,51 @@ class Instrument: quote_currency: str | None = None +PRICE_BAR_COLUMNS = ( + "timestamp", + "open", + "high", + "low", + "close", + "adjusted_close", + "volume", +) + +YFINANCE_PRICE_BAR_MAPPING = { + "Date": "timestamp", + "Open": "open", + "High": "high", + "Low": "low", + "Close": "close", + "Adj Close": "adjusted_close", + "Volume": "volume", +} + + @dataclass -class PriceBar: +class MarketDataRequest: source: DataSource instrument: Instrument - timestamp: date timeframe: str - close: float - open: float | None = None - high: float | None = None - low: float | None = None - adjusted_close: float | None = None - volume: float | None = None + start: date + end: date + + +@dataclass +class MarketDataResponse: + source: DataSource + instrument: Instrument + bars: pd.DataFrame + message: str = "" + + def __post_init__(self) -> None: + if not isinstance(self.bars, pd.DataFrame): + raise TypeError("bars must be a pandas DataFrame") + if self.message == "": + missing_cols = [ + col for col in PRICE_BAR_COLUMNS if col not in self.bars.columns + ] + if missing_cols: + raise ValueError( + f"Missing required columns in bars DataFrame: {missing_cols}" + ) diff --git a/src/argus/gui/app.py b/src/argus/gui/app.py index 9a19523..02f4c1f 100644 --- a/src/argus/gui/app.py +++ b/src/argus/gui/app.py @@ -1,6 +1,7 @@ import tkinter as tk +import pandas as pd from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg -from argus.analytics.charts.trend_chart import create_trendchart +from argus.services.market_data_service import prepare_trend_analysis from argus.services.calculator_service import calc, check_op from argus.services.conversion_service import convert, check_currency from argus.domain.validation import parse_amount @@ -76,11 +77,6 @@ def show_trend() -> None: global trend_canvas global trend_chart_widget - curr_symbol = "EURUSD=X" - start = "2024-01-01" - end = "2025-01-01" - interval = "1d" - calc_frame.pack_forget() conv_frame.pack_forget() menu_frame.pack_forget() @@ -90,7 +86,8 @@ def show_trend() -> None: content.pack(side="top", fill=tk.BOTH, expand=True) if trend_canvas is None: - fig = create_trendchart(curr_symbol, start, end, interval) + df = pd.DataFrame() + fig = prepare_trend_analysis(df) if fig is None: return None fig.set_size_inches(7, 4) diff --git a/src/argus/main.py b/src/argus/main.py index 7130dbf..105de61 100644 --- a/src/argus/main.py +++ b/src/argus/main.py @@ -1,11 +1,14 @@ from argus.gui.app import app +from argus.storage.database import initialize_database -def main() -> None: +def main(db) -> None: """ The main function that starts the application. """ + initialize_database(db) app() -main() +db = "" +main(db) diff --git a/src/argus/services/market_data_service.py b/src/argus/services/market_data_service.py new file mode 100644 index 0000000..9328141 --- /dev/null +++ b/src/argus/services/market_data_service.py @@ -0,0 +1,47 @@ +from argus.domain.internal_models import MarketDataRequest, MarketDataResponse +from argus.clients.yfinance_client import get_timeseries +from argus.storage.database import read_price_bars, insert_price_bar +import pandas as pd + + +def get_market_data( + db: str, + request: MarketDataRequest, +) -> MarketDataResponse: + """ + Get a time series either from local stroage or client with first-storage-workflow + + Args: + curr_symbol (str): Currency symbol used by Yahoo Finance, for example + "EURUSD=X". + start (str): Start date of the requested time range in YYYY-MM-DD format. + end (str): End date of the requested time range in YYYY-MM-DD format. + intervall (str): Data interval supported by Yahoo Finance, for example + "1d", "1h", or "15m". + + Returns: + pd.DataFrame | None: A + DataFrame with dates and rates. Returns + ``None`` if no time-series data could be fetched. + """ + bars = read_price_bars(db, request) + if not (bars.empty): + db_response = MarketDataResponse( + source=request.source, instrument=request.instrument, bars=bars, message="" + ) + return db_response + + try: + bars = get_timeseries(request) + api_response = MarketDataResponse( + source=request.source, instrument=request.instrument, bars=bars + ) + insert_price_bar(db, api_response) + return api_response + except (ConnectionError, ValueError) as e: + return MarketDataResponse( + source=request.source, + instrument=request.instrument, + bars=pd.DataFrame(), + message=str(e), + ) diff --git a/src/argus/services/timeseries_service.py b/src/argus/services/timeseries_service.py deleted file mode 100644 index b6251bb..0000000 --- a/src/argus/services/timeseries_service.py +++ /dev/null @@ -1,41 +0,0 @@ -import pandas as pd -from argus.clients.yfinance_client import get_timeseries -from argus.analytics.metrics.trend_metrics import ( - add_rolling_average, - add_daily_percentage_change, - get_min_max_rates, -) - - -def prepare_trend_analysis( - curr_symbol: str, start: str, end: str, intervall: str -) -> tuple[pd.DataFrame, dict] | None: - """ - Prepare time-series data for trend analysis. - - Fetches historical exchange-rate data for the given currency symbol and - enriches it with daily percentage changes and a rolling average. It also - calculates the minimum and maximum exchange rates for the resulting time - series. - - Args: - curr_symbol (str): Currency symbol used by Yahoo Finance, for example - "EURUSD=X". - start (str): Start date of the requested time range in YYYY-MM-DD format. - end (str): End date of the requested time range in YYYY-MM-DD format. - intervall (str): Data interval supported by Yahoo Finance, for example - "1d", "1h", or "15m". - - Returns: - tuple[pd.DataFrame, dict] | None: A tuple containing the prepared - DataFrame and a dictionary with minimum and maximum rates. Returns - ``None`` if no time-series data could be fetched. - """ - - df = get_timeseries(curr_symbol, start, end, intervall) - if df is None: - return None - df = add_daily_percentage_change(df) - df = add_rolling_average(df) - min_max_rates = get_min_max_rates(df) - return df, min_max_rates diff --git a/src/argus/services/trend_analysis_service.py b/src/argus/services/trend_analysis_service.py new file mode 100644 index 0000000..a8428ae --- /dev/null +++ b/src/argus/services/trend_analysis_service.py @@ -0,0 +1,37 @@ +from argus.analytics.metrics.trend_metrics import ( + add_rolling_average, + add_daily_percentage_change, + get_min_max_rates, +) +from matplotlib.figure import Figure +from argus.analytics.charts.trend_chart import create_trendchart +from argus.services.market_data_service import get_market_data +from argus.domain.internal_models import MarketDataRequest + + +def prepare_trend_analysis(db: str, request: MarketDataRequest) -> Figure | str: + """ + Prepare time-series data and generate a trend analysis chart. + + Enriches the historical exchange-rate DataFrame with daily percentage changes + and a rolling average, calculates the minimum and maximum rates, and uses + the result to build a trend visualization chart. + + Args: + df (pd.DataFrame): A DataFrame containing market data time-series. + + Returns: + plotly.graph_objects.Figure: A figure object representing the + generated trend chart. + """ + response = get_market_data(db, request) + + if response.message: + return response.message + + df = response.bars.copy() + df = add_daily_percentage_change(df) + df = add_rolling_average(df) + min_max_rates = get_min_max_rates(df) + fig = create_trendchart(df, min_max_rates) + return fig diff --git a/src/argus/storage/__init__.py b/src/argus/storage/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index ea7128c..aa00c78 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -1,7 +1,11 @@ import duckdb -from datetime import date import pandas as pd -from argus.domain.internal_models import DataSource, PriceBar, Instrument +from argus.domain.internal_models import ( + DataSource, + Instrument, + MarketDataRequest, + MarketDataResponse, +) def initialize_database(database_path: str) -> None: @@ -47,7 +51,6 @@ def initialize_database(database_path: str) -> None: source_id INTEGER NOT NULL, instrument_id INTEGER NOT NULL, timestamp DATE NOT NULL, - timeframe TEXT NOT NULL, close DOUBLE NOT NULL, open DOUBLE, high DOUBLE, @@ -56,7 +59,7 @@ def initialize_database(database_path: str) -> None: volume DOUBLE, FOREIGN KEY (source_id) REFERENCES data_sources (id), FOREIGN KEY (instrument_id) REFERENCES instruments (id), - UNIQUE (source_id, instrument_id, timestamp, timeframe) + UNIQUE (source_id, instrument_id, timestamp,close) ); """, ] @@ -186,13 +189,27 @@ def get_or_create_instrument(connection, instrument: Instrument) -> int: return result[0] -def insert_price_bar(db: str, price_bar: PriceBar) -> None: +def insert_price_bar(db: str, marketdata: MarketDataResponse) -> None: + """ + Insert a price bar into the database. + + Ensures that the related data source and instrument exist, then inserts + the price bar into the ``price_bars`` table. Duplicate price bars are + ignored through the table's unique constraint. + + Args: + db (str): Path to the DuckDB database file. + price_bar (PriceBar): Price bar model containing source, + instrument, timestamp and market values. + + Returns: + None + """ insert_query = """ INSERT INTO price_bars ( source_id, instrument_id, timestamp, - timeframe, close, open, high, @@ -200,46 +217,58 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: adjusted_close, volume ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING; """ connection = duckdb.connect(db) try: - source_id = get_or_create_source(connection, price_bar.source) - instrument_id = get_or_create_instrument(connection, price_bar.instrument) - connection.execute( - query=insert_query, - parameters=[ - source_id, - instrument_id, - price_bar.timestamp, - price_bar.timeframe, - price_bar.close, - price_bar.open, - price_bar.high, - price_bar.low, - price_bar.adjusted_close, - price_bar.volume, - ], - ) + source_id = get_or_create_source(connection, marketdata.source) + instrument_id = get_or_create_instrument(connection, marketdata.instrument) + if marketdata.bars is None: + return None + for _, row in marketdata.bars.iterrows(): + connection.execute( + query=insert_query, + parameters=[ + source_id, + instrument_id, + row["timestamp"], + row["close"], + row["open"], + row["high"], + row["low"], + row["adjusted_close"], + row["volume"], + ], + ) finally: connection.close() -def read_price_bars( - db: str, - source: DataSource, - instrument: Instrument, - start_date: date, - end_date: date, -) -> pd.DataFrame: +def read_price_bars(db: str, request: MarketDataRequest) -> pd.DataFrame: + """ + Read price bars for a source, instrument, and date range. + + Queries stored price bars joined with their data source and instrument + metadata. Results are ordered by timestamp and returned as a pandas + DataFrame. + + Args: + db (str): Path to the DuckDB database file. + source (DataSource): Data source used to filter the stored price bars. + instrument (Instrument): Instrument used to filter the stored price bars. + start_date (date): Inclusive start date of the requested time range. + end_date (date): Inclusive end date of the requested time range. + + Returns: + pd.DataFrame: DataFrame containing matching price bars and metadata. + """ search_query = """ SELECT data_sources.name AS source_name, instruments.symbol AS instrument_symbol, price_bars.timestamp, - price_bars.timeframe, price_bars.open, price_bars.high, price_bars.low, @@ -260,10 +289,10 @@ def read_price_bars( result = connection.execute( query=search_query, parameters=[ - source.name, - instrument.symbol, - start_date, - end_date, + request.source.name, + request.instrument.symbol, + request.start, + request.end, ], ).df() finally: diff --git a/tests/test_exchangerate_client.py b/tests/test_exchangerate_client.py index 132a8a8..faf0864 100644 --- a/tests/test_exchangerate_client.py +++ b/tests/test_exchangerate_client.py @@ -3,7 +3,7 @@ from argus.clients.exchangerate_client import get_rates, check_error -def test_check_currency_timeout(monkeypatch, capsys): +def test_check_currency_timeout(monkeypatch): def test_get_resp(url, timeout): raise req.exceptions.Timeout() @@ -12,11 +12,8 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured = capsys.readouterr() - assert "API hat zu lange gebraucht." in captured.out - -def test_check_currency_connection_error(monkeypatch, capsys): +def test_check_currency_connection_error(monkeypatch): def test_get_resp(url, timeout): raise req.exceptions.ConnectionError() @@ -25,11 +22,8 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured = capsys.readouterr() - assert "Keine Verbindung zur API." in captured.out - -def test_check_currency_request_exception(monkeypatch, capsys): +def test_check_currency_request_exception(monkeypatch): def test_get_resp(url, timeout): raise req.exceptions.RequestException("Testfehler") @@ -38,11 +32,8 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured = capsys.readouterr() - assert "Request fehlgeschlagen:" in captured.out - -def test_check_currency_value_error(monkeypatch, capsys): +def test_check_currency_value_error(monkeypatch): test_resp = Mock() test_resp.raise_for_status.return_value = None test_resp.json.side_effect = ValueError("Ungültige JSON-Antwort") @@ -55,16 +46,14 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured = capsys.readouterr() - assert "Fehler beim Verarbeiten der API-Antwort." in captured.out - -def test_check_currency_key_error(monkeypatch, capsys): +def test_check_currency_key_error(monkeypatch): test_resp = Mock() test_resp.raise_for_status.return_value = None test_resp.json.return_value = { - "result": "success", # not passing "success" bypases the "conversion_rate" checking + "result": "", "error_type": "", + # "conversion_rate" fehlt absichtlich } def test_get_resp(url, timeout): @@ -75,9 +64,6 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured = capsys.readouterr() - assert "Unerwartete API-Antwortstruktur." in captured.out - def test_check_currency_valid(monkeypatch): test_resp = Mock() @@ -97,7 +83,7 @@ def test_get_resp(url, timeout): assert data == {"result": "success", "error_type": "", "conversion_rate": 1.2} -def test_check_currency_invalid(monkeypatch, capsys): +def test_check_currency_invalid(monkeypatch): test_resp = Mock() test_resp.raise_for_status.return_value = None test_resp.json.return_value = { @@ -114,9 +100,6 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured = capsys.readouterr() - assert "Invalid request! Please try again later." in captured.out - def test_check_error(capsys): check_error("unsupported-code") @@ -140,7 +123,3 @@ def test_check_error(capsys): captured.out == "Request limit reached! Please try again later or upgrade to exchangerate-api.com.\n" ) - - check_error("Some unknown Error") - captured = capsys.readouterr() - assert captured.out == "" diff --git a/tests/test_internal_models.py b/tests/test_internal_models.py index 97df4c6..0fc7f0c 100644 --- a/tests/test_internal_models.py +++ b/tests/test_internal_models.py @@ -1,101 +1,66 @@ -from argus.domain.internal_models import DataSource, Instrument, PriceBar +import pytest +import pandas as pd from datetime import date +from argus.domain.internal_models import DataSource, Instrument, MarketDataResponse -def test_data_source_can_be_created() -> None: - source = DataSource( - name="yfinance", - provider_kind="fx_rates", - ) - - assert source.name == "yfinance" - assert source.provider_kind == "fx_rates" - assert source.requires_api_key is False +@pytest.fixture +def valid_source(): + return DataSource(name="Yahoo", provider_kind="yfinance_api") -def test_instrument_can_be_created() -> None: - instrument = Instrument( - symbol="EUR/USD", - name="Euro / US Dollar", - asset_class="fx", - base_currency="EUR", - quote_currency="USD", - ) +@pytest.fixture +def valid_instrument(): + return Instrument(symbol="AAPL", name="Apple Inc.", asset_class="stock") - assert instrument.symbol == "EUR/USD" - assert instrument.name == "Euro / US Dollar" - assert instrument.asset_class == "fx" - assert instrument.base_currency == "EUR" - assert instrument.quote_currency == "USD" - assert instrument.currency is None - assert instrument.exchange is None +def test_market_data_response_accepts_valid_dataframe( + valid_source, valid_instrument +) -> None: + valid_bar = { + "timestamp": [date(2026, 1, 1)], + "open": [150.0], + "high": [155.0], + "low": [149.0], + "close": [153.5], + "adjusted_close": [153.5], + "volume": [1000000.0], + } + df = pd.DataFrame(valid_bar) -def test_rate_bar_can_be_created() -> None: - source = DataSource( - name="yfinance", - provider_kind="fx_rates", + resp = MarketDataResponse( + source=valid_source, instrument=valid_instrument, bars=df, message="" ) + assert resp.bars.equals(df) - instrument_rate = Instrument( - symbol="EUR/USD", - name="Euro / US Dollar", - asset_class="fx", - base_currency="EUR", - quote_currency="USD", - ) - price_bar = PriceBar( - source=source, - instrument=instrument_rate, - timestamp=date(2026, 1, 1), - timeframe="1d", - close=1.89, - ) +def test_market_data_response_raises_error_on_missing_columns( + valid_source, valid_instrument +) -> None: + incomplete_bar = { + "timestamp": [date(2026, 1, 1)], + "close": [153.5], + } + df = pd.DataFrame(incomplete_bar) - assert price_bar.source == source - assert price_bar.instrument == instrument_rate - assert price_bar.timestamp == date(2026, 1, 1) - assert price_bar.timeframe == "1d" - assert price_bar.close == 1.89 - assert price_bar.open is None - assert price_bar.high is None - assert price_bar.low is None - assert price_bar.adjusted_close is None - assert price_bar.volume is None + with pytest.raises(ValueError) as exc_info: + MarketDataResponse( + source=valid_source, instrument=valid_instrument, bars=df, message="" + ) + assert "Missing required columns" in str(exc_info.value) -def test_stock_ohlcv_bar_can_be_created() -> None: - source = DataSource( - name="yfinance", - provider_kind="market_prices", - ) - instrument = Instrument( - symbol="AAPL", - name="Apple Inc.", - asset_class="stock", - currency="USD", - exchange="NASDAQ", - ) +def test_market_data_response_raises_error_if_not_a_dataframe( + valid_source, valid_instrument +) -> None: + invalid_input = "I'm just a string :D" - price_bar = PriceBar( - source=source, - instrument=instrument, - timestamp=date(2026, 1, 1), - timeframe="1d", - open=187.15, - high=188.44, - low=183.89, - close=185.64, - adjusted_close=184.25, - volume=50_200_000, - ) + with pytest.raises(TypeError) as exc_info: + MarketDataResponse( + source=valid_source, + instrument=valid_instrument, + bars=invalid_input, # type: ignore + ) - assert price_bar.instrument.symbol == "AAPL" - assert price_bar.open == 187.15 - assert price_bar.high == 188.44 - assert price_bar.low == 183.89 - assert price_bar.close == 185.64 - assert price_bar.adjusted_close == 184.25 - assert price_bar.volume == 50_200_000 + assert "must be a pandas DataFrame" in str(exc_info.value) diff --git a/tests/test_market_data_series.py b/tests/test_market_data_series.py new file mode 100644 index 0000000..b7832a2 --- /dev/null +++ b/tests/test_market_data_series.py @@ -0,0 +1,138 @@ +import pytest +import pandas as pd +from datetime import date +from unittest.mock import Mock +from argus.domain.internal_models import ( + DataSource, + Instrument, + MarketDataRequest, + MarketDataResponse, +) +from argus.services.market_data_service import get_market_data + + +@pytest.fixture +def sample_source(): + return DataSource(name="Yahoo", provider_kind="yfinance_api") + + +@pytest.fixture +def sample_instrument(): + return Instrument(symbol="AAPL", name="Apple Inc.", asset_class="stock") + + +@pytest.fixture +def sample_response(sample_source, sample_instrument): + test_bar = { + "timestamp": date(2026, 1, 1), + "open": None, + "high": None, + "low": None, + "close": 1.89, + "adjusted_close": None, + "volume": None, + } + return MarketDataResponse( + source=sample_source, + instrument=sample_instrument, + bars=pd.DataFrame(test_bar, index=[0]), + ) + + +def test_get_market_data_storage_hit( + monkeypatch, sample_source, sample_instrument, sample_response +): + req = MarketDataRequest( + source=sample_source, + instrument=sample_instrument, + timeframe="1d", + start=date(2026, 1, 1), + end=date(2026, 1, 1), + ) + + monkeypatch.setattr( + "argus.services.market_data_service.read_price_bars", + lambda db, r: sample_response.bars, + ) + + mock_get_timeseries = Mock() + monkeypatch.setattr( + "argus.services.market_data_service.get_timeseries", mock_get_timeseries + ) + + res = get_market_data("mock_db_path", req) + + assert res is not None + assert not res.bars.empty + mock_get_timeseries.assert_not_called() + + +def test_get_market_data_storage_miss( + monkeypatch, sample_source, sample_instrument, sample_response +): + req = MarketDataRequest( + source=sample_source, + instrument=sample_instrument, + timeframe="1d", + start=date(2026, 1, 1), + end=date(2026, 1, 1), + ) + + monkeypatch.setattr( + "argus.services.market_data_service.read_price_bars", + lambda db, r: pd.DataFrame(), + ) + monkeypatch.setattr( + "argus.services.market_data_service.get_timeseries", + lambda r: sample_response.bars, + ) + + mock_insert = Mock() + monkeypatch.setattr( + "argus.services.market_data_service.insert_price_bar", mock_insert + ) + + res = get_market_data("mock_db_path", req) + + assert res is not None + mock_insert.assert_called_once() + + +def test_get_market_data_handles_client_exceptions( + monkeypatch, sample_source, sample_instrument +): + req = MarketDataRequest( + source=sample_source, + instrument=sample_instrument, + timeframe="1d", + start=date(2026, 1, 1), + end=date(2026, 1, 1), + ) + + monkeypatch.setattr( + "argus.services.market_data_service.read_price_bars", + lambda db, r: pd.DataFrame(), + ) + + def mock_value_error(request): + raise ValueError("Quote not found or no data available for symbol") + + monkeypatch.setattr( + "argus.services.market_data_service.get_timeseries", mock_value_error + ) + + res_val = get_market_data("mock_db_path", req) + assert res_val.bars.empty + assert res_val.message == "Quote not found or no data available for symbol" + + # Fall 2: ConnectionError testen (z.B. Internet weg) + def mock_connection_error(request): + raise ConnectionError("Network error or connection timeout") + + monkeypatch.setattr( + "argus.services.market_data_service.get_timeseries", mock_connection_error + ) + + res_conn = get_market_data("mock_db_path", req) + assert res_conn.bars.empty + assert res_conn.message == "Network error or connection timeout" diff --git a/tests/test_storage_database.py b/tests/test_storage_database.py index d513008..9a25ac3 100644 --- a/tests/test_storage_database.py +++ b/tests/test_storage_database.py @@ -1,8 +1,13 @@ from datetime import date - +import pytest import duckdb - -from argus.domain.internal_models import DataSource, Instrument, PriceBar +import pandas as pd +from argus.domain.internal_models import ( + DataSource, + Instrument, + MarketDataRequest, + MarketDataResponse, +) from argus.storage.database import ( initialize_database, insert_price_bar, @@ -10,26 +15,16 @@ ) -def test_initialize_database_creates_required_tables(tmp_path): - db = tmp_path / "test.duckdb" - - initialize_database(db) - connection = duckdb.connect(db) - tables = connection.execute("SHOW TABLES;").fetchall() - connection.close() - table_names = {row[0] for row in tables} - - assert "data_sources" in table_names - assert "instruments" in table_names - assert "price_bars" in table_names - - -def test_data_is_inserted(tmp_path): - source = DataSource( +@pytest.fixture +def sample_source(): + return DataSource( name="Yahoo", provider_kind="yfinance_api", requires_api_key=False ) - instrument = Instrument( + +@pytest.fixture +def sample_instrument(): + return Instrument( symbol="EUR/USD", name="EUR - USD Rate", asset_class="fx", @@ -37,25 +32,52 @@ def test_data_is_inserted(tmp_path): quote_currency="USD", ) - pricebar = PriceBar( - source=source, - instrument=instrument, - timestamp=date(2026, 1, 1), - timeframe="1d", - close=1.89, + +@pytest.fixture +def sample_response(sample_source, sample_instrument): + test_bar = { + "timestamp": date(2026, 1, 1), + "open": None, + "high": None, + "low": None, + "close": 1.89, + "adjusted_close": None, + "volume": None, + } + return MarketDataResponse( + source=sample_source, + instrument=sample_instrument, + bars=pd.DataFrame(test_bar, index=[0]), ) + +@pytest.fixture +def db_path(tmp_path): db = tmp_path / "test.duckdb" initialize_database(db) - insert_price_bar(db, pricebar) - connection = duckdb.connect(db) + return db + + +def test_initialize_database_creates_required_tables(db_path): + connection = duckdb.connect(str(db_path)) + tables = connection.execute("SHOW TABLES;").fetchall() + connection.close() + table_names = {row[0] for row in tables} + + assert "data_sources" in table_names + assert "instruments" in table_names + assert "price_bars" in table_names + + +def test_data_is_inserted(db_path, sample_response) -> None: + insert_price_bar(db_path, sample_response) + + connection = duckdb.connect(str(db_path)) instrument_count = connection.execute( "SELECT COUNT(*) FROM instruments;" ).fetchone() - source_count = connection.execute("SELECT COUNT(*) FROM data_sources;").fetchone() - price_bar_count = connection.execute("SELECT COUNT(*) FROM price_bars;").fetchone() assert instrument_count is not None @@ -66,157 +88,71 @@ def test_data_is_inserted(tmp_path): assert price_bar_count[0] == 1 -def test_fx_has_correct_format(tmp_path): - source = DataSource( - name="Yahoo", provider_kind="yfinance_api", requires_api_key=False - ) +def test_fx_has_correct_format(db_path, sample_response) -> None: + insert_price_bar(db_path, sample_response) - instrument = Instrument( - symbol="EUR/USD", - name="EUR - USD Rate", - asset_class="fx", - base_currency="EUR", - quote_currency="USD", - ) - - pricebar = PriceBar( - source=source, - instrument=instrument, - timestamp=date(2026, 1, 1), - timeframe="1d", - close=1.89, - ) - - db = tmp_path / "test.duckdb" - initialize_database(db) - insert_price_bar(db, pricebar) - connection = duckdb.connect(db) - - price_bar_fx = connection.execute("SELECT * FROM price_bars;").fetchone() - connection.close() + connection = duckdb.connect(str(db_path)) + try: + price_bar_fx = connection.execute("SELECT * FROM price_bars;").fetchone() + finally: + connection.close() assert price_bar_fx is not None assert price_bar_fx[0] == 1 assert price_bar_fx[1] == 1 assert price_bar_fx[2] == 1 assert price_bar_fx[3] == date(2026, 1, 1) - assert price_bar_fx[4] == "1d" - assert price_bar_fx[5] == 1.89 + assert price_bar_fx[4] == 1.89 + assert price_bar_fx[5] is None assert price_bar_fx[6] is None assert price_bar_fx[7] is None assert price_bar_fx[8] is None assert price_bar_fx[9] is None - assert price_bar_fx[10] is None - -def test_duplicates_are_ignored(tmp_path): - source = DataSource( - name="Yahoo", provider_kind="yfinance_api", requires_api_key=False - ) - - instrument = Instrument( - symbol="EUR/USD", - name="EUR - USD Rate", - asset_class="fx", - base_currency="EUR", - quote_currency="USD", - ) - pricebar = PriceBar( - source=source, - instrument=instrument, - timestamp=date(2026, 1, 1), - timeframe="1d", - close=1.89, - ) +def test_duplicates_are_ignored(db_path, sample_response) -> None: + insert_price_bar(db_path, sample_response) + insert_price_bar(db_path, sample_response) # Erneuter Insert des Duplikats - db = tmp_path / "test.duckdb" - initialize_database(db) - insert_price_bar(db, pricebar) - insert_price_bar(db, pricebar) - connection = duckdb.connect(db) - count = connection.execute("SELECT COUNT(*) FROM price_bars;").fetchone() + connection = duckdb.connect(str(db_path)) + try: + count = connection.execute("SELECT COUNT(*) FROM price_bars;").fetchone() + finally: + connection.close() assert count is not None assert count[0] == 1 -def test_read_price_bars_returns_matching_data(tmp_path): - source = DataSource( - name="Yahoo", - provider_kind="yfinance_api", - requires_api_key=False, - ) - - instrument = Instrument( - symbol="EUR/USD", - name="EUR - USD Rate", - asset_class="fx", - base_currency="EUR", - quote_currency="USD", - ) - - pricebar = PriceBar( - source=source, - instrument=instrument, - timestamp=date(2026, 1, 1), +def test_read_price_bars_returns_matching_data( + db_path, sample_source, sample_instrument, sample_response +) -> None: + req = MarketDataRequest( + source=sample_source, + instrument=sample_instrument, timeframe="1d", - close=1.89, + start=date(2026, 1, 1), + end=date(2026, 1, 1), ) + insert_price_bar(db_path, sample_response) - db = tmp_path / "test.duckdb" - initialize_database(db) - insert_price_bar(db, pricebar) - - result = read_price_bars( - db=db, - source=source, - instrument=instrument, - start_date=date(2026, 1, 1), - end_date=date(2026, 1, 31), - ) + result = read_price_bars(db_path, req) - assert result.empty is False assert len(result) == 1 - assert result.iloc[0]["source_name"] == "Yahoo" - assert result.iloc[0]["instrument_symbol"] == "EUR/USD" - assert result.iloc[0]["timeframe"] == "1d" assert result.iloc[0]["close"] == 1.89 -def test_read_price_bars_returns_empty_dataframe_for_missing_range(tmp_path): - source = DataSource( - name="Yahoo", - provider_kind="yfinance_api", - requires_api_key=False, - ) - - instrument = Instrument( - symbol="EUR/USD", - name="EUR - USD Rate", - asset_class="fx", - base_currency="EUR", - quote_currency="USD", - ) - - pricebar = PriceBar( - source=source, - instrument=instrument, - timestamp=date(2026, 1, 1), +def test_read_price_bars_returns_empty_dataframe_for_missing_range( + db_path, sample_source, sample_instrument +) -> None: + req = MarketDataRequest( + source=sample_source, + instrument=sample_instrument, timeframe="1d", - close=1.89, + start=date(2026, 1, 1), + end=date(2026, 1, 1), ) - db = tmp_path / "test.duckdb" - initialize_database(db) - insert_price_bar(db, pricebar) - - result = read_price_bars( - db=db, - source=source, - instrument=instrument, - start_date=date(2027, 1, 1), - end_date=date(2027, 1, 31), - ) + result = read_price_bars(db_path, req) assert result.empty is True diff --git a/tests/test_timeseries_service.py b/tests/test_timeseries_service.py deleted file mode 100644 index cd5c97a..0000000 --- a/tests/test_timeseries_service.py +++ /dev/null @@ -1,36 +0,0 @@ -import pandas as pd -import pandas.testing as pdt -import numpy as np -from argus.services.timeseries_service import prepare_trend_analysis - - -def test_get_a_full_timeseries(): - test_curr = "EURUSD=X" - test_start = "2024-01-01" - test_end = "2024-01-04" - test_interval = "1d" - - expect_result = { - "date": ["2024-01-01", "2024-01-02", "2024-01-03"], - "rate": [1.1055831909179688, 1.1038745641708374, 1.0941756963729858], - "daily_pct_change": [np.nan, -0.1545452898675692, -0.8786204622023064], - "roll_avg": [1.1055831909179688, 1.104728877544403, 1.101211150487264], - } - expect_dict = { - "min_date": ["2024-01-03 00:00:00"], - "min_rate": [1.0941756963729858], - "max_date": ["2024-01-01 00:00:00"], - "max_rate": [1.1055831909179688], - } - result = prepare_trend_analysis(test_curr, test_start, test_end, test_interval) - - assert result is not None - - result_df, result_dict = result - result_df["date"] = result_df["date"].astype("str") - result_dict["min_date"] = [str(result_dict["min_date"][0])] - result_dict["max_date"] = [str(result_dict["max_date"][0])] - expect_df = pd.DataFrame(expect_result) - - pdt.assert_frame_equal(result_df, expect_df) - assert result_dict == expect_dict diff --git a/tests/test_trend_analysis_service.py b/tests/test_trend_analysis_service.py new file mode 100644 index 0000000..84f8ccd --- /dev/null +++ b/tests/test_trend_analysis_service.py @@ -0,0 +1,95 @@ +""" +import pytest +import pandas as pd +from datetime import date +from matplotlib.figure import Figure +from argus.domain.internal_models import ( + DataSource, + Instrument, + MarketDataRequest, + MarketDataResponse, +) +from argus.services.trend_analysis_service import prepare_trend_analysis +from argus.services.market_data_service import get_market_data +from argus.storage.database import initialize_database + + +@pytest.fixture +def sample_source(): + return DataSource( + name="Yahoo", provider_kind="yfinance_api", requires_api_key=False + ) + + +@pytest.fixture +def sample_instrument(): + return Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD", + ) + + +@pytest.fixture +def sample_response(sample_source, sample_instrument): + test_bar = { + "timestamp": [date(2024, 1, 1), date(2024, 1, 2), date(2024, 1, 3)], + "open": [None, None, None], + "high": [None, None, None], + "low": [None, None, None], + "close": [1.10, 1.12, 1.11], + "adjusted_close": [None, None, None], + "volume": [None, None, None], + } + return MarketDataResponse( + source=sample_source, instrument=sample_instrument, bars=pd.DataFrame(test_bar) + ) + + +def test_prepare_trend_analysis_success( + sample_source, sample_instrument, sample_response, monkeypatch +): + req = MarketDataRequest( + source=sample_source, + instrument=sample_instrument, + timeframe="1d", + start=date(2024, 1, 1), + end=date(2024, 1, 4), + ) + + monkeypatch.setattr( + "argus.services.trend_analysis_service.get_market_data", + lambda db, r: sample_response, + ) + + res = prepare_trend_analysis("mock_db_path", req) + assert isinstance(res, Figure) + + +def test_prepare_trend_analysis_failure(sample_source, sample_instrument, monkeypatch): + req = MarketDataRequest( + source=sample_source, + instrument=sample_instrument, + timeframe="1d", + start=date(2024, 1, 1), + end=date(2024, 1, 4), + ) + + error_response = MarketDataResponse( + source=sample_source, + instrument=sample_instrument, + bars=pd.DataFrame(), + message="Quote not found or no data available for symbol", + ) + + monkeypatch.setattr( + "argus.services.trend_analysis_service.get_market_data", + lambda db, r: error_response, + ) + + res = prepare_trend_analysis("mock_db_path", req) + assert isinstance(res, str) + assert res == "Quote not found or no data available for symbol" +""" diff --git a/tests/test_validation_domain.py b/tests/test_validation_domain.py index 0166741..a5bd41f 100644 --- a/tests/test_validation_domain.py +++ b/tests/test_validation_domain.py @@ -7,13 +7,9 @@ def test_op_is_valid(): + data = is_valid_op("+") - assert is_valid_op("+") is True - assert is_valid_op("-") is True - assert is_valid_op("*") is True - assert is_valid_op("/") is True - assert is_valid_op("%") is True - assert is_valid_op("**") is True + assert data is True def test_op_is_not_valid(): diff --git a/tests/test_yfinance_client.py b/tests/test_yfinance_client.py index 6201b19..419d999 100644 --- a/tests/test_yfinance_client.py +++ b/tests/test_yfinance_client.py @@ -1,80 +1,118 @@ from argus.clients.yfinance_client import get_timeseries +from argus.domain.internal_models import DataSource, Instrument, MarketDataRequest +from datetime import date +import pytest import pandas as pd import pandas.testing as pdt -def test_get_dataframe(monkeypatch): +@pytest.fixture +def sample_source(): + return DataSource( + name="Yahoo", provider_kind="yfinance_api", requires_api_key=False + ) + + +@pytest.fixture +def sample_instrument(): + return Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD", + ) + + +def test_get_dataframe(monkeypatch, sample_source, sample_instrument): test_resp = pd.DataFrame( { + "Open": [None, None, None], + "High": [None, None, None], + "Low": [None, None, None], "Close": [1.105583, 1.103875, 1.094176], + "Adj Close": [None, None, None], + "Volume": [None, None, None], }, index=pd.to_datetime(["2024-01-01", "2024-01-02", "2024-01-03"]), ) test_resp.index.name = "Date" - test_curr = "EURUSD=X" - test_start = "2024-01-01" - test_end = "2024-01-04" - test_interval = "1d" + req = MarketDataRequest( + source=sample_source, + instrument=sample_instrument, + timeframe="1d", + start=date(2024, 1, 1), + end=date(2024, 1, 4), + ) def fake_yfinance_download(*args, **kwargs): return test_resp monkeypatch.setattr("yfinance.download", fake_yfinance_download) - - result = get_timeseries(test_curr, test_start, test_end, test_interval) + resp = get_timeseries(req) expected = pd.DataFrame( { - "date": pd.to_datetime(["2024-01-01", "2024-01-02", "2024-01-03"]), - "rate": [1.105583, 1.103875, 1.094176], + "timestamp": pd.to_datetime(["2024-01-01", "2024-01-02", "2024-01-03"]), + "open": [None, None, None], + "high": [None, None, None], + "low": [None, None, None], + "close": [1.105583, 1.103875, 1.094176], + "adjusted_close": [None, None, None], + "volume": [None, None, None], } ) - assert result is not None - pdt.assert_frame_equal(result, expected) + assert resp is not None + pdt.assert_frame_equal(resp, expected) -def test_get_none(monkeypatch): - test_curr = "EURUSD=X" - test_start = "2024-01-01" - test_end = "2024-01-04" - test_interval = "1d" - - def fake_yfinance_download(*args, **kwargs): - return None - monkeypatch.setattr("yfinance.download", fake_yfinance_download) +def test_client_network_error(monkeypatch, sample_source, sample_instrument): + req = MarketDataRequest( + source=sample_source, + instrument=sample_instrument, + timeframe="1d", + start=date(2024, 1, 1), + end=date(2024, 1, 4), + ) - result = get_timeseries(test_curr, test_start, test_end, test_interval) - assert result is None + def mock_crash(*args, **kwargs): + raise Exception() + monkeypatch.setattr("yfinance.download", mock_crash) -def test_get_empty_frame(monkeypatch): - test_curr = "EURUSD=X" - test_start = "2024-01-01" - test_end = "2024-01-01" - test_interval = "1d" + with pytest.raises(ConnectionError, match="Network error or connection timeout"): + get_timeseries(req) - def fake_yfinance_download(*args, **kwargs): - return pd.DataFrame() - monkeypatch.setattr("yfinance.download", fake_yfinance_download) +def test_client_invalid_response(monkeypatch, sample_source, sample_instrument): + req = MarketDataRequest( + source=sample_source, + instrument=sample_instrument, + timeframe="1d", + start=date(2024, 1, 1), + end=date(2024, 1, 4), + ) - result = get_timeseries(test_curr, test_start, test_end, test_interval) - assert result is None + monkeypatch.setattr("yfinance.download", lambda *args, **kwargs: None) + with pytest.raises( + ConnectionError, match="Yahoo Finance API returned an invalid response" + ): + get_timeseries(req) -def test_error_raise(monkeypatch): - test_curr = "EURUSD=X" - # start date is inclusiv and end date is exclusiv - the range 2024-01-01-2024-01-01 is not possible - test_start = "2024-01-04" - test_end = "2024-01-02" - test_interval = "1d" - def fake_yfinance_download( - tickers=test_curr, start=test_start, end=test_end, interval=test_interval - ): - raise Exception("fake yfinance error") +def test_client_quote_not_found(monkeypatch, sample_source, sample_instrument): + req = MarketDataRequest( + source=sample_source, + instrument=sample_instrument, + timeframe="1d", + start=date(2024, 1, 1), + end=date(2024, 1, 4), + ) - monkeypatch.setattr("yfinance.download", fake_yfinance_download) + monkeypatch.setattr("yfinance.download", lambda *args, **kwargs: pd.DataFrame()) - result = get_timeseries(test_curr, test_start, test_end, test_interval) - assert result is None + with pytest.raises( + ValueError, match="Quote not found or no data available for symbol" + ): + get_timeseries(req)